From c7667c2d7f602aecfd8a39f26d8151a363ba0b5e Mon Sep 17 00:00:00 2001
From: SyxbEaEQ2 <SyxbEaEQ2@users.noreply.github.com>
Date: Thu, 31 Jul 2014 03:08:24 +0200
Subject: [PATCH 0001/1937] [downloader/(common/http)] Changes calculation of
 the rate-limit. (Fix #2297, fix #2140, fix #595, fix #2370)

---
 youtube_dl/downloader/common.py | 15 +++++++++------
 youtube_dl/downloader/http.py   | 31 ++++++++++++++++++++++---------
 2 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 917f3450e..6404e1928 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -77,8 +77,10 @@ class FileDownloader(object):
     def calc_eta(start, now, total, current):
         if total is None:
             return None
+        if now is None:
+            now = time.time()
         dif = now - start
-        if current == 0 or dif < 0.001: # One millisecond
+        if current == 0 or dif < 0.001:  # One millisecond
             return None
         rate = float(current) / dif
         return int((float(total) - float(current)) / rate)
@@ -92,7 +94,7 @@ class FileDownloader(object):
     @staticmethod
     def calc_speed(start, now, bytes):
         dif = now - start
-        if bytes == 0 or dif < 0.001: # One millisecond
+        if bytes == 0 or dif < 0.001:  # One millisecond
             return None
         return float(bytes) / dif
 
@@ -105,7 +107,7 @@ class FileDownloader(object):
     @staticmethod
     def best_block_size(elapsed_time, bytes):
         new_min = max(bytes / 2.0, 1.0)
-        new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
+        new_max = min(max(bytes * 2.0, 1.0), 4194304)  # Do not surpass 4 MB
         if elapsed_time < 0.001:
             return int(new_max)
         rate = bytes / elapsed_time
@@ -143,18 +145,19 @@ class FileDownloader(object):
     def report_error(self, *args, **kargs):
         self.ydl.report_error(*args, **kargs)
 
-    def slow_down(self, start_time, byte_counter):
+    def slow_down(self, start_time, now, byte_counter):
         """Sleep if the download speed is over the rate limit."""
         rate_limit = self.params.get('ratelimit', None)
         if rate_limit is None or byte_counter == 0:
             return
-        now = time.time()
+        if now is None:
+            now = time.time()
         elapsed = now - start_time
         if elapsed <= 0.0:
             return
         speed = float(byte_counter) / elapsed
         if speed > rate_limit:
-            time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
+            time.sleep((byte_counter / rate_limit) - elapsed)
 
     def temp_name(self, filename):
         """Returns a temporary filename for the given filename."""
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index f79e6a995..462be2739 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -128,16 +128,21 @@ class HttpFD(FileDownloader):
         byte_counter = 0 + resume_len
         block_size = self.params.get('buffersize', 1024)
         start = time.time()
+
+        # measure time over whole while-loop, so slow_down() and best_block_size() work together properly
+        now = None  # needed for slow_down() in the first loop run
+        before = start  # start measuring
         while True:
+
             # Download and write
-            before = time.time()
             data_block = data.read(block_size if not is_test else min(block_size, data_len - byte_counter))
-            after = time.time()
-            if len(data_block) == 0:
-                break
             byte_counter += len(data_block)
 
-            # Open file just in time
+            # exit loop when download is finished
+            if len(data_block) == 0:
+                break
+
+            # Open destination file just in time
             if stream is None:
                 try:
                     (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
@@ -153,11 +158,22 @@ class HttpFD(FileDownloader):
                 self.to_stderr(u"\n")
                 self.report_error(u'unable to write data: %s' % str(err))
                 return False
+
+            # Apply rate limit
+            self.slow_down(start, now, byte_counter - resume_len)
+
+            # end measuring of one loop run
+            now = time.time()
+            after = now
+
+            # Adjust block size
             if not self.params.get('noresizebuffer', False):
                 block_size = self.best_block_size(after - before, len(data_block))
 
+            before = after
+
             # Progress message
-            speed = self.calc_speed(start, time.time(), byte_counter - resume_len)
+            speed = self.calc_speed(start, now, byte_counter - resume_len)
             if data_len is None:
                 eta = percent = None
             else:
@@ -178,9 +194,6 @@ class HttpFD(FileDownloader):
             if is_test and byte_counter == data_len:
                 break
 
-            # Apply rate limit
-            self.slow_down(start, byte_counter - resume_len)
-
         if stream is None:
             self.to_stderr(u"\n")
             self.report_error(u'Did not get any data blocks')

From 03359e9864bfb925f577fa5b16c3ef22884127aa Mon Sep 17 00:00:00 2001
From: rupertbaxter2 <rupertbaxter2@yahoo.com>
Date: Sun, 3 Aug 2014 07:34:04 -0700
Subject: [PATCH 0002/1937] Added --sleep-interval option

---
 youtube_dl/__init__.py          | 8 ++++++++
 youtube_dl/downloader/common.py | 3 +++
 youtube_dl/utils.py             | 3 +++
 3 files changed, 14 insertions(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 429630ce5..2bd5ec33b 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -351,6 +351,8 @@ def parseOpts(overrideArguments=None):
 
     downloader.add_option('-r', '--rate-limit',
             dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
+    downloader.add_option('--sleep-interval',
+            dest='sleepinterval', metavar='SLEEPINTERVAL', help='number of seconds to sleep between downloads (default is %default)', default="0")
     downloader.add_option('-R', '--retries',
             dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
     downloader.add_option('--buffer-size',
@@ -671,6 +673,11 @@ def _real_main(argv=None):
         if numeric_limit is None:
             parser.error(u'invalid rate limit specified')
         opts.ratelimit = numeric_limit
+    if opts.sleepinterval is not None:
+        try:
+            opts.sleepinterval = abs(int(opts.sleepinterval))
+        except ValueError:
+            parser.error(u'invalid sleep interval specified')
     if opts.min_filesize is not None:
         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
         if numeric_limit is None:
@@ -767,6 +774,7 @@ def _real_main(argv=None):
         'restrictfilenames': opts.restrictfilenames,
         'ignoreerrors': opts.ignoreerrors,
         'ratelimit': opts.ratelimit,
+        'sleepinterval': opts.sleepinterval,
         'nooverwrites': opts.nooverwrites,
         'retries': opts.retries,
         'buffersize': opts.buffersize,
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 917f3450e..8e0e386bf 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -278,6 +278,9 @@ class FileDownloader(object):
         """Download to a filename using the info from info_dict
         Return True on success and False otherwise
         """
+        sleep_interval = self.params.get('sleepinterval', 0)
+        self.to_screen(u'[download] Sleeping %d seconds...' %sleep_interval)
+        time.sleep(sleep_interval)
         # Check file already present
         if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
             self.report_file_already_downloaded(filename)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e40b367c2..d199d26d2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -6,6 +6,7 @@ import codecs
 import contextlib
 import ctypes
 import datetime
+import time
 import email.utils
 import errno
 import getpass
@@ -747,6 +748,8 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
                 del req.headers['User-agent']
             req.headers['User-agent'] = req.headers['Youtubedl-user-agent']
             del req.headers['Youtubedl-user-agent']
+        #print("sleeping\n")
+        #time.sleep(1)
         return req
 
     def http_response(self, req, resp):

From 2f61fe4cccc1ef4186943f4eed2e89f8fe2e2c23 Mon Sep 17 00:00:00 2001
From: rupertbaxter2 <rupertbaxter2@yahoo.com>
Date: Sun, 3 Aug 2014 07:38:04 -0700
Subject: [PATCH 0003/1937] Removed unneccesary changes to utils.py

---
 youtube_dl/utils.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d199d26d2..e40b367c2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -6,7 +6,6 @@ import codecs
 import contextlib
 import ctypes
 import datetime
-import time
 import email.utils
 import errno
 import getpass
@@ -748,8 +747,6 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
                 del req.headers['User-agent']
             req.headers['User-agent'] = req.headers['Youtubedl-user-agent']
             del req.headers['Youtubedl-user-agent']
-        #print("sleeping\n")
-        #time.sleep(1)
         return req
 
     def http_response(self, req, resp):

From 00cf122d7a79e81a2b328b7352d23eb0bdb17e52 Mon Sep 17 00:00:00 2001
From: SyxbEaEQ2 <SyxbEaEQ2@users.noreply.github.com>
Date: Wed, 6 Aug 2014 20:53:04 +0200
Subject: [PATCH 0004/1937]  [downloader/common] Fix possible negative sleep
 time in slow_down()

---
 youtube_dl/downloader/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 6404e1928..33ebbf6b4 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -157,7 +157,7 @@ class FileDownloader(object):
             return
         speed = float(byte_counter) / elapsed
         if speed > rate_limit:
-            time.sleep((byte_counter / rate_limit) - elapsed)
+            time.sleep(max((byte_counter / rate_limit) - elapsed, 0))
 
     def temp_name(self, filename):
         """Returns a temporary filename for the given filename."""

From a42c9215983c4d62d1c000c9dede6e0850dbb5e4 Mon Sep 17 00:00:00 2001
From: rupertbaxter2 <rupertbaxter2@yahoo.com>
Date: Wed, 13 Aug 2014 04:38:40 -0700
Subject: [PATCH 0005/1937] Removed sleep and sleep output when interval is
 zero

---
 youtube_dl/downloader/common.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 48e829deb..c1da065b5 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -279,8 +279,9 @@ class FileDownloader(object):
         Return True on success and False otherwise
         """
         sleep_interval = self.params.get('sleepinterval', 0)
-        self.to_screen(u'[download] Sleeping %d seconds...' %sleep_interval)
-        time.sleep(sleep_interval)
+        if sleep_interval > 0:
+            self.to_screen(u'[download] Sleeping %d seconds...' %sleep_interval)
+            time.sleep(sleep_interval)
         # Check file already present
         if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
             self.report_file_already_downloaded(filename)

From b8874d4d4ea3becfde813d451d884bce558fe213 Mon Sep 17 00:00:00 2001
From: megustamucho <megustamucho@localhost>
Date: Tue, 9 Sep 2014 12:46:58 +1000
Subject: [PATCH 0006/1937] [tube8] Improved regex

---
 youtube_dl/extractor/tube8.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index 08a48c05a..39f20c546 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -14,7 +14,7 @@ from ..aes import aes_decrypt_text
 
 
 class Tube8IE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tube8\.com/(?:[^/]+/){2}(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?tube8\.com/(?:gay/|shemale/)?(?:[^/]+/){2}(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.tube8.com/teen/kasia-music-video/229795/',
         'md5': '44bf12b98313827dd52d35b8706a4ea0',

From 94b539d15505daf5213d5c4de7c2fde08b5d2f40 Mon Sep 17 00:00:00 2001
From: megustamucho <megustamucho@users.noreply.github.com>
Date: Tue, 9 Sep 2014 12:46:58 +1000
Subject: [PATCH 0007/1937] [tube8] Improved regex

---
 youtube_dl/extractor/tube8.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index 08a48c05a..39f20c546 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -14,7 +14,7 @@ from ..aes import aes_decrypt_text
 
 
 class Tube8IE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tube8\.com/(?:[^/]+/){2}(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?tube8\.com/(?:gay/|shemale/)?(?:[^/]+/){2}(?P<id>\d+)'
     _TEST = {
         'url': 'http://www.tube8.com/teen/kasia-music-video/229795/',
         'md5': '44bf12b98313827dd52d35b8706a4ea0',

From 67abbe95273f59f4a04486172e6d422a10b6afb3 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 17 Sep 2014 22:57:01 +0300
Subject: [PATCH 0008/1937] [videomega] Add new extractor. Closes #3775

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/videomega.py | 59 +++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 youtube_dl/extractor/videomega.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f715c3310..75831b40a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -393,6 +393,7 @@ from .videobam import VideoBamIE
 from .videodetective import VideoDetectiveIE
 from .videolecturesnet import VideoLecturesNetIE
 from .videofyme import VideofyMeIE
+from .videomega import VideoMegaIE
 from .videopremium import VideoPremiumIE
 from .videott import VideoTtIE
 from .videoweed import VideoWeedIE
diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
new file mode 100644
index 000000000..1b6b65839
--- /dev/null
+++ b/youtube_dl/extractor/videomega.py
@@ -0,0 +1,59 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    compat_urllib_parse,
+    remove_start,
+)
+
+
+class VideoMegaIE(InfoExtractor):
+    _VALID_URL = r'''(?x)https?://
+        (?:www\.)?videomega\.tv/
+        (?:iframe\.php)?\?ref=(?P<id>[A-Za-z0-9]+)
+        '''
+    _TEST = {
+        'url': 'http://videomega.tv/?ref=GKeGPVedBe',
+        'md5': '240fb5bcf9199961f48eb17839b084d6',
+        'info_dict': {
+            'id': 'GKeGPVedBe',
+            'ext': 'mp4',
+            'title': 'XXL - All Sports United',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        url = 'http://videomega.tv/iframe.php?ref={0:}'.format(video_id)
+        webpage = self._download_webpage(url, video_id)
+
+        escaped_data = self._search_regex(
+            'unescape\("([^"]+)"\)', webpage, 'escaped data')
+        playlist = compat_urllib_parse.unquote(escaped_data)
+
+        thumbnail = self._search_regex(
+            r'image:\s*"([^"]+)"', playlist, 'thumbnail', fatal=False)
+        url = self._search_regex(r'file:\s*"([^"]+)"', playlist, 'URL')
+        title = self._html_search_regex(
+            r'<title>(.*?)</title>', webpage, 'title')
+        if title:
+            title = remove_start(title, 'VideoMega.tv - ')
+
+        formats = []
+        formats.append({
+            'format_id': 'sd',
+            'url': url,
+        })
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+        }

From 0e59b9fffb12255a16577dca7710b7738feca75c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 18 Sep 2014 00:18:27 +0200
Subject: [PATCH 0009/1937] [videomega] Simplify (#3786)

* Use raw strings (r'foo') for regular expressions (enables highlighting and avoids some errors).
* title is always true-ish
---
 youtube_dl/extractor/videomega.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 1b6b65839..29c4e0101 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -34,22 +34,20 @@ class VideoMegaIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         escaped_data = self._search_regex(
-            'unescape\("([^"]+)"\)', webpage, 'escaped data')
+            r'unescape\("([^"]+)"\)', webpage, 'escaped data')
         playlist = compat_urllib_parse.unquote(escaped_data)
 
         thumbnail = self._search_regex(
             r'image:\s*"([^"]+)"', playlist, 'thumbnail', fatal=False)
         url = self._search_regex(r'file:\s*"([^"]+)"', playlist, 'URL')
-        title = self._html_search_regex(
-            r'<title>(.*?)</title>', webpage, 'title')
-        if title:
-            title = remove_start(title, 'VideoMega.tv - ')
+        title = remove_start(self._html_search_regex(
+            r'<title>(.*?)</title>', webpage, 'title'), 'VideoMega.tv - ')
 
-        formats = []
-        formats.append({
+        formats = [{
             'format_id': 'sd',
             'url': url,
-        })
+        }]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From 9296738f20c1335498a78c99a86767e9bae4f6d2 Mon Sep 17 00:00:00 2001
From: dequis <dx@dxzone.com.ar>
Date: Thu, 18 Sep 2014 03:02:03 -0300
Subject: [PATCH 0010/1937] [soundcloud] Support api urls with secret_token,
 Closes #3707

---
 youtube_dl/extractor/soundcloud.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index b78aed7f0..129f587ec 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -31,7 +31,8 @@ class SoundcloudIE(InfoExtractor):
                             (?!sets/|likes/?(?:$|[?#]))
                             (?P<title>[\w\d-]+)/?
                             (?P<token>[^?]+?)?(?:[?].*)?$)
-                       |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+))
+                       |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)
+                          (?:/?\?secret_token=(?P<secret_token>[^&]+?))?$)
                        |(?P<player>(?:w|player|p.)\.soundcloud\.com/player/?.*?url=.*)
                     )
                     '''
@@ -80,6 +81,20 @@ class SoundcloudIE(InfoExtractor):
                 'duration': 9,
             },
         },
+        # private link (alt format)
+        {
+            'url': 'https://api.soundcloud.com/tracks/123998367?secret_token=s-8Pjrp',
+            'md5': 'aa0dd32bfea9b0c5ef4f02aacd080604',
+            'info_dict': {
+                'id': '123998367',
+                'ext': 'mp3',
+                'title': 'Youtube - Dl Test Video \'\' Ä↭',
+                'uploader': 'jaimeMF',
+                'description': 'test chars:  \"\'/\\ä↭',
+                'upload_date': '20131209',
+                'duration': 9,
+            },
+        },
         # downloadable song
         {
             'url': 'https://soundcloud.com/oddsamples/bus-brakes',
@@ -197,6 +212,9 @@ class SoundcloudIE(InfoExtractor):
         if track_id is not None:
             info_json_url = 'http://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID
             full_title = track_id
+            token = mobj.group('secret_token')
+            if token:
+                info_json_url += "&secret_token=" + token
         elif mobj.group('player'):
             query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
             return self.url_result(query['url'][0])

From 2f834e938192a61fd4a32fa98bffb5e1b614bc29 Mon Sep 17 00:00:00 2001
From: dequis <dx@dxzone.com.ar>
Date: Thu, 18 Sep 2014 06:35:11 -0300
Subject: [PATCH 0011/1937] [soundcloud] Secret playlists and sets

Closes #3707 again.

No test cases because I don't know what urls to use that won't be turned
into public eventually (as it happened with the first one in that
ticket)
---
 youtube_dl/extractor/soundcloud.py | 30 +++++++++++++++++++++---------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 129f587ec..2bed3c350 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -238,7 +238,7 @@ class SoundcloudIE(InfoExtractor):
 
 
 class SoundcloudSetIE(SoundcloudIE):
-    _VALID_URL = r'https?://(?:www\.)?soundcloud\.com/([\w\d-]+)/sets/([\w\d-]+)'
+    _VALID_URL = r'https?://(?:www\.)?soundcloud\.com/(?P<uploader>[\w\d-]+)/sets/(?P<slug_title>[\w\d-]+)(?:/(?P<token>[^?/]+))?'
     IE_NAME = 'soundcloud:set'
     _TESTS = [{
         'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep',
@@ -252,14 +252,19 @@ class SoundcloudSetIE(SoundcloudIE):
         mobj = re.match(self._VALID_URL, url)
 
         # extract uploader (which is in the url)
-        uploader = mobj.group(1)
+        uploader = mobj.group('uploader')
         # extract simple title (uploader + slug of song title)
-        slug_title = mobj.group(2)
+        slug_title = mobj.group('slug_title')
         full_title = '%s/sets/%s' % (uploader, slug_title)
+        url = 'http://soundcloud.com/%s/sets/%s' % (uploader, slug_title)
+
+        token = mobj.group('token')
+        if token:
+            full_title += '/' + token
+            url += '/' + token
 
         self.report_resolve(full_title)
 
-        url = 'http://soundcloud.com/%s/sets/%s' % (uploader, slug_title)
         resolv_url = self._resolv_url(url)
         info = self._download_json(resolv_url, full_title)
 
@@ -270,7 +275,7 @@ class SoundcloudSetIE(SoundcloudIE):
 
         return {
             '_type': 'playlist',
-            'entries': [self._extract_info_dict(track) for track in info['tracks']],
+            'entries': [self._extract_info_dict(track, secret_token=token) for track in info['tracks']],
             'id': info['id'],
             'title': info['title'],
         }
@@ -333,7 +338,7 @@ class SoundcloudUserIE(SoundcloudIE):
 
 
 class SoundcloudPlaylistIE(SoundcloudIE):
-    _VALID_URL = r'https?://api\.soundcloud\.com/playlists/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://api\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))$'
     IE_NAME = 'soundcloud:playlist'
     _TESTS = [
 
@@ -353,14 +358,21 @@ class SoundcloudPlaylistIE(SoundcloudIE):
         playlist_id = mobj.group('id')
         base_url = '%s//api.soundcloud.com/playlists/%s.json?' % (self.http_scheme(), playlist_id)
 
-        data = compat_urllib_parse.urlencode({
+        data_dict = {
             'client_id': self._CLIENT_ID,
-        })
+        }
+        token = mobj.group('token')
+
+        if token:
+            data_dict['secret_token'] = token
+
+        data = compat_urllib_parse.urlencode(data_dict)
         data = self._download_json(
             base_url + data, playlist_id, 'Downloading playlist')
 
         entries = [
-            self._extract_info_dict(t, quiet=True) for t in data['tracks']]
+            self._extract_info_dict(t, quiet=True, secret_token=token)
+                for t in data['tracks']]
 
         return {
             '_type': 'playlist',

From 2914e5f00f6ebcc59712b7091a87988408ff3c88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Sep 2014 20:56:54 +0700
Subject: [PATCH 0012/1937] [drtuber] Fix categories

---
 youtube_dl/extractor/drtuber.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/drtuber.py b/youtube_dl/extractor/drtuber.py
index d5bfd7f22..ca274dff6 100644
--- a/youtube_dl/extractor/drtuber.py
+++ b/youtube_dl/extractor/drtuber.py
@@ -19,7 +19,7 @@ class DrTuberIE(InfoExtractor):
             'like_count': int,
             'dislike_count': int,
             'comment_count': int,
-            'categories': list,  # NSFW
+            'categories': ['Babe', 'Blonde', 'Erotic', 'Outdoor', 'Softcore', 'Solo'],
             'thumbnail': 're:https?://.*\.jpg$',
             'age_limit': 18,
         }
@@ -52,9 +52,9 @@ class DrTuberIE(InfoExtractor):
             r'<span class="comments_count">([\d,\.]+)</span>',
             webpage, 'comment count', fatal=False))
 
-        cats_str = self._html_search_regex(
-            r'<meta name="keywords" content="([^"]+)"', webpage, 'categories', fatal=False)
-        categories = None if cats_str is None else cats_str.split(' ')
+        cats_str = self._search_regex(
+            r'<span>Categories:</span><div>(.+?)</div>', webpage, 'categories', fatal=False)
+        categories = [] if not cats_str else re.findall(r'<a title="([^"]+)"', cats_str)
 
         return {
             'id': video_id,

From 109a540e7a4c5741fa77b68b4f346f42dc1cda97 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 18 Sep 2014 16:57:34 +0200
Subject: [PATCH 0013/1937] [ign] Fix extraction

---
 youtube_dl/extractor/ign.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index 9e8b69f57..ac7804ad9 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -71,6 +71,7 @@ class IGNIE(InfoExtractor):
 
     def _find_video_id(self, webpage):
         res_id = [
+            r'"video_id"\s*:\s*"(.*?)"',
             r'data-video-id="(.+?)"',
             r'<object id="vid_(.+?)"',
             r'<meta name="og:image" content=".*/(.+?)-(.+?)/.+.jpg"',
@@ -85,7 +86,7 @@ class IGNIE(InfoExtractor):
         webpage = self._download_webpage(url, name_or_id)
         if page_type != 'video':
             multiple_urls = re.findall(
-                '<param name="flashvars" value="[^"]*?url=(https?://www\.ign\.com/videos/.*?)["&]',
+                '<param name="flashvars"[^>]*value="[^"]*?url=(https?://www\.ign\.com/videos/.*?)["&]',
                 webpage)
             if multiple_urls:
                 return [self.url_result(u, ie='IGN') for u in multiple_urls]

From 09b23c902b5ab4a4ca9607128128d110a3c41875 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 18 Sep 2014 17:02:53 +0200
Subject: [PATCH 0014/1937] [1up.com] Urls end now with '.html'

---
 youtube_dl/extractor/ign.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index ac7804ad9..12e9e61c4 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -112,13 +112,13 @@ class IGNIE(InfoExtractor):
 
 
 class OneUPIE(IGNIE):
-    _VALID_URL = r'https?://gamevideos\.1up\.com/(?P<type>video)/id/(?P<name_or_id>.+)'
+    _VALID_URL = r'https?://gamevideos\.1up\.com/(?P<type>video)/id/(?P<name_or_id>.+)\.html'
     IE_NAME = '1up.com'
 
     _DESCRIPTION_RE = r'<div id="vid_summary">(.+?)</div>'
 
     _TESTS = [{
-        'url': 'http://gamevideos.1up.com/video/id/34976',
+        'url': 'http://gamevideos.1up.com/video/id/34976.html',
         'md5': '68a54ce4ebc772e4b71e3123d413163d',
         'info_dict': {
             'id': '34976',

From e2e5dae64da60c37af65c7cffd18475a30fcbad3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 18 Sep 2014 18:40:19 +0200
Subject: [PATCH 0015/1937] Add -f m4a

---
 youtube_dl/YoutubeDL.py | 2 +-
 youtube_dl/options.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 9519594c9..eaba40bf2 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -708,7 +708,7 @@ class YoutubeDL(object):
             if video_formats:
                 return video_formats[0]
         else:
-            extensions = ['mp4', 'flv', 'webm', '3gp']
+            extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a']
             if format_spec in extensions:
                 filter_f = lambda f: f['ext'] == format_spec
             else:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 31baab469..7df20ae61 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -218,7 +218,7 @@ def parseOpts(overrideArguments=None):
 
     video_format.add_option('-f', '--format',
             action='store', dest='format', metavar='FORMAT', default=None,
-            help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.')
+            help='video format code, specify the order of preference using slashes: -f 22/17/18 .  -f mp4 , -f m4a and  -f flv  are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.')
     video_format.add_option('--all-formats',
             action='store_const', dest='format', help='download all available video formats', const='all')
     video_format.add_option('--prefer-free-formats',

From 1de33fafd94c7e0d4ccede711ef7f13bd3e2301b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 18 Sep 2014 18:43:49 +0200
Subject: [PATCH 0016/1937] [YoutubeDL] Allow downloading multiple formats with
 ,

---
 youtube_dl/YoutubeDL.py | 43 +++++++++++++++++++++--------------------
 youtube_dl/options.py   |  2 +-
 2 files changed, 23 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index eaba40bf2..a1713dc5a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -809,28 +809,29 @@ class YoutubeDL(object):
         if req_format in ('-1', 'all'):
             formats_to_download = formats
         else:
-            # We can accept formats requested in the format: 34/5/best, we pick
-            # the first that is available, starting from left
-            req_formats = req_format.split('/')
-            for rf in req_formats:
-                if re.match(r'.+?\+.+?', rf) is not None:
-                    # Two formats have been requested like '137+139'
-                    format_1, format_2 = rf.split('+')
-                    formats_info = (self.select_format(format_1, formats),
-                        self.select_format(format_2, formats))
-                    if all(formats_info):
-                        selected_format = {
-                            'requested_formats': formats_info,
-                            'format': rf,
-                            'ext': formats_info[0]['ext'],
-                        }
+            for rfstr in req_format.split(','):
+                # We can accept formats requested in the format: 34/5/best, we pick
+                # the first that is available, starting from left
+                req_formats = rfstr.split('/')
+                for rf in req_formats:
+                    if re.match(r'.+?\+.+?', rf) is not None:
+                        # Two formats have been requested like '137+139'
+                        format_1, format_2 = rf.split('+')
+                        formats_info = (self.select_format(format_1, formats),
+                            self.select_format(format_2, formats))
+                        if all(formats_info):
+                            selected_format = {
+                                'requested_formats': formats_info,
+                                'format': rf,
+                                'ext': formats_info[0]['ext'],
+                            }
+                        else:
+                            selected_format = None
                     else:
-                        selected_format = None
-                else:
-                    selected_format = self.select_format(rf, formats)
-                if selected_format is not None:
-                    formats_to_download = [selected_format]
-                    break
+                        selected_format = self.select_format(rf, formats)
+                    if selected_format is not None:
+                        formats_to_download.append(selected_format)
+                        break
         if not formats_to_download:
             raise ExtractorError('requested format not available',
                                  expected=True)
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 7df20ae61..44dcb1e34 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -218,7 +218,7 @@ def parseOpts(overrideArguments=None):
 
     video_format.add_option('-f', '--format',
             action='store', dest='format', metavar='FORMAT', default=None,
-            help='video format code, specify the order of preference using slashes: -f 22/17/18 .  -f mp4 , -f m4a and  -f flv  are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality.')
+            help='video format code, specify the order of preference using slashes: -f 22/17/18 .  -f mp4 , -f m4a and  -f flv  are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality. Use commas to download multiple audio formats, such as  -f  136/137/mp4/bestvideo,140/m4a/bestaudio')
     video_format.add_option('--all-formats',
             action='store_const', dest='format', help='download all available video formats', const='all')
     video_format.add_option('--prefer-free-formats',

From fd78a4d3e63f191e0774584d9b71bf25a2d8dbcf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 18 Sep 2014 18:43:59 +0200
Subject: [PATCH 0017/1937] release 2014.09.18

---
 README.md             | 15 +++++++++------
 youtube_dl/version.py |  2 +-
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 5cc959ac5..5d15decb5 100644
--- a/README.md
+++ b/README.md
@@ -227,12 +227,15 @@ which means you can modify it, redistribute it or use it however you like.
 
 ## Video Format Options:
     -f, --format FORMAT              video format code, specify the order of
-                                     preference using slashes: "-f 22/17/18".
-                                     "-f mp4" and "-f flv" are also supported.
-                                     You can also use the special names "best",
-                                     "bestvideo", "bestaudio", "worst",
-                                     "worstvideo" and "worstaudio". By default,
-                                     youtube-dl will pick the best quality.
+                                     preference using slashes: -f 22/17/18 .  -f
+                                     mp4 , -f m4a and  -f flv  are also
+                                     supported. You can also use the special
+                                     names "best", "bestvideo", "bestaudio",
+                                     "worst", "worstvideo" and "worstaudio". By
+                                     default, youtube-dl will pick the best
+                                     quality. Use commas to download multiple
+                                     audio formats, such as  -f
+                                     136/137/mp4/bestvideo,140/m4a/bestaudio
     --all-formats                    download all available video formats
     --prefer-free-formats            prefer free video formats unless a specific
                                      one is requested
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 23892a8bd..430509ba3 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.09.16.1'
+__version__ = '2014.09.18'

From 0529eef5a4513d8f3c042f09fe5485e1c41e2f08 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 18 Sep 2014 18:54:03 +0200
Subject: [PATCH 0018/1937] [hypestat] Unify allmyvideos and vidspot (Closes
 #3788)

---
 youtube_dl/extractor/__init__.py               |  2 +-
 .../extractor/{allmyvideos.py => hypestat.py}  | 18 +++++++++++++-----
 2 files changed, 14 insertions(+), 6 deletions(-)
 rename youtube_dl/extractor/{allmyvideos.py => hypestat.py} (77%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 75831b40a..97693018f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -6,7 +6,6 @@ from .aftonbladet import AftonbladetIE
 from .anitube import AnitubeIE
 from .anysex import AnySexIE
 from .aol import AolIE
-from .allmyvideos import AllmyvideosIE
 from .allocine import AllocineIE
 from .aparat import AparatIE
 from .appletrailers import AppleTrailersIE
@@ -151,6 +150,7 @@ from .howcast import HowcastIE
 from .howstuffworks import HowStuffWorksIE
 from .huffpost import HuffPostIE
 from .hypem import HypemIE
+from .hypestat import HypestatIE
 from .iconosquare import IconosquareIE
 from .ign import IGNIE, OneUPIE
 from .imdb import (
diff --git a/youtube_dl/extractor/allmyvideos.py b/youtube_dl/extractor/hypestat.py
similarity index 77%
rename from youtube_dl/extractor/allmyvideos.py
rename to youtube_dl/extractor/hypestat.py
index e6c60e7e4..8b8db30ae 100644
--- a/youtube_dl/extractor/allmyvideos.py
+++ b/youtube_dl/extractor/hypestat.py
@@ -11,11 +11,11 @@ from ..utils import (
 )
 
 
-class AllmyvideosIE(InfoExtractor):
-    IE_NAME = 'allmyvideos.net'
-    _VALID_URL = r'https?://allmyvideos\.net/(?P<id>[a-zA-Z0-9_-]+)'
+class HypestatIE(InfoExtractor):
+    IE_DESC = 'allmyvideos.net and vidspot.net'
+    _VALID_URL = r'https?://(?:allmyvideos|vidspot)\.net/(?P<id>[a-zA-Z0-9_-]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://allmyvideos.net/jih3nce3x6wn',
         'md5': '710883dee1bfc370ecf9fa6a89307c88',
         'info_dict': {
@@ -23,7 +23,15 @@ class AllmyvideosIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'youtube-dl test video',
         },
-    }
+    }, {
+        'url': 'http://vidspot.net/l2ngsmhs8ci5',
+        'md5': '710883dee1bfc370ecf9fa6a89307c88',
+        'info_dict': {
+            'id': 'l2ngsmhs8ci5',
+            'ext': 'mp4',
+            'title': 'youtube-dl test video',
+        },
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 37bfe8ace4dcd1b476a54aedb7f39b88e7bb527e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 18 Sep 2014 18:56:02 +0200
Subject: [PATCH 0019/1937] [hypestat] Match URLs with www. and https://

---
 youtube_dl/extractor/hypestat.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/hypestat.py b/youtube_dl/extractor/hypestat.py
index 8b8db30ae..e1a142268 100644
--- a/youtube_dl/extractor/hypestat.py
+++ b/youtube_dl/extractor/hypestat.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 class HypestatIE(InfoExtractor):
     IE_DESC = 'allmyvideos.net and vidspot.net'
-    _VALID_URL = r'https?://(?:allmyvideos|vidspot)\.net/(?P<id>[a-zA-Z0-9_-]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:allmyvideos|vidspot)\.net/(?P<id>[a-zA-Z0-9_-]+)'
 
     _TESTS = [{
         'url': 'http://allmyvideos.net/jih3nce3x6wn',
@@ -31,6 +31,9 @@ class HypestatIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'youtube-dl test video',
         },
+    }, {
+        'url': 'https://www.vidspot.net/l2ngsmhs8ci5',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 46f74bcf5c5fc876e3a966408cb8bde6d6ef15e0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 18 Sep 2014 18:57:04 +0200
Subject: [PATCH 0020/1937] [soundcloud] Fix non-secret playlists

---
 youtube_dl/extractor/soundcloud.py | 23 ++++++++++-------------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 2bed3c350..4719ba45c 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -338,20 +338,17 @@ class SoundcloudUserIE(SoundcloudIE):
 
 
 class SoundcloudPlaylistIE(SoundcloudIE):
-    _VALID_URL = r'https?://api\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))$'
+    _VALID_URL = r'https?://api\.soundcloud\.com/playlists/(?P<id>[0-9]+)(?:/?\?secret_token=(?P<token>[^&]+?))?$'
     IE_NAME = 'soundcloud:playlist'
-    _TESTS = [
-
-        {
-            'url': 'http://api.soundcloud.com/playlists/4110309',
-            'info_dict': {
-                'id': '4110309',
-                'title': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]',
-                'description': 're:.*?TILT Brass - Bowery Poetry Club',
-            },
-            'playlist_count': 6,
-        }
-    ]
+    _TESTS = [{
+        'url': 'http://api.soundcloud.com/playlists/4110309',
+        'info_dict': {
+            'id': '4110309',
+            'title': 'TILT Brass - Bowery Poetry Club, August \'03 [Non-Site SCR 02]',
+            'description': 're:.*?TILT Brass - Bowery Poetry Club',
+        },
+        'playlist_count': 6,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 589d3d7c7ae18875060caa15f5547c0194932e55 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 18 Sep 2014 21:37:09 +0200
Subject: [PATCH 0021/1937] [moniker] rename from hypestat (#3788)

---
 youtube_dl/extractor/__init__.py                 | 2 +-
 youtube_dl/extractor/{hypestat.py => moniker.py} | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename youtube_dl/extractor/{hypestat.py => moniker.py} (98%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 97693018f..a9a33c40f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -150,7 +150,6 @@ from .howcast import HowcastIE
 from .howstuffworks import HowStuffWorksIE
 from .huffpost import HuffPostIE
 from .hypem import HypemIE
-from .hypestat import HypestatIE
 from .iconosquare import IconosquareIE
 from .ign import IGNIE, OneUPIE
 from .imdb import (
@@ -209,6 +208,7 @@ from .mpora import MporaIE
 from .moevideo import MoeVideoIE
 from .mofosex import MofosexIE
 from .mojvideo import MojvideoIE
+from .moniker import MonikerIE
 from .mooshare import MooshareIE
 from .morningstar import MorningstarIE
 from .motherless import MotherlessIE
diff --git a/youtube_dl/extractor/hypestat.py b/youtube_dl/extractor/moniker.py
similarity index 98%
rename from youtube_dl/extractor/hypestat.py
rename to youtube_dl/extractor/moniker.py
index e1a142268..79bb2ca59 100644
--- a/youtube_dl/extractor/hypestat.py
+++ b/youtube_dl/extractor/moniker.py
@@ -11,7 +11,7 @@ from ..utils import (
 )
 
 
-class HypestatIE(InfoExtractor):
+class MonikerIE(InfoExtractor):
     IE_DESC = 'allmyvideos.net and vidspot.net'
     _VALID_URL = r'https?://(?:www\.)?(?:allmyvideos|vidspot)\.net/(?P<id>[a-zA-Z0-9_-]+)'
 

From 7267bd536fb81cb1bdcc6554219a0b66a75b31a6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 19 Sep 2014 09:57:53 +0200
Subject: [PATCH 0022/1937] [muenchentv] Add support (Fixes #3507)

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/common.py     |  2 +
 youtube_dl/extractor/muenchentv.py | 77 ++++++++++++++++++++++++++++++
 3 files changed, 80 insertions(+)
 create mode 100644 youtube_dl/extractor/muenchentv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a9a33c40f..625666acb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -221,6 +221,7 @@ from .mtv import (
     MTVServicesEmbeddedIE,
     MTVIggyIE,
 )
+from .muenchentv import MuenchenTVIE
 from .musicplayon import MusicPlayOnIE
 from .musicvault import MusicVaultIE
 from .muzu import MuzuTVIE
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 929dd1e97..9c30a1d33 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -130,6 +130,8 @@ class InfoExtractor(object):
                     by YoutubeDL if it's missing)
     categories:     A list of categories that the video falls in, for example
                     ["Sports", "Berlin"]
+    is_live:        True, False, or None (=unknown). Whether this video is a
+                    live stream that goes on instead of a fixed-length video.
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 
diff --git a/youtube_dl/extractor/muenchentv.py b/youtube_dl/extractor/muenchentv.py
new file mode 100644
index 000000000..3a938861b
--- /dev/null
+++ b/youtube_dl/extractor/muenchentv.py
@@ -0,0 +1,77 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import datetime
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    js_to_json,
+)
+
+
+class MuenchenTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?muenchen\.tv/livestream'
+    IE_DESC = 'münchen.tv'
+    _TEST = {
+        'url': 'http://www.muenchen.tv/livestream/',
+        'info_dict': {
+            'id': '5334',
+            'display_id': 'live',
+            'ext': 'mp4',
+            'title': 're:^münchen.tv-Livestream [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = 'live'
+        webpage = self._download_webpage(url, display_id)
+
+        now = datetime.datetime.now()
+        now_str = now.strftime("%Y-%m-%d %H:%M")
+        title = self._og_search_title(webpage) + ' ' + now_str
+
+        data_js = self._search_regex(
+            r'(?s)\nplaylist:\s*(\[.*?}\]),related:',
+            webpage, 'playlist configuration')
+        data_json = js_to_json(data_js)
+        data = json.loads(data_json)[0]
+
+        video_id = data['mediaid']
+        thumbnail = data.get('image')
+
+        formats = []
+        for format_num, s in enumerate(data['sources']):
+            ext = determine_ext(s['file'], None)
+            label_str = s.get('label')
+            if label_str is None:
+                label_str = '_%d' % format_num
+
+            if ext is None:
+                format_id = label_str
+            else:
+                format_id = '%s-%s' % (ext, label_str)
+
+            formats.append({
+                'url': s['file'],
+                'tbr': int_or_none(s.get('label')),
+                'ext': 'mp4',
+                'format_id': format_id,
+                'preference': -100 if '.smil' in s['file'] else 0,
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'formats': formats,
+            'is_live': True,
+        }
+

From f566d9f1d54a61497a17c5ed62a32ee1387483bd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 19 Sep 2014 09:58:01 +0200
Subject: [PATCH 0023/1937] release 2014.09.19

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 430509ba3..940e9c8cf 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.09.18'
+__version__ = '2014.09.19'

From 532f5bff70cc32f54f38fbce9233a88faf4423b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 Sep 2014 20:58:50 +0700
Subject: [PATCH 0024/1937] [franceinter] Fix extraction and modernize

---
 youtube_dl/extractor/franceinter.py | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/franceinter.py b/youtube_dl/extractor/franceinter.py
index deb1b0b9d..6613ee17a 100644
--- a/youtube_dl/extractor/franceinter.py
+++ b/youtube_dl/extractor/franceinter.py
@@ -4,16 +4,21 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import int_or_none
 
 
 class FranceInterIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?franceinter\.fr/player/reecouter\?play=(?P<id>[0-9]{6})'
+    _VALID_URL = r'http://(?:www\.)?franceinter\.fr/player/reecouter\?play=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.franceinter.fr/player/reecouter?play=793962',
-        'file': '793962.mp3',
         'md5': '4764932e466e6f6c79c317d2e74f6884',
         "info_dict": {
-            "title": "L’Histoire dans les jeux vidéo",
+            'id': '793962',
+            'ext': 'mp3',
+            'title': 'L’Histoire dans les jeux vidéo',
+            'description': 'md5:7e93ddb4451e7530022792240a3049c7',
+            'timestamp': 1387369800,
+            'upload_date': '20131218',
         },
     }
 
@@ -22,17 +27,26 @@ class FranceInterIE(InfoExtractor):
         video_id = mobj.group('id')
 
         webpage = self._download_webpage(url, video_id)
-        title = self._html_search_regex(
-            r'<span class="roll_overflow">(.*?)</span></h1>', webpage, 'title')
+
         path = self._search_regex(
-            r'&urlAOD=(.*?)&startTime', webpage, 'video url')
+            r'<a id="player".+?href="([^"]+)"', webpage, 'video url')
         video_url = 'http://www.franceinter.fr/' + path
 
+        title = self._html_search_regex(
+            r'<span class="title">(.+?)</span>', webpage, 'title')
+        description = self._html_search_regex(
+            r'<span class="description">(.*?)</span>',
+            webpage, 'description', fatal=False)
+        timestamp = int_or_none(self._search_regex(
+            r'data-date="(\d+)"', webpage, 'upload date', fatal=False))
+
         return {
             'id': video_id,
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
             'formats': [{
                 'url': video_url,
                 'vcodec': 'none',
             }],
-            'title': title,
         }

From 5aa38e75b27b428b67f9f7083c44051881c98fd8 Mon Sep 17 00:00:00 2001
From: Carlos Ramos <carlos.ramos1@alu.uclm.es>
Date: Fri, 19 Sep 2014 22:46:57 +0200
Subject: [PATCH 0025/1937] [played] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/played.py   | 57 ++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+)
 create mode 100644 youtube_dl/extractor/played.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 625666acb..9ee3f9190 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -266,6 +266,7 @@ from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
 from .photobucket import PhotobucketIE
+from .played import PlayedIE
 from .playfm import PlayFMIE
 from .playvid import PlayvidIE
 from .podomatic import PodomaticIE
diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
new file mode 100644
index 000000000..a396e62e5
--- /dev/null
+++ b/youtube_dl/extractor/played.py
@@ -0,0 +1,57 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import time
+import os.path
+
+from .common import InfoExtractor
+from ..utils import (
+    compat_urllib_parse,
+    compat_urllib_request,
+)
+
+
+class PlayedIE(InfoExtractor):
+    IE_NAME = 'played.to'
+    _VALID_URL = r'https?://played\.to/(?P<id>[a-zA-Z0-9_-]+)'
+
+    _TEST = {
+        'url': 'http://played.to/j2f2sfiiukgt',
+        'md5': 'c2bd75a368e82980e7257bf500c00637',
+        'info_dict': {
+            'id': 'j2f2sfiiukgt',
+            'ext': 'flv',
+            'title': 'youtube-dl_test_video.mp4',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        orig_webpage = self._download_webpage(url, video_id)
+        fields = re.findall(r'type="hidden" name="(.+?)"\s* value="?(.+?)">', orig_webpage)
+        data = dict(fields)
+
+        self.to_screen('%s: Waiting for timeout' % video_id)
+        time.sleep(2)
+
+        post = compat_urllib_parse.urlencode(data)
+        headers = {
+            b'Content-Type': b'application/x-www-form-urlencoded',
+        }
+        req = compat_urllib_request.Request(url, post, headers)
+        webpage = self._download_webpage(
+            req, video_id, note='Downloading video page ...')
+
+        title = os.path.splitext(data['fname'])[0]
+
+        video_url = self._search_regex(
+            r'file: "?(.+?)",', webpage, 'video URL')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+        }
\ No newline at end of file

From 746c67d72f760f2805dbc125e5a3863aa0d569e3 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 20 Sep 2014 03:02:11 +0300
Subject: [PATCH 0026/1937] [wistia] Use API and make more generic

---
 youtube_dl/extractor/generic.py | 23 +++++++++++++++++++++++
 youtube_dl/extractor/wistia.py  | 15 +++++++++------
 2 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 40eeaad16..2d77f604a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -382,6 +382,19 @@ class GenericIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
+        # Wistia embed
+        {
+            'url': 'http://education-portal.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
+            'md5': '8788b683c777a5cf25621eaf286d0c23',
+            'info_dict': {
+                'id': '1cfaf6b7ea',
+                'ext': 'mov',
+                'title': 'md5:51364a8d3d009997ba99656004b5e20d',
+                'duration': 643.0,
+                'filesize': 182808282,
+                'uploader': 'education-portal.com',
+            },
+        },
     ]
 
     def report_download_webpage(self, video_id):
@@ -654,6 +667,16 @@ class GenericIE(InfoExtractor):
                 'title': video_title,
                 'id': video_id,
             }
+        match = re.search(r'(?:id=["\']wistia_|data-wistiaid=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
+        if match:
+            return {
+                '_type': 'url_transparent',
+                'url': 'http://fast.wistia.net/embed/iframe/{0:}'.format(match.group('id')),
+                'ie_key': 'Wistia',
+                'uploader': video_uploader,
+                'title': video_title,
+                'id': match.group('id')
+            }
 
         # Look for embedded blip.tv player
         mobj = re.search(r'<meta\s[^>]*https?://api\.blip\.tv/\w+/redirect/\w+/(\d+)', webpage)
diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index e6bfa9e14..748443f81 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -1,13 +1,14 @@
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor
+from ..utils import ExtractorError, compat_urllib_request
 
 
 class WistiaIE(InfoExtractor):
     _VALID_URL = r'https?://(?:fast\.)?wistia\.net/embed/iframe/(?P<id>[a-z0-9]+)'
+    _API_URL = 'http://fast.wistia.com/embed/medias/{0:}.json'
 
     _TEST = {
         'url': 'http://fast.wistia.net/embed/iframe/sh7fpupwlt',
@@ -24,11 +25,13 @@ class WistiaIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        webpage = self._download_webpage(url, video_id)
-        data_json = self._html_search_regex(
-            r'Wistia\.iframeInit\((.*?), {}\);', webpage, 'video data')
-
-        data = json.loads(data_json)
+        request = compat_urllib_request.Request(self._API_URL.format(video_id))
+        request.add_header('Referer', url)  # Some videos require this.
+        data_json = self._download_json(request, video_id)
+        if data_json.get('error'):
+            raise ExtractorError('Error while getting the playlist',
+                                 expected=True)
+        data = data_json['media']
 
         formats = []
         thumbnails = []

From 3e8fcd9fa1ae23ee3f0370dd948411a5f74c03dc Mon Sep 17 00:00:00 2001
From: Marco Schuster <marco+github@m-s-d.eu>
Date: Sat, 20 Sep 2014 02:32:41 +0200
Subject: [PATCH 0027/1937] [divxstage] added .to TLD

Example video "http://www.divxstage.eu/video/930c52709d2" which gets redirected to .to TLD
---
 youtube_dl/extractor/divxstage.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/divxstage.py b/youtube_dl/extractor/divxstage.py
index 4ca3f37a2..b88379e06 100644
--- a/youtube_dl/extractor/divxstage.py
+++ b/youtube_dl/extractor/divxstage.py
@@ -7,7 +7,7 @@ class DivxStageIE(NovaMovIE):
     IE_NAME = 'divxstage'
     IE_DESC = 'DivxStage'
 
-    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'divxstage\.(?:eu|net|ch|co|at|ag)'}
+    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'divxstage\.(?:eu|net|ch|co|at|ag|to)'}
 
     _HOST = 'www.divxstage.eu'
 
@@ -24,4 +24,4 @@ class DivxStageIE(NovaMovIE):
             'title': 'youtubedl test video',
             'description': 'This is a test video for youtubedl.',
         }
-    }
\ No newline at end of file
+    }

From 752297631ffd9a51535e650f4444a36f820f01f4 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 21 Sep 2014 06:20:42 +0700
Subject: [PATCH 0028/1937] [noco] Adapt to API v1.1 (Closes #3797)

---
 youtube_dl/extractor/noco.py | 73 ++++++++++++++++++++++++++----------
 1 file changed, 53 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 959fdf590..e3ec9ed15 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -2,6 +2,8 @@
 from __future__ import unicode_literals
 
 import re
+import time
+import hashlib
 
 from .common import InfoExtractor
 from ..utils import (
@@ -17,6 +19,7 @@ from ..utils import (
 class NocoIE(InfoExtractor):
     _VALID_URL = r'http://(?:(?:www\.)?noco\.tv/emission/|player\.noco\.tv/\?idvideo=)(?P<id>\d+)'
     _LOGIN_URL = 'http://noco.tv/do.php'
+    _API_URL_TEMPLATE = 'https://api.noco.tv/1.1/%s?ts=%s&tk=%s'
     _NETRC_MACHINE = 'noco'
 
     _TEST = {
@@ -55,33 +58,52 @@ class NocoIE(InfoExtractor):
         login = self._download_json(request, None, 'Logging in as %s' % username)
 
         if 'erreur' in login:
-            raise  ExtractorError('Unable to login: %s' % clean_html(login['erreur']), expected=True)
+            raise ExtractorError('Unable to login: %s' % clean_html(login['erreur']), expected=True)
+
+    def _call_api(self, path, video_id, note):
+        ts = compat_str(int(time.time() * 1000))
+        tk = hashlib.md5(hashlib.md5(ts).hexdigest() + '#8S?uCraTedap6a').hexdigest()
+        url = self._API_URL_TEMPLATE % (path, ts, tk)
+
+        resp = self._download_json(url, video_id, note)
+
+        if isinstance(resp, dict) and resp.get('error'):
+            self._raise_error(resp['error'], resp['description'])
+
+        return resp
+
+    def _raise_error(self, error, description):
+        raise ExtractorError(
+            '%s returned error: %s - %s' % (self.IE_NAME, error, description),
+            expected=True)
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        medias = self._download_json(
-            'https://api.noco.tv/1.0/video/medias/%s' % video_id, video_id, 'Downloading video JSON')
+        medias = self._call_api(
+            'shows/%s/medias' % video_id,
+            video_id, 'Downloading video JSON')
+
+        qualities = self._call_api(
+            'qualities',
+            video_id, 'Downloading qualities JSON')
 
         formats = []
 
-        for fmt in medias['fr']['video_list']['default']['quality_list']:
-            format_id = fmt['quality_key']
+        for format_id, fmt in medias['fr']['video_list']['none']['quality_list'].items():
 
-            file = self._download_json(
-                'https://api.noco.tv/1.0/video/file/%s/fr/%s' % (format_id.lower(), video_id),
+            video = self._call_api(
+                'shows/%s/video/%s/fr' % (video_id, format_id.lower()),
                 video_id, 'Downloading %s video JSON' % format_id)
 
-            file_url = file['file']
+            file_url = video['file']
             if not file_url:
                 continue
 
-            if file_url == 'forbidden':
-                raise ExtractorError(
-                    '%s returned error: %s - %s' % (
-                        self.IE_NAME, file['popmessage']['title'], file['popmessage']['message']),
-                    expected=True)
+            if file_url in ['forbidden', 'not found']:
+                popmessage = video['popmessage']
+                self._raise_error(popmessage['title'], popmessage['message'])
 
             formats.append({
                 'url': file_url,
@@ -91,20 +113,31 @@ class NocoIE(InfoExtractor):
                 'abr': fmt['audiobitrate'],
                 'vbr': fmt['videobitrate'],
                 'filesize': fmt['filesize'],
-                'format_note': fmt['quality_name'],
-                'preference': fmt['priority'],
+                'format_note': qualities[format_id]['quality_name'],
+                'preference': qualities[format_id]['priority'],
             })
 
         self._sort_formats(formats)
 
-        show = self._download_json(
-            'https://api.noco.tv/1.0/shows/show/%s' % video_id, video_id, 'Downloading show JSON')[0]
+        show = self._call_api(
+            'shows/by_id/%s' % video_id,
+            video_id, 'Downloading show JSON')[0]
 
-        upload_date = unified_strdate(show['indexed'])
+        upload_date = unified_strdate(show['online_date_start_utc'])
         uploader = show['partner_name']
         uploader_id = show['partner_key']
         duration = show['duration_ms'] / 1000.0
-        thumbnail = show['screenshot']
+
+        thumbnails = []
+        for thumbnail_key, thumbnail_url in show.items():
+            m = re.search(r'^screenshot_(?P<width>\d+)x(?P<height>\d+)$', thumbnail_key)
+            if not m:
+                continue
+            thumbnails.append({
+                'url': thumbnail_url,
+                'width': int(m.group('width')),
+                'height': int(m.group('height')),
+            })
 
         episode = show.get('show_TT') or show.get('show_OT')
         family = show.get('family_TT') or show.get('family_OT')
@@ -124,7 +157,7 @@ class NocoIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'description': description,
-            'thumbnail': thumbnail,
+            'thumbnails': thumbnails,
             'upload_date': upload_date,
             'uploader': uploader,
             'uploader_id': uploader_id,

From 58e7071a2ced491a6ecd8a8bcb1b4533a2b0fd8f Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 21 Sep 2014 06:37:11 +0700
Subject: [PATCH 0029/1937] [tube8] Improve _VALID_URL and add display_id

---
 youtube_dl/extractor/tube8.py | 37 ++++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index 39f20c546..64a1e9030 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -14,27 +14,35 @@ from ..aes import aes_decrypt_text
 
 
 class Tube8IE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tube8\.com/(?:gay/|shemale/)?(?:[^/]+/){2}(?P<id>\d+)'
-    _TEST = {
-        'url': 'http://www.tube8.com/teen/kasia-music-video/229795/',
-        'md5': '44bf12b98313827dd52d35b8706a4ea0',
-        'info_dict': {
-            'id': '229795',
-            'ext': 'mp4',
-            'description': 'hot teen Kasia grinding',
-            'uploader': 'unknown',
-            'title': 'Kasia music video',
-            'age_limit': 18,
-        }
-    }
+    _VALID_URL = r'https?://(?:www\.)?tube8\.com/(?:[^/]+/)+(?P<display_id>[^/]+)/(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'http://www.tube8.com/teen/kasia-music-video/229795/',
+            'md5': '44bf12b98313827dd52d35b8706a4ea0',
+            'info_dict': {
+                'id': '229795',
+                'display_id': 'kasia-music-video',
+                'ext': 'mp4',
+                'description': 'hot teen Kasia grinding',
+                'uploader': 'unknown',
+                'title': 'Kasia music video',
+                'age_limit': 18,
+            }
+        },
+        {
+            'url': 'http://www.tube8.com/shemale/teen/blonde-cd-gets-kidnapped-by-two-blacks-and-punished-for-being-a-slutty-girl/19569151/',
+            'only_matching': True,
+        },
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
         req = compat_urllib_request.Request(url)
         req.add_header('Cookie', 'age_verified=1')
-        webpage = self._download_webpage(req, video_id)
+        webpage = self._download_webpage(req, display_id)
 
         flashvars = json.loads(self._html_search_regex(
             r'var flashvars\s*=\s*({.+?})', webpage, 'flashvars'))
@@ -70,6 +78,7 @@ class Tube8IE(InfoExtractor):
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'url': video_url,
             'title': title,
             'description': description,

From 522c55b7f2622b6138a2781db362d822b4fed32d Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 21 Sep 2014 03:26:12 +0300
Subject: [PATCH 0030/1937] [mgoon] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/mgoon.py    | 87 ++++++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+)
 create mode 100644 youtube_dl/extractor/mgoon.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 625666acb..fb546eeae 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -199,6 +199,7 @@ from .malemotion import MalemotionIE
 from .mdr import MDRIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
+from .mgoon import MgoonIE
 from .ministrygrid import MinistryGridIE
 from .mit import TechTVMITIE, MITIE, OCWMITIE
 from .mitele import MiTeleIE
diff --git a/youtube_dl/extractor/mgoon.py b/youtube_dl/extractor/mgoon.py
new file mode 100644
index 000000000..94bc87b00
--- /dev/null
+++ b/youtube_dl/extractor/mgoon.py
@@ -0,0 +1,87 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    qualities,
+    unified_strdate,
+)
+
+
+class MgoonIE(InfoExtractor):
+    _VALID_URL = r'''(?x)https?://(?:www\.)?
+    (?:(:?m\.)?mgoon\.com/(?:ch/(?:.+)/v|play/view)|
+        video\.mgoon\.com)/(?P<id>[0-9]+)'''
+    _API_URL = 'http://mpos.mgoon.com/player/video?id={0:}'
+    _TESTS = [
+        {
+            'url': 'http://m.mgoon.com/ch/hi6618/v/5582148',
+            'md5': 'dd46bb66ab35cf6d51cc812fd82da79d',
+            'info_dict': {
+                'id': '5582148',
+                'uploader_id': 'hi6618',
+                'duration': 240.419,
+                'upload_date': '20131220',
+                'ext': 'mp4',
+                'title': 'md5:543aa4c27a4931d371c3f433e8cebebc',
+                'thumbnail': 're:^https?://.*\.jpg$',
+            }
+        },
+        {
+            'url': 'http://www.mgoon.com/play/view/5582148',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://video.mgoon.com/5582148',
+            'only_matching': True,
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        data = self._download_json(self._API_URL.format(video_id), video_id)
+
+        if data.get('errorInfo', {}).get('code') != 'NONE':
+            raise ExtractorError('%s encountered an error: %s' % (
+                self.IE_NAME, data['errorInfo']['message']), expected=True)
+
+        v_info = data['videoInfo']
+        title = v_info.get('v_title')
+        thumbnail = v_info.get('v_thumbnail')
+        duration = v_info.get('v_duration')
+        upload_date = unified_strdate(v_info.get('v_reg_date'))
+        uploader_id = data.get('userInfo', {}).get('u_alias')
+        if duration:
+            duration /= 1000.0
+
+        age_limit = None
+        if data.get('accessInfo', {}).get('code') == 'VIDEO_STATUS_ADULT':
+            age_limit = 18
+
+        formats = []
+        get_quality = qualities(['360p', '480p', '720p', '1080p'])
+        for fmt in data['videoFiles']:
+            formats.append({
+                'format_id': fmt['label'],
+                'quality': get_quality(fmt['label']),
+                'url': fmt['url'],
+                'ext': fmt['format'],
+
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'upload_date': upload_date,
+            'uploader_id': uploader_id,
+            'age_limit': age_limit,
+        }

From 72e450c5550ae26b5b36216be1c001f64479c773 Mon Sep 17 00:00:00 2001
From: Anton Larionov <diffident.cat@gmail.com>
Date: Sun, 21 Sep 2014 13:21:29 +0400
Subject: [PATCH 0031/1937] [thvideo] Add support for THVideo

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/thvideo.py  | 55 ++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/thvideo.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fb546eeae..ae5b4e9e6 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -362,6 +362,7 @@ from .thisav import ThisAVIE
 from .tinypic import TinyPicIE
 from .tlc import TlcIE, TlcDeIE
 from .tnaflix import TNAFlixIE
+from .thvideo import THVideoIE
 from .toutv import TouTvIE
 from .toypics import ToypicsUserIE, ToypicsIE
 from .traileraddict import TrailerAddictIE
diff --git a/youtube_dl/extractor/thvideo.py b/youtube_dl/extractor/thvideo.py
new file mode 100644
index 000000000..9fa14d3c4
--- /dev/null
+++ b/youtube_dl/extractor/thvideo.py
@@ -0,0 +1,55 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate
+)
+
+
+class THVideoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?thvideo\.tv/v/th(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://thvideo.tv/v/th1987/',
+        'md5': 'fa107b1f73817e325e9433505a70db50',
+        'info_dict': {
+            'id': '1987',
+            'ext': 'mp4',
+            'title': '【动画】秘封活动记录 ～ The Sealed Esoteric History.分镜稿预览',
+            'display_id': 'th1987',
+            'thumbnail': 'http://thvideo.tv/uploadfile/2014/0722/20140722013459856.jpg',
+            'description': '社团京都幻想剧团的第一个东方二次同人动画作品「秘封活动记录 ～ The Sealed Esoteric History.」 本视频是该动画第一期的分镜草稿...',
+            'upload_date': '20140722'
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        # extract download link from mobile player page
+        webpage_player = self._download_webpage('http://thvideo.tv/mobile.php?cid=%s-0' % video_id, video_id)
+        video_url = self._html_search_regex(r'<source src="(.*?)" type', webpage_player, 'video url')
+
+        # extract video info from main page
+        webpage = self._download_webpage(url, video_id)
+        title = self._og_search_title(webpage)
+        display_id = 'th%s' % video_id
+        thumbnail = self._og_search_thumbnail(webpage)
+        description = self._og_search_description(webpage)
+        upload_date_raw = self._html_search_regex(r'span itemprop="datePublished" content="(.*?)">', webpage,
+                                                  'upload date', fatal=False)
+        upload_date = unified_strdate(upload_date_raw)
+
+        return {
+            'id': video_id,
+            'ext': 'mp4',
+            'url': video_url,
+            'title': title,
+            'display_id': display_id,
+            'thumbnail': thumbnail,
+            'description': description,
+            'upload_date': upload_date
+        }
\ No newline at end of file

From 7bd4b4229a126a9f47035beec8a13eff08804850 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Sep 2014 13:40:22 +0200
Subject: [PATCH 0032/1937] [dropbox] Recognize 'https://www.dropbox.com/sh/*'
 urls (fixes #3795)

And extract the title from the url last path component.
---
 youtube_dl/extractor/dropbox.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/dropbox.py b/youtube_dl/extractor/dropbox.py
index 1e1763abf..817a9bd61 100644
--- a/youtube_dl/extractor/dropbox.py
+++ b/youtube_dl/extractor/dropbox.py
@@ -5,24 +5,29 @@ import os.path
 import re
 
 from .common import InfoExtractor
-from ..utils import compat_urllib_parse_unquote
+from ..utils import compat_urllib_parse_unquote, url_basename
 
 
 class DropboxIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dropbox[.]com/s/(?P<id>[a-zA-Z0-9]{15})/(?P<title>[^?#]*)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?dropbox[.]com/sh?/(?P<id>[a-zA-Z0-9]{15})/.*'
+    _TESTS = [{
         'url': 'https://www.dropbox.com/s/nelirfsxnmcfbfh/youtube-dl%20test%20video%20%27%C3%A4%22BaW_jenozKc.mp4?dl=0',
         'info_dict': {
             'id': 'nelirfsxnmcfbfh',
             'ext': 'mp4',
             'title': 'youtube-dl test video \'ä"BaW_jenozKc'
         }
-    }
+    },
+    {
+        'url': 'https://www.dropbox.com/sh/662glsejgzoj9sr/AAByil3FGH9KFNZ13e08eSa1a/Pregame%20Ceremony%20Program%20PA%2020140518.m4v',
+        'only_matching': True,
+    },
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        fn = compat_urllib_parse_unquote(mobj.group('title'))
+        fn = compat_urllib_parse_unquote(url_basename(url))
         title = os.path.splitext(fn)[0]
         video_url = (
             re.sub(r'[?&]dl=0', '', url) +

From b28c8403b2c1ef51f04520e8116176b1fee12dcb Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 21 Sep 2014 15:13:35 +0300
Subject: [PATCH 0033/1937] [yourupload] Add new extractor. Fixes #3085

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/yourupload.py | 58 ++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+)
 create mode 100644 youtube_dl/extractor/yourupload.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fb546eeae..1a6033320 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -448,6 +448,7 @@ from .yahoo import (
 from .youjizz import YouJizzIE
 from .youku import YoukuIE
 from .youporn import YouPornIE
+from .yourupload import YourUploadIE
 from .youtube import (
     YoutubeIE,
     YoutubeChannelIE,
diff --git a/youtube_dl/extractor/yourupload.py b/youtube_dl/extractor/yourupload.py
new file mode 100644
index 000000000..40fc4165f
--- /dev/null
+++ b/youtube_dl/extractor/yourupload.py
@@ -0,0 +1,58 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class YourUploadIE(InfoExtractor):
+    _VALID_URL = r'''(?x)https?://(?:www\.)?
+        (?:yourupload\.com/watch|
+           embed\.yourupload\.com|
+           embed\.yucache\.net
+        )/(?P<id>[A-Za-z0-9]+)
+        '''
+    _TESTS = [
+        {
+            'url': 'http://yourupload.com/watch/14i14h',
+            'md5': 'bf5c2f95c4c917536e80936af7bc51e1',
+            'info_dict': {
+                'id': '14i14h',
+                'ext': 'mp4',
+                'title': 'BigBuckBunny_320x180.mp4',
+                'thumbnail': 're:^https?://.*\.jpe?g',
+            }
+        },
+        {
+            'url': 'http://embed.yourupload.com/14i14h',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://embed.yucache.net/14i14h?client_file_id=803349',
+            'only_matching': True,
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        url = 'http://embed.yucache.net/{0:}'.format(video_id)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+        url = self._og_search_video_url(webpage)
+
+        formats = [{
+            'format_id': 'sd',
+            'url': url,
+        }]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+        }

From b509a4b17643422b750e5258f538894105c58d42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Sep 2014 15:43:09 +0200
Subject: [PATCH 0034/1937] [downloader/f4m] If <pv-2.0> is in the manifest,
 add it to the fragments urls query (fixes #3176)

It's used in some akamai videos (for example for theplatform.com).
---
 youtube_dl/downloader/f4m.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 71353f607..b3be16ff1 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -16,6 +16,7 @@ from ..utils import (
     format_bytes,
     encodeFilename,
     sanitize_open,
+    xpath_text,
 )
 
 
@@ -251,6 +252,8 @@ class F4mFD(FileDownloader):
             # We only download the first fragment
             fragments_list = fragments_list[:1]
         total_frags = len(fragments_list)
+        # For some akamai manifests we'll need to add a query to the fragment url
+        akamai_pv = xpath_text(doc, _add_ns('pv-2.0'))
 
         tmpfilename = self.temp_name(filename)
         (dest_stream, tmpfilename) = sanitize_open(tmpfilename, 'wb')
@@ -290,6 +293,8 @@ class F4mFD(FileDownloader):
         for (seg_i, frag_i) in fragments_list:
             name = 'Seg%d-Frag%d' % (seg_i, frag_i)
             url = base_url + name
+            if akamai_pv:
+                url += '?' + akamai_pv.strip(';')
             frag_filename = '%s-%s' % (tmpfilename, name)
             success = http_dl.download(frag_filename, {'url': url})
             if not success:

From dd41e8c82bb14bda0c407f9f0865cfb112e8fc30 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Sep 2014 15:47:58 +0200
Subject: [PATCH 0035/1937] [theplatform] Extract all formats for f4m videos

---
 youtube_dl/extractor/theplatform.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index b6b2dba9c..031a958fa 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -62,10 +62,7 @@ class ThePlatformIE(InfoExtractor):
             # the parameters are from syfy.com, other sites may use others,
             # they also work for nbc.com
             f4m_url += '&g=UXWGVKRWHFSP&hdcore=3.0.3'
-            formats = [{
-                'ext': 'flv',
-                'url': f4m_url,
-            }]
+            formats = self._extract_f4m_formats(f4m_url, video_id)
         else:
             base_url = head.find(_x('smil:meta')).attrib['base']
             switch = body.find(_x('smil:switch'))

From 224ce0d87299cf54469baccb9922e78f9594d029 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Sep 2014 15:49:04 +0200
Subject: [PATCH 0036/1937] [nbc] Update test

---
 youtube_dl/extractor/nbc.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index d2e4acbad..e75ab7c39 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -16,9 +16,9 @@ class NBCIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.nbc.com/chicago-fire/video/i-am-a-firefighter/2734188',
-        'md5': '54d0fbc33e0b853a65d7b4de5c06d64e',
+        # md5 checksum is not stable
         'info_dict': {
-            'id': 'u1RInQZRN7QJ',
+            'id': 'bTmnLCvIbaaH',
             'ext': 'flv',
             'title': 'I Am a Firefighter',
             'description': 'An emergency puts Dawson\'sf irefighter skills to the ultimate test in this four-part digital series.',

From e35cb78c4099263c26f717669463a3c025c30d17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Sep 2014 16:08:38 +0200
Subject: [PATCH 0037/1937] [theplatform] Correctly extract videos that don't
 use f4m or rtmp (reported in #3176)

---
 youtube_dl/extractor/sbs.py         |  2 +-
 youtube_dl/extractor/theplatform.py | 48 ++++++++++++++++++-----------
 2 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index 34058fd4b..214990e7a 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -21,7 +21,7 @@ class SBSIE(InfoExtractor):
         'md5': '3150cf278965eeabb5b4cea1c963fe0a',
         'info_dict': {
             'id': '320403011771',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Dingo Conservation',
             'description': 'Dingoes are on the brink of extinction; most of the animals we think are dingoes are in fact crossbred with wild dogs. This family run a dingo conservation park to prevent their extinction',
             'thumbnail': 're:http://.*\.jpg',
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 031a958fa..0be793b1c 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -5,6 +5,7 @@ import json
 
 from .common import InfoExtractor
 from ..utils import (
+    compat_str,
     ExtractorError,
     xpath_with_ns,
 )
@@ -55,7 +56,7 @@ class ThePlatformIE(InfoExtractor):
         body = meta.find(_x('smil:body'))
 
         f4m_node = body.find(_x('smil:seq//smil:video'))
-        if f4m_node is not None:
+        if f4m_node is not None and '.f4m' in f4m_node.attrib['src']:
             f4m_url = f4m_node.attrib['src']
             if 'manifest.f4m?' not in f4m_url:
                 f4m_url += '?'
@@ -64,24 +65,35 @@ class ThePlatformIE(InfoExtractor):
             f4m_url += '&g=UXWGVKRWHFSP&hdcore=3.0.3'
             formats = self._extract_f4m_formats(f4m_url, video_id)
         else:
-            base_url = head.find(_x('smil:meta')).attrib['base']
-            switch = body.find(_x('smil:switch'))
             formats = []
-            for f in switch.findall(_x('smil:video')):
-                attr = f.attrib
-                width = int(attr['width'])
-                height = int(attr['height'])
-                vbr = int(attr['system-bitrate']) // 1000
-                format_id = '%dx%d_%dk' % (width, height, vbr)
-                formats.append({
-                    'format_id': format_id,
-                    'url': base_url,
-                    'play_path': 'mp4:' + attr['src'],
-                    'ext': 'flv',
-                    'width': width,
-                    'height': height,
-                    'vbr': vbr,
-                })
+            switch = body.find(_x('smil:switch'))
+            if switch is not None:
+                base_url = head.find(_x('smil:meta')).attrib['base']
+                for f in switch.findall(_x('smil:video')):
+                    attr = f.attrib
+                    width = int(attr['width'])
+                    height = int(attr['height'])
+                    vbr = int(attr['system-bitrate']) // 1000
+                    format_id = '%dx%d_%dk' % (width, height, vbr)
+                    formats.append({
+                        'format_id': format_id,
+                        'url': base_url,
+                        'play_path': 'mp4:' + attr['src'],
+                        'ext': 'flv',
+                        'width': width,
+                        'height': height,
+                        'vbr': vbr,
+                    })
+            else:
+                switch = body.find(_x('smil:seq//smil:switch'))
+                for f in switch.findall(_x('smil:video')):
+                    attr = f.attrib
+                    vbr = int(attr['system-bitrate']) // 1000
+                    formats.append({
+                        'format_id': compat_str(vbr),
+                        'url': attr['src'],
+                        'vbr': vbr,
+                    })
             self._sort_formats(formats)
 
         return {

From df8f53f752c0f01577dcc5d63c6d9a81d924770b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 21 Sep 2014 16:32:38 +0200
Subject: [PATCH 0038/1937] [thvideo] Support mobile URLs as well

---
 youtube_dl/extractor/thvideo.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/thvideo.py b/youtube_dl/extractor/thvideo.py
index 9fa14d3c4..607e947bb 100644
--- a/youtube_dl/extractor/thvideo.py
+++ b/youtube_dl/extractor/thvideo.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class THVideoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?thvideo\.tv/v/th(?P<id>[0-9]+)'
+    _VALID_URL = r'http://(?:www\.)?thvideo\.tv/(?:v/th|mobile\.php\?cid=)(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://thvideo.tv/v/th1987/',
         'md5': 'fa107b1f73817e325e9433505a70db50',
@@ -30,18 +30,22 @@ class THVideoIE(InfoExtractor):
         video_id = mobj.group('id')
 
         # extract download link from mobile player page
-        webpage_player = self._download_webpage('http://thvideo.tv/mobile.php?cid=%s-0' % video_id, video_id)
-        video_url = self._html_search_regex(r'<source src="(.*?)" type', webpage_player, 'video url')
+        webpage_player = self._download_webpage(
+            'http://thvideo.tv/mobile.php?cid=%s-0' % (video_id),
+            video_id, note='Downloading video source page')
+        video_url = self._html_search_regex(
+            r'<source src="(.*?)" type', webpage_player, 'video url')
 
         # extract video info from main page
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(
+            'http://thvideo.tv/v/th%s' % (video_id), video_id)
         title = self._og_search_title(webpage)
         display_id = 'th%s' % video_id
         thumbnail = self._og_search_thumbnail(webpage)
         description = self._og_search_description(webpage)
-        upload_date_raw = self._html_search_regex(r'span itemprop="datePublished" content="(.*?)">', webpage,
-                                                  'upload date', fatal=False)
-        upload_date = unified_strdate(upload_date_raw)
+        upload_date = unified_strdate(self._html_search_regex(
+            r'span itemprop="datePublished" content="(.*?)">', webpage,
+            'upload date', fatal=False))
 
         return {
             'id': video_id,
@@ -52,4 +56,4 @@ class THVideoIE(InfoExtractor):
             'thumbnail': thumbnail,
             'description': description,
             'upload_date': upload_date
-        }
\ No newline at end of file
+        }

From d0df92928bc099775e18f6413e387713839012ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Sep 2014 16:53:00 +0200
Subject: [PATCH 0039/1937] [npo] Add extractor for tegenlicht.vpro.nl (closes
 #3778)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/npo.py      | 30 ++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1a6033320..bca34ae73 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -249,7 +249,10 @@ from .nosvideo import NosVideoIE
 from .novamov import NovaMovIE
 from .nowness import NownessIE
 from .nowvideo import NowVideoIE
-from .npo import NPOIE
+from .npo import (
+    NPOIE,
+    TegenlichtVproIE,
+)
 from .nrk import (
     NRKIE,
     NRKTVIE,
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 7a154e94a..f36d446d2 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -7,6 +7,7 @@ from ..utils import (
     unified_strdate,
     parse_duration,
     qualities,
+    url_basename,
 )
 
 
@@ -55,7 +56,9 @@ class NPOIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
+        return self._get_info(video_id)
 
+    def _get_info(self, video_id):
         metadata = self._download_json(
             'http://e.omroep.nl/metadata/aflevering/%s' % video_id,
             video_id,
@@ -106,3 +109,30 @@ class NPOIE(InfoExtractor):
             'duration': parse_duration(metadata.get('tijdsduur')),
             'formats': formats,
         }
+
+
+class TegenlichtVproIE(NPOIE):
+    IE_NAME = 'tegenlicht.vpro.nl'
+    _VALID_URL = r'https?://tegenlicht\.vpro\.nl/afleveringen/.*?'
+
+    _TESTS = [
+        {
+            'url': 'http://tegenlicht.vpro.nl/afleveringen/2012-2013/de-toekomst-komt-uit-afrika.html',
+            'md5': 'f8065e4e5a7824068ed3c7e783178f2c',
+            'info_dict': {
+                'id': 'VPWON_1169289',
+                'ext': 'm4v',
+                'title': 'Tegenlicht',
+                'description': 'md5:d6476bceb17a8c103c76c3b708f05dd1',
+                'upload_date': '20130225',
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        name = url_basename(url)
+        webpage = self._download_webpage(url, name)
+        urn = self._html_search_meta('mediaurn', webpage)
+        info_page = self._download_json(
+            'http://rs.vpro.nl/v2/api/media/%s.json' % urn, name)
+        return self._get_info(info_page['mid'])

From f90d95edeb981481834f4b092b4c2ac793f225f9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 22 Sep 2014 13:07:23 +0200
Subject: [PATCH 0040/1937] release 2014.09.22

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 940e9c8cf..4bf208b67 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.09.19'
+__version__ = '2014.09.22'

From 273dea42487461884926b4d810ebf74e541dc8b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Sep 2014 18:58:22 +0700
Subject: [PATCH 0041/1937] [playfm] Fix view count and add comment count

---
 youtube_dl/extractor/playfm.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/playfm.py b/youtube_dl/extractor/playfm.py
index 72df4d842..ebc046804 100644
--- a/youtube_dl/extractor/playfm.py
+++ b/youtube_dl/extractor/playfm.py
@@ -10,6 +10,7 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     int_or_none,
+    str_to_int,
 )
 
 
@@ -29,6 +30,7 @@ class PlayFMIE(InfoExtractor):
             'duration': 5627.428,
             'upload_date': '20140712',
             'view_count': int,
+            'comment_count': int,
             'thumbnail': 're:^https?://.*\.jpg$',
         },
     }
@@ -51,7 +53,8 @@ class PlayFMIE(InfoExtractor):
 
         recording = rec_doc.find('./recording')
         title = recording.find('./title').text
-        view_count = int_or_none(recording.find('./stats/playcount').text)
+        view_count = str_to_int(recording.find('./stats/playcount').text)
+        comment_count = str_to_int(recording.find('./stats/comments').text)
         duration = float_or_none(recording.find('./duration').text, scale=1000)
         thumbnail = recording.find('./image').text
 
@@ -75,6 +78,7 @@ class PlayFMIE(InfoExtractor):
             'title': title,
             'upload_date': upload_date,
             'view_count': view_count,
+            'comment_count': comment_count,
             'duration': duration,
             'thumbnail': thumbnail,
             'uploader': uploader,

From 63cddb6477e785ca2bfb6e3bb1ac2af20aa9842c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 22 Sep 2014 14:11:08 +0200
Subject: [PATCH 0042/1937] [sbs] Recognize urls with format
 'http://www.sbs.com.au/ondemand/video/<id>' (#3811)

---
 youtube_dl/extractor/sbs.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index 214990e7a..409f8540a 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class SBSIE(InfoExtractor):
     IE_DESC = 'sbs.com.au'
-    _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/ondemand/video/single/(?P<id>[0-9]+)/'
+    _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/ondemand/video/(?:single/)?(?P<id>[0-9]+)'
 
     _TESTS = [{
         # Original URL is handled by the generic IE which finds the iframe:
@@ -27,6 +27,10 @@ class SBSIE(InfoExtractor):
             'thumbnail': 're:http://.*\.jpg',
         },
         'add_ies': ['generic'],
+    },
+    {
+        'url': 'http://www.sbs.com.au/ondemand/video/320403011771/Dingo-Conservation-The-Feed',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 094d42fe443c8f7ad5bd9049d63317195ab8fd3a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 22 Sep 2014 18:15:07 +0200
Subject: [PATCH 0043/1937] release 2014.09.22.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4bf208b67..2853c79c9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.09.22'
+__version__ = '2014.09.22.1'

From 632e5684ce797eb8a7372eb25dd4ce299f2e66de Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 23 Sep 2014 00:28:19 +0300
Subject: [PATCH 0044/1937] [nfl] Add new extractor. (Closes #3815)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/nfl.py      | 103 +++++++++++++++++++++++++++++++
 2 files changed, 104 insertions(+)
 create mode 100644 youtube_dl/extractor/nfl.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 244d22297..1f1fc0eb2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -240,6 +240,7 @@ from .ndtv import NDTVIE
 from .newgrounds import NewgroundsIE
 from .newstube import NewstubeIE
 from .nfb import NFBIE
+from .nfl import NFLIE
 from .nhl import NHLIE, NHLVideocenterIE
 from .niconico import NiconicoIE
 from .ninegag import NineGagIE
diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
new file mode 100644
index 000000000..f53596f5e
--- /dev/null
+++ b/youtube_dl/extractor/nfl.py
@@ -0,0 +1,103 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    remove_end,
+)
+
+
+class NFLIE(InfoExtractor):
+    IE_NAME = 'nfl.com'
+    _VALID_URL = r'(?x)https?://(?:www\.)?nfl\.com/(?:videos/(?:.+)/|.*?\#video=)(?P<id>\d..[0-9]+)'
+    _PLAYER_CONFIG_URL = 'http://www.nfl.com/static/content/static/config/video/config.json'
+    _TEST = {
+        'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights',
+        'skip_download': True,  # md5 sum fluctuates
+        'info_dict': {
+            'id': '0ap3000000398478',
+            'ext': 'mp4',
+            'title': 'Week 3: Washington Redskins vs. Philadelphia Eagles highlights',
+            'description': 'md5:56323bfb0ac4ee5ab24bd05fdf3bf478',
+            'upload_date': '20140921',
+            'timestamp': 1411337580,
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        config = self._download_json(self._PLAYER_CONFIG_URL, video_id,
+                                     note='Downloading player config')
+        url_template = 'http://nfl.com{contentURLTemplate:s}'.format(**config)
+        video_data = self._download_json(url_template.format(id=video_id), video_id)
+
+        cdns = config.get('cdns')
+        if not cdns:
+            raise ExtractorError('Failed to get CDN data', expected=True)
+
+        formats = []
+        streams = video_data.get('cdnData', {}).get('bitrateInfo', [])
+        for name, cdn in cdns.items():
+            # LimeLight streams don't seem to work
+            if cdn.get('name') == 'LIMELIGHT':
+                continue
+
+            protocol = cdn.get('protocol')
+            host = remove_end(cdn.get('host', ''), '/')
+            if not (protocol and host):
+                continue
+
+            path_prefix = cdn.get('pathprefix', '')
+            if path_prefix and not path_prefix.endswith('/'):
+                path_prefix = '%s/' % path_prefix
+
+            get_url = lambda p: '{protocol:s}://{host:s}/{prefix:s}{path:}'.format(
+                protocol=protocol,
+                host=host,
+                prefix=path_prefix,
+                path=p,
+            )
+
+            if protocol == 'rtmp':
+                preference = -2
+            elif 'prog' in name.lower():
+                preference = -1
+            else:
+                preference = 0
+
+            for stream in streams:
+                path = stream.get('path')
+                if not path:
+                    continue
+
+                formats.append({
+                    'url': get_url(path),
+                    'vbr': int_or_none(stream.get('rate', 0), 1000),
+                    'preference': preference,
+                    'format_note': name,
+                })
+
+        self._sort_formats(formats)
+
+        thumbnail = None
+        for q in ('xl', 'l', 'm', 's', 'xs'):
+            thumbnail = video_data.get('imagePaths', {}).get(q)
+            if thumbnail:
+                break
+
+        return {
+            'id': video_id,
+            'title': video_data.get('storyHeadline'),
+            'formats': formats,
+            'description': video_data.get('caption'),
+            'duration': video_data.get('duration'),
+            'thumbnail': thumbnail,
+            'timestamp': int_or_none(video_data.get('posted'), 1000),
+        }

From f7d159cf953bd1884ca45f535327f3016998270c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 Sep 2014 19:13:11 +0700
Subject: [PATCH 0045/1937] [noco] Encode before passing to hashlib.md5 (Closes
 #3816)

---
 youtube_dl/extractor/noco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index e3ec9ed15..7f1bc6377 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -62,7 +62,7 @@ class NocoIE(InfoExtractor):
 
     def _call_api(self, path, video_id, note):
         ts = compat_str(int(time.time() * 1000))
-        tk = hashlib.md5(hashlib.md5(ts).hexdigest() + '#8S?uCraTedap6a').hexdigest()
+        tk = hashlib.md5((hashlib.md5(ts.encode('ascii')).hexdigest() + '#8S?uCraTedap6a').encode('ascii')).hexdigest()
         url = self._API_URL_TEMPLATE % (path, ts, tk)
 
         resp = self._download_json(url, video_id, note)

From 86916dae4b8604431205d11ccfa5f9796c0798dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 Sep 2014 19:58:35 +0700
Subject: [PATCH 0046/1937] [wat] Capture and output error message

---
 youtube_dl/extractor/wat.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py
index 46b4d9133..268e2f618 100644
--- a/youtube_dl/extractor/wat.py
+++ b/youtube_dl/extractor/wat.py
@@ -5,7 +5,10 @@ import re
 import hashlib
 
 from .common import InfoExtractor
-from ..utils import unified_strdate
+from ..utils import (
+    ExtractorError,
+    unified_strdate,
+)
 
 
 class WatIE(InfoExtractor):
@@ -57,6 +60,11 @@ class WatIE(InfoExtractor):
 
         video_info = self.download_video_info(real_id)
 
+        error_desc = video_info.get('error_desc')
+        if error_desc:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error_desc), expected=True)
+
         geo_list = video_info.get('geoList')
         country = geo_list[0] if geo_list else ''
 

From bd5650ac64fedd1c1ad7b90c4ec4ff5d4c053bc0 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 23 Sep 2014 20:42:28 +0300
Subject: [PATCH 0047/1937] [nfl] Fix test case - download, but don't check md5

---
 youtube_dl/extractor/nfl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index f53596f5e..963c4587c 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -17,7 +17,7 @@ class NFLIE(InfoExtractor):
     _PLAYER_CONFIG_URL = 'http://www.nfl.com/static/content/static/config/video/config.json'
     _TEST = {
         'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights',
-        'skip_download': True,  # md5 sum fluctuates
+        # 'md5': '5eb8c40a727dda106d510e5d6ffa79e5',  # md5 checksum fluctuates
         'info_dict': {
             'id': '0ap3000000398478',
             'ext': 'mp4',

From 4bc3a23ec5d2c1bbcdc5289393881606604922c7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 24 Sep 2014 09:49:53 +0200
Subject: [PATCH 0048/1937] [youtube] Modernize

---
 youtube_dl/extractor/youtube.py | 103 +++++++++++++++++---------------
 1 file changed, 54 insertions(+), 49 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b54c69122..602be9859 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -307,69 +307,74 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
     IE_NAME = 'youtube'
     _TESTS = [
         {
-            u"url":  u"http://www.youtube.com/watch?v=BaW_jenozKc",
-            u"file":  u"BaW_jenozKc.mp4",
-            u"info_dict": {
-                u"title": u"youtube-dl test video \"'/\\ä↭𝕐",
-                u"uploader": u"Philipp Hagemeister",
-                u"uploader_id": u"phihag",
-                u"upload_date": u"20121002",
-                u"description": u"test chars:  \"'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .",
-                u"categories": [u'Science & Technology'],
+            'url': 'http://www.youtube.com/watch?v=BaW_jenozKc',
+            'info_dict': {
+                'id': 'BaW_jenozKc',
+                'ext': 'mp4',
+                'title': 'youtube-dl test video "\'/\\ä↭𝕐',
+                'uploader': 'Philipp Hagemeister',
+                'uploader_id': 'phihag',
+                'upload_date': '20121002',
+                'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
+                'categories': ['Science & Technology'],
                 'like_count': int,
                 'dislike_count': int,
             }
         },
         {
-            u"url":  u"http://www.youtube.com/watch?v=UxxajLWwzqY",
-            u"file":  u"UxxajLWwzqY.mp4",
-            u"note": u"Test generic use_cipher_signature video (#897)",
-            u"info_dict": {
-                u"upload_date": u"20120506",
-                u"title": u"Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]",
-                u"description": u"md5:fea86fda2d5a5784273df5c7cc994d9f",
-                u"uploader": u"Icona Pop",
-                u"uploader_id": u"IconaPop"
+            'url': 'http://www.youtube.com/watch?v=UxxajLWwzqY',
+            'note': 'Test generic use_cipher_signature video (#897)',
+            'info_dict': {
+                'id': 'UxxajLWwzqY',
+                'ext': 'mp4',
+                'upload_date': '20120506',
+                'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
+                'description': 'md5:fea86fda2d5a5784273df5c7cc994d9f',
+                'uploader': 'Icona Pop',
+                'uploader_id': 'IconaPop',
             }
         },
         {
-            u"url":  u"https://www.youtube.com/watch?v=07FYdnEawAQ",
-            u"file":  u"07FYdnEawAQ.mp4",
-            u"note": u"Test VEVO video with age protection (#956)",
-            u"info_dict": {
-                u"upload_date": u"20130703",
-                u"title": u"Justin Timberlake - Tunnel Vision (Explicit)",
-                u"description": u"md5:64249768eec3bc4276236606ea996373",
-                u"uploader": u"justintimberlakeVEVO",
-                u"uploader_id": u"justintimberlakeVEVO"
+            'url': 'https://www.youtube.com/watch?v=07FYdnEawAQ',
+            'note': 'Test VEVO video with age protection (#956)',
+            'info_dict': {
+                'id': '07FYdnEawAQ',
+                'ext': 'mp4',
+                'upload_date': '20130703',
+                'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
+                'description': 'md5:64249768eec3bc4276236606ea996373',
+                'uploader': 'justintimberlakeVEVO',
+                'uploader_id': 'justintimberlakeVEVO',
             }
         },
         {
-            u"url":  u"//www.YouTube.com/watch?v=yZIXLfi8CZQ",
-            u"file":  u"yZIXLfi8CZQ.mp4",
-            u"note": u"Embed-only video (#1746)",
-            u"info_dict": {
-                u"upload_date": u"20120608",
-                u"title": u"Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012",
-                u"description": u"md5:09b78bd971f1e3e289601dfba15ca4f7",
-                u"uploader": u"SET India",
-                u"uploader_id": u"setindia"
+            'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
+            'note': 'Embed-only video (#1746)',
+            'info_dict': {
+                'id': 'yZIXLfi8CZQ',
+                'ext': 'mp4',
+                'upload_date': '20120608',
+                'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
+                'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
+                'uploader': 'SET India',
+                'uploader_id': 'setindia'
             }
         },
         {
-            u"url": u"http://www.youtube.com/watch?v=a9LDPn-MO4I",
-            u"file": u"a9LDPn-MO4I.m4a",
-            u"note": u"256k DASH audio (format 141) via DASH manifest",
-            u"info_dict": {
-                u"upload_date": "20121002",
-                u"uploader_id": "8KVIDEO",
-                u"description": '',
-                u"uploader": "8KVIDEO",
-                u"title": "UHDTV TEST 8K VIDEO.mp4"
+            'url': 'http://www.youtube.com/watch?v=a9LDPn-MO4I',
+            'note': '256k DASH audio (format 141) via DASH manifest',
+            'info_dict': {
+                'id': 'a9LDPn-MO4I',
+                'ext': 'm4a',
+                'upload_date': '20121002',
+                'uploader_id': '8KVIDEO',
+                'description': '',
+                'uploader': '8KVIDEO',
+                'title': 'UHDTV TEST 8K VIDEO.mp4'
             },
-            u"params": {
-                u"youtube_include_dash_manifest": True,
-                u"format": "141",
+            'params': {
+                'youtube_include_dash_manifest': True,
+                'format': '141',
             },
         },
         # DASH manifest with encrypted signature
@@ -384,7 +389,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'uploader_id': 'AfrojackVEVO',
                 'upload_date': '20131011',
             },
-            u"params": {
+            'params': {
                 'youtube_include_dash_manifest': True,
                 'format': '141',
             },

From 69ea8ca42cd4fc62fdd4e7f18defb3b23da618d2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 24 Sep 2014 09:51:45 +0200
Subject: [PATCH 0049/1937] [youtube] Remove superfluous unicode specifiers

---
 youtube_dl/extractor/youtube.py | 136 ++++++++++++++++----------------
 1 file changed, 68 insertions(+), 68 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 602be9859..0257ee2f9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -46,7 +46,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     def _set_language(self):
         return bool(self._download_webpage(
             self._LANG_URL, None,
-            note=u'Setting language', errnote='unable to set language',
+            note='Setting language', errnote='unable to set language',
             fatal=False))
 
     def _login(self):
@@ -61,13 +61,13 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         # No authentication to be performed
         if username is None:
             if self._LOGIN_REQUIRED:
-                raise ExtractorError(u'No login info available, needed for using %s.' % self.IE_NAME, expected=True)
+                raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
             return True
 
         login_page = self._download_webpage(
             self._LOGIN_URL, None,
-            note=u'Downloading login page',
-            errnote=u'unable to fetch login page', fatal=False)
+            note='Downloading login page',
+            errnote='unable to fetch login page', fatal=False)
         if login_page is False:
             return
 
@@ -105,12 +105,12 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         req = compat_urllib_request.Request(self._LOGIN_URL, login_data)
         login_results = self._download_webpage(
             req, None,
-            note=u'Logging in', errnote=u'unable to log in', fatal=False)
+            note='Logging in', errnote='unable to log in', fatal=False)
         if login_results is False:
             return False
 
         if re.search(r'id="errormsg_0_Passwd"', login_results) is not None:
-            raise ExtractorError(u'Please use your account password and a two-factor code instead of an application-specific password.', expected=True)
+            raise ExtractorError('Please use your account password and a two-factor code instead of an application-specific password.', expected=True)
 
         # Two-Factor
         # TODO add SMS and phone call support - these require making a request and then prompting the user
@@ -119,19 +119,19 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             tfa_code = self._get_tfa_info()
 
             if tfa_code is None:
-                self._downloader.report_warning(u'Two-factor authentication required. Provide it with --twofactor <code>')
-                self._downloader.report_warning(u'(Note that only TOTP (Google Authenticator App) codes work at this time.)')
+                self._downloader.report_warning('Two-factor authentication required. Provide it with --twofactor <code>')
+                self._downloader.report_warning('(Note that only TOTP (Google Authenticator App) codes work at this time.)')
                 return False
 
             # Unlike the first login form, secTok and timeStmp are both required for the TFA form
 
             match = re.search(r'id="secTok"\n\s+value=\'(.+)\'/>', login_results, re.M | re.U)
             if match is None:
-                self._downloader.report_warning(u'Failed to get secTok - did the page structure change?')
+                self._downloader.report_warning('Failed to get secTok - did the page structure change?')
             secTok = match.group(1)
             match = re.search(r'id="timeStmp"\n\s+value=\'(.+)\'/>', login_results, re.M | re.U)
             if match is None:
-                self._downloader.report_warning(u'Failed to get timeStmp - did the page structure change?')
+                self._downloader.report_warning('Failed to get timeStmp - did the page structure change?')
             timeStmp = match.group(1)
 
             tfa_form_strs = {
@@ -155,23 +155,23 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             tfa_req = compat_urllib_request.Request(self._TWOFACTOR_URL, tfa_data)
             tfa_results = self._download_webpage(
                 tfa_req, None,
-                note=u'Submitting TFA code', errnote=u'unable to submit tfa', fatal=False)
+                note='Submitting TFA code', errnote='unable to submit tfa', fatal=False)
 
             if tfa_results is False:
                 return False
 
             if re.search(r'(?i)<form[^>]* id="gaia_secondfactorform"', tfa_results) is not None:
-                self._downloader.report_warning(u'Two-factor code expired. Please try again, or use a one-use backup code instead.')
+                self._downloader.report_warning('Two-factor code expired. Please try again, or use a one-use backup code instead.')
                 return False
             if re.search(r'(?i)<form[^>]* id="gaia_loginform"', tfa_results) is not None:
-                self._downloader.report_warning(u'unable to log in - did the page structure change?')
+                self._downloader.report_warning('unable to log in - did the page structure change?')
                 return False
             if re.search(r'smsauth-interstitial-reviewsettings', tfa_results) is not None:
-                self._downloader.report_warning(u'Your Google account has a security notice. Please log in on your web browser, resolve the notice, and try again.')
+                self._downloader.report_warning('Your Google account has a security notice. Please log in on your web browser, resolve the notice, and try again.')
                 return False
 
         if re.search(r'(?i)<form[^>]* id="gaia_loginform"', login_results) is not None:
-            self._downloader.report_warning(u'unable to log in: bad username or password')
+            self._downloader.report_warning('unable to log in: bad username or password')
             return False
         return True
 
@@ -185,7 +185,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
         self._download_webpage(
             req, None,
-            note=u'Confirming age', errnote=u'Unable to confirm age')
+            note='Confirming age', errnote='Unable to confirm age')
         return True
 
     def _real_initialize(self):
@@ -402,19 +402,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
     def report_video_info_webpage_download(self, video_id):
         """Report attempt to download video info webpage."""
-        self.to_screen(u'%s: Downloading video info webpage' % video_id)
+        self.to_screen('%s: Downloading video info webpage' % video_id)
 
     def report_information_extraction(self, video_id):
         """Report attempt to extract video information."""
-        self.to_screen(u'%s: Extracting video information' % video_id)
+        self.to_screen('%s: Extracting video information' % video_id)
 
     def report_unavailable_format(self, video_id, format):
         """Report extracted video URL."""
-        self.to_screen(u'%s: Format %s not available' % (video_id, format))
+        self.to_screen('%s: Format %s not available' % (video_id, format))
 
     def report_rtmp_download(self):
         """Indicate the download will use the RTMP protocol."""
-        self.to_screen(u'RTMP download detected')
+        self.to_screen('RTMP download detected')
 
     def _signature_cache_id(self, example_sig):
         """ Return a string representation of a signature """
@@ -434,21 +434,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             player_type, player_id, self._signature_cache_id(example_sig))
         assert os.path.basename(func_id) == func_id
 
-        cache_spec = self._downloader.cache.load(u'youtube-sigfuncs', func_id)
+        cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
         if cache_spec is not None:
             return lambda s: ''.join(s[i] for i in cache_spec)
 
         if player_type == 'js':
             code = self._download_webpage(
                 player_url, video_id,
-                note=u'Downloading %s player %s' % (player_type, player_id),
-                errnote=u'Download of %s failed' % player_url)
+                note='Downloading %s player %s' % (player_type, player_id),
+                errnote='Download of %s failed' % player_url)
             res = self._parse_sig_js(code)
         elif player_type == 'swf':
             urlh = self._request_webpage(
                 player_url, video_id,
-                note=u'Downloading %s player %s' % (player_type, player_id),
-                errnote=u'Download of %s failed' % player_url)
+                note='Downloading %s player %s' % (player_type, player_id),
+                errnote='Download of %s failed' % player_url)
             code = urlh.read()
             res = self._parse_sig_swf(code)
         else:
@@ -459,15 +459,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             cache_res = res(test_string)
             cache_spec = [ord(c) for c in cache_res]
 
-        self._downloader.cache.store(u'youtube-sigfuncs', func_id, cache_spec)
+        self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
         return res
 
     def _print_sig_code(self, func, example_sig):
         def gen_sig_code(idxs):
             def _genslice(start, end, step):
                 starts = '' if start == 0 else str(start)
-                ends = (u':%d' % (end+step)) if end + step >= 0 else ':'
-                steps = '' if step == 1 else (u':%d' % step)
+                ends = (':%d' % (end+step)) if end + step >= 0 else ':'
+                steps = '' if step == 1 else (':%d' % step)
                 return 's[%s%s%s]' % (starts, ends, steps)
 
             step = None
@@ -497,9 +497,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         expr_code = ' + '.join(gen_sig_code(cache_spec))
         signature_id_tuple = '(%s)' % (
             ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
-        code = (u'if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
+        code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
                 '    return %s\n') % (signature_id_tuple, expr_code)
-        self.to_screen(u'Extracted signature function:\n' + code)
+        self.to_screen('Extracted signature function:\n' + code)
 
     def _parse_sig_js(self, jscode):
         funcname = self._search_regex(
@@ -521,9 +521,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         """Turn the encrypted s field into a working signature"""
 
         if player_url is None:
-            raise ExtractorError(u'Cannot decrypt signature without player_url')
+            raise ExtractorError('Cannot decrypt signature without player_url')
 
-        if player_url.startswith(u'//'):
+        if player_url.startswith('//'):
             player_url = 'https:' + player_url
         try:
             player_id = (player_url, self._signature_cache_id(s))
@@ -547,7 +547,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
                 video_id, note=False)
         except ExtractorError as err:
-            self._downloader.report_warning(u'unable to download video subtitles: %s' % compat_str(err))
+            self._downloader.report_warning('unable to download video subtitles: %s' % compat_str(err))
             return {}
         lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list)
 
@@ -565,7 +565,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             url = 'https://www.youtube.com/api/timedtext?' + params
             sub_lang_list[lang] = url
         if not sub_lang_list:
-            self._downloader.report_warning(u'video doesn\'t have subtitles')
+            self._downloader.report_warning('video doesn\'t have subtitles')
             return {}
         return sub_lang_list
 
@@ -573,7 +573,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         """We need the webpage for getting the captions url, pass it as an
            argument to speed up the process."""
         sub_format = self._downloader.params.get('subtitlesformat', 'srt')
-        self.to_screen(u'%s: Looking for automatic captions' % video_id)
+        self.to_screen('%s: Looking for automatic captions' % video_id)
         mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
         err_msg = 'Couldn\'t find automatic captions for %s' % video_id
         if mobj is None:
@@ -594,7 +594,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             caption_list = self._download_xml(list_url, video_id)
             original_lang_node = caption_list.find('track')
             if original_lang_node is None or original_lang_node.attrib.get('kind') != 'asr' :
-                self._downloader.report_warning(u'Video doesn\'t have automatic captions')
+                self._downloader.report_warning('Video doesn\'t have automatic captions')
                 return {}
             original_lang = original_lang_node.attrib['lang_code']
 
@@ -620,7 +620,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
     def extract_id(cls, url):
         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
         if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
+            raise ExtractorError('Invalid URL: %s' % url)
         video_id = mobj.group(2)
         return video_id
 
@@ -640,7 +640,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
     def _extract_annotations(self, video_id):
         url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
-        return self._download_webpage(url, video_id, note=u'Searching for annotations.', errnote=u'Unable to download video annotations.')
+        return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
 
     def _real_extract(self, url):
         proto = (
@@ -710,14 +710,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
         # Check for "rental" videos
         if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
-            raise ExtractorError(u'"rental" videos not supported')
+            raise ExtractorError('"rental" videos not supported')
 
         # Start extracting information
         self.report_information_extraction(video_id)
 
         # uploader
         if 'author' not in video_info:
-            raise ExtractorError(u'Unable to extract uploader name')
+            raise ExtractorError('Unable to extract uploader name')
         video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
 
         # uploader_id
@@ -726,13 +726,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         if mobj is not None:
             video_uploader_id = mobj.group(1)
         else:
-            self._downloader.report_warning(u'unable to extract uploader nickname')
+            self._downloader.report_warning('unable to extract uploader nickname')
 
         # title
         if 'title' in video_info:
             video_title = video_info['title'][0]
         else:
-            self._downloader.report_warning(u'Unable to extract video title')
+            self._downloader.report_warning('Unable to extract video title')
             video_title = '_'
 
         # thumbnail image
@@ -742,7 +742,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         if m_thumb is not None:
             video_thumbnail = m_thumb.group(1)
         elif 'thumbnail_url' not in video_info:
-            self._downloader.report_warning(u'unable to extract video thumbnail')
+            self._downloader.report_warning('unable to extract video thumbnail')
             video_thumbnail = None
         else:   # don't panic if we can't find it
             video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
@@ -796,8 +796,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             if count is not None:
                 return int(count.replace(',', ''))
             return None
-        like_count = _extract_count(u'like')
-        dislike_count = _extract_count(u'dislike')
+        like_count = _extract_count('like')
+        dislike_count = _extract_count('dislike')
 
         # subtitles
         video_subtitles = self.extract_subtitles(video_id, video_webpage)
@@ -807,7 +807,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             return
 
         if 'length_seconds' not in video_info:
-            self._downloader.report_warning(u'unable to extract video duration')
+            self._downloader.report_warning('unable to extract video duration')
             video_duration = None
         else:
             video_duration = int(compat_urllib_parse.unquote_plus(video_info['length_seconds'][0]))
@@ -828,11 +828,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             # Easy way to know if the 's' value is in url_encoded_fmt_stream_map
             # this signatures are encrypted
             if 'url_encoded_fmt_stream_map' not in args:
-                raise ValueError(u'No stream_map present')  # caught below
+                raise ValueError('No stream_map present')  # caught below
             re_signature = re.compile(r'[&,]s=')
             m_s = re_signature.search(args['url_encoded_fmt_stream_map'])
             if m_s is not None:
-                self.to_screen(u'%s: Encrypted signatures detected.' % video_id)
+                self.to_screen('%s: Encrypted signatures detected.' % video_id)
                 video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']]
             m_s = re_signature.search(args.get('adaptive_fmts', ''))
             if m_s is not None:
@@ -910,7 +910,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                                 player_desc = 'html5 player %s' % player_version
 
                         parts_sizes = self._signature_cache_id(encrypted_sig)
-                        self.to_screen(u'{%s} signature length %s, %s' %
+                        self.to_screen('{%s} signature length %s, %s' %
                             (format_id, parts_sizes, player_desc))
 
                     signature = self._decrypt_signature(
@@ -925,7 +925,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             url_map = self._extract_from_m3u8(manifest_url, video_id)
             formats = _map_to_format_list(url_map)
         else:
-            raise ExtractorError(u'no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
+            raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
 
         # Look for the DASH manifest
         if (self._downloader.params.get('youtube_include_dash_manifest', False)):
@@ -946,9 +946,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 dash_manifest_url = re.sub(r'/s/([\w\.]+)', decrypt_sig, dash_manifest_url)
                 dash_doc = self._download_xml(
                     dash_manifest_url, video_id,
-                    note=u'Downloading DASH manifest',
-                    errnote=u'Could not download DASH manifest')
-                for r in dash_doc.findall(u'.//{urn:mpeg:DASH:schema:MPD:2011}Representation'):
+                    note='Downloading DASH manifest',
+                    errnote='Could not download DASH manifest')
+                for r in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}Representation'):
                     url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
                     if url_el is None:
                         continue
@@ -974,7 +974,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                         existing_format.update(f)
 
             except (ExtractorError, KeyError) as e:
-                self.report_warning(u'Skipping DASH manifest: %s' % e, video_id)
+                self.report_warning('Skipping DASH manifest: %s' % e, video_id)
 
         self._sort_formats(formats)
 
@@ -1095,7 +1095,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         # Extract playlist id
         mobj = re.match(self._VALID_URL, url)
         if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
+            raise ExtractorError('Invalid URL: %s' % url)
         playlist_id = mobj.group(1) or mobj.group(2)
 
         # Check if it's a video-specific URL
@@ -1103,16 +1103,16 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         if 'v' in query_dict:
             video_id = query_dict['v'][0]
             if self._downloader.params.get('noplaylist'):
-                self.to_screen(u'Downloading just video %s because of --no-playlist' % video_id)
+                self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
                 return self.url_result(video_id, 'Youtube', video_id=video_id)
             else:
-                self.to_screen(u'Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
+                self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
 
         if playlist_id.startswith('RD'):
             # Mixes require a custom extraction process
             return self._extract_mix(playlist_id)
         if playlist_id.startswith('TL'):
-            raise ExtractorError(u'For downloading YouTube.com top lists, use '
+            raise ExtractorError('For downloading YouTube.com top lists, use '
                 'the "yttoplist" keyword, for example "youtube-dl \'yttoplist:music:Top Tracks\'"', expected=True)
 
         url = self._TEMPLATE_URL % playlist_id
@@ -1157,7 +1157,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
 
 class YoutubeTopListIE(YoutubePlaylistIE):
     IE_NAME = 'youtube:toplist'
-    IE_DESC = (u'YouTube.com top lists, "yttoplist:{channel}:{list title}"'
+    IE_DESC = ('YouTube.com top lists, "yttoplist:{channel}:{list title}"'
         ' (Example: "yttoplist:music:Top Tracks")')
     _VALID_URL = r'yttoplist:(?P<chann>.*?):(?P<title>.*?)$'
     _TESTS = []
@@ -1207,7 +1207,7 @@ class YoutubeChannelIE(InfoExtractor):
         # Extract channel id
         mobj = re.match(self._VALID_URL, url)
         if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
+            raise ExtractorError('Invalid URL: %s' % url)
 
         # Download channel page
         channel_id = mobj.group(1)
@@ -1229,7 +1229,7 @@ class YoutubeChannelIE(InfoExtractor):
             for pagenum in itertools.count(1):
                 url = self._MORE_PAGES_URL % (pagenum, channel_id)
                 page = self._download_json(
-                    url, channel_id, note=u'Downloading page #%s' % pagenum,
+                    url, channel_id, note='Downloading page #%s' % pagenum,
                     transform_source=uppercase_escape)
 
                 ids_in_page = self.extract_videos_from_page(page['content_html'])
@@ -1238,7 +1238,7 @@ class YoutubeChannelIE(InfoExtractor):
                 if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
                     break
 
-        self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
+        self._downloader.to_screen('[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
 
         url_entries = [self.url_result(video_id, 'Youtube', video_id=video_id)
                        for video_id in video_ids]
@@ -1265,7 +1265,7 @@ class YoutubeUserIE(InfoExtractor):
         # Extract username
         mobj = re.match(self._VALID_URL, url)
         if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
+            raise ExtractorError('Invalid URL: %s' % url)
 
         username = mobj.group(1)
 
@@ -1286,7 +1286,7 @@ class YoutubeUserIE(InfoExtractor):
             try:
                 response = json.loads(page)
             except ValueError as err:
-                raise ExtractorError(u'Invalid JSON in API response: ' + compat_str(err))
+                raise ExtractorError('Invalid JSON in API response: ' + compat_str(err))
             if 'entry' not in response['feed']:
                 return
 
@@ -1327,9 +1327,9 @@ class YoutubeSearchIE(SearchInfoExtractor):
                 compat_urllib_parse.quote_plus(query.encode('utf-8')),
                 (PAGE_SIZE * pagenum) + 1)
             data_json = self._download_webpage(
-                result_url, video_id=u'query "%s"' % query,
-                note=u'Downloading page %s' % (pagenum + 1),
-                errnote=u'Unable to download API page')
+                result_url, video_id='query "%s"' % query,
+                note='Downloading page %s' % (pagenum + 1),
+                errnote='Unable to download API page')
             data = json.loads(data_json)
             api_response = data['data']
 
@@ -1404,7 +1404,7 @@ class YoutubeShowIE(InfoExtractor):
         webpage = self._download_webpage(url, show_name, 'Downloading show webpage')
         # There's one playlist for each season of the show
         m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
-        self.to_screen(u'%s: Found %s seasons' % (show_name, len(m_seasons)))
+        self.to_screen('%s: Found %s seasons' % (show_name, len(m_seasons)))
         return [self.url_result('https://www.youtube.com' + season.group(1), 'YoutubePlaylist') for season in m_seasons]
 
 

From cdc628a498b8f2198d057ba1ba78e86d8915e3aa Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 24 Sep 2014 10:25:47 +0200
Subject: [PATCH 0050/1937] [youtube] Move more tests to extractors

---
 test/test_youtube_lists.py      | 39 ------------------
 youtube_dl/extractor/youtube.py | 71 ++++++++++++++++++++++++++++-----
 2 files changed, 62 insertions(+), 48 deletions(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index 1fa99f88b..410f9edc2 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -10,7 +10,6 @@ from test.helper import FakeYDL
 
 
 from youtube_dl.extractor import (
-    YoutubeUserIE,
     YoutubePlaylistIE,
     YoutubeIE,
     YoutubeChannelIE,
@@ -43,28 +42,6 @@ class TestYoutubeLists(unittest.TestCase):
         self.assertEqual(len(entries), 25)
         self.assertEqual(YoutubeIE().extract_id(entries[-1]['url']), 'rYefUsYuEp0')
 
-    def test_youtube_channel(self):
-        dl = FakeYDL()
-        ie = YoutubeChannelIE(dl)
-        #test paginated channel
-        result = ie.extract('https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w')
-        self.assertTrue(len(result['entries']) > 90)
-        #test autogenerated channel
-        result = ie.extract('https://www.youtube.com/channel/HCtnHdj3df7iM/videos')
-        self.assertTrue(len(result['entries']) >= 18)
-
-    def test_youtube_user(self):
-        dl = FakeYDL()
-        ie = YoutubeUserIE(dl)
-        result = ie.extract('https://www.youtube.com/user/TheLinuxFoundation')
-        self.assertTrue(len(result['entries']) >= 320)
-
-    def test_youtube_show(self):
-        dl = FakeYDL()
-        ie = YoutubeShowIE(dl)
-        result = ie.extract('http://www.youtube.com/show/airdisasters')
-        self.assertTrue(len(result) >= 3)
-
     def test_youtube_mix(self):
         dl = FakeYDL()
         ie = YoutubePlaylistIE(dl)
@@ -83,21 +60,5 @@ class TestYoutubeLists(unittest.TestCase):
         entries = result['entries']
         self.assertEqual(len(entries), 100)
 
-    def test_youtube_toplist(self):
-        dl = FakeYDL()
-        ie = YoutubeTopListIE(dl)
-        result = ie.extract('yttoplist:music:Trending')
-        entries = result['entries']
-        self.assertTrue(len(entries) >= 5)
-
-    def test_youtube_search_url(self):
-        dl = FakeYDL()
-        ie = YoutubeSearchURLIE(dl)
-        result = ie.extract('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video')
-        entries = result['entries']
-        self.assertIsPlaylist(result)
-        self.assertEqual(result['title'], 'youtube-dl test video')
-        self.assertTrue(len(entries) >= 5)
-
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0257ee2f9..2ef76b69b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1160,16 +1160,25 @@ class YoutubeTopListIE(YoutubePlaylistIE):
     IE_DESC = ('YouTube.com top lists, "yttoplist:{channel}:{list title}"'
         ' (Example: "yttoplist:music:Top Tracks")')
     _VALID_URL = r'yttoplist:(?P<chann>.*?):(?P<title>.*?)$'
-    _TESTS = []
+    _TESTS = [{
+        'url': 'yttoplist:music:Trending',
+        'playlist_mincount': 5,
+        'skip': 'Only works for logged-in users',
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         channel = mobj.group('chann')
         title = mobj.group('title')
         query = compat_urllib_parse.urlencode({'title': title})
-        playlist_re = 'href="([^"]+?%s.*?)"' % re.escape(query)
-        channel_page = self._download_webpage('https://www.youtube.com/%s' % channel, title)
-        link = self._html_search_regex(playlist_re, channel_page, 'list')
+        channel_page = self._download_webpage(
+            'https://www.youtube.com/%s' % channel, title)
+        link = self._html_search_regex(
+            r'''(?x)
+                <a\s+href="([^"]+)".*?>\s*
+                <span\s+class="branded-page-module-title-text">\s*
+                <span[^>]*>.*?%s.*?</span>''' % re.escape(query),
+            channel_page, 'list')
         url = compat_urlparse.urljoin('https://www.youtube.com/', link)
         
         video_re = r'data-index="\d+".*?data-video-id="([0-9A-Za-z_-]{11})"'
@@ -1195,6 +1204,11 @@ class YoutubeChannelIE(InfoExtractor):
     _MORE_PAGES_INDICATOR = 'yt-uix-load-more'
     _MORE_PAGES_URL = 'https://www.youtube.com/c4_browse_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
     IE_NAME = 'youtube:channel'
+    _TESTS = [{
+        'note': 'paginated channel',
+        'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
+        'playlist_mincount': 91,
+    }]
 
     def extract_videos_from_page(self, page):
         ids_in_page = []
@@ -1253,6 +1267,17 @@ class YoutubeUserIE(InfoExtractor):
     _GDATA_URL = 'https://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d&alt=json'
     IE_NAME = 'youtube:user'
 
+    _TESTS = [{
+        'url': 'https://www.youtube.com/user/TheLinuxFoundation',
+        'playlist_mincount': 320,
+        'info_dict': {
+            'title': 'TheLinuxFoundation',
+        }
+    }, {
+        'url': 'ytuser:phihag',
+        'only_matching': True,
+    }]
+
     @classmethod
     def suitable(cls, url):
         # Don't return True if the url can be extracted with other youtube
@@ -1361,6 +1386,13 @@ class YoutubeSearchURLIE(InfoExtractor):
     IE_DESC = 'YouTube.com search URLs'
     IE_NAME = 'youtube:search_url'
     _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?search_query=(?P<query>[^&]+)(?:[&]|$)'
+    _TESTS = [{
+        'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
+        'playlist_mincount': 5,
+        'info_dict': {
+            'title': 'youtube-dl test video',
+        }
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -1395,17 +1427,38 @@ class YoutubeSearchURLIE(InfoExtractor):
 
 class YoutubeShowIE(InfoExtractor):
     IE_DESC = 'YouTube.com (multi-season) shows'
-    _VALID_URL = r'https?://www\.youtube\.com/show/(.*)'
+    _VALID_URL = r'https?://www\.youtube\.com/show/(?P<id>[^?#]*)'
     IE_NAME = 'youtube:show'
+    _TESTS = [{
+        'url': 'http://www.youtube.com/show/airdisasters',
+        'playlist_mincount': 3,
+        'info_dict': {
+            'id': 'airdisasters',
+            'title': 'Air Disasters',
+        }
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        show_name = mobj.group(1)
-        webpage = self._download_webpage(url, show_name, 'Downloading show webpage')
+        playlist_id = mobj.group('id')
+        webpage = self._download_webpage(
+            url, playlist_id, 'Downloading show webpage')
         # There's one playlist for each season of the show
         m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
-        self.to_screen('%s: Found %s seasons' % (show_name, len(m_seasons)))
-        return [self.url_result('https://www.youtube.com' + season.group(1), 'YoutubePlaylist') for season in m_seasons]
+        self.to_screen('%s: Found %s seasons' % (playlist_id, len(m_seasons)))
+        entries = [
+            self.url_result(
+                'https://www.youtube.com' + season.group(1), 'YoutubePlaylist')
+            for season in m_seasons
+        ]
+        title = self._og_search_title(webpage, fatal=False)
+
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'title': title,
+            'entries': entries,
+        }
 
 
 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):

From ac7553d031ffa6cdcdb109330467eb7c423ffd13 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 24 Sep 2014 10:34:29 +0200
Subject: [PATCH 0051/1937] [youtube] Support embed/videoseries URLs (#3821)

---
 youtube_dl/extractor/youtube.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2ef76b69b..ae9564862 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -211,7 +211,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                             youtube\.googleapis\.com/)                        # the various hostnames, with wildcard subdomains
                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
                          (?:                                                  # the various things that can precede the ID:
-                             (?:(?:v|embed|e)/)                               # v/ or embed/ or e/
+                             (?:(?:v|embed|e)/(?!videoseries))                # v/ or embed/ or e/
                              |(?:                                             # or the v= param in all its forms
                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
@@ -1005,7 +1005,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
                         (?:\w+\.)?
                         youtube\.com/
                         (?:
-                           (?:course|view_play_list|my_playlists|artist|playlist|watch)
+                           (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/videoseries)
                            \? (?:.*?&)*? (?:p|a|list)=
                         |  p/
                         )
@@ -1061,6 +1061,13 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
             'title': 'YDL_safe_search',
         },
         'playlist_count': 2,
+    }, {
+        'note': 'embedded',
+        'url': 'http://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
+        'playlist_count': 4,
+        'info_dict': {
+            'title': 'JODA15',
+        }
     }]
 
     def _real_initialize(self):

From cc746841e76a0ab6a1bb65400ca496a105f65821 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 24 Sep 2014 10:46:33 +0200
Subject: [PATCH 0052/1937] [flickr] Modernize

---
 youtube_dl/extractor/flickr.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 21ea5ec2b..e09982e88 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -10,13 +10,13 @@ from ..utils import (
 
 
 class FlickrIE(InfoExtractor):
-    """Information Extractor for Flickr videos"""
-    _VALID_URL = r'(?:https?://)?(?:www\.|secure\.)?flickr\.com/photos/(?P<uploader_id>[\w\-_@]+)/(?P<id>\d+).*'
+    _VALID_URL = r'https?://(?:www\.|secure\.)?flickr\.com/photos/(?P<uploader_id>[\w\-_@]+)/(?P<id>\d+).*'
     _TEST = {
         'url': 'http://www.flickr.com/photos/forestwander-nature-pictures/5645318632/in/photostream/',
-        'file': '5645318632.mp4',
         'md5': '6fdc01adbc89d72fc9c4f15b4a4ba87b',
         'info_dict': {
+            'id': '5645318632',
+            'ext': 'mp4',
             "description": "Waterfalls in the Springtime at Dark Hollow Waterfalls. These are located just off of Skyline Drive in Virginia. They are only about 6/10 of a mile hike but it is a pretty steep hill and a good climb back up.", 
             "uploader_id": "forestwander-nature-pictures", 
             "title": "Dark Hollow Waterfalls"
@@ -49,12 +49,12 @@ class FlickrIE(InfoExtractor):
             raise ExtractorError('Unable to extract video url')
         video_url = mobj.group(1) + unescapeHTML(mobj.group(2))
 
-        return [{
-            'id':          video_id,
-            'url':         video_url,
-            'ext':         'mp4',
-            'title':       self._og_search_title(webpage),
+        return {
+            'id': video_id,
+            'url': video_url,
+            'ext': 'mp4',
+            'title': self._og_search_title(webpage),
             'description': self._og_search_description(webpage),
-            'thumbnail':   self._og_search_thumbnail(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
             'uploader_id': video_uploader_id,
-        }]
+        }

From 3b2f933b01c30a8b3a6bd7fb8418b44167ca30c5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 24 Sep 2014 11:05:14 +0200
Subject: [PATCH 0053/1937] [generic] Allow embedded YoutubePlaylists (Fixes
 #3821)

---
 youtube_dl/extractor/generic.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 40eeaad16..a3bfeb174 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -584,7 +584,9 @@ class GenericIE(InfoExtractor):
 
         # Helper method
         def _playlist_from_matches(matches, getter, ie=None):
-            urlrs = orderedSet(self.url_result(getter(m), ie) for m in matches)
+            urlrs = orderedSet(
+                self.url_result(self._proto_relative_url(getter(m)), ie)
+                for m in matches)
             return self.playlist_result(
                 urlrs, playlist_id=video_id, playlist_title=video_title)
 
@@ -633,7 +635,7 @@ class GenericIE(InfoExtractor):
             \1''', webpage)
         if matches:
             return _playlist_from_matches(
-                matches, lambda m: unescapeHTML(m[1]), ie='Youtube')
+                matches, lambda m: unescapeHTML(m[1]))
 
         # Look for embedded Dailymotion player
         matches = re.findall(

From 2f771f6c99480684522f2ccdfac25d69c1470ea5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 24 Sep 2014 11:06:46 +0200
Subject: [PATCH 0054/1937] release 2014.09.24

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2853c79c9..960fd59a3 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.09.22.1'
+__version__ = '2014.09.24'

From f0b5d6af74469d8216aebfe8079dbe1516188b89 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 24 Sep 2014 14:16:56 +0200
Subject: [PATCH 0055/1937] [vevo] Support 1080p videos (Fixes #3656)

---
 youtube_dl/downloader/__init__.py |  3 ++
 youtube_dl/downloader/hls.py      | 47 +++++++++++++++++++++++++++++++
 youtube_dl/extractor/common.py    | 16 +++++++++--
 youtube_dl/extractor/vevo.py      | 40 +++++++++++++++++++++++++-
 4 files changed, 102 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 4ea5811a5..3f941596e 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 from .common import FileDownloader
 from .hls import HlsFD
+from .hls import NativeHlsFD
 from .http import HttpFD
 from .mplayer import MplayerFD
 from .rtmp import RtmpFD
@@ -19,6 +20,8 @@ def get_suitable_downloader(info_dict):
 
     if url.startswith('rtmp'):
         return RtmpFD
+    if protocol == 'm3u8_native':
+        return NativeHlsFD
     if (protocol == 'm3u8') or (protocol is None and determine_ext(url) == 'm3u8'):
         return HlsFD
     if url.startswith('mms') or url.startswith('rtsp'):
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 32852f333..8040bdf08 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -1,8 +1,12 @@
+from __future__ import unicode_literals
+
 import os
+import re
 import subprocess
 
 from .common import FileDownloader
 from ..utils import (
+    compat_urlparse,
     check_executable,
     encodeFilename,
 )
@@ -43,3 +47,46 @@ class HlsFD(FileDownloader):
             self.to_stderr(u"\n")
             self.report_error(u'%s exited with code %d' % (program, retval))
             return False
+
+
+class NativeHlsFD(FileDownloader):
+    """ A more limited implementation that does not require ffmpeg """
+
+    def real_download(self, filename, info_dict):
+        url = info_dict['url']
+        self.report_destination(filename)
+        tmpfilename = self.temp_name(filename)
+
+        self.to_screen(
+            '[hlsnative] %s: Downloading m3u8 manifest' % info_dict['id'])
+        data = self.ydl.urlopen(url).read()
+        s = data.decode('utf-8', 'ignore')
+        segment_urls = []
+        for line in s.splitlines():
+            line = line.strip()
+            if line and not line.startswith('#'):
+                segment_url = (
+                    line
+                    if re.match(r'^https?://', line)
+                    else compat_urlparse.urljoin(url, line))
+                segment_urls.append(segment_url)
+
+        byte_counter = 0
+        with open(tmpfilename, 'wb') as outf:
+            for i, segurl in enumerate(segment_urls):
+                segment = self.ydl.urlopen(segurl).read()
+                outf.write(segment)
+                byte_counter += len(segment)
+                self.to_screen(
+                    '[hlsnative] %s: Downloading segment %d / %d' %
+                    (info_dict['id'], i + 1, len(segment_urls)))
+
+        self._hook_progress({
+            'downloaded_bytes': byte_counter,
+            'total_bytes': byte_counter,
+            'filename': filename,
+            'status': 'finished',
+        })
+        self.try_rename(tmpfilename, filename)
+        return True
+
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9c30a1d33..60cab6f4e 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -15,6 +15,7 @@ from ..utils import (
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse_urlparse,
+    compat_urlparse,
     compat_str,
 
     clean_html,
@@ -640,7 +641,9 @@ class InfoExtractor(object):
 
         return formats
 
-    def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None):
+    def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
+                              entry_protocol='m3u8', preference=None):
+
         formats = [{
             'format_id': 'm3u8-meta',
             'url': m3u8_url,
@@ -651,6 +654,11 @@ class InfoExtractor(object):
             'format_note': 'Quality selection URL',
         }]
 
+        format_url = lambda u: (
+            u
+            if re.match(r'^https?://', u)
+            else compat_urlparse.urljoin(m3u8_url, u))
+
         m3u8_doc = self._download_webpage(m3u8_url, video_id)
         last_info = None
         kv_rex = re.compile(
@@ -667,15 +675,17 @@ class InfoExtractor(object):
                 continue
             else:
                 if last_info is None:
-                    formats.append({'url': line})
+                    formats.append({'url': format_url(line)})
                     continue
                 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
 
                 f = {
                     'format_id': 'm3u8-%d' % (tbr if tbr else len(formats)),
-                    'url': line.strip(),
+                    'url': format_url(line.strip()),
                     'tbr': tbr,
                     'ext': ext,
+                    'protocol': entry_protocol,
+                    'preference': preference,
                 }
                 codecs = last_info.get('CODECS')
                 if codecs:
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index d2ffd1b6b..5e54a35d4 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -6,6 +6,7 @@ import xml.etree.ElementTree
 from .common import InfoExtractor
 from ..utils import (
     compat_HTTPError,
+    compat_urllib_request,
     ExtractorError,
 )
 
@@ -69,6 +70,21 @@ class VevoIE(InfoExtractor):
     }]
     _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/'
 
+    def _real_initialize(self):
+        req = compat_urllib_request.Request(
+            'http://www.vevo.com/auth', data=b'')
+        webpage = self._download_webpage(
+            req, None,
+            note='Retrieving oauth token',
+            errnote='Unable to retrieve oauth token',
+            fatal=False)
+        if webpage is False:
+            self._oauth_token = None
+        else:
+            self._oauth_token = self._search_regex(
+                r'access_token":\s*"([^"]+)"',
+                webpage, 'access token', fatal=False)
+
     def _formats_from_json(self, video_info):
         last_version = {'version': -1}
         for version in video_info['videoVersions']:
@@ -129,6 +145,26 @@ class VevoIE(InfoExtractor):
             })
         return formats
 
+    def _download_api_formats(self, video_id):
+        if not self._oauth_token:
+            self._downloader.report_warning(
+                'No oauth token available, skipping API HLS download')
+            return []
+
+        api_url = 'https://apiv2.vevo.com/video/%s/streams/hls?token=%s' % (
+            video_id, self._oauth_token)
+        api_data = self._download_json(
+            api_url, video_id,
+            note='Downloading HLS formats',
+            errnote='Failed to download HLS format list', fatal=False)
+        if api_data is None:
+            return []
+
+        m3u8_url = api_data[0]['url']
+        return self._extract_m3u8_formats(
+            m3u8_url, video_id, entry_protocol='m3u8_native', ext='mp4',
+            preference=0)
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -152,6 +188,9 @@ class VevoIE(InfoExtractor):
         else:
             age_limit = None
 
+        # Download via HLS API
+        formats.extend(self._download_api_formats(video_id))
+
         # Download SMIL
         smil_blocks = sorted((
             f for f in video_info['videoVersions']
@@ -166,7 +205,6 @@ class VevoIE(InfoExtractor):
                 fatal=False)
             if smil_url_m is not None:
                 smil_url = smil_url_m
-
         try:
             smil_xml = self._download_webpage(smil_url, video_id,
                                               'Downloading SMIL info')

From eb73f2649f41e80063d6f2f3e4b6345eb90f9777 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 24 Sep 2014 14:17:33 +0200
Subject: [PATCH 0056/1937] [vevo] Skip SMIL download

---
 youtube_dl/extractor/vevo.py | 24 ------------------------
 1 file changed, 24 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 5e54a35d4..1edeece3f 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -191,30 +191,6 @@ class VevoIE(InfoExtractor):
         # Download via HLS API
         formats.extend(self._download_api_formats(video_id))
 
-        # Download SMIL
-        smil_blocks = sorted((
-            f for f in video_info['videoVersions']
-            if f['sourceType'] == 13),
-            key=lambda f: f['version'])
-
-        smil_url = '%s/Video/V2/VFILE/%s/%sr.smil' % (
-            self._SMIL_BASE_URL, video_id, video_id.lower())
-        if smil_blocks:
-            smil_url_m = self._search_regex(
-                r'url="([^"]+)"', smil_blocks[-1]['data'], 'SMIL URL',
-                fatal=False)
-            if smil_url_m is not None:
-                smil_url = smil_url_m
-        try:
-            smil_xml = self._download_webpage(smil_url, video_id,
-                                              'Downloading SMIL info')
-            formats.extend(self._formats_from_smil(smil_xml))
-        except ExtractorError as ee:
-            if not isinstance(ee.cause, compat_HTTPError):
-                raise
-            self._downloader.report_warning(
-                'Cannot download SMIL information, falling back to JSON ..')
-
         self._sort_formats(formats)
         timestamp_ms = int(self._search_regex(
             r'/Date\((\d+)\)/', video_info['launchDate'], 'launch date'))

From 0b97f3a93690ea5449790acc1274df8900d141aa Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 24 Sep 2014 14:17:42 +0200
Subject: [PATCH 0057/1937] release 2014.09.24.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 960fd59a3..ecbd578db 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.09.24'
+__version__ = '2014.09.24.1'

From b686fc18dacaa6994c646c171368b99e168b619a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 24 Sep 2014 14:38:40 +0200
Subject: [PATCH 0058/1937] [hlsnative] Support test parameter

---
 youtube_dl/downloader/common.py |  1 +
 youtube_dl/downloader/hls.py    | 18 +++++++++++++++---
 youtube_dl/downloader/http.py   |  2 --
 youtube_dl/extractor/vevo.py    |  4 ++--
 4 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 9ce97f5fe..f85f0c94e 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -42,6 +42,7 @@ class FileDownloader(object):
     Subclasses of this one must re-define the real_download method.
     """
 
+    _TEST_FILE_SIZE = 10241
     params = None
 
     def __init__(self, ydl, params):
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 8040bdf08..56cce2811 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -7,6 +7,7 @@ import subprocess
 from .common import FileDownloader
 from ..utils import (
     compat_urlparse,
+    compat_urllib_request,
     check_executable,
     encodeFilename,
 )
@@ -71,15 +72,26 @@ class NativeHlsFD(FileDownloader):
                     else compat_urlparse.urljoin(url, line))
                 segment_urls.append(segment_url)
 
+        is_test = self.params.get('test', False)
+        remaining_bytes = self._TEST_FILE_SIZE if is_test else None
         byte_counter = 0
         with open(tmpfilename, 'wb') as outf:
             for i, segurl in enumerate(segment_urls):
-                segment = self.ydl.urlopen(segurl).read()
-                outf.write(segment)
-                byte_counter += len(segment)
                 self.to_screen(
                     '[hlsnative] %s: Downloading segment %d / %d' %
                     (info_dict['id'], i + 1, len(segment_urls)))
+                seg_req = compat_urllib_request.Request(segurl)
+                if remaining_bytes:
+                    seg_req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1))
+
+                segment = self.ydl.urlopen(seg_req).read()
+                if remaining_bytes:
+                    segment = segment[:remaining_bytes]
+                    remaining_bytes -= len(segment)
+                outf.write(segment)
+                byte_counter += len(segment)
+                if remaining_bytes <= 0:
+                    break
 
         self._hook_progress({
             'downloaded_bytes': byte_counter,
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 6caf7451e..f62555ce0 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -14,8 +14,6 @@ from ..utils import (
 
 
 class HttpFD(FileDownloader):
-    _TEST_FILE_SIZE = 10241
-
     def real_download(self, filename, info_dict):
         url = info_dict['url']
         tmpfilename = self.temp_name(filename)
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 1edeece3f..ebab8b86c 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -25,7 +25,7 @@ class VevoIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280',
-        "md5": "06bea460acb744eab74a9d7dcb4bfd61",
+        "md5": "95ee28ee45e70130e3ab02b0f579ae23",
         'info_dict': {
             'id': 'GB1101300280',
             'ext': 'mp4',
@@ -41,7 +41,7 @@ class VevoIE(InfoExtractor):
     }, {
         'note': 'v3 SMIL format',
         'url': 'http://www.vevo.com/watch/cassadee-pope/i-wish-i-could-break-your-heart/USUV71302923',
-        'md5': '893ec0e0d4426a1d96c01de8f2bdff58',
+        'md5': 'f6ab09b034f8c22969020b042e5ac7fc',
         'info_dict': {
             'id': 'USUV71302923',
             'ext': 'mp4',

From 6b08cdf626afc71740e539a83ef570999df2c50b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 25 Sep 2014 01:58:49 +0200
Subject: [PATCH 0059/1937] [youtube] Support for embedded /p players (Fixes
 #3821)

---
 youtube_dl/extractor/generic.py | 2 +-
 youtube_dl/extractor/youtube.py | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a3bfeb174..0dcbb39db 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -631,7 +631,7 @@ class GenericIE(InfoExtractor):
             )
             (["\'])
                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/
-                (?:embed|v)/.+?)
+                (?:embed|v|p)/.+?)
             \1''', webpage)
         if matches:
             return _playlist_from_matches(
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ae9564862..99198e380 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1068,6 +1068,13 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'info_dict': {
             'title': 'JODA15',
         }
+    }, {
+        'note': 'Embedded SWF player',
+        'url': 'http://www.youtube.com/p/YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ?hl=en_US&fs=1&rel=0',
+        'playlist_count': 4,
+        'info_dict': {
+            'title': 'JODA7',
+        }
     }]
 
     def _real_initialize(self):

From 4bbf157794084e1ca076b63c402bc5aab4a5ad0a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 25 Sep 2014 01:59:45 +0200
Subject: [PATCH 0060/1937] release 2014.09.25

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index ecbd578db..c17701d6a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.09.24.1'
+__version__ = '2014.09.25'

From fec02bcc90ad26ac5bbd11173fa83db91b3858bb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 25 Sep 2014 09:21:45 +0200
Subject: [PATCH 0061/1937] [hlsnative] Correct handling when remaining_bytes
 is None

---
 youtube_dl/downloader/hls.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 56cce2811..68eafa403 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -81,16 +81,16 @@ class NativeHlsFD(FileDownloader):
                     '[hlsnative] %s: Downloading segment %d / %d' %
                     (info_dict['id'], i + 1, len(segment_urls)))
                 seg_req = compat_urllib_request.Request(segurl)
-                if remaining_bytes:
+                if remaining_bytes is not None:
                     seg_req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1))
 
                 segment = self.ydl.urlopen(seg_req).read()
-                if remaining_bytes:
+                if remaining_bytes is not None:
                     segment = segment[:remaining_bytes]
                     remaining_bytes -= len(segment)
                 outf.write(segment)
                 byte_counter += len(segment)
-                if remaining_bytes <= 0:
+                if remaining_bytes is not None and remaining_bytes <= 0:
                     break
 
         self._hook_progress({

From 8a32b82e46b73680a9287336e455e6e38894bff3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 25 Sep 2014 09:58:09 +0200
Subject: [PATCH 0062/1937] [youku] Modernize somewhat

---
 youtube_dl/extractor/youku.py | 89 ++++++++++++++++++-----------------
 1 file changed, 45 insertions(+), 44 deletions(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index a8fd40c83..07ed7cbd1 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -1,5 +1,7 @@
 # coding: utf-8
 
+from __future__ import unicode_literals
+
 import json
 import math
 import random
@@ -13,18 +15,25 @@ from ..utils import (
 
 
 class YoukuIE(InfoExtractor):
-    _VALID_URL =  r'(?:(?:http://)?(?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)|youku:)(?P<ID>[A-Za-z0-9]+)(?:\.html|/v\.swf|)'
-    _TEST =   {
-        u"url": u"http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html",
-        u"file": u"XNDgyMDQ2NTQw_part00.flv",
-        u"md5": u"ffe3f2e435663dc2d1eea34faeff5b5b",
-        u"params": {u"test": False},
-        u"info_dict": {
-            u"title": u"youtube-dl test video \"'/\\ä↭𝕐"
+    _VALID_URL = r'''(?x)
+        (?:
+            http://(?:v|player)\.youku\.com/(?:v_show/id_|player\.php/sid/)|
+            youku:)
+        (?P<id>[A-Za-z0-9]+)(?:\.html|/v\.swf|)
+    '''
+    _TEST = {
+        'url': 'http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html',
+        'md5': 'ffe3f2e435663dc2d1eea34faeff5b5b',
+        'params': {
+            'test': False
+        },
+        'info_dict': {
+            'id': 'XNDgyMDQ2NTQw_part00',
+            'ext': 'flv',
+            'title': 'youtube-dl test video "\'/\\ä↭𝕐'
         }
     }
 
-
     def _gen_sid(self):
         nowTime = int(time.time() * 1000)
         random1 = random.randint(1000,1998)
@@ -55,49 +64,42 @@ class YoukuIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
-        video_id = mobj.group('ID')
+        video_id = mobj.group('id')
 
         info_url = 'http://v.youku.com/player/getPlayList/VideoIDS/' + video_id
 
-        jsondata = self._download_webpage(info_url, video_id)
+        config = self._download_json(info_url, video_id)
 
-        self.report_extraction(video_id)
-        try:
-            config = json.loads(jsondata)
-            error_code = config['data'][0].get('error_code')
-            if error_code:
-                # -8 means blocked outside China.
-                error = config['data'][0].get('error')  # Chinese and English, separated by newline.
-                raise ExtractorError(error or u'Server reported error %i' % error_code,
-                    expected=True)
+        error_code = config['data'][0].get('error_code')
+        if error_code:
+            # -8 means blocked outside China.
+            error = config['data'][0].get('error')  # Chinese and English, separated by newline.
+            raise ExtractorError(error or 'Server reported error %i' % error_code,
+                expected=True)
 
-            video_title =  config['data'][0]['title']
-            seed = config['data'][0]['seed']
+        video_title = config['data'][0]['title']
+        seed = config['data'][0]['seed']
 
-            format = self._downloader.params.get('format', None)
-            supported_format = list(config['data'][0]['streamfileids'].keys())
+        format = self._downloader.params.get('format', None)
+        supported_format = list(config['data'][0]['streamfileids'].keys())
 
-            if format is None or format == 'best':
-                if 'hd2' in supported_format:
-                    format = 'hd2'
-                else:
-                    format = 'flv'
-                ext = u'flv'
-            elif format == 'worst':
-                format = 'mp4'
-                ext = u'mp4'
+        # TODO proper format selection
+        if format is None or format == 'best':
+            if 'hd2' in supported_format:
+                format = 'hd2'
             else:
                 format = 'flv'
-                ext = u'flv'
+            ext = 'flv'
+        elif format == 'worst':
+            format = 'mp4'
+            ext = 'mp4'
+        else:
+            format = 'flv'
+            ext = 'flv'
 
-
-            fileid = config['data'][0]['streamfileids'][format]
-            keys = [s['k'] for s in config['data'][0]['segs'][format]]
-            # segs is usually a dictionary, but an empty *list* if an error occured.
-        except (UnicodeDecodeError, ValueError, KeyError):
-            raise ExtractorError(u'Unable to extract info section')
+        fileid = config['data'][0]['streamfileids'][format]
+        keys = [s['k'] for s in config['data'][0]['segs'][format]]
+        # segs is usually a dictionary, but an empty *list* if an error occured.
 
         files_info=[]
         sid = self._gen_sid()
@@ -106,9 +108,8 @@ class YoukuIE(InfoExtractor):
         #column 8,9 of fileid represent the segment number
         #fileid[7:9] should be changed
         for index, key in enumerate(keys):
-
             temp_fileid = '%s%02X%s' % (fileid[0:8], index, fileid[10:])
-            download_url = 'http://f.youku.com/player/getFlvPath/sid/%s_%02X/st/flv/fileid/%s?k=%s' % (sid, index, temp_fileid, key)
+            download_url = 'http://k.youku.com/player/getFlvPath/sid/%s_%02X/st/flv/fileid/%s?k=%s' % (sid, index, temp_fileid, key)
 
             info = {
                 'id': '%s_part%02d' % (video_id, index),

From 54e9a4af951f26edd7719f1a1b56e0a92d2791ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Sep 2014 20:33:11 +0700
Subject: [PATCH 0063/1937] [wat] Skip test

---
 youtube_dl/extractor/wat.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py
index 268e2f618..bf9e40bad 100644
--- a/youtube_dl/extractor/wat.py
+++ b/youtube_dl/extractor/wat.py
@@ -40,6 +40,7 @@ class WatIE(InfoExtractor):
                 'upload_date': '20140816',
                 'duration': 2910,
             },
+            'skip': "Ce contenu n'est pas disponible pour l'instant.",
         },
     ]
 

From fbd3162e4918a2e1321ebdcec47ac84a8b121fbe Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Thu, 25 Sep 2014 20:48:54 +0700
Subject: [PATCH 0064/1937] [vube] Add DMCA notice

---
 youtube_dl/extractor/vube.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/vube.py b/youtube_dl/extractor/vube.py
index 2544c24bd..bcca4897a 100644
--- a/youtube_dl/extractor/vube.py
+++ b/youtube_dl/extractor/vube.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     compat_str,
+    ExtractorError,
 )
 
 
@@ -102,6 +103,11 @@ class VubeIE(InfoExtractor):
 
         self._sort_formats(formats)
 
+        if not formats and video.get('vst') == 'dmca':
+            raise ExtractorError(
+                'This video has been removed in response to a complaint received under the US Digital Millennium Copyright Act.',
+                expected=True)
+
         title = video['title']
         description = video.get('description')
         thumbnail = self._proto_relative_url(video.get('thumbnail_src'), scheme='http:')

From 9a0d98bb401a809eaed68623a8534b3874d079e8 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Thu, 25 Sep 2014 20:57:18 +0700
Subject: [PATCH 0065/1937] [vube] Update tests

---
 youtube_dl/extractor/vube.py | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vube.py b/youtube_dl/extractor/vube.py
index bcca4897a..1b2f731e9 100644
--- a/youtube_dl/extractor/vube.py
+++ b/youtube_dl/extractor/vube.py
@@ -17,6 +17,24 @@ class VubeIE(InfoExtractor):
 
     _TESTS = [
         {
+            'url': 'http://vube.com/trending/William+Wei/Y8NUZ69Tf7?t=s',
+            'md5': 'e7aabe1f8f1aa826b9e4735e1f9cee42',
+            'info_dict': {
+                'id': 'Y8NUZ69Tf7',
+                'ext': 'mp4',
+                'title': 'Best Drummer Ever [HD]',
+                'description': 'md5:2d63c4b277b85c2277761c2cf7337d71',
+                'thumbnail': 're:^https?://.*\.jpg',
+                'uploader': 'William',
+                'timestamp': 1406876915,
+                'upload_date': '20140801',
+                'duration': 258.051,
+                'like_count': int,
+                'dislike_count': int,
+                'comment_count': int,
+                'categories': ['amazing', 'hd', 'best drummer ever', 'william wei', 'bucket drumming', 'street drummer', 'epic street drumming'],
+            },
+        }, {
             'url': 'http://vube.com/Chiara+Grispo+Video+Channel/YL2qNPkqon',
             'md5': 'db7aba89d4603dadd627e9d1973946fe',
             'info_dict': {
@@ -33,7 +51,8 @@ class VubeIE(InfoExtractor):
                 'dislike_count': int,
                 'comment_count': int,
                 'categories': ['pop', 'music', 'cover', 'singing', 'jessie j', 'price tag', 'chiara grispo'],
-            }
+            },
+            'skip': 'Removed due to DMCA',
         },
         {
             'url': 'http://vube.com/SerainaMusic/my-7-year-old-sister-and-i-singing-alive-by-krewella/UeBhTudbfS?t=s&n=1',
@@ -52,7 +71,8 @@ class VubeIE(InfoExtractor):
                 'dislike_count': int,
                 'comment_count': int,
                 'categories': ['seraina', 'jessica', 'krewella', 'alive'],
-            }
+            },
+            'skip': 'Removed due to DMCA',
         }, {
             'url': 'http://vube.com/vote/Siren+Gene/0nmsMY5vEq?n=2&t=s',
             'md5': '0584fc13b50f887127d9d1007589d27f',
@@ -70,7 +90,8 @@ class VubeIE(InfoExtractor):
                 'dislike_count': int,
                 'comment_count': int,
                 'categories': ['let it go', 'cover', 'idina menzel', 'frozen', 'singing', 'disney', 'siren gene'],
-            }
+            },
+            'skip': 'Removed due to DMCA',
         }
     ]
 

From 6a5af6acb9131d702b0d206242053b202440dbb9 Mon Sep 17 00:00:00 2001
From: Mats <d912e3@gmail.com>
Date: Thu, 25 Sep 2014 16:25:53 +0200
Subject: [PATCH 0066/1937] [golem] Add new extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/golem.py    | 131 +++++++++++++++++++++++++++++++
 2 files changed, 132 insertions(+)
 create mode 100644 youtube_dl/extractor/golem.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1f1fc0eb2..71fe38ca0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -135,6 +135,7 @@ from .gametrailers import GametrailersIE
 from .gdcvault import GDCVaultIE
 from .generic import GenericIE
 from .godtube import GodTubeIE
+from .golem import GolemIE
 from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
 from .gorillavid import GorillaVidIE
diff --git a/youtube_dl/extractor/golem.py b/youtube_dl/extractor/golem.py
new file mode 100644
index 000000000..afb620b1c
--- /dev/null
+++ b/youtube_dl/extractor/golem.py
@@ -0,0 +1,131 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import compat_urlparse
+
+
+class GolemIE(InfoExtractor):
+    _VALID_URL = r'^https?://video\.golem\.de/.+?/(?P<id>.+?)/'
+    _TEST = {
+        'url': 'http://video.golem.de/handy/14095/iphone-6-und-6-plus-test.html',
+        'md5': 'c1a2c0a3c863319651c7c992c5ee29bf',
+        'info_dict': {
+            'id': '14095',
+            'format_id': 'high',
+            'ext': 'mp4',
+            'title': 'iPhone 6 und 6 Plus - Test',
+            'duration': 300,
+            'filesize': 65309548,
+        }
+    }
+
+    _CONFIG = 'https://video.golem.de/xml/{}.xml'
+    _PREFIX = 'http://video.golem.de'
+
+    def _warn(self, fmt, *args):
+        self.report_warning(fmt.format(*args), self._id)
+
+    def _extract_format(self, elem):
+        format_id = elem.tag
+
+        url = elem.findtext('./url')
+        if url == '':
+            self._warn("{}: url: empty, skipping", format_id)
+            return None
+
+        fmt = {
+            'format_id': format_id,
+            'url': compat_urlparse.urljoin(self._PREFIX, url)
+        }
+
+        try:
+            _, ext = elem.findtext('./filename', '').rsplit('.', 1)
+        except ValueError:
+            self._warn('{}: ext: missing extension', format_id)
+        else:
+            fmt['ext'] = ext
+
+        filesize = elem.findtext('./filesize')
+        if filesize is not None:
+            try:
+                fmt['filesize'] = int(filesize)
+            except ValueError as e:
+                self._warn('{}: filesize: {}', format_id, e)
+
+        width = elem.get('width')
+        if width is not None:
+            try:
+                fmt['width'] = int(width)
+            except ValueError as e:
+                self._warn('{}: width: {}', format_id, e)
+
+        height = elem.get('height')
+        if height is not None:
+            try:
+                fmt['height'] = int(height)
+            except ValueError as e:
+                self._warn('{}: height: {}', format_id, e)
+
+        return fmt
+
+    def _extract_thumbnail(self, elem):
+        url = elem.findtext('./url')
+        if url == '':
+            return None
+        thumb = {
+            'url': compat_urlparse.urljoin(self._PREFIX, url)
+        }
+
+        width = elem.get('width')
+        if width is not None:
+            try:
+                thumb['width'] = int(width)
+            except ValueError as e:
+                self._warn('thumbnail: width: {}', e)
+
+        height = elem.get('height')
+        if height is not None:
+            try:
+                thumb['height'] = int(height)
+            except ValueError as e:
+                self._warn('thumbnail: height: {}', e)
+
+        return thumb
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        self._id = mobj.group('id')
+
+        config = self._download_xml(self._CONFIG.format(self._id), self._id)
+
+        info = {
+            'id': self._id,
+            'title': config.findtext('./title', 'golem')
+        }
+
+        formats = []
+        for e in config.findall('./*[url]'):
+            fmt = self._extract_format(e)
+            if fmt is not None:
+                formats.append(fmt)
+        self._sort_formats(formats)
+        info['formats'] = formats
+
+        thumbnails = []
+        for e in config.findall('.//teaser[url]'):
+            thumb = self._extract_thumbnail(e)
+            if thumb is not None:
+                thumbnails.append(thumb)
+        info['thumbnails'] = thumbnails
+
+        playtime = config.findtext('./playtime')
+        if playtime is not None:
+            try:
+                info['duration'] = round(float(playtime))
+            except ValueError as e:
+                self._warn('duration: {}', e)
+
+        return info

From 11b3ce85097430e1d26ddff0f51aa895c9d5af43 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 25 Sep 2014 17:57:38 +0300
Subject: [PATCH 0067/1937] [crunchyroll] Allow to list subtitles (fixes #3805)

---
 youtube_dl/extractor/crunchyroll.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 4903764f7..f99888ecc 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -9,7 +9,7 @@ import xml.etree.ElementTree
 
 from hashlib import sha1
 from math import pow, sqrt, floor
-from .common import InfoExtractor
+from .subtitles import SubtitlesInfoExtractor
 from ..utils import (
     ExtractorError,
     compat_urllib_parse,
@@ -26,7 +26,7 @@ from ..aes import (
 )
 
 
-class CrunchyrollIE(InfoExtractor):
+class CrunchyrollIE(SubtitlesInfoExtractor):
     _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
     _TEST = {
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
@@ -271,6 +271,10 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             else:
                 subtitles[lang_code] = self._convert_subtitles_to_srt(subtitle)
 
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, subtitles)
+            return
+
         return {
             'id':          video_id,
             'title':       video_title,

From 8e6f8051f084f445015140f1f88ac770f3c0f43d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 27 Sep 2014 10:53:02 +0200
Subject: [PATCH 0068/1937] [vbox7] Don't set the extension to 'flv' (fixes
 #3836)

---
 youtube_dl/extractor/vbox7.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index df115d251..ebd64f0f5 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -19,7 +19,7 @@ class Vbox7IE(InfoExtractor):
         'md5': '99f65c0c9ef9b682b97313e052734c3f',
         'info_dict': {
             'id': '249bb972c2',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Смях! Чудо - чист за секунди - Скрита камера',
         },
     }
@@ -50,7 +50,6 @@ class Vbox7IE(InfoExtractor):
         return {
             'id': video_id,
             'url': final_url,
-            'ext': 'flv',
             'title': title,
             'thumbnail': thumbnail_url,
         }

From 497339fa0e633c8b1dcebf3f70670f6d96ee2d62 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Sep 2014 22:29:27 +0700
Subject: [PATCH 0069/1937] [anysex] Fix extraction

---
 youtube_dl/extractor/anysex.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/anysex.py b/youtube_dl/extractor/anysex.py
index bc64423a3..ad86d6e58 100644
--- a/youtube_dl/extractor/anysex.py
+++ b/youtube_dl/extractor/anysex.py
@@ -35,7 +35,7 @@ class AnySexIE(InfoExtractor):
 
         title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
         description = self._html_search_regex(
-            r'<div class="description">([^<]+)</div>', webpage, 'description', fatal=False)
+            r'<div class="description"[^>]*>([^<]+)</div>', webpage, 'description', fatal=False)
         thumbnail = self._html_search_regex(
             r'preview_url\s*:\s*\'(.*?)\'', webpage, 'thumbnail', fatal=False)
 
@@ -43,7 +43,7 @@ class AnySexIE(InfoExtractor):
             r'<a href="http://anysex\.com/categories/[^"]+" title="[^"]*">([^<]+)</a>', webpage)
 
         duration = parse_duration(self._search_regex(
-            r'<b>Duration:</b> (\d+:\d+)', webpage, 'duration', fatal=False))
+            r'<b>Duration:</b> (?:<q itemprop="duration">)?(\d+:\d+)', webpage, 'duration', fatal=False))
         view_count = int_or_none(self._html_search_regex(
             r'<b>Views:</b> (\d+)', webpage, 'view count', fatal=False))
 

From 2f9e8776df664e21aee18b05c468a56b03fe4417 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Sep 2014 22:36:53 +0700
Subject: [PATCH 0070/1937] [extremetube] Fix extraction

---
 youtube_dl/extractor/extremetube.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
index 14a196ffc..aacbf1414 100644
--- a/youtube_dl/extractor/extremetube.py
+++ b/youtube_dl/extractor/extremetube.py
@@ -7,6 +7,7 @@ from ..utils import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urllib_parse,
+    str_to_int,
 )
 
 
@@ -20,6 +21,7 @@ class ExtremeTubeIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Music Video 14 british euro brit european cumshots swallow',
             'uploader': 'unknown',
+            'view_count': int,
             'age_limit': 18,
         }
     }, {
@@ -39,8 +41,12 @@ class ExtremeTubeIE(InfoExtractor):
         video_title = self._html_search_regex(
             r'<h1 [^>]*?title="([^"]+)"[^>]*>', webpage, 'title')
         uploader = self._html_search_regex(
-            r'>Posted by:(?=<)(?:\s|<[^>]*>)*(.+?)\|', webpage, 'uploader',
-            fatal=False)
+            r'Uploaded by:\s*</strong>\s*(.+?)\s*</div>',
+            webpage, 'uploader', fatal=False)
+        view_count = str_to_int(self._html_search_regex(
+            r'Views:\s*</strong>\s*<span>([\d,\.]+)</span>',
+            webpage, 'view count', fatal=False))
+
         video_url = compat_urllib_parse.unquote(self._html_search_regex(
             r'video_url=(.+?)&amp;', webpage, 'video_url'))
         path = compat_urllib_parse_urlparse(video_url).path
@@ -51,6 +57,7 @@ class ExtremeTubeIE(InfoExtractor):
             'id': video_id,
             'title': video_title,
             'uploader': uploader,
+            'view_count': view_count,
             'url': video_url,
             'format': format,
             'format_id': format,

From 2a1325fdde5c88fc052710b3c42fcc0d73153901 Mon Sep 17 00:00:00 2001
From: net <net@h-MacBook-Pro-sl-net.local>
Date: Sat, 27 Sep 2014 20:11:22 +0300
Subject: [PATCH 0071/1937] [ynet] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ynet.py     | 47 ++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 youtube_dl/extractor/ynet.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1f1fc0eb2..944e356ae 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -450,6 +450,7 @@ from .yahoo import (
     YahooNewsIE,
     YahooSearchIE,
 )
+from .ynet import YnetIE
 from .youjizz import YouJizzIE
 from .youku import YoukuIE
 from .youporn import YouPornIE
diff --git a/youtube_dl/extractor/ynet.py b/youtube_dl/extractor/ynet.py
new file mode 100644
index 000000000..94d253679
--- /dev/null
+++ b/youtube_dl/extractor/ynet.py
@@ -0,0 +1,47 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import base64
+import json
+
+from .common import InfoExtractor
+from youtube_dl.utils import compat_urllib_parse_urlparse, compat_urllib_parse
+
+class YnetIE(InfoExtractor):
+    _VALID_URL = r'http://.*ynet\.co\.il/.*/0,7340,(?P<id>L(-[0-9]+)+),00\.html'
+    _TEST = {
+        'url': 'http://hot.ynet.co.il/home/0,7340,L-11659-99244,00.html',
+        'info_dict': {
+            'id': 'L-11659-99244',
+            'ext': 'flv',
+            'title': 'md5:3dba12d2837ee2ad9652cc64af652b16',
+            'thumbnail': 'http://hot.ynet.co.il/PicServer4/2014/09/23/5606015/AMERICAN_COMMUNE1_T.jpg',
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        
+        id = mobj.group('id')
+        
+        webpage = self._download_webpage(url, id)
+
+        content = compat_urllib_parse.unquote_plus(self._og_search_video_url(webpage).decode('utf-8'))
+
+        player_url = re.match('(http.*\.swf)\?' ,content).group(1)
+        
+        config = json.loads(re.match('.*config\=(.*)' ,content).group(1))
+                
+        f4m_url = config['clip']['url']    
+
+        title = re.sub(': Video$', '', self._og_search_title(webpage))
+
+        return {
+            'id': id,
+            'title': title,
+            'formats': self._extract_f4m_formats(f4m_url, id),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'player_url': player_url,
+        }
+

From b66745288e50cff42ff711e63242b5d97e80cd4f Mon Sep 17 00:00:00 2001
From: net <net@h-MacBook-Pro-sl-net.local>
Date: Sat, 27 Sep 2014 20:21:46 +0300
Subject: [PATCH 0072/1937] [sport5] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/sport5.py   | 70 ++++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 youtube_dl/extractor/sport5.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1f1fc0eb2..c3a4d3c9a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -340,6 +340,7 @@ from .spiegel import SpiegelIE, SpiegelArticleIE
 from .spiegeltv import SpiegeltvIE
 from .spike import SpikeIE
 from .sportdeutschland import SportDeutschlandIE
+from .sport5 import Sport5IE
 from .stanfordoc import StanfordOpenClassroomIE
 from .steam import SteamIE
 from .streamcloud import StreamcloudIE
diff --git a/youtube_dl/extractor/sport5.py b/youtube_dl/extractor/sport5.py
new file mode 100644
index 000000000..9a4e39a43
--- /dev/null
+++ b/youtube_dl/extractor/sport5.py
@@ -0,0 +1,70 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from youtube_dl.utils import compat_str, compat_urlretrieve
+
+
+
+class Sport5IE(InfoExtractor):
+    _VALID_URL = r'http://.*sport5\.co\.il'
+    _TESTS = [{
+            'url': 'http://vod.sport5.co.il/?Vc=147&Vi=176331&Page=1',
+            'info_dict': {
+                'id': 's5-Y59xx1-GUh2',
+                'ext': 'mp4',
+                'title': 'md5:4a2a5eba7e7dc88fdc446cbca8a41c79',
+            }
+        }, {
+            'url': 'http://www.sport5.co.il/articles.aspx?FolderID=3075&docID=176372&lang=HE',
+            'info_dict': {
+                'id': 's5-SiXxx1-hKh2',
+                'ext': 'mp4',
+                'title': 'md5:5cb1c6bfc0f16086e59f6683013f8e02',
+            }
+        }
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        webpage = self._download_webpage(url, '')
+
+        media_id = self._html_search_regex('clipId=(s5-\w+-\w+)', webpage, 'media id')
+
+        xml = self._download_xml(
+            'http://sport5-metadata-rr-d.nsacdn.com/vod/vod/%s/HDS/metadata.xml' % media_id,
+            media_id, 'Downloading media XML')
+
+        title = xml.find('./Title').text
+        duration = xml.find('./Duration').text
+        description = xml.find('./Description').text
+        thumbnail = xml.find('./PosterLinks/PosterIMG').text
+        player_url = xml.find('./PlaybackLinks/PlayerUrl').text
+        file_els = xml.findall('./PlaybackLinks/FileURL')
+
+        formats = []
+
+        for file_el in file_els:
+            bitrate = file_el.attrib.get('bitrate')
+            width = int(file_el.attrib.get('width'))
+            height = int(file_el.attrib.get('height'))
+            formats.append({
+                'url': compat_str(file_el.text),
+                'ext': 'mp4',
+                'height': height,
+                'width': width
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': media_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
+            'player_url': player_url,
+        }
\ No newline at end of file

From 0155549d6cec6f49279ebe4a5a73cf6dcc6716fe Mon Sep 17 00:00:00 2001
From: Mats <d912e3@gmail.com>
Date: Sat, 27 Sep 2014 19:28:01 +0200
Subject: [PATCH 0073/1937] [heise] Add new extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/heise.py    | 120 +++++++++++++++++++++++++++++++
 2 files changed, 121 insertions(+)
 create mode 100644 youtube_dl/extractor/heise.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1f1fc0eb2..d0417a1f2 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -141,6 +141,7 @@ from .gorillavid import GorillaVidIE
 from .goshgay import GoshgayIE
 from .grooveshark import GroovesharkIE
 from .hark import HarkIE
+from .heise import HeiseIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
 from .hornbunny import HornBunnyIE
diff --git a/youtube_dl/extractor/heise.py b/youtube_dl/extractor/heise.py
new file mode 100644
index 000000000..b3cb10fde
--- /dev/null
+++ b/youtube_dl/extractor/heise.py
@@ -0,0 +1,120 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    compat_urllib_parse,
+    get_meta_content,
+    parse_iso8601,
+)
+
+
+class HeiseIE(InfoExtractor):
+    _VALID_URL = (
+        r'^https?://(?:www\.)?heise\.de/video/artikel/' +
+        r'.+?(?P<id>[0-9]+)\.html$'
+    )
+    _TEST = {
+        'url': (
+            'http://www.heise.de/video/artikel/Podcast-c-t-uplink-3-3-' +
+            'Owncloud-Tastaturen-Peilsender-Smartphone-2404147.html'
+        ),
+        'md5': 'ffed432483e922e88545ad9f2f15d30e',
+        'info_dict': {
+            'id': '2404147',
+            'ext': 'mp4',
+            'title': (
+                "Podcast: c't uplink 3.3 – Owncloud / Tastaturen / " +
+                "Peilsender Smartphone"
+            ),
+            'format_id': 'mp4_720',
+            'timestamp': 1411812600,
+            'upload_date': '20140927',
+        }
+    }
+
+    _CONFIG = (
+        r'".+?\?sequenz=(?P<sequenz>.+?)&container=(?P<container>.+?)' +
+        r'(?:&hd=(?P<hd>.+?))?(?:&signature=(?P<signature>.+?))?&callback=\?"'
+    )
+    _PREFIX = 'http://www.heise.de/videout/info?'
+
+    def _warn(self, fmt, *args):
+        self.report_warning(fmt.format(*args), self._id)
+
+    def _parse_config_url(self, html):
+        m = re.search(self._CONFIG, html)
+        if not m:
+            raise ExtractorError('No config found')
+
+        qs = compat_urllib_parse.urlencode(dict((k, v) for k, v
+                                                in m.groupdict().items()
+                                                if v is not None))
+        return self._PREFIX + qs
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        self._id = mobj.group('id')
+
+        html = self._download_webpage(url, self._id)
+        config = self._download_json(self._parse_config_url(html), self._id)
+
+        info = {
+            'id': self._id
+        }
+
+        title = get_meta_content('fulltitle', html)
+        if title:
+            info['title'] = title
+        elif config.get('title'):
+            info['title'] = config['title']
+        else:
+            self._warn('title: not found')
+            info['title'] = 'heise'
+
+        if (not config.get('formats') or
+                not hasattr(config['formats'], 'items')):
+            raise ExtractorError('No formats found')
+
+        formats = []
+        for t, rs in config['formats'].items():
+            if not rs or not hasattr(rs, 'items'):
+                self._warn('formats: {0}: no resolutions', t)
+                continue
+
+            for res, obj in rs.items():
+                format_id = '{0}_{1}'.format(t, res)
+
+                if (not obj or not obj.get('url') or
+                        not isinstance(obj['url'], str)):
+                    self._warn('formats: {0}: no url', format_id)
+                    continue
+
+                fmt = {
+                    'url': obj['url'],
+                    'format_id': format_id
+                }
+                try:
+                    fmt['height'] = int(res)
+                except ValueError as e:
+                    self._warn('formats: {0}: height: {1}', t, e)
+
+                formats.append(fmt)
+
+        self._sort_formats(formats)
+        info['formats'] = formats
+
+        if config.get('poster') and isinstance(config['poster'], str):
+            info['thumbnail'] = config['poster']
+
+        date = get_meta_content('date', html)
+        if date and isinstance(date, str):
+            try:
+                info['timestamp'] = parse_iso8601(date)
+            except ValueError as e:
+                self._warn('timestamp: {0}', e)
+
+        return info

From 70752ccefd2dcb54d131644aea38c324c81ff168 Mon Sep 17 00:00:00 2001
From: Mats <d912e3@gmail.com>
Date: Sat, 27 Sep 2014 19:35:55 +0200
Subject: [PATCH 0074/1937] [golem] Don't omit positional argument specifiers

Required by Python 2.6.
---
 youtube_dl/extractor/golem.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/golem.py b/youtube_dl/extractor/golem.py
index afb620b1c..6a64b5d95 100644
--- a/youtube_dl/extractor/golem.py
+++ b/youtube_dl/extractor/golem.py
@@ -22,7 +22,7 @@ class GolemIE(InfoExtractor):
         }
     }
 
-    _CONFIG = 'https://video.golem.de/xml/{}.xml'
+    _CONFIG = 'https://video.golem.de/xml/{0}.xml'
     _PREFIX = 'http://video.golem.de'
 
     def _warn(self, fmt, *args):
@@ -33,7 +33,7 @@ class GolemIE(InfoExtractor):
 
         url = elem.findtext('./url')
         if url == '':
-            self._warn("{}: url: empty, skipping", format_id)
+            self._warn("{0}: url: empty, skipping", format_id)
             return None
 
         fmt = {
@@ -44,7 +44,7 @@ class GolemIE(InfoExtractor):
         try:
             _, ext = elem.findtext('./filename', '').rsplit('.', 1)
         except ValueError:
-            self._warn('{}: ext: missing extension', format_id)
+            self._warn('{0}: ext: missing extension', format_id)
         else:
             fmt['ext'] = ext
 
@@ -53,21 +53,21 @@ class GolemIE(InfoExtractor):
             try:
                 fmt['filesize'] = int(filesize)
             except ValueError as e:
-                self._warn('{}: filesize: {}', format_id, e)
+                self._warn('{0}: filesize: {1}', format_id, e)
 
         width = elem.get('width')
         if width is not None:
             try:
                 fmt['width'] = int(width)
             except ValueError as e:
-                self._warn('{}: width: {}', format_id, e)
+                self._warn('{0}: width: {1}', format_id, e)
 
         height = elem.get('height')
         if height is not None:
             try:
                 fmt['height'] = int(height)
             except ValueError as e:
-                self._warn('{}: height: {}', format_id, e)
+                self._warn('{0}: height: {1}', format_id, e)
 
         return fmt
 
@@ -84,14 +84,14 @@ class GolemIE(InfoExtractor):
             try:
                 thumb['width'] = int(width)
             except ValueError as e:
-                self._warn('thumbnail: width: {}', e)
+                self._warn('thumbnail: width: {0}', e)
 
         height = elem.get('height')
         if height is not None:
             try:
                 thumb['height'] = int(height)
             except ValueError as e:
-                self._warn('thumbnail: height: {}', e)
+                self._warn('thumbnail: height: {0}', e)
 
         return thumb
 
@@ -126,6 +126,6 @@ class GolemIE(InfoExtractor):
             try:
                 info['duration'] = round(float(playtime))
             except ValueError as e:
-                self._warn('duration: {}', e)
+                self._warn('duration: {0}', e)
 
         return info

From 68b09730461de20395cee9427dc469fa9edc4022 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Sep 2014 02:07:42 +0700
Subject: [PATCH 0075/1937] [YoutubeDL] Expect all kind of strings in urlopen

Now it doesn't fail if req is python2's str
---
 youtube_dl/YoutubeDL.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index a1713dc5a..b485dbdf1 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1250,12 +1250,13 @@ class YoutubeDL(object):
         # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
         # To work around aforementioned issue we will replace request's original URL with
         # percent-encoded one
-        url = req if isinstance(req, compat_str) else req.get_full_url()
+        req_is_string = isinstance(req, basestring)
+        url = req if req_is_string else req.get_full_url()
         url_escaped = escape_url(url)
 
         # Substitute URL if any change after escaping
         if url != url_escaped:
-            if isinstance(req, compat_str):
+            if req_is_string:
                 req = url_escaped
             else:
                 req = compat_urllib_request.Request(

From 7b7518124ee433484b485502671e011017bc1897 Mon Sep 17 00:00:00 2001
From: Mats <d912e3@gmail.com>
Date: Sat, 27 Sep 2014 21:12:23 +0200
Subject: [PATCH 0076/1937] [heise] Don't check string type

Before Python 3 could be unicode, so don't check at all.
---
 youtube_dl/extractor/heise.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/heise.py b/youtube_dl/extractor/heise.py
index b3cb10fde..73c953181 100644
--- a/youtube_dl/extractor/heise.py
+++ b/youtube_dl/extractor/heise.py
@@ -88,8 +88,7 @@ class HeiseIE(InfoExtractor):
             for res, obj in rs.items():
                 format_id = '{0}_{1}'.format(t, res)
 
-                if (not obj or not obj.get('url') or
-                        not isinstance(obj['url'], str)):
+                if not obj or not obj.get('url'):
                     self._warn('formats: {0}: no url', format_id)
                     continue
 
@@ -107,11 +106,11 @@ class HeiseIE(InfoExtractor):
         self._sort_formats(formats)
         info['formats'] = formats
 
-        if config.get('poster') and isinstance(config['poster'], str):
+        if config.get('poster'):
             info['thumbnail'] = config['poster']
 
         date = get_meta_content('date', html)
-        if date and isinstance(date, str):
+        if date:
             try:
                 info['timestamp'] = parse_iso8601(date)
             except ValueError as e:

From 0b75c2a88ba56a84322db6cc1a298d7e52b44b2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Sep 2014 02:31:14 +0700
Subject: [PATCH 0077/1937] [sport5] Capture error message and improve

---
 youtube_dl/extractor/sport5.py | 88 +++++++++++++++++++++-------------
 1 file changed, 55 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/sport5.py b/youtube_dl/extractor/sport5.py
index 9a4e39a43..3f680bfc6 100644
--- a/youtube_dl/extractor/sport5.py
+++ b/youtube_dl/extractor/sport5.py
@@ -4,67 +4,89 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from youtube_dl.utils import compat_str, compat_urlretrieve
-
+from ..utils import ExtractorError
 
 
 class Sport5IE(InfoExtractor):
-    _VALID_URL = r'http://.*sport5\.co\.il'
-    _TESTS = [{
+    _VALID_URL = r'http://(?:www|vod)?\.sport5\.co\.il/.*\b(?:Vi|docID)=(?P<id>\d+)'
+    _TESTS = [
+        {
             'url': 'http://vod.sport5.co.il/?Vc=147&Vi=176331&Page=1',
             'info_dict': {
                 'id': 's5-Y59xx1-GUh2',
                 'ext': 'mp4',
-                'title': 'md5:4a2a5eba7e7dc88fdc446cbca8a41c79',
-            }
+                'title': 'ולנסיה-קורדובה 0:3',
+                'description': 'אלקאסר, גאייה ופגולי סידרו לקבוצה של נונו ניצחון על קורדובה ואת המקום הראשון בליגה',
+                'duration': 228,
+                'categories': list,
+            },
+            'skip': 'Blocked outside of Israel',
         }, {
             'url': 'http://www.sport5.co.il/articles.aspx?FolderID=3075&docID=176372&lang=HE',
             'info_dict': {
                 'id': 's5-SiXxx1-hKh2',
                 'ext': 'mp4',
-                'title': 'md5:5cb1c6bfc0f16086e59f6683013f8e02',
-            }
+                'title': 'GOALS_CELTIC_270914.mp4',
+                'description': '',
+                'duration': 87,
+                'categories': list,
+            },
+            'skip': 'Blocked outside of Israel',
         }
     ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
+        media_id = mobj.group('id')
 
-        webpage = self._download_webpage(url, '')
+        webpage = self._download_webpage(url, media_id)
 
-        media_id = self._html_search_regex('clipId=(s5-\w+-\w+)', webpage, 'media id')
+        video_id = self._html_search_regex('clipId=([\w-]+)', webpage, 'video id')
 
-        xml = self._download_xml(
-            'http://sport5-metadata-rr-d.nsacdn.com/vod/vod/%s/HDS/metadata.xml' % media_id,
-            media_id, 'Downloading media XML')
+        metadata = self._download_xml(
+            'http://sport5-metadata-rr-d.nsacdn.com/vod/vod/%s/HDS/metadata.xml' % video_id,
+            video_id)
 
-        title = xml.find('./Title').text
-        duration = xml.find('./Duration').text
-        description = xml.find('./Description').text
-        thumbnail = xml.find('./PosterLinks/PosterIMG').text
-        player_url = xml.find('./PlaybackLinks/PlayerUrl').text
-        file_els = xml.findall('./PlaybackLinks/FileURL')
+        error = metadata.find('./Error')
+        if error is not None:
+            raise ExtractorError(
+                '%s returned error: %s - %s' % (
+                    self.IE_NAME,
+                    error.find('./Name').text,
+                    error.find('./Description').text),
+                expected=True)
 
-        formats = []
+        title = metadata.find('./Title').text
+        description = metadata.find('./Description').text
+        duration = int(metadata.find('./Duration').text)
 
-        for file_el in file_els:
-            bitrate = file_el.attrib.get('bitrate')
-            width = int(file_el.attrib.get('width'))
-            height = int(file_el.attrib.get('height'))
-            formats.append({
-                'url': compat_str(file_el.text),
-                'ext': 'mp4',
-                'height': height,
-                'width': width
-            })
+        posters_el = metadata.find('./PosterLinks')
+        thumbnails = [{
+            'url': thumbnail.text,
+            'width': int(thumbnail.get('width')),
+            'height': int(thumbnail.get('height')),
+        } for thumbnail in posters_el.findall('./PosterIMG')] if posters_el is not None else []
 
+        categories_el = metadata.find('./Categories')
+        categories = [
+            cat.get('name') for cat in categories_el.findall('./Category')
+        ] if categories_el is not None else []
+
+        formats = [{
+            'url': fmt.text,
+            'ext': 'mp4',
+            'vbr': int(fmt.get('bitrate')),
+            'width': int(fmt.get('width')),
+            'height': int(fmt.get('height')),
+        } for fmt in metadata.findall('./PlaybackLinks/FileURL')]
         self._sort_formats(formats)
 
         return {
-            'id': media_id,
+            'id': video_id,
             'title': title,
-            'thumbnail': thumbnail,
+            'description': description,
+            'thumbnails': thumbnails,
             'duration': duration,
+            'categories': categories,
             'formats': formats,
-            'player_url': player_url,
         }
\ No newline at end of file

From f776d8f6081b305ba7ccc1bda323aa510a01db7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Sep 2014 02:35:46 +0700
Subject: [PATCH 0078/1937] [sport5] Keep alphanumeric order

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c3a4d3c9a..5a14540a3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -339,8 +339,8 @@ from .spankwire import SpankwireIE
 from .spiegel import SpiegelIE, SpiegelArticleIE
 from .spiegeltv import SpiegeltvIE
 from .spike import SpikeIE
-from .sportdeutschland import SportDeutschlandIE
 from .sport5 import Sport5IE
+from .sportdeutschland import SportDeutschlandIE
 from .stanfordoc import StanfordOpenClassroomIE
 from .steam import SteamIE
 from .streamcloud import StreamcloudIE

From ee0d90707a38537355bab8527edd9a42d6514aa9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Sep 2014 02:48:41 +0700
Subject: [PATCH 0079/1937] [YoutubeDL] Fix string check for python3

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b485dbdf1..4a9610355 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1250,7 +1250,7 @@ class YoutubeDL(object):
         # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
         # To work around aforementioned issue we will replace request's original URL with
         # percent-encoded one
-        req_is_string = isinstance(req, basestring)
+        req_is_string = isinstance(req, basestring if sys.version_info < (3, 0) else compat_str)
         url = req if req_is_string else req.get_full_url()
         url_escaped = escape_url(url)
 

From c6641823238ac70091520fe9b4b02ec3d41cb1a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Sep 2014 03:26:38 +0700
Subject: [PATCH 0080/1937] [ynet] Remove unused stuff, simplify and improve

---
 youtube_dl/extractor/ynet.py | 63 ++++++++++++++++++++----------------
 1 file changed, 35 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/ynet.py b/youtube_dl/extractor/ynet.py
index 94d253679..66d53962a 100644
--- a/youtube_dl/extractor/ynet.py
+++ b/youtube_dl/extractor/ynet.py
@@ -2,46 +2,53 @@
 from __future__ import unicode_literals
 
 import re
-import base64
 import json
 
 from .common import InfoExtractor
-from youtube_dl.utils import compat_urllib_parse_urlparse, compat_urllib_parse
+from ..utils import compat_urllib_parse
+
 
 class YnetIE(InfoExtractor):
-    _VALID_URL = r'http://.*ynet\.co\.il/.*/0,7340,(?P<id>L(-[0-9]+)+),00\.html'
-    _TEST = {
-        'url': 'http://hot.ynet.co.il/home/0,7340,L-11659-99244,00.html',
-        'info_dict': {
-            'id': 'L-11659-99244',
-            'ext': 'flv',
-            'title': 'md5:3dba12d2837ee2ad9652cc64af652b16',
-            'thumbnail': 'http://hot.ynet.co.il/PicServer4/2014/09/23/5606015/AMERICAN_COMMUNE1_T.jpg',
+    _VALID_URL = r'http://.*ynet\.co\.il/.*/0,7340,(?P<id>L(?:-[0-9]+)+),00\.html'
+    _TESTS = [
+        {
+            'url': 'http://hot.ynet.co.il/home/0,7340,L-11659-99244,00.html',
+            'md5': '002b44ee2f33d50363a1c153bed524cf',
+            'info_dict': {
+                'id': 'L-11659-99244',
+                'ext': 'flv',
+                'title': 'איש לא יודע מאיפה באנו',
+                'thumbnail': 're:^https?://.*\.jpg',
+            }
+        }, {
+            'url': 'http://hot.ynet.co.il/home/0,7340,L-8859-84418,00.html',
+            'md5': '6455046ae1b48cf7e2b7cae285e53a16',
+            'info_dict': {
+                'id': 'L-8859-84418',
+                'ext': 'flv',
+                'title': "צפו: הנשיקה הלוהטת של תורגי' ויוליה פלוטקין",
+                'thumbnail': 're:^https?://.*\.jpg',
+            }
         }
-    }
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
         
-        id = mobj.group('id')
-        
-        webpage = self._download_webpage(url, id)
+        webpage = self._download_webpage(url, video_id)
 
-        content = compat_urllib_parse.unquote_plus(self._og_search_video_url(webpage).decode('utf-8'))
-
-        player_url = re.match('(http.*\.swf)\?' ,content).group(1)
-        
-        config = json.loads(re.match('.*config\=(.*)' ,content).group(1))
-                
-        f4m_url = config['clip']['url']    
-
-        title = re.sub(': Video$', '', self._og_search_title(webpage))
+        content = compat_urllib_parse.unquote_plus(self._og_search_video_url(webpage))
+        config = json.loads(self._search_regex(r'config=({.+?})$', content, 'video config'))
+        f4m_url = config['clip']['url']
+        title = self._og_search_title(webpage)
+        m = re.search(r'ynet - HOT -- (["\']+)(?P<title>.+?)\1', title)
+        if m:
+            title = m.group('title')
 
         return {
-            'id': id,
+            'id': video_id,
             'title': title,
-            'formats': self._extract_f4m_formats(f4m_url, id),
+            'formats': self._extract_f4m_formats(f4m_url, video_id),
             'thumbnail': self._og_search_thumbnail(webpage),
-            'player_url': player_url,
-        }
-
+        }
\ No newline at end of file

From a89435a7a8e0574239531bfeedc437ae14b13902 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Sep 2014 03:30:41 +0700
Subject: [PATCH 0081/1937] [ynet] Improve _VALID_URL

---
 youtube_dl/extractor/ynet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ynet.py b/youtube_dl/extractor/ynet.py
index 66d53962a..24872861a 100644
--- a/youtube_dl/extractor/ynet.py
+++ b/youtube_dl/extractor/ynet.py
@@ -9,7 +9,7 @@ from ..utils import compat_urllib_parse
 
 
 class YnetIE(InfoExtractor):
-    _VALID_URL = r'http://.*ynet\.co\.il/.*/0,7340,(?P<id>L(?:-[0-9]+)+),00\.html'
+    _VALID_URL = r'http://(?:.+?\.)?ynet\.co\.il/(?:.+?/)?0,7340,(?P<id>L(?:-[0-9]+)+),00\.html'
     _TESTS = [
         {
             'url': 'http://hot.ynet.co.il/home/0,7340,L-11659-99244,00.html',

From 5e43e3803c462d7a0f5ac85f8b54ab24f271cb0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Sep 2014 03:45:15 +0700
Subject: [PATCH 0082/1937] Credit @lenaten for ynet (#3840) and sport5 (#3841)

---
 youtube_dl/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 20d7a57ce..7f2b4dfcc 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -78,6 +78,7 @@ __authors__  = (
     'Hari Padmanaban',
     'Carlos Ramos',
     '5moufl',
+    'lenaten',
 )
 
 __license__ = 'Public Domain'

From c95eeb7b80e5007259df260b64874b675a802431 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 28 Sep 2014 08:49:03 +0200
Subject: [PATCH 0083/1937] [eitb] Modernize

---
 youtube_dl/extractor/eitb.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
index 4ba323148..2cba82532 100644
--- a/youtube_dl/extractor/eitb.py
+++ b/youtube_dl/extractor/eitb.py
@@ -1,4 +1,6 @@
 # encoding: utf-8
+from __future__ import unicode_literals
+
 import re
 
 from .common import InfoExtractor
@@ -7,20 +9,20 @@ from ..utils import ExtractorError
 
 
 class EitbIE(InfoExtractor):
-    IE_NAME = u'eitb.tv'
+    IE_NAME = 'eitb.tv'
     _VALID_URL = r'https?://www\.eitb\.tv/(eu/bideoa|es/video)/[^/]+/(?P<playlist_id>\d+)/(?P<chapter_id>\d+)'
 
     _TEST = {
-        u'add_ie': ['Brightcove'],
-        u'url': u'http://www.eitb.tv/es/video/60-minutos-60-minutos-2013-2014/2677100210001/2743577154001/lasa-y-zabala-30-anos/',
-        u'md5': u'edf4436247185adee3ea18ce64c47998',
-        u'info_dict': {
-            u'id': u'2743577154001',
-            u'ext': u'mp4',
-            u'title': u'60 minutos (Lasa y Zabala, 30 años)',
+        'add_ie': ['Brightcove'],
+        'url': 'http://www.eitb.tv/es/video/60-minutos-60-minutos-2013-2014/2677100210001/2743577154001/lasa-y-zabala-30-anos/',
+        'md5': 'edf4436247185adee3ea18ce64c47998',
+        'info_dict': {
+            'id': '2743577154001',
+            'ext': 'mp4',
+            'title': '60 minutos (Lasa y Zabala, 30 años)',
             # All videos from eitb has this description in the brightcove info
-            u'description': u'.',
-            u'uploader': u'Euskal Telebista',
+            'description': '.',
+            'uploader': 'Euskal Telebista',
         },
     }
 
@@ -30,7 +32,7 @@ class EitbIE(InfoExtractor):
         webpage = self._download_webpage(url, chapter_id)
         bc_url = BrightcoveIE._extract_brightcove_url(webpage)
         if bc_url is None:
-            raise ExtractorError(u'Could not extract the Brightcove url')
+            raise ExtractorError('Could not extract the Brightcove url')
         # The BrightcoveExperience object doesn't contain the video id, we set
         # it manually
         bc_url += '&%40videoPlayer={0}'.format(chapter_id)

From f4b1c7adb81555fde0dff390b48e4139438b4071 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 28 Sep 2014 08:53:52 +0200
Subject: [PATCH 0084/1937] [muenchentv] Move live title generation to common

---
 youtube_dl/extractor/common.py     | 7 +++++++
 youtube_dl/extractor/muenchentv.py | 5 +----
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 60cab6f4e..403791e6b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import base64
+import datetime
 import hashlib
 import json
 import netrc
@@ -705,6 +706,12 @@ class InfoExtractor(object):
         self._sort_formats(formats)
         return formats
 
+    def _live_title(self, name):
+        """ Generate the title for a live video """
+        now = datetime.datetime.now()
+        now_str = now.strftime("%Y-%m-%d %H:%M")
+        return name + ' ' + now_str
+
 
 class SearchInfoExtractor(InfoExtractor):
     """
diff --git a/youtube_dl/extractor/muenchentv.py b/youtube_dl/extractor/muenchentv.py
index 3a938861b..7cb6749be 100644
--- a/youtube_dl/extractor/muenchentv.py
+++ b/youtube_dl/extractor/muenchentv.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import datetime
 import json
 
 from .common import InfoExtractor
@@ -33,9 +32,7 @@ class MuenchenTVIE(InfoExtractor):
         display_id = 'live'
         webpage = self._download_webpage(url, display_id)
 
-        now = datetime.datetime.now()
-        now_str = now.strftime("%Y-%m-%d %H:%M")
-        title = self._og_search_title(webpage) + ' ' + now_str
+        title = self._live_title(self._og_search_title(webpage))
 
         data_js = self._search_regex(
             r'(?s)\nplaylist:\s*(\[.*?}\]),related:',

From ed9266db90023500e687aa634b55e11742c2e18c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 28 Sep 2014 09:31:58 +0200
Subject: [PATCH 0085/1937] [common] Add new helper function _match_id

---
 youtube_dl/extractor/abc.py    | 3 +--
 youtube_dl/extractor/common.py | 8 ++++++++
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index 7d89f44ee..69f89320c 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -22,8 +22,7 @@ class ABCIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         urls_info_json = self._search_regex(
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 403791e6b..8d6a6f601 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -165,6 +165,14 @@ class InfoExtractor(object):
             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
         return cls._VALID_URL_RE.match(url) is not None
 
+    @classmethod
+    def _match_id(cls, url):
+        if '_VALID_URL_RE' not in cls.__dict__:
+            cls._VALID_URL_RE = re.compile(cls._VALID_URL)
+        m = cls._VALID_URL_RE.match(url)
+        assert m
+        return m.group('id')
+
     @classmethod
     def working(cls):
         """Getter method for _WORKING."""

From 394599f422b11f54efa78123296867efa45a1a2c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 28 Sep 2014 09:48:51 +0200
Subject: [PATCH 0086/1937] [oktoberfesttv] Add new extractor (Fixes #3845)

---
 youtube_dl/extractor/__init__.py      |  1 +
 youtube_dl/extractor/oktoberfesttv.py | 47 +++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 youtube_dl/extractor/oktoberfesttv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 079221567..629280215 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -261,6 +261,7 @@ from .nrk import (
 from .ntv import NTVIE
 from .nytimes import NYTimesIE
 from .nuvid import NuvidIE
+from .oktoberfesttv import OktoberfestTVIE
 from .ooyala import OoyalaIE
 from .orf import (
     ORFTVthekIE,
diff --git a/youtube_dl/extractor/oktoberfesttv.py b/youtube_dl/extractor/oktoberfesttv.py
new file mode 100644
index 000000000..4a41c0542
--- /dev/null
+++ b/youtube_dl/extractor/oktoberfesttv.py
@@ -0,0 +1,47 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class OktoberfestTVIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.oktoberfest-tv\.de/[^/]+/[^/]+/video/(?P<id>[^/?#]+)'
+
+    _TEST = {
+        'url': 'http://www.oktoberfest-tv.de/de/kameras/video/hb-zelt',
+        'info_dict': {
+            'id': 'hb-zelt',
+            'ext': 'mp4',
+            'title': 're:^Live-Kamera: Hofbräuzelt [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._live_title(self._html_search_regex(
+            r'<h1><strong>.*?</strong>(.*?)</h1>', webpage, 'title'))
+
+        clip = self._search_regex(
+            r"clip:\s*\{\s*url:\s*'([^']+)'", webpage, 'clip')
+        ncurl = self._search_regex(
+            r"netConnectionUrl:\s*'([^']+)'", webpage, 'rtmp base')
+        video_url = ncurl + clip
+        thumbnail = self._search_regex(
+            r"canvas:\s*\{\s*backgroundImage:\s*'url\(([^)]+)\)'", webpage,
+            'thumbnail', fatal=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+            'ext': 'mp4',
+            'is_live': True,
+            'thumbnail': thumbnail,
+        }

From 88fbe4c2ccdae7f917b6c9a2655f0878b6e4308c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 28 Sep 2014 09:49:42 +0200
Subject: [PATCH 0087/1937] release 2014.09.28

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c17701d6a..e62bef2cf 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.09.25'
+__version__ = '2014.09.28'

From b14f3a4c1da00cbee8775904c24c4d0547018ae0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 28 Sep 2014 10:34:55 +0200
Subject: [PATCH 0088/1937] [golem] Simplify (#3828)

---
 youtube_dl/extractor/common.py |  23 ++++++
 youtube_dl/extractor/golem.py  | 124 +++++++++------------------------
 2 files changed, 56 insertions(+), 91 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8d6a6f601..f43a0a569 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -22,6 +22,7 @@ from ..utils import (
     clean_html,
     compiled_regex_type,
     ExtractorError,
+    float_or_none,
     int_or_none,
     RegexNotFoundError,
     sanitize_filename,
@@ -720,6 +721,28 @@ class InfoExtractor(object):
         now_str = now.strftime("%Y-%m-%d %H:%M")
         return name + ' ' + now_str
 
+    def _int(self, v, name, fatal=False, **kwargs):
+        res = int_or_none(v, **kwargs)
+        if 'get_attr' in kwargs:
+            print(getattr(v, kwargs['get_attr']))
+        if res is None:
+            msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
+            if fatal:
+                raise ExtractorError(msg)
+            else:
+                self._downloader.report_warning(msg)
+        return res
+
+    def _float(self, v, name, fatal=False, **kwargs):
+        res = float_or_none(v, **kwargs)
+        if res is None:
+            msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
+            if fatal:
+                raise ExtractorError(msg)
+            else:
+                self._downloader.report_warning(msg)
+        return res
+
 
 class SearchInfoExtractor(InfoExtractor):
     """
diff --git a/youtube_dl/extractor/golem.py b/youtube_dl/extractor/golem.py
index 6a64b5d95..a237f19ee 100644
--- a/youtube_dl/extractor/golem.py
+++ b/youtube_dl/extractor/golem.py
@@ -4,7 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import compat_urlparse
+from ..utils import (
+    compat_urlparse,
+    determine_ext,
+)
 
 
 class GolemIE(InfoExtractor):
@@ -17,115 +20,54 @@ class GolemIE(InfoExtractor):
             'format_id': 'high',
             'ext': 'mp4',
             'title': 'iPhone 6 und 6 Plus - Test',
-            'duration': 300,
+            'duration': 300.44,
             'filesize': 65309548,
         }
     }
 
-    _CONFIG = 'https://video.golem.de/xml/{0}.xml'
     _PREFIX = 'http://video.golem.de'
 
-    def _warn(self, fmt, *args):
-        self.report_warning(fmt.format(*args), self._id)
-
-    def _extract_format(self, elem):
-        format_id = elem.tag
-
-        url = elem.findtext('./url')
-        if url == '':
-            self._warn("{0}: url: empty, skipping", format_id)
-            return None
-
-        fmt = {
-            'format_id': format_id,
-            'url': compat_urlparse.urljoin(self._PREFIX, url)
-        }
-
-        try:
-            _, ext = elem.findtext('./filename', '').rsplit('.', 1)
-        except ValueError:
-            self._warn('{0}: ext: missing extension', format_id)
-        else:
-            fmt['ext'] = ext
-
-        filesize = elem.findtext('./filesize')
-        if filesize is not None:
-            try:
-                fmt['filesize'] = int(filesize)
-            except ValueError as e:
-                self._warn('{0}: filesize: {1}', format_id, e)
-
-        width = elem.get('width')
-        if width is not None:
-            try:
-                fmt['width'] = int(width)
-            except ValueError as e:
-                self._warn('{0}: width: {1}', format_id, e)
-
-        height = elem.get('height')
-        if height is not None:
-            try:
-                fmt['height'] = int(height)
-            except ValueError as e:
-                self._warn('{0}: height: {1}', format_id, e)
-
-        return fmt
-
-    def _extract_thumbnail(self, elem):
-        url = elem.findtext('./url')
-        if url == '':
-            return None
-        thumb = {
-            'url': compat_urlparse.urljoin(self._PREFIX, url)
-        }
-
-        width = elem.get('width')
-        if width is not None:
-            try:
-                thumb['width'] = int(width)
-            except ValueError as e:
-                self._warn('thumbnail: width: {0}', e)
-
-        height = elem.get('height')
-        if height is not None:
-            try:
-                thumb['height'] = int(height)
-            except ValueError as e:
-                self._warn('thumbnail: height: {0}', e)
-
-        return thumb
-
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        self._id = mobj.group('id')
+        video_id = self._match_id(url)
 
-        config = self._download_xml(self._CONFIG.format(self._id), self._id)
+        config = self._download_xml(
+            'https://video.golem.de/xml/{0}.xml'.format(video_id), video_id)
 
         info = {
-            'id': self._id,
-            'title': config.findtext('./title', 'golem')
+            'id': video_id,
+            'title': config.findtext('./title', 'golem'),
+            'duration': self._float(config.findtext('./playtime'), 'duration'),
         }
 
         formats = []
         for e in config.findall('./*[url]'):
-            fmt = self._extract_format(e)
-            if fmt is not None:
-                formats.append(fmt)
+            url = e.findtext('./url')
+            if not url:
+                self._downloader.report_warning(
+                    "{0}: url: empty, skipping".format(e.tag))
+                continue
+
+            formats.append({
+                'format_id': e.tag,
+                'url': compat_urlparse.urljoin(self._PREFIX, url),
+                'height': self._int(e.get('height'), 'height'),
+                'width': self._int(e.get('width'), 'width'),
+                'filesize': self._int(e.findtext('filesize'), 'filesize'),
+                'ext': determine_ext(e.findtext('./filename')),
+            })
         self._sort_formats(formats)
         info['formats'] = formats
 
         thumbnails = []
         for e in config.findall('.//teaser[url]'):
-            thumb = self._extract_thumbnail(e)
-            if thumb is not None:
-                thumbnails.append(thumb)
+            url = e.findtext('./url')
+            if not url:
+                continue
+            thumbnails.append({
+                'url': compat_urlparse.urljoin(self._PREFIX, url),
+                'width': self._int(e.get('width'), 'thumbnail width'),
+                'height': self._int(e.get('height'), 'thumbnail height'),
+            })
         info['thumbnails'] = thumbnails
 
-        playtime = config.findtext('./playtime')
-        if playtime is not None:
-            try:
-                info['duration'] = round(float(playtime))
-            except ValueError as e:
-                self._warn('duration: {0}', e)
-
         return info

From 5a8b77551d930d4672159a015f553e64be111492 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 28 Sep 2014 10:40:49 +0200
Subject: [PATCH 0089/1937] [heise] Simplify (#3842)

---
 youtube_dl/extractor/heise.py | 92 ++++++++++-------------------------
 1 file changed, 26 insertions(+), 66 deletions(-)

diff --git a/youtube_dl/extractor/heise.py b/youtube_dl/extractor/heise.py
index 73c953181..05d4efb8c 100644
--- a/youtube_dl/extractor/heise.py
+++ b/youtube_dl/extractor/heise.py
@@ -1,34 +1,28 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
-    compat_urllib_parse,
     get_meta_content,
     parse_iso8601,
 )
 
 
 class HeiseIE(InfoExtractor):
-    _VALID_URL = (
-        r'^https?://(?:www\.)?heise\.de/video/artikel/' +
-        r'.+?(?P<id>[0-9]+)\.html$'
-    )
+    _VALID_URL = r'''(?x)
+        https?://(?:www\.)?heise\.de/video/artikel/
+        .+?(?P<id>[0-9]+)\.html(?:$|[?#])
+    '''
     _TEST = {
         'url': (
-            'http://www.heise.de/video/artikel/Podcast-c-t-uplink-3-3-' +
-            'Owncloud-Tastaturen-Peilsender-Smartphone-2404147.html'
+            'http://www.heise.de/video/artikel/Podcast-c-t-uplink-3-3-Owncloud-Tastaturen-Peilsender-Smartphone-2404147.html'
         ),
         'md5': 'ffed432483e922e88545ad9f2f15d30e',
         'info_dict': {
             'id': '2404147',
             'ext': 'mp4',
             'title': (
-                "Podcast: c't uplink 3.3 – Owncloud / Tastaturen / " +
-                "Peilsender Smartphone"
+                "Podcast: c't uplink 3.3 – Owncloud / Tastaturen / Peilsender Smartphone"
             ),
             'format_id': 'mp4_720',
             'timestamp': 1411812600,
@@ -36,84 +30,50 @@ class HeiseIE(InfoExtractor):
         }
     }
 
-    _CONFIG = (
-        r'".+?\?sequenz=(?P<sequenz>.+?)&container=(?P<container>.+?)' +
-        r'(?:&hd=(?P<hd>.+?))?(?:&signature=(?P<signature>.+?))?&callback=\?"'
-    )
-    _PREFIX = 'http://www.heise.de/videout/info?'
-
-    def _warn(self, fmt, *args):
-        self.report_warning(fmt.format(*args), self._id)
-
-    def _parse_config_url(self, html):
-        m = re.search(self._CONFIG, html)
-        if not m:
-            raise ExtractorError('No config found')
-
-        qs = compat_urllib_parse.urlencode(dict((k, v) for k, v
-                                                in m.groupdict().items()
-                                                if v is not None))
-        return self._PREFIX + qs
-
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        self._id = mobj.group('id')
+        video_id = self._match_id(url)
 
-        html = self._download_webpage(url, self._id)
-        config = self._download_json(self._parse_config_url(html), self._id)
+        webpage = self._download_webpage(url, video_id)
+        json_url = self._search_regex(
+            r'json_url:\s*"([^"]+)"', webpage, 'json URL')
+        config = self._download_json(json_url, video_id)
 
         info = {
-            'id': self._id
+            'id': video_id,
+            'thumbnail': config.get('poster'),
+            'timestamp': parse_iso8601(get_meta_content('date', webpage)),
         }
 
-        title = get_meta_content('fulltitle', html)
+        title = get_meta_content('fulltitle', webpage)
         if title:
             info['title'] = title
         elif config.get('title'):
             info['title'] = config['title']
         else:
-            self._warn('title: not found')
-            info['title'] = 'heise'
-
-        if (not config.get('formats') or
-                not hasattr(config['formats'], 'items')):
-            raise ExtractorError('No formats found')
+            info['title'] = self._og_search_title(webpage)
 
         formats = []
         for t, rs in config['formats'].items():
             if not rs or not hasattr(rs, 'items'):
-                self._warn('formats: {0}: no resolutions', t)
+                self._downloader.report_warning(
+                    'formats: {0}: no resolutions'.format(t))
                 continue
 
-            for res, obj in rs.items():
-                format_id = '{0}_{1}'.format(t, res)
+            for height_str, obj in rs.items():
+                format_id = '{0}_{1}'.format(t, height_str)
 
                 if not obj or not obj.get('url'):
-                    self._warn('formats: {0}: no url', format_id)
+                    self._downloader.report_warning(
+                        'formats: {0}: no url'.format(format_id))
                     continue
 
-                fmt = {
+                formats.append({
                     'url': obj['url'],
-                    'format_id': format_id
-                }
-                try:
-                    fmt['height'] = int(res)
-                except ValueError as e:
-                    self._warn('formats: {0}: height: {1}', t, e)
-
-                formats.append(fmt)
+                    'format_id': format_id,
+                    'height': self._int(height_str, 'height'),
+                })
 
         self._sort_formats(formats)
         info['formats'] = formats
 
-        if config.get('poster'):
-            info['thumbnail'] = config['poster']
-
-        date = get_meta_content('date', html)
-        if date:
-            try:
-                info['timestamp'] = parse_iso8601(date)
-            except ValueError as e:
-                self._warn('timestamp: {0}', e)
-
         return info

From c121a75b368a0c75de7416cbb36d9b9f40a7f1a7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 28 Sep 2014 10:49:12 +0200
Subject: [PATCH 0090/1937] [heise] Add support for description

---
 youtube_dl/extractor/heise.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/heise.py b/youtube_dl/extractor/heise.py
index 05d4efb8c..f97b1e085 100644
--- a/youtube_dl/extractor/heise.py
+++ b/youtube_dl/extractor/heise.py
@@ -27,6 +27,7 @@ class HeiseIE(InfoExtractor):
             'format_id': 'mp4_720',
             'timestamp': 1411812600,
             'upload_date': '20140927',
+            'description': 'In uplink-Episode 3.3 geht es darum, wie man sich von Cloud-Anbietern emanzipieren kann, worauf man beim Kauf einer Tastatur achten sollte und was Smartphones über uns verraten.',
         }
     }
 
@@ -42,6 +43,7 @@ class HeiseIE(InfoExtractor):
             'id': video_id,
             'thumbnail': config.get('poster'),
             'timestamp': parse_iso8601(get_meta_content('date', webpage)),
+            'description': self._og_search_description(webpage),
         }
 
         title = get_meta_content('fulltitle', webpage)

From c84178977268df1fb705bc8fd8cf3aa73158139a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 28 Sep 2014 10:49:58 +0200
Subject: [PATCH 0091/1937] [muenchentv] Add thumbnail

---
 youtube_dl/extractor/muenchentv.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/muenchentv.py b/youtube_dl/extractor/muenchentv.py
index 7cb6749be..c7f6beb9c 100644
--- a/youtube_dl/extractor/muenchentv.py
+++ b/youtube_dl/extractor/muenchentv.py
@@ -22,6 +22,7 @@ class MuenchenTVIE(InfoExtractor):
             'ext': 'mp4',
             'title': 're:^münchen.tv-Livestream [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'is_live': True,
+            'thumbnail': 're:^https?://.*\.jpg$'
         },
         'params': {
             'skip_download': True,
@@ -70,5 +71,6 @@ class MuenchenTVIE(InfoExtractor):
             'title': title,
             'formats': formats,
             'is_live': True,
+            'thumbnail': thumbnail,
         }
 

From 51ee08c4bb044bb670c8a6b855ba48a91892d27b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 28 Sep 2014 10:50:43 +0200
Subject: [PATCH 0092/1937] Remove unused imports

---
 youtube_dl/extractor/ard.py   | 2 --
 youtube_dl/extractor/golem.py | 2 --
 youtube_dl/extractor/vevo.py  | 1 -
 youtube_dl/extractor/youku.py | 1 -
 4 files changed, 6 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 54cec1c2f..8de9c11ea 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -8,8 +8,6 @@ from ..utils import (
     determine_ext,
     ExtractorError,
     qualities,
-    compat_urllib_parse_urlparse,
-    compat_urllib_parse,
     int_or_none,
     parse_duration,
     unified_strdate,
diff --git a/youtube_dl/extractor/golem.py b/youtube_dl/extractor/golem.py
index a237f19ee..bebfe8568 100644
--- a/youtube_dl/extractor/golem.py
+++ b/youtube_dl/extractor/golem.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     compat_urlparse,
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index ebab8b86c..5b1a3ec78 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -5,7 +5,6 @@ import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from ..utils import (
-    compat_HTTPError,
     compat_urllib_request,
     ExtractorError,
 )
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 07ed7cbd1..48d47a245 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -2,7 +2,6 @@
 
 from __future__ import unicode_literals
 
-import json
 import math
 import random
 import re

From 38c4d41b744660463abbb333737e031d9c87243e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 28 Sep 2014 10:55:12 +0200
Subject: [PATCH 0093/1937] [played] Simplify (#3798)

---
 youtube_dl/extractor/played.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
index a396e62e5..db40da43b 100644
--- a/youtube_dl/extractor/played.py
+++ b/youtube_dl/extractor/played.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 class PlayedIE(InfoExtractor):
     IE_NAME = 'played.to'
-    _VALID_URL = r'https?://played\.to/(?P<id>[a-zA-Z0-9_-]+)'
+    _VALID_URL = r'https?://(?:www\.)?played\.to/(?P<id>[a-zA-Z0-9_-]+)'
 
     _TEST = {
         'url': 'http://played.to/j2f2sfiiukgt',
@@ -27,15 +27,14 @@ class PlayedIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         orig_webpage = self._download_webpage(url, video_id)
-        fields = re.findall(r'type="hidden" name="(.+?)"\s* value="?(.+?)">', orig_webpage)
+        fields = re.findall(
+            r'type="hidden" name="([^"]+)"\s+value="([^"]+)">', orig_webpage)
         data = dict(fields)
 
-        self.to_screen('%s: Waiting for timeout' % video_id)
-        time.sleep(2)
+        self._sleep(2, video_id)
 
         post = compat_urllib_parse.urlencode(data)
         headers = {
@@ -54,4 +53,4 @@ class PlayedIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'url': video_url,
-        }
\ No newline at end of file
+        }

From 76e7d1e74b10b99ed9289b0c30c5f4933f9d841e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 28 Sep 2014 10:56:36 +0200
Subject: [PATCH 0094/1937] [played] Remove unused import

---
 youtube_dl/extractor/played.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
index db40da43b..645a1e06d 100644
--- a/youtube_dl/extractor/played.py
+++ b/youtube_dl/extractor/played.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import time
 import os.path
 
 from .common import InfoExtractor

From d6e6a4225650ff220c7fe0687d883552e4b45bde Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 28 Sep 2014 12:14:16 +0200
Subject: [PATCH 0095/1937] [vimeo:likes] Add new extractor (Fixes #3835)

---
 test/test_download.py            |  4 +++-
 youtube_dl/extractor/__init__.py |  5 +++--
 youtube_dl/extractor/generic.py  | 10 ++++------
 youtube_dl/extractor/vimeo.py    | 33 ++++++++++++++++++++++++++++++++
 4 files changed, 43 insertions(+), 9 deletions(-)

diff --git a/test/test_download.py b/test/test_download.py
index 2b8ac6975..8178015ea 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -139,7 +139,9 @@ def generator(test_case):
 
             if is_playlist:
                 self.assertEqual(res_dict['_type'], 'playlist')
+                self.assertTrue('entries' in res_dict)
                 expect_info_dict(self, test_case.get('info_dict', {}), res_dict)
+
             if 'playlist_mincount' in test_case:
                 assertGreaterEqual(
                     self,
@@ -188,7 +190,7 @@ def generator(test_case):
                 expect_info_dict(self, tc.get('info_dict', {}), info_dict)
         finally:
             try_rm_tcs_files()
-            if is_playlist and res_dict is not None:
+            if is_playlist and res_dict is not None and res_dict.get('entries'):
                 # Remove all other files that may have been extracted if the
                 # extractor returns full results even with extract_flat
                 res_tcs = [{'info_dict': e} for e in res_dict['entries']]
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6ab3eeaf5..86bff185b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -412,11 +412,12 @@ from .videoweed import VideoWeedIE
 from .vidme import VidmeIE
 from .vimeo import (
     VimeoIE,
-    VimeoChannelIE,
-    VimeoUserIE,
     VimeoAlbumIE,
+    VimeoChannelIE,
     VimeoGroupsIE,
+    VimeoLikesIE,
     VimeoReviewIE,
+    VimeoUserIE,
     VimeoWatchLaterIE,
 )
 from .vimple import VimpleIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 367f930dd..0dfa4853d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -397,12 +397,6 @@ class GenericIE(InfoExtractor):
         },
     ]
 
-    def report_download_webpage(self, video_id):
-        """Report webpage download."""
-        if not self._downloader.params.get('test', False):
-            self._downloader.report_warning('Falling back on generic information extractor.')
-        super(GenericIE, self).report_download_webpage(video_id)
-
     def report_following_redirect(self, new_url):
         """Report information extraction."""
         self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
@@ -502,6 +496,7 @@ class GenericIE(InfoExtractor):
 
         url, smuggled_data = unsmuggle_url(url)
         force_videoid = None
+        is_intentional = smuggled_data and smuggled_data.get('to_generic')
         if smuggled_data and 'force_videoid' in smuggled_data:
             force_videoid = smuggled_data['force_videoid']
             video_id = force_videoid
@@ -544,6 +539,9 @@ class GenericIE(InfoExtractor):
                     'upload_date': upload_date,
                 }
 
+        if not self._downloader.params.get('test', False) and not is_intentional:
+            self._downloader.report_warning('Falling back on generic information extractor.')
+
         try:
             webpage = self._download_webpage(url, video_id)
         except ValueError:
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index bc01d7fbf..4be1b8785 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -15,6 +15,7 @@ from ..utils import (
     get_element_by_attribute,
     ExtractorError,
     RegexNotFoundError,
+    smuggle_url,
     std_headers,
     unsmuggle_url,
     urlencode_postdata,
@@ -529,3 +530,35 @@ class VimeoWatchLaterIE(VimeoBaseInfoExtractor, VimeoChannelIE):
 
     def _real_extract(self, url):
         return self._extract_videos('watchlater', 'https://vimeo.com/home/watchlater')
+
+
+class VimeoLikesIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?vimeo\.com/user(?P<id>[0-9]+)/likes(?:$|[?#])'
+    IE_NAME = 'vimeo:likes'
+    IE_DESC = 'Vimeo user likes'
+    _TEST = {
+        'url': 'https://vimeo.com/user20132939/likes',
+        'playlist_mincount': 4,
+        'add_ies': ['Generic'],
+        "info_dict": {
+            "description": "Videos Philipp Hagemeister likes on Vimeo.",
+            "title": "Vimeo / Philipp Hagemeister's likes",
+        },
+        'params': {
+            'extract_flat': False,
+        },
+    }
+
+    def _real_extract(self, url):
+        user_id = self._match_id(url)
+        rss_url = '%s//vimeo.com/user%s/likes/rss' % (
+            self.http_scheme(), user_id)
+        surl = smuggle_url(rss_url, {
+            'force_videoid': '%s_likes' % user_id,
+            'to_generic': True,
+        })
+
+        return {
+            '_type': 'url',
+            'url': surl,
+        }

From 22dd3fad8623472cfe681fdfbaa346e0c8f5fb84 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 28 Sep 2014 12:14:25 +0200
Subject: [PATCH 0096/1937] release 2014.09.28.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e62bef2cf..eb4356811 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.09.28'
+__version__ = '2014.09.28.1'

From 4bc77c8417ca0340d09dcebb311d06aa7d5ba0ac Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 28 Sep 2014 13:52:21 +0200
Subject: [PATCH 0097/1937] [README] Use _match_id helper function

---
 README.md | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 5d15decb5..5e0d07997 100644
--- a/README.md
+++ b/README.md
@@ -442,8 +442,6 @@ If you want to add support for a new site, you can follow this quick list (assum
     # coding: utf-8
     from __future__ import unicode_literals
 
-    import re
-
     from .common import InfoExtractor
 
 
@@ -466,8 +464,7 @@ If you want to add support for a new site, you can follow this quick list (assum
         }
 
         def _real_extract(self, url):
-            mobj = re.match(self._VALID_URL, url)
-            video_id = mobj.group('id')
+            video_id = self._match_id(url)
 
             # TODO more code goes here, for example ...
             webpage = self._download_webpage(url, video_id)

From 7f5c0c4a19cf72b6ede80ee0fea4611d8bd45010 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Sep 2014 22:10:20 +0700
Subject: [PATCH 0098/1937] [README] Clarify test's md5 filesize (#3846)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5e0d07997..0f7442906 100644
--- a/README.md
+++ b/README.md
@@ -449,7 +449,7 @@ If you want to add support for a new site, you can follow this quick list (assum
         _VALID_URL = r'https?://(?:www\.)?yourextractor\.com/watch/(?P<id>[0-9]+)'
         _TEST = {
             'url': 'http://yourextractor.com/watch/42',
-            'md5': 'TODO: md5 sum of the first 10KiB of the video file',
+            'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
             'info_dict': {
                 'id': '42',
                 'ext': 'mp4',

From dfee83234b642a94255d52d992295b980ce2a5f7 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 28 Sep 2014 19:25:28 +0300
Subject: [PATCH 0099/1937] [nfl] Prefer progressive downloads

---
 youtube_dl/extractor/nfl.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index 963c4587c..668d99512 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -17,11 +17,11 @@ class NFLIE(InfoExtractor):
     _PLAYER_CONFIG_URL = 'http://www.nfl.com/static/content/static/config/video/config.json'
     _TEST = {
         'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights',
-        # 'md5': '5eb8c40a727dda106d510e5d6ffa79e5',  # md5 checksum fluctuates
+        'md5': '394ef771ddcd1354f665b471d78ec4c6',
         'info_dict': {
             'id': '0ap3000000398478',
             'ext': 'mp4',
-            'title': 'Week 3: Washington Redskins vs. Philadelphia Eagles highlights',
+            'title': 'Week 3: Redskins vs. Eagles highlights',
             'description': 'md5:56323bfb0ac4ee5ab24bd05fdf3bf478',
             'upload_date': '20140921',
             'timestamp': 1411337580,
@@ -66,9 +66,9 @@ class NFLIE(InfoExtractor):
             )
 
             if protocol == 'rtmp':
-                preference = -2
-            elif 'prog' in name.lower():
                 preference = -1
+            elif 'prog' in name.lower():
+                preference = 1
             else:
                 preference = 0
 
@@ -94,7 +94,7 @@ class NFLIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': video_data.get('storyHeadline'),
+            'title': video_data.get('headline'),
             'formats': formats,
             'description': video_data.get('caption'),
             'duration': video_data.get('duration'),

From 5f4c318844180d51745303979682a0a482f05328 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 28 Sep 2014 21:48:26 +0300
Subject: [PATCH 0100/1937] [nfl] Support team micro-sites (fixes #3831)

---
 youtube_dl/extractor/nfl.py | 159 +++++++++++++++++++++++-------------
 1 file changed, 100 insertions(+), 59 deletions(-)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index 668d99512..4832b3ce4 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    compat_urllib_parse,
     int_or_none,
     remove_end,
 )
@@ -13,76 +14,116 @@ from ..utils import (
 
 class NFLIE(InfoExtractor):
     IE_NAME = 'nfl.com'
-    _VALID_URL = r'(?x)https?://(?:www\.)?nfl\.com/(?:videos/(?:.+)/|.*?\#video=)(?P<id>\d..[0-9]+)'
-    _PLAYER_CONFIG_URL = 'http://www.nfl.com/static/content/static/config/video/config.json'
-    _TEST = {
-        'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights',
-        'md5': '394ef771ddcd1354f665b471d78ec4c6',
-        'info_dict': {
-            'id': '0ap3000000398478',
-            'ext': 'mp4',
-            'title': 'Week 3: Redskins vs. Eagles highlights',
-            'description': 'md5:56323bfb0ac4ee5ab24bd05fdf3bf478',
-            'upload_date': '20140921',
-            'timestamp': 1411337580,
-            'thumbnail': 're:^https?://.*\.jpg$',
+    _VALID_URL = r'''(?x)https?://
+        (?P<host>(?:www\.)?(?:nfl\.com|.*?\.clubs\.nfl\.com))/
+        (?:.+?/)*
+        (?P<id>(?:\d[a-z]{2}\d{13}|\w{8}\-(?:\w{4}\-){3}\w{12}))'''
+    _TESTS = [
+        {
+            'url': 'http://www.nfl.com/videos/nfl-game-highlights/0ap3000000398478/Week-3-Redskins-vs-Eagles-highlights',
+            'md5': '394ef771ddcd1354f665b471d78ec4c6',
+            'info_dict': {
+                'id': '0ap3000000398478',
+                'ext': 'mp4',
+                'title': 'Week 3: Redskins vs. Eagles highlights',
+                'description': 'md5:56323bfb0ac4ee5ab24bd05fdf3bf478',
+                'upload_date': '20140921',
+                'timestamp': 1411337580,
+                'thumbnail': 're:^https?://.*\.jpg$',
+            }
+        },
+        {
+            'url': 'http://prod.www.steelers.clubs.nfl.com/video-and-audio/videos/LIVE_Post_Game_vs_Browns/9d72f26a-9e2b-4718-84d3-09fb4046c266',
+            'md5': 'cf85bdb4bc49f6e9d3816d130c78279c',
+            'info_dict': {
+                'id': '9d72f26a-9e2b-4718-84d3-09fb4046c266',
+                'ext': 'mp4',
+                'title': 'LIVE: Post Game vs. Browns',
+                'description': 'md5:6a97f7e5ebeb4c0e69a418a89e0636e8',
+                'upload_date': '20131229',
+                'timestamp': 1388354455,
+                'thumbnail': 're:^https?://.*\.jpg$',
+            }
+        }
+    ]
+
+    @staticmethod
+    def prepend_host(host, url):
+        if not url.startswith('http'):
+            if not url.startswith('/'):
+                url = '/%s' % url
+            url = 'http://{0:}{1:}'.format(host, url)
+        return url
+
+    @staticmethod
+    def format_from_stream(stream, protocol, host, path_prefix='',
+                           preference=0, note=None):
+        url = '{protocol:}://{host:}/{prefix:}{path:}'.format(
+            protocol=protocol,
+            host=host,
+            prefix=path_prefix,
+            path=stream.get('path'),
+        )
+        return {
+            'url': url,
+            'vbr': int_or_none(stream.get('rate', 0), 1000),
+            'preference': preference,
+            'format_note': note,
         }
-    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id, host = mobj.group('id'), mobj.group('host')
 
-        config = self._download_json(self._PLAYER_CONFIG_URL, video_id,
+        webpage = self._download_webpage(url, video_id)
+
+        config_url = NFLIE.prepend_host(host, self._search_regex(
+            r'(?:config|configURL)\s*:\s*"([^"]+)"', webpage, 'config URL'))
+        config = self._download_json(config_url, video_id,
                                      note='Downloading player config')
-        url_template = 'http://nfl.com{contentURLTemplate:s}'.format(**config)
-        video_data = self._download_json(url_template.format(id=video_id), video_id)
-
-        cdns = config.get('cdns')
-        if not cdns:
-            raise ExtractorError('Failed to get CDN data', expected=True)
+        url_template = NFLIE.prepend_host(
+            host, '{contentURLTemplate:}'.format(**config))
+        video_data = self._download_json(
+            url_template.format(id=video_id), video_id)
 
         formats = []
-        streams = video_data.get('cdnData', {}).get('bitrateInfo', [])
-        for name, cdn in cdns.items():
-            # LimeLight streams don't seem to work
-            if cdn.get('name') == 'LIMELIGHT':
-                continue
-
-            protocol = cdn.get('protocol')
-            host = remove_end(cdn.get('host', ''), '/')
-            if not (protocol and host):
-                continue
-
-            path_prefix = cdn.get('pathprefix', '')
-            if path_prefix and not path_prefix.endswith('/'):
-                path_prefix = '%s/' % path_prefix
-
-            get_url = lambda p: '{protocol:s}://{host:s}/{prefix:s}{path:}'.format(
-                protocol=protocol,
-                host=host,
-                prefix=path_prefix,
-                path=p,
-            )
-
-            if protocol == 'rtmp':
-                preference = -1
-            elif 'prog' in name.lower():
-                preference = 1
-            else:
-                preference = 0
-
+        cdn_data = video_data.get('cdnData', {})
+        streams = cdn_data.get('bitrateInfo', [])
+        if cdn_data.get('format') == 'EXTERNAL_HTTP_STREAM':
+            parts = compat_urllib_parse.urlparse(cdn_data.get('uri'))
+            protocol, host = parts.scheme, parts.netloc
             for stream in streams:
-                path = stream.get('path')
-                if not path:
+                formats.append(
+                    NFLIE.format_from_stream(stream, protocol, host))
+        else:
+            cdns = config.get('cdns')
+            if not cdns:
+                raise ExtractorError('Failed to get CDN data', expected=True)
+
+            for name, cdn in cdns.items():
+                # LimeLight streams don't seem to work
+                if cdn.get('name') == 'LIMELIGHT':
                     continue
 
-                formats.append({
-                    'url': get_url(path),
-                    'vbr': int_or_none(stream.get('rate', 0), 1000),
-                    'preference': preference,
-                    'format_note': name,
-                })
+                protocol = cdn.get('protocol')
+                host = remove_end(cdn.get('host', ''), '/')
+                if not (protocol and host):
+                    continue
+
+                prefix = cdn.get('pathprefix', '')
+                if prefix and not prefix.endswith('/'):
+                    prefix = '%s/' % prefix
+
+                preference = 0
+                if protocol == 'rtmp':
+                    preference = -2
+                elif 'prog' in name.lower():
+                    preference = 1
+
+                for stream in streams:
+                    formats.append(
+                        NFLIE.format_from_stream(stream, protocol, host,
+                                                 prefix, preference, name))
 
         self._sort_formats(formats)
 

From 67077b182b698ac56cec9525a2669d5cee394226 Mon Sep 17 00:00:00 2001
From: Anton Larionov <diffident.cat@gmail.com>
Date: Sun, 28 Sep 2014 23:36:55 +0400
Subject: [PATCH 0101/1937] [thvideo] Add support for playlists

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/thvideo.py  | 24 ++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 86bff185b..89a9d8106 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -371,7 +371,10 @@ from .thisav import ThisAVIE
 from .tinypic import TinyPicIE
 from .tlc import TlcIE, TlcDeIE
 from .tnaflix import TNAFlixIE
-from .thvideo import THVideoIE
+from .thvideo import (
+    THVideoIE,
+    THVideoPlaylistIE
+)
 from .toutv import TouTvIE
 from .toypics import ToypicsUserIE, ToypicsIE
 from .traileraddict import TrailerAddictIE
diff --git a/youtube_dl/extractor/thvideo.py b/youtube_dl/extractor/thvideo.py
index 607e947bb..0ae20ea30 100644
--- a/youtube_dl/extractor/thvideo.py
+++ b/youtube_dl/extractor/thvideo.py
@@ -57,3 +57,27 @@ class THVideoIE(InfoExtractor):
             'description': description,
             'upload_date': upload_date
         }
+
+
+class THVideoPlaylistIE(InfoExtractor):
+    _VALID_URL = r'http?://(?:www\.)?thvideo\.tv/mylist(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://thvideo.tv/mylist2',
+        'info_dict': {
+            'id': '2',
+            'title': '幻想万華鏡',
+        },
+        'playlist_mincount': 23,
+    }
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(url, 'playlist')
+        mobj = re.match(self._VALID_URL, url)
+        list_id = mobj.group('id')
+        list_title = self._html_search_regex(r'<h1 class="show_title">(.*?)<b id', webpage, 'playlist title')
+
+        entries = [
+            self.url_result('http://thvideo.tv/v/th' + id, 'THVideo')
+            for id in re.findall(r'<dd><a href="http://thvideo.tv/v/th(\d+)/" target=', webpage)]
+
+        return self.playlist_result(entries, list_id, list_title)
\ No newline at end of file

From d2e32f7df56ab497175437bffdcdfedbd71ca8d9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 00:23:41 +0200
Subject: [PATCH 0102/1937] Do not use HTML characters in output

This messes up the format when people paste it outside of code tags.
---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 44dcb1e34..f651337ad 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -87,7 +87,7 @@ def parseOpts(overrideArguments=None):
         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
             try:
                 i = opts.index(private_opt)
-                opts[i+1] = '<PRIVATE>'
+                opts[i+1] = 'PRIVATE'
             except ValueError:
                 pass
         return opts

From 9c44d2429b90dece734df778c63b04c15e91c1ca Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 00:36:06 +0200
Subject: [PATCH 0103/1937] [vimeo:likes] Support large like lists (Fixes
 #3847)

---
 test/test_utils.py              |  9 ++++-
 youtube_dl/extractor/vimeo.py   | 66 ++++++++++++++++++++++-----------
 youtube_dl/extractor/youtube.py |  4 +-
 youtube_dl/utils.py             | 39 +++++++++++++++++--
 4 files changed, 89 insertions(+), 29 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 3efbed29d..6419b3ca9 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -22,7 +22,8 @@ from youtube_dl.utils import (
     fix_xml_ampersands,
     get_meta_content,
     orderedSet,
-    PagedList,
+    OnDemandPagedList,
+    InAdvancePagedList,
     parse_duration,
     read_batch_urls,
     sanitize_filename,
@@ -246,10 +247,14 @@ class TestUtil(unittest.TestCase):
                 for i in range(firstid, upto):
                     yield i
 
-            pl = PagedList(get_page, pagesize)
+            pl = OnDemandPagedList(get_page, pagesize)
             got = pl.getslice(*sliceargs)
             self.assertEqual(got, expected)
 
+            iapl = InAdvancePagedList(get_page, size // pagesize + 1, pagesize)
+            got = iapl.getslice(*sliceargs)
+            self.assertEqual(got, expected)
+
         testPL(5, 2, (), [0, 1, 2, 3, 4])
         testPL(5, 2, (1,), [1, 2, 3, 4])
         testPL(5, 2, (2,), [2, 3, 4])
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 4be1b8785..403d0bb28 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -8,18 +8,19 @@ import itertools
 from .common import InfoExtractor
 from .subtitles import SubtitlesInfoExtractor
 from ..utils import (
+    clean_html,
     compat_HTTPError,
     compat_urllib_parse,
     compat_urllib_request,
-    clean_html,
-    get_element_by_attribute,
+    compat_urlparse,
     ExtractorError,
+    get_element_by_attribute,
+    InAdvancePagedList,
+    int_or_none,
     RegexNotFoundError,
-    smuggle_url,
     std_headers,
     unsmuggle_url,
     urlencode_postdata,
-    int_or_none,
 )
 
 
@@ -533,32 +534,55 @@ class VimeoWatchLaterIE(VimeoBaseInfoExtractor, VimeoChannelIE):
 
 
 class VimeoLikesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vimeo\.com/user(?P<id>[0-9]+)/likes(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?vimeo\.com/user(?P<id>[0-9]+)/likes/?(?:$|[?#]|sort:)'
     IE_NAME = 'vimeo:likes'
     IE_DESC = 'Vimeo user likes'
     _TEST = {
-        'url': 'https://vimeo.com/user20132939/likes',
-        'playlist_mincount': 4,
-        'add_ies': ['Generic'],
+        'url': 'https://vimeo.com/user755559/likes/',
+        'playlist_mincount': 293,
         "info_dict": {
-            "description": "Videos Philipp Hagemeister likes on Vimeo.",
-            "title": "Vimeo / Philipp Hagemeister's likes",
-        },
-        'params': {
-            'extract_flat': False,
+            "description": "See all the videos urza likes",
+            "title": 'Videos urza likes',
         },
     }
 
     def _real_extract(self, url):
         user_id = self._match_id(url)
-        rss_url = '%s//vimeo.com/user%s/likes/rss' % (
-            self.http_scheme(), user_id)
-        surl = smuggle_url(rss_url, {
-            'force_videoid': '%s_likes' % user_id,
-            'to_generic': True,
-        })
+        webpage = self._download_webpage(url, user_id)
+        page_count = self._int(
+            self._search_regex(
+                r'''(?x)<li><a\s+href="[^"]+"\s+data-page="([0-9]+)">
+                    .*?</a></li>\s*<li\s+class="pagination_next">
+                ''', webpage, 'page count'),
+            'page count', fatal=True)
+        PAGE_SIZE = 12
+        title = self._html_search_regex(
+            r'(?s)<h1>(.+?)</h1>', webpage, 'title', fatal=False)
+        description = self._html_search_meta('description', webpage)
+
+        def _get_page(idx):
+            page_url = '%s//vimeo.com/user%s/likes/page:%d/sort:date' % (
+                self.http_scheme(), user_id, idx + 1)
+            webpage = self._download_webpage(
+                page_url, user_id,
+                note='Downloading page %d/%d' % (idx + 1, page_count))
+            video_list = self._search_regex(
+                r'(?s)<ol class="js-browse_list[^"]+"[^>]*>(.*?)</ol>',
+                webpage, 'video content')
+            paths = re.findall(
+                r'<li[^>]*>\s*<a\s+href="([^"]+)"', video_list)
+            for path in paths:
+                yield {
+                    '_type': 'url',
+                    'url': compat_urlparse.urljoin(page_url, path),
+                }
+
+        pl = InAdvancePagedList(_get_page, page_count, PAGE_SIZE)
 
         return {
-            '_type': 'url',
-            'url': surl,
+            '_type': 'playlist',
+            'id': 'user%s_likes' % user_id,
+            'title': title,
+            'description': description,
+            'entries': pl,
         }
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 99198e380..045507bc7 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -26,7 +26,7 @@ from ..utils import (
     get_element_by_attribute,
     ExtractorError,
     int_or_none,
-    PagedList,
+    OnDemandPagedList,
     unescapeHTML,
     unified_strdate,
     orderedSet,
@@ -1341,7 +1341,7 @@ class YoutubeUserIE(InfoExtractor):
                     'id': video_id,
                     'title': title,
                 }
-        url_results = PagedList(download_page, self._GDATA_PAGE_SIZE)
+        url_results = OnDemandPagedList(download_page, self._GDATA_PAGE_SIZE)
 
         return self.playlist_result(url_results, playlist_title=username)
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b644f4e92..9f49507c1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1384,14 +1384,16 @@ def check_executable(exe, args=[]):
 
 
 class PagedList(object):
-    def __init__(self, pagefunc, pagesize):
-        self._pagefunc = pagefunc
-        self._pagesize = pagesize
-
     def __len__(self):
         # This is only useful for tests
         return len(self.getslice())
 
+
+class OnDemandPagedList(PagedList):
+    def __init__(self, pagefunc, pagesize):
+        self._pagefunc = pagefunc
+        self._pagesize = pagesize
+
     def getslice(self, start=0, end=None):
         res = []
         for pagenum in itertools.count(start // self._pagesize):
@@ -1430,6 +1432,35 @@ class PagedList(object):
         return res
 
 
+class InAdvancePagedList(PagedList):
+    def __init__(self, pagefunc, pagecount, pagesize):
+        self._pagefunc = pagefunc
+        self._pagecount = pagecount
+        self._pagesize = pagesize
+
+    def getslice(self, start=0, end=None):
+        res = []
+        start_page = start // self._pagesize
+        end_page = (
+            self._pagecount if end is None else (end // self._pagesize + 1))
+        skip_elems = start - start_page * self._pagesize
+        only_more = None if end is None else end - start
+        for pagenum in range(start_page, end_page):
+            page = list(self._pagefunc(pagenum))
+            if skip_elems:
+                page = page[skip_elems:]
+                skip_elems = None
+            if only_more is not None:
+                if len(page) < only_more:
+                    only_more -= len(page)
+                else:
+                    page = page[:only_more]
+                    res.extend(page)
+                    break
+            res.extend(page)
+        return res
+
+
 def uppercase_escape(s):
     unicode_escape = codecs.getdecoder('unicode_escape')
     return re.sub(

From 1770ed9e86a147eceb86210dec0aefcf0d94ab52 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 00:38:37 +0200
Subject: [PATCH 0104/1937] [thvideo] Simplify (#3848)

---
 youtube_dl/extractor/thvideo.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/thvideo.py b/youtube_dl/extractor/thvideo.py
index 0ae20ea30..496f15d80 100644
--- a/youtube_dl/extractor/thvideo.py
+++ b/youtube_dl/extractor/thvideo.py
@@ -26,8 +26,7 @@ class THVideoIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         # extract download link from mobile player page
         webpage_player = self._download_webpage(
@@ -71,13 +70,15 @@ class THVideoPlaylistIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        webpage = self._download_webpage(url, 'playlist')
-        mobj = re.match(self._VALID_URL, url)
-        list_id = mobj.group('id')
-        list_title = self._html_search_regex(r'<h1 class="show_title">(.*?)<b id', webpage, 'playlist title')
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+        list_title = self._html_search_regex(
+            r'<h1 class="show_title">(.*?)<b id', webpage, 'playlist title',
+            fatal=False)
 
         entries = [
             self.url_result('http://thvideo.tv/v/th' + id, 'THVideo')
             for id in re.findall(r'<dd><a href="http://thvideo.tv/v/th(\d+)/" target=', webpage)]
 
-        return self.playlist_result(entries, list_id, list_title)
\ No newline at end of file
+        return self.playlist_result(entries, playlist_id, list_title)

From e2dce5378191e315ad86785aac7e786c86a1a121 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 01:39:26 +0200
Subject: [PATCH 0105/1937] [youtube] Always request webpage in English (Fixes
 #3844)

---
 youtube_dl/extractor/youtube.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 045507bc7..61228817e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -655,7 +655,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
         # Get video webpage
         url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id
-        video_webpage = self._download_webpage(url, video_id)
+        req = compat_urllib_request.Request(url)
+        req.add_header('Cookie', 'PREF=hl=en')
+        video_webpage = self._download_webpage(req, video_id)
 
         # Attempt to extract SWF player URL
         mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)

From a43ee88c6f888196b47cb1e12463a64ada0ead12 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 01:51:53 +0200
Subject: [PATCH 0106/1937] release 2014.09.29

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index eb4356811..17e5ea8e2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.09.28.1'
+__version__ = '2014.09.29'

From a1f934b171dcc8e1215ee30d0715ce562eb220e3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 02:04:16 +0200
Subject: [PATCH 0107/1937] [youtube] Correct language cookie handling

---
 youtube_dl/extractor/youtube.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 61228817e..9041cfa87 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -655,9 +655,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
         # Get video webpage
         url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id
-        req = compat_urllib_request.Request(url)
-        req.add_header('Cookie', 'PREF=hl=en')
-        video_webpage = self._download_webpage(req, video_id)
+        pref_cookies = [
+            c for c in self._downloader.cookiejar
+            if c.domain == '.youtube.com' and c.name == 'PREF']
+        for pc in pref_cookies:
+            if 'hl=' in pc.value:
+                pc.value = re.sub(r'hl=[^&]+', 'hl=en', pc.value)
+            else:
+                if pc.value:
+                    pc.value += '&'
+                pc.value += 'hl=en'
+        video_webpage = self._download_webpage(url, video_id)
 
         # Attempt to extract SWF player URL
         mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)

From f5b7e6a842b00bab8320d30608dd4a10a4752a17 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 02:04:28 +0200
Subject: [PATCH 0108/1937] release 2014.09.29.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 17e5ea8e2..885df83c0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.09.29'
+__version__ = '2014.09.29.1'

From 27aede907436fa58600cd46bb04e7eae6e1e9279 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 04:48:50 +0200
Subject: [PATCH 0109/1937] [pbs] Add support for series/jwplayer type video
 (Fixes #3849)

---
 youtube_dl/extractor/pbs.py | 39 +++++++++++++++++++++++++++++--------
 1 file changed, 31 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 2adfde909..8f140d626 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -4,6 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    unified_strdate,
     US_RATINGS,
 )
 
@@ -11,10 +12,10 @@ from ..utils import (
 class PBSIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://
         (?:
-            # Direct video URL
-            video\.pbs\.org/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
-            # Article with embedded player
-           (?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P<presumptive_id>[^/]+)/?(?:$|[?\#]) |
+           # Direct video URL
+           video\.pbs\.org/(?:viralplayer|video)/(?P<id>[0-9]+)/? |
+           # Article with embedded player (or direct video)
+           (?:www\.)?pbs\.org/(?:[^/]+/){2,5}(?P<presumptive_id>[^/]+?)(?:\.html)?/?(?:$|[?\#]) |
            # Player
            video\.pbs\.org/(?:widget/)?partnerplayer/(?P<player_id>[^/]+)/
         )
@@ -65,10 +66,25 @@ class PBSIE(InfoExtractor):
                 'duration': 6559,
                 'thumbnail': 're:^https?://.*\.jpg$',
             }
+        },
+        {
+            'url': 'http://www.pbs.org/wgbh/nova/earth/killer-typhoon.html',
+            'md5': '908f3e5473a693b266b84e25e1cf9703',
+            'info_dict': {
+                'id': '2365160389',
+                'display_id': 'killer-typhoon',
+                'ext': 'mp4',
+                'description': 'md5:c741d14e979fc53228c575894094f157',
+                'title': 'Killer Typhoon',
+                'duration': 3172,
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'upload_date': '20140122',
+            }
         }
+
     ]
 
-    def _extract_ids(self, url):
+    def _extract_webpage(self, url):
         mobj = re.match(self._VALID_URL, url)
 
         presumptive_id = mobj.group('presumptive_id')
@@ -76,15 +92,20 @@ class PBSIE(InfoExtractor):
         if presumptive_id:
             webpage = self._download_webpage(url, display_id)
 
+            upload_date = unified_strdate(self._search_regex(
+                r'<input type="hidden" id="air_date_[0-9]+" value="([^"]+)"',
+                webpage, 'upload date', default=None))
+
             MEDIA_ID_REGEXES = [
                 r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'",  # frontline video embed
                 r'class="coveplayerid">([^<]+)<',                       # coveplayer
+                r'<input type="hidden" id="pbs_video_id_[0-9]+" value="([0-9]+)"/>',  # jwplayer
             ]
 
             media_id = self._search_regex(
                 MEDIA_ID_REGEXES, webpage, 'media ID', fatal=False, default=None)
             if media_id:
-                return media_id, presumptive_id
+                return media_id, presumptive_id, upload_date
 
             url = self._search_regex(
                 r'<iframe\s+(?:class|id)=["\']partnerPlayer["\'].*?\s+src=["\'](.*?)["\']>',
@@ -104,10 +125,10 @@ class PBSIE(InfoExtractor):
             video_id = mobj.group('id')
             display_id = video_id
 
-        return video_id, display_id
+        return video_id, display_id, None
 
     def _real_extract(self, url):
-        video_id, display_id = self._extract_ids(url)
+        video_id, display_id, upload_date = self._extract_webpage(url)
 
         info_url = 'http://video.pbs.org/videoInfo/%s?format=json' % video_id
         info = self._download_json(info_url, display_id)
@@ -119,6 +140,7 @@ class PBSIE(InfoExtractor):
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': info['title'],
             'url': info['alternate_encoding']['url'],
             'ext': 'mp4',
@@ -126,4 +148,5 @@ class PBSIE(InfoExtractor):
             'thumbnail': info.get('image_url'),
             'duration': info.get('duration'),
             'age_limit': age_limit,
+            'upload_date': upload_date,
         }

From 35d3e63d24c524922cb39ba36cb5f6de12400504 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 04:49:11 +0200
Subject: [PATCH 0110/1937] release 2014.09.29.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 885df83c0..1384b496b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.09.29.1'
+__version__ = '2014.09.29.2'

From 25930395225c45a9e5045ada291d37817371b086 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 04:58:29 +0200
Subject: [PATCH 0111/1937] [vimeo] Use regexps to find description

This fixes descriptions on 2.6 and makes the code simpler.
---
 youtube_dl/extractor/vimeo.py | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 403d0bb28..a002555a9 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -275,18 +275,9 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
                 _, video_thumbnail = sorted((int(width if width.isdigit() else 0), t_url) for (width, t_url) in video_thumbs.items())[-1]
 
         # Extract video description
-        video_description = None
-        try:
-            video_description = get_element_by_attribute("class", "description_wrapper", webpage)
-            if video_description:
-                video_description = clean_html(video_description)
-        except AssertionError as err:
-            # On some pages like (http://player.vimeo.com/video/54469442) the
-            # html tags are not closed, python 2.6 cannot handle it
-            if err.args[0] == 'we should not get here!':
-                pass
-            else:
-                raise
+        video_description = self._html_search_regex(
+            r'(?s)<div class="[^"]*description"[^>]*>(.*?)</div>',
+            webpage, 'description', fatal=False)
 
         # Extract video duration
         video_duration = int_or_none(config["video"].get("duration"))

From 12548cd9330222848f6b49fe9eac91aaff897325 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 05:02:58 +0200
Subject: [PATCH 0112/1937] [worldstarhiphop] Correct title extraction

---
 youtube_dl/extractor/worldstarhiphop.py | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/worldstarhiphop.py b/youtube_dl/extractor/worldstarhiphop.py
index 4e89acd81..bda3870db 100644
--- a/youtube_dl/extractor/worldstarhiphop.py
+++ b/youtube_dl/extractor/worldstarhiphop.py
@@ -13,37 +13,35 @@ class WorldStarHipHopIE(InfoExtractor):
         "info_dict": {
             "id": "wshh6a7q1ny0G34ZwuIO",
             "ext": "mp4",
-            "title": "Video: KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!"
+            "title": "KO Of The Week: MMA Fighter Gets Knocked Out By Swift Head Kick!"
         }
     }
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('id')
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
 
-        webpage_src = self._download_webpage(url, video_id)
-
-        m_vevo_id = re.search(r'videoId=(.*?)&amp?',
-                              webpage_src)
+        m_vevo_id = re.search(r'videoId=(.*?)&amp?', webpage)
         if m_vevo_id is not None:
             return self.url_result('vevo:%s' % m_vevo_id.group(1), ie='Vevo')
 
         video_url = self._search_regex(
-            r'so\.addVariable\("file","(.*?)"\)', webpage_src, 'video URL')
+            r'so\.addVariable\("file","(.*?)"\)', webpage, 'video URL')
 
         if 'youtube' in video_url:
             return self.url_result(video_url, ie='Youtube')
 
         video_title = self._html_search_regex(
-            r"<title>(.*)</title>", webpage_src, 'title')
+            r'(?s)<div class="content-heading">\s*<h1>(.*?)</h1>',
+            webpage, 'title')
 
         # Getting thumbnail and if not thumbnail sets correct title for WSHH candy video.
         thumbnail = self._html_search_regex(
-            r'rel="image_src" href="(.*)" />', webpage_src, 'thumbnail',
+            r'rel="image_src" href="(.*)" />', webpage, 'thumbnail',
             fatal=False)
         if not thumbnail:
-            _title = r"""candytitles.*>(.*)</span>"""
-            mobj = re.search(_title, webpage_src)
+            _title = r'candytitles.*>(.*)</span>'
+            mobj = re.search(_title, webpage)
             if mobj is not None:
                 video_title = mobj.group(1)
 

From 6043f1df4e4f74bd0ade52b3fc0938ff241366dc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 05:05:06 +0200
Subject: [PATCH 0113/1937] [ign] Return proper playlist object

---
 youtube_dl/extractor/ign.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index 12e9e61c4..c80185b53 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -89,7 +89,12 @@ class IGNIE(InfoExtractor):
                 '<param name="flashvars"[^>]*value="[^"]*?url=(https?://www\.ign\.com/videos/.*?)["&]',
                 webpage)
             if multiple_urls:
-                return [self.url_result(u, ie='IGN') for u in multiple_urls]
+                entries = [self.url_result(u, ie='IGN') for u in multiple_urls]
+                return {
+                    '_type': 'playlist',
+                    'id': name_or_id,
+                    'entries': entries,
+                }
 
         video_id = self._find_video_id(webpage)
         result = self._get_video_info(video_id)

From a8eb5a8e610a2b90eac2789d5b5f3cda81f543bb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 05:12:57 +0200
Subject: [PATCH 0114/1937] [generic] Fix testcases

---
 youtube_dl/extractor/generic.py | 18 +++++++++---------
 youtube_dl/extractor/ted.py     |  2 +-
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 0dfa4853d..263aa8579 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -180,13 +180,13 @@ class GenericIE(InfoExtractor):
         # Embedded TED video
         {
             'url': 'http://en.support.wordpress.com/videos/ted-talks/',
-            'md5': 'deeeabcc1085eb2ba205474e7235a3d5',
+            'md5': '65fdff94098e4a607385a60c5177c638',
             'info_dict': {
-                'id': '981',
+                'id': '1969',
                 'ext': 'mp4',
-                'title': 'My web playroom',
-                'uploader': 'Ze Frank',
-                'description': 'md5:ddb2a40ecd6b6a147e400e535874947b',
+                'title': 'Hidden miracles of the natural world',
+                'uploader': 'Louie Schwartzberg',
+                'description': 'md5:8145d19d320ff3e52f28401f4c4283b9',
             }
         },
         # Embeded Ustream video
@@ -295,13 +295,13 @@ class GenericIE(InfoExtractor):
         {
             'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM',
             'info_dict': {
-                'id': 'jpSGZsgga_I',
+                'id': '4vAffPZIT44',
                 'ext': 'mp4',
-                'title': 'Asphalt 8: Airborne - Launch Trailer',
+                'title': 'Asphalt 8: Airborne - Update - Welcome to Dubai!',
                 'uploader': 'Gameloft',
                 'uploader_id': 'gameloft',
-                'upload_date': '20130821',
-                'description': 'md5:87bd95f13d8be3e7da87a5f2c443106a',
+                'upload_date': '20140828',
+                'description': 'md5:c80da9ed3d83ae6d1876c834de03e1c4',
             },
             'params': {
                 'skip_download': True,
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 1cca47771..d5e28efad 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -149,7 +149,7 @@ class TEDIE(SubtitlesInfoExtractor):
             thumbnail = 'http://' + thumbnail
         return {
             'id': video_id,
-            'title': talk_info['title'],
+            'title': talk_info['title'].strip(),
             'uploader': talk_info['speaker'],
             'thumbnail': thumbnail,
             'description': self._og_search_description(webpage),

From dbe3043cd6d6bf468495df1e9f927a8c512e82a0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 05:15:42 +0200
Subject: [PATCH 0115/1937] [ynet] Fix test checksums

---
 youtube_dl/extractor/ynet.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/ynet.py b/youtube_dl/extractor/ynet.py
index 24872861a..944d7da38 100644
--- a/youtube_dl/extractor/ynet.py
+++ b/youtube_dl/extractor/ynet.py
@@ -13,7 +13,7 @@ class YnetIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://hot.ynet.co.il/home/0,7340,L-11659-99244,00.html',
-            'md5': '002b44ee2f33d50363a1c153bed524cf',
+            'md5': '4b29cb57c3dddd57642b3f051f535b07',
             'info_dict': {
                 'id': 'L-11659-99244',
                 'ext': 'flv',
@@ -22,7 +22,7 @@ class YnetIE(InfoExtractor):
             }
         }, {
             'url': 'http://hot.ynet.co.il/home/0,7340,L-8859-84418,00.html',
-            'md5': '6455046ae1b48cf7e2b7cae285e53a16',
+            'md5': '8194c2ea221e9a639cac96b6b0753dc5',
             'info_dict': {
                 'id': 'L-8859-84418',
                 'ext': 'flv',
@@ -33,9 +33,7 @@ class YnetIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         content = compat_urllib_parse.unquote_plus(self._og_search_video_url(webpage))

From 8ff14175e228a30f9940a69a4e72ca3a2a99aaf6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 05:17:16 +0200
Subject: [PATCH 0116/1937] [sportdeutschland] Fix testcase

---
 youtube_dl/extractor/sportdeutschland.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sportdeutschland.py b/youtube_dl/extractor/sportdeutschland.py
index 185353bef..abb827783 100644
--- a/youtube_dl/extractor/sportdeutschland.py
+++ b/youtube_dl/extractor/sportdeutschland.py
@@ -17,11 +17,11 @@ class SportDeutschlandIE(InfoExtractor):
         'info_dict': {
             'id': 'live-li-ning-badminton-weltmeisterschaft-2014-kopenhagen',
             'ext': 'mp4',
-            'title': 'LIVE: Li-Ning Badminton Weltmeisterschaft 2014 Kopenhagen',
+            'title': 're:Li-Ning Badminton Weltmeisterschaft 2014 Kopenhagen',
             'categories': ['Badminton'],
             'view_count': int,
             'thumbnail': 're:^https?://.*\.jpg$',
-            'description': 're:^Die Badminton-WM 2014 aus Kopenhagen LIVE',
+            'description': 're:Die Badminton-WM 2014 aus Kopenhagen bei Sportdeutschland\.TV',
             'timestamp': int,
             'upload_date': 're:^201408[23][0-9]$',
         },

From 761e1645e075ff9f8c5aeb8d0f2a4cfac71fb528 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 05:18:45 +0200
Subject: [PATCH 0117/1937] [generic] Remove unstable test checksum

---
 youtube_dl/extractor/generic.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 263aa8579..742bc2856 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -155,7 +155,6 @@ class GenericIE(InfoExtractor):
         # funnyordie embed
         {
             'url': 'http://www.theguardian.com/world/2014/mar/11/obama-zach-galifianakis-between-two-ferns',
-            'md5': '7cf780be104d40fea7bae52eed4a470e',
             'info_dict': {
                 'id': '18e820ec3f',
                 'ext': 'mp4',

From 5e4f06197f3d949bf89ee7e156391ca78121bf16 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 05:19:56 +0200
Subject: [PATCH 0118/1937] [facebook] Fix test case

---
 youtube_dl/extractor/facebook.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 60e68d98a..3ad993751 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -35,7 +35,7 @@ class FacebookIE(InfoExtractor):
             'id': '637842556329505',
             'ext': 'mp4',
             'duration': 38,
-            'title': 'Did you know Kei Nishikori is the first Asian man to ever reach a Grand Slam fin...',
+            'title': 're:Did you know Kei Nishikori is the first Asian man to ever reach a Grand Slam',
         }
     }, {
         'note': 'Video without discernible title',

From 6be451f422090601e25b6a9b1f801f521f1ca41f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 05:23:58 +0200
Subject: [PATCH 0119/1937] [youtube] Remove swf signature test cases

These files are now 0 Bytes
---
 test/test_youtube_signature.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 604e76ab6..df2cb09f2 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -47,18 +47,6 @@ _TESTS = [
         '2ACFC7A61CA478CD21425E5A57EBD73DDC78E22A.2094302436B2D377D14A3BBA23022D023B8BC25AA',
         'A52CB8B320D22032ABB3A41D773D2B6342034902.A22E87CDD37DBE75A5E52412DC874AC16A7CFCA2',
     ),
-    (
-        'http://s.ytimg.com/yts/swfbin/player-vfl5vIhK2/watch_as3.swf',
-        'swf',
-        86,
-        'O1I3456789abcde0ghijklmnopqrstuvwxyzABCDEFGHfJKLMN2PQRSTUVWXY\\!"#$%&\'()*+,-./:;<=>?'
-    ),
-    (
-        'http://s.ytimg.com/yts/swfbin/player-vflmDyk47/watch_as3.swf',
-        'swf',
-        'F375F75BF2AFDAAF2666E43868D46816F83F13E81C46.3725A8218E446A0DECD33F79DC282994D6AA92C92C9',
-        '9C29AA6D499282CD97F33DCED0A644E8128A5273.64C18E31F38361864D86834E6662FAADFA2FB57F'
-    ),
     (
         'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflBb0OQx.js',
         'js',

From e50e2fcd4deaab50da506a0abf1bafed16085cd7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 05:40:20 +0200
Subject: [PATCH 0120/1937] [br] fix test case

---
 youtube_dl/extractor/br.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py
index 4e2960c62..2e277c8c3 100644
--- a/youtube_dl/extractor/br.py
+++ b/youtube_dl/extractor/br.py
@@ -26,6 +26,8 @@ class BRIE(InfoExtractor):
                 'title': 'Wenn das Traditions-Theater wackelt',
                 'description': 'Heimatsound-Festival 2014: Wenn das Traditions-Theater wackelt',
                 'duration': 34,
+                'uploader': 'BR',
+                'upload_date': '20140802',
             }
         },
         {
@@ -66,8 +68,7 @@ class BRIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('id')
+        display_id = self._match_id(url)
         page = self._download_webpage(url, display_id)
         xml_url = self._search_regex(
             r"return BRavFramework\.register\(BRavFramework\('avPlayer_(?:[a-f0-9-]{36})'\)\.setup\({dataURL:'(/(?:[a-z0-9\-]+/)+[a-z0-9/~_.-]+)'}\)\);", page, 'XMLURL')

From 8157ae39042298831afc8f8e5d67619d21e3e00b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 05:48:56 +0200
Subject: [PATCH 0121/1937] [golem] Fix under 2.6

It's a sad story; 2.6 does not support any non-trivial xpaths.
---
 youtube_dl/extractor/golem.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/golem.py b/youtube_dl/extractor/golem.py
index bebfe8568..53714f47f 100644
--- a/youtube_dl/extractor/golem.py
+++ b/youtube_dl/extractor/golem.py
@@ -38,11 +38,9 @@ class GolemIE(InfoExtractor):
         }
 
         formats = []
-        for e in config.findall('./*[url]'):
+        for e in config:
             url = e.findtext('./url')
             if not url:
-                self._downloader.report_warning(
-                    "{0}: url: empty, skipping".format(e.tag))
                 continue
 
             formats.append({
@@ -57,7 +55,7 @@ class GolemIE(InfoExtractor):
         info['formats'] = formats
 
         thumbnails = []
-        for e in config.findall('.//teaser[url]'):
+        for e in config.findall('.//teaser'):
             url = e.findtext('./url')
             if not url:
                 continue

From 2a7b4681c6628e2a17b9b980333af1011e482058 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 05:51:41 +0200
Subject: [PATCH 0122/1937] [godtube] Fix on Python 2.6

---
 youtube_dl/extractor/godtube.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/godtube.py b/youtube_dl/extractor/godtube.py
index 73bd6d890..363dc6608 100644
--- a/youtube_dl/extractor/godtube.py
+++ b/youtube_dl/extractor/godtube.py
@@ -36,16 +36,16 @@ class GodTubeIE(InfoExtractor):
             'http://www.godtube.com/resource/mediaplayer/%s.xml' % video_id.lower(),
             video_id, 'Downloading player config XML')
 
-        video_url = config.find('.//file').text
-        uploader = config.find('.//author').text
-        timestamp = parse_iso8601(config.find('.//date').text)
-        duration = parse_duration(config.find('.//duration').text)
-        thumbnail = config.find('.//image').text
+        video_url = config.find('file').text
+        uploader = config.find('author').text
+        timestamp = parse_iso8601(config.find('date').text)
+        duration = parse_duration(config.find('duration').text)
+        thumbnail = config.find('image').text
 
         media = self._download_xml(
             'http://www.godtube.com/media/xml/?v=%s' % video_id, video_id, 'Downloading media XML')
 
-        title = media.find('.//title').text
+        title = media.find('title').text
 
         return {
             'id': video_id,

From 989b4b2b86588c011314200c0d30db965f79105e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 06:15:46 +0200
Subject: [PATCH 0123/1937] [utils:YoutubeDLHandler] Work around brain-dead
 Python 2.6 httplib

In 2.6, the httplib sends fragments! Remove those (fixes generic_26 on 2.6).
---
 youtube_dl/utils.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9f49507c1..950cd1a7a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -799,6 +799,12 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
                 del req.headers['User-agent']
             req.headers['User-agent'] = req.headers['Youtubedl-user-agent']
             del req.headers['Youtubedl-user-agent']
+
+        if sys.version_info < (2, 7) and '#' in req.get_full_url():
+            # Python 2.6 is brain-dead when it comes to fragments
+            req._Request__original = req._Request__original.partition('#')[0]
+            req._Request__r_type = req._Request__r_type.partition('#')[0]
+
         return req
 
     def http_response(self, req, resp):

From 8c23945c727dab01eabbc6e134cbb80db34d3120 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 06:19:18 +0200
Subject: [PATCH 0124/1937] [eporner] Adapt to changed default format

---
 youtube_dl/extractor/eporner.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/eporner.py b/youtube_dl/extractor/eporner.py
index 522aa3d63..bb231ecb1 100644
--- a/youtube_dl/extractor/eporner.py
+++ b/youtube_dl/extractor/eporner.py
@@ -14,11 +14,11 @@ class EpornerIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?eporner\.com/hd-porn/(?P<id>\d+)/(?P<display_id>[\w-]+)'
     _TEST = {
         'url': 'http://www.eporner.com/hd-porn/95008/Infamous-Tiffany-Teen-Strip-Tease-Video/',
-        'md5': '3b427ae4b9d60619106de3185c2987cd',
+        'md5': '39d486f046212d8e1b911c52ab4691f8',
         'info_dict': {
             'id': '95008',
             'display_id': 'Infamous-Tiffany-Teen-Strip-Tease-Video',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Infamous Tiffany Teen Strip Tease Video',
             'duration': 194,
             'view_count': int,

From 80bcefcd77415eff62b722c0a432e5c217a1d64f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 06:22:54 +0200
Subject: [PATCH 0125/1937] [cliphunter] Remove duration

---
 youtube_dl/extractor/cliphunter.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/youtube_dl/extractor/cliphunter.py b/youtube_dl/extractor/cliphunter.py
index 65c12136a..d4227e6eb 100644
--- a/youtube_dl/extractor/cliphunter.py
+++ b/youtube_dl/extractor/cliphunter.py
@@ -35,7 +35,6 @@ class CliphunterIE(InfoExtractor):
             'title': 'Fun Jynx Maze solo',
             'thumbnail': 're:^https?://.*\.jpg$',
             'age_limit': 18,
-            'duration': 1317,
         }
     }
 
@@ -86,14 +85,11 @@ class CliphunterIE(InfoExtractor):
         thumbnail = self._search_regex(
             r"var\s+mov_thumb\s*=\s*'([^']+)';",
             webpage, 'thumbnail', fatal=False)
-        duration = int_or_none(self._search_regex(
-            r'pl_dur\s*=\s*([0-9]+)', webpage, 'duration', fatal=False))
 
         return {
             'id': video_id,
             'title': video_title,
             'formats': formats,
-            'duration': duration,
             'age_limit': self._rta_search(webpage),
             'thumbnail': thumbnail,
         }

From 937f935db0932fcbd6402068c0147f07f78af4ed Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 12:15:23 +0200
Subject: [PATCH 0126/1937] [jukebox] Remove md5 sum, it fluctuates

---
 youtube_dl/extractor/jukebox.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/jukebox.py b/youtube_dl/extractor/jukebox.py
index 9b553b9fa..5aa32bf09 100644
--- a/youtube_dl/extractor/jukebox.py
+++ b/youtube_dl/extractor/jukebox.py
@@ -11,10 +11,9 @@ from ..utils import (
 
 
 class JukeboxIE(InfoExtractor):
-    _VALID_URL = r'^http://www\.jukebox?\..+?\/.+[,](?P<video_id>[a-z0-9\-]+)\.html'
+    _VALID_URL = r'^http://www\.jukebox?\..+?\/.+[,](?P<id>[a-z0-9\-]+)\.html'
     _TEST = {
         'url': 'http://www.jukebox.es/kosheen/videoclip,pride,r303r.html',
-        'md5': '1574e9b4d6438446d5b7dbcdf2786276',
         'info_dict': {
             'id': 'r303r',
             'ext': 'flv',
@@ -24,8 +23,7 @@ class JukeboxIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
+        video_id = self._match_id(url)
 
         html = self._download_webpage(url, video_id)
         iframe_url = unescapeHTML(self._search_regex(r'<iframe .*src="([^"]*)"', html, 'iframe url'))

From 457749a703acd73857096544057640925fc546bb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 12:18:49 +0200
Subject: [PATCH 0127/1937] [prosiebensat1] Fix test case

---
 youtube_dl/extractor/prosiebensat1.py | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 5b2a723c1..619496de7 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -144,7 +144,7 @@ class ProSiebenSat1IE(InfoExtractor):
                 'id': '2156342',
                 'ext': 'mp4',
                 'title': 'Kurztrips zum Valentinstag',
-                'description': 'md5:8ba6301e70351ae0bedf8da00f7ba528',
+                'description': 'Romantischer Kurztrip zum Valentinstag? Wir verraten, was sich hier wirklich lohnt.',
                 'duration': 307.24,
             },
             'params': {
@@ -180,12 +180,10 @@ class ProSiebenSat1IE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
 
-        page = self._download_webpage(url, video_id, 'Downloading page')
-
-        clip_id = self._html_search_regex(self._CLIPID_REGEXES, page, 'clip id')
+        clip_id = self._html_search_regex(self._CLIPID_REGEXES, webpage, 'clip id')
 
         access_token = 'testclient'
         client_name = 'kolibri-1.2.5'
@@ -234,12 +232,12 @@ class ProSiebenSat1IE(InfoExtractor):
 
         urls = self._download_json(url_api_url, clip_id, 'Downloading urls JSON')
 
-        title = self._html_search_regex(self._TITLE_REGEXES, page, 'title')
-        description = self._html_search_regex(self._DESCRIPTION_REGEXES, page, 'description', fatal=False)
-        thumbnail = self._og_search_thumbnail(page)
+        title = self._html_search_regex(self._TITLE_REGEXES, webpage, 'title')
+        description = self._html_search_regex(self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False)
+        thumbnail = self._og_search_thumbnail(webpage)
 
         upload_date = unified_strdate(self._html_search_regex(
-            self._UPLOAD_DATE_REGEXES, page, 'upload date', default=None))
+            self._UPLOAD_DATE_REGEXES, webpage, 'upload date', default=None))
 
         formats = []
 

From 1e60e5546e437197bd9a0eabe7e8d23c5470a4db Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 12:20:20 +0200
Subject: [PATCH 0128/1937] [funnyordie] Fix test case md5

---
 youtube_dl/extractor/funnyordie.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index 721e5fce0..d966e8403 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -21,7 +21,7 @@ class FunnyOrDieIE(InfoExtractor):
         },
     }, {
         'url': 'http://www.funnyordie.com/embed/e402820827',
-        'md5': 'ff4d83318f89776ed0250634cfaa8d36',
+        'md5': '29f4c5e5a61ca39dfd7e8348a75d0aad',
         'info_dict': {
             'id': 'e402820827',
             'ext': 'mp4',

From 9e1e67fc156e0d8b059796d3781a12793be66410 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 12:23:52 +0200
Subject: [PATCH 0129/1937] [internetvideoarchive] Modernize

---
 youtube_dl/extractor/internetvideoarchive.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/internetvideoarchive.py b/youtube_dl/extractor/internetvideoarchive.py
index 4ddda2f1b..21c77978d 100644
--- a/youtube_dl/extractor/internetvideoarchive.py
+++ b/youtube_dl/extractor/internetvideoarchive.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import re
 
 from .common import InfoExtractor
@@ -12,12 +14,13 @@ class InternetVideoArchiveIE(InfoExtractor):
     _VALID_URL = r'https?://video\.internetvideoarchive\.net/flash/players/.*?\?.*?publishedid.*?'
 
     _TEST = {
-        u'url': u'http://video.internetvideoarchive.net/flash/players/flashconfiguration.aspx?customerid=69249&publishedid=452693&playerid=247',
-        u'file': u'452693.mp4',
-        u'info_dict': {
-            u'title': u'SKYFALL',
-            u'description': u'In SKYFALL, Bond\'s loyalty to M is tested as her past comes back to haunt her. As MI6 comes under attack, 007 must track down and destroy the threat, no matter how personal the cost.',
-            u'duration': 153,
+        'url': 'http://video.internetvideoarchive.net/flash/players/flashconfiguration.aspx?customerid=69249&publishedid=452693&playerid=247',
+        'info_dict': {
+            'id': '452693',
+            'ext': 'mp4',
+            'title': 'SKYFALL',
+            'description': 'In SKYFALL, Bond\'s loyalty to M is tested as her past comes back to haunt her. As MI6 comes under attack, 007 must track down and destroy the threat, no matter how personal the cost.',
+            'duration': 153,
         },
     }
 
@@ -42,7 +45,7 @@ class InternetVideoArchiveIE(InfoExtractor):
         url = self._build_url(query)
 
         flashconfiguration = self._download_xml(url, video_id,
-            u'Downloading flash configuration')
+            'Downloading flash configuration')
         file_url = flashconfiguration.find('file').text
         file_url = file_url.replace('/playlist.aspx', '/mrssplaylist.aspx')
         # Replace some of the parameters in the query to get the best quality
@@ -51,7 +54,7 @@ class InternetVideoArchiveIE(InfoExtractor):
             lambda m: self._clean_query(m.group()),
             file_url)
         info = self._download_xml(file_url, video_id,
-            u'Downloading video info')
+            'Downloading video info')
         item = info.find('channel/item')
 
         def _bp(p):

From 72d53356f68cf73e6f2a88650b56f801790ea5f3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 12:24:48 +0200
Subject: [PATCH 0130/1937] [internetvideoarchive] Fix test case

---
 youtube_dl/extractor/internetvideoarchive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/internetvideoarchive.py b/youtube_dl/extractor/internetvideoarchive.py
index 21c77978d..53f9a5f75 100644
--- a/youtube_dl/extractor/internetvideoarchive.py
+++ b/youtube_dl/extractor/internetvideoarchive.py
@@ -20,7 +20,7 @@ class InternetVideoArchiveIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'SKYFALL',
             'description': 'In SKYFALL, Bond\'s loyalty to M is tested as her past comes back to haunt her. As MI6 comes under attack, 007 must track down and destroy the threat, no matter how personal the cost.',
-            'duration': 153,
+            'duration': 149,
         },
     }
 

From 89294b5f50462ede8ba83463ff262eb2c5219e1b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 12:33:29 +0200
Subject: [PATCH 0131/1937] [README] Explain updating in detail (Fixes #3850)

---
 README.md | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 0f7442906..cabc5eb9a 100644
--- a/README.md
+++ b/README.md
@@ -348,21 +348,34 @@ $ youtube-dl --dateafter 20000101 --datebefore 20091231
 
 # FAQ
 
-### I'm getting an error `Unable to extract OpenGraph title` on YouTube playlists
+### How do I update youtube-dl?
 
-YouTube changed their playlist format in March 2014 and later on, so you'll need at least youtube-dl 2014.07.25 to download all YouTube videos.
+If you've followed [our manual installation instructions](http://rg3.github.io/youtube-dl/download.html), you can simply run `youtube-dl -U` (or, on Linux, `sudo youtube-dl -U`).
 
-If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to report bugs to the Ubuntu packaging guys - all they have to do is update the package to a somewhat recent version.
+If you have used pip, a simple `sudo pip install -U youtube-dl` is sufficient to update.
 
-Alternatively, uninstall the youtube-dl package and follow [our manual installation instructions](http://rg3.github.io/youtube-dl/download.html). In a pinch, this should do if you used `apt-get` before to install youtube-dl:
+If you have installed youtube-dl using a package manager like *apt-get* or *yum*, use the standard system update mechanism to update. Note that distribution packages are often outdated. As a rule of thumb, youtube-dl releases at least once a month, and often weekly or even daily. Simply go to http://yt-dl.org/ to find out the current version. Unfortunately, there is nothing we youtube-dl developers can do if your distributions serves a really outdated version. You can (and should) complain to your distribution in their bugtracker or support forum.
+
+As a last resort, you can also uninstall the version installed by your package manager and follow our manual installation instructions. For that, remove the distribution's package, with a line like
+
+    sudo apt-get remove -y youtube-dl
+
+Afterwards, simply follow [our manual installation instructions](http://rg3.github.io/youtube-dl/download.html):
 
 ```
-sudo apt-get remove -y youtube-dl
 sudo wget https://yt-dl.org/latest/youtube-dl -O /usr/local/bin/youtube-dl
 sudo chmod a+x /usr/local/bin/youtube-dl
 hash -r
 ```
 
+Again, from then on you'll be able to update with `sudo youtube-dl -U`.
+
+### I'm getting an error `Unable to extract OpenGraph title` on YouTube playlists
+
+YouTube changed their playlist format in March 2014 and later on, so you'll need at least youtube-dl 2014.07.25 to download all YouTube videos.
+
+If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to report bugs to the Ubuntu packaging guys - all they have to do is update the package to a somewhat recent version. See above for a way to update.
+
 ### Do I always have to pass in `--max-quality FORMAT`, or `-citw`?
 
 By default, youtube-dl intends to have the best options (incidentally, if you have a convincing case that these should be different, [please file an issue where you explain that](https://yt-dl.org/bug)). Therefore, it is unnecessary and sometimes harmful to copy long option strings from webpages. In particular, `--max-quality` *limits* the video quality (so if you want the best quality, do NOT pass it in), and the only option out of `-citw` that is regularly useful is `-i`.

From 99b67fecc5ab6c57eada1e1678034dd71c57e338 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 12:45:18 +0200
Subject: [PATCH 0132/1937] [arte] Fix upload date extraction

---
 test/test_utils.py           | 1 +
 youtube_dl/extractor/arte.py | 6 +++++-
 youtube_dl/utils.py          | 1 +
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 6419b3ca9..97551ce9c 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -138,6 +138,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_strdate('Dec 14, 2012'), '20121214')
         self.assertEqual(unified_strdate('2012/10/11 01:56:38 +0000'), '20121011')
         self.assertEqual(unified_strdate('1968-12-10'), '19681210')
+        self.assertEqual(unified_strdate('28/01/2014 21:00:00 +0100'), '20140128')
 
     def test_find_xpath_attr(self):
         testxml = '''<root>
diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 957d35979..c3d02f85e 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -86,11 +86,15 @@ class ArteTVPlus7IE(InfoExtractor):
         info = self._download_json(json_url, video_id)
         player_info = info['videoJsonPlayer']
 
+        upload_date_str = player_info.get('shootingDate')
+        if not upload_date_str:
+            upload_date_str = player_info.get('VDA', '').split(' ')[0]
+
         info_dict = {
             'id': player_info['VID'],
             'title': player_info['VTI'],
             'description': player_info.get('VDE'),
-            'upload_date': unified_strdate(player_info.get('VDA', '').split(' ')[0]),
+            'upload_date': unified_strdate(upload_date_str),
             'thumbnail': player_info.get('programImage') or player_info.get('VTU', {}).get('IUR'),
         }
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 950cd1a7a..f05747097 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -890,6 +890,7 @@ def unified_strdate(date_str):
         '%d/%m/%Y',
         '%d/%m/%y',
         '%Y/%m/%d %H:%M:%S',
+        '%d/%m/%Y %H:%M:%S',
         '%Y-%m-%d %H:%M:%S',
         '%d.%m.%Y %H:%M',
         '%d.%m.%Y %H.%M',

From e7db97332859f17dbcf71fabc21927670da07553 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 12:45:57 +0200
Subject: [PATCH 0133/1937] [yahoo] Remove test case

This video seems to have been removed entirely
---
 youtube_dl/extractor/yahoo.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 3ab6017cd..221341c13 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -37,16 +37,6 @@ class YahooIE(InfoExtractor):
                 'description': 'Agent Topple\'s mustache does its dirty work, and Nicole brokers a deal for peace. But why is the NSA collecting millions of Instagram brunch photos? And if your waffles have nothing to hide, what are they so worried about?',
             },
         },
-        {
-            'url': 'https://movies.yahoo.com/video/world-loves-spider-man-190819223.html',
-            'md5': '410b7104aa9893b765bc22787a22f3d9',
-            'info_dict': {
-                'id': '516ed8e2-2c4f-339f-a211-7a8b49d30845',
-                'ext': 'mp4',
-                'title': 'The World Loves Spider-Man',
-                'description': '''People all over the world are celebrating the release of \"The Amazing Spider-Man 2.\" We're taking a look at the enthusiastic response Spider-Man has received from viewers all over the world.''',
-            }
-        },
         {
             'url': 'https://screen.yahoo.com/community/community-sizzle-reel-203225340.html?format=embed',
             'md5': '60e8ac193d8fb71997caa8fce54c6460',

From 59c03a9bfb59699ea76801f5b964a18b0cc5ee3d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 13:07:58 +0200
Subject: [PATCH 0134/1937] [vuclip] Fix extraction

---
 youtube_dl/extractor/vuclip.py | 38 +++++++++++++++++++++++++---------
 1 file changed, 28 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vuclip.py b/youtube_dl/extractor/vuclip.py
index fb0600f1a..c2aa497bf 100644
--- a/youtube_dl/extractor/vuclip.py
+++ b/youtube_dl/extractor/vuclip.py
@@ -5,6 +5,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     compat_urllib_parse_urlparse,
+    ExtractorError,
     parse_duration,
     qualities,
 )
@@ -14,13 +15,12 @@ class VuClipIE(InfoExtractor):
     _VALID_URL = r'http://(?:m\.)?vuclip\.com/w\?.*?cid=(?P<id>[0-9]+)'
 
     _TEST = {
-        'url': 'http://m.vuclip.com/w?cid=843902317&fid=63532&z=1007&nvar&frm=index.html&bu=4757321434',
-        'md5': '92ac9d1ccefec4f0bb474661ab144fcf',
+        'url': 'http://m.vuclip.com/w?cid=922692425&fid=70295&z=1010&nvar&frm=index.html',
         'info_dict': {
-            'id': '843902317',
+            'id': '922692425',
             'ext': '3gp',
-            'title': 'Movie Trailer: Noah',
-            'duration': 139,
+            'title': 'The Toy Soldiers - Hollywood Movie Trailer',
+            'duration': 180,
         }
     }
 
@@ -28,7 +28,9 @@ class VuClipIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        webpage = self._download_webpage(url, video_id)
+        #webpage = self._download_webpage(url, video_id)
+        import io
+        webpage = io.open('922692425_http_-_m.vuclip.com_wcid=922692425_fid=70295_z=1010_nvar_frm=index.html.dump', encoding='utf-8').read()
         ad_m = re.search(
             r'''value="No.*?" onClick="location.href='([^"']+)'"''', webpage)
         if ad_m:
@@ -37,16 +39,32 @@ class VuClipIE(InfoExtractor):
             webpage = self._download_webpage(
                 adfree_url, video_id, note='Download post-ad page')
 
+        error_msg = self._html_search_regex(
+            r'<p class="message">(.*?)</p>', webpage, 'error message',
+            default=None)
+        if error_msg:
+            raise ExtractorError(
+                '%s said: %s' % (self.IE_NAME, error_msg), expected=True)
+
+        # These clowns alternate between two page types
         links_code = self._search_regex(
-            r'(?s)<div class="social align_c".*?>(.*?)<hr\s*/?>', webpage,
-            'links')
+            r'''(?xs)
+                (?:
+                    <img\s+src="/im/play.gif".*?>|
+                    <!--\ player\ end\ -->\s*</div><!--\ thumb\ end-->
+                )
+                (.*?)
+                (?:
+                    <a\s+href="fblike'|<div\s+class="social">
+                )
+            ''', webpage, 'links')
         title = self._html_search_regex(
             r'<title>(.*?)-\s*Vuclip</title>', webpage, 'title').strip()
 
         quality_order = qualities(['Reg', 'Hi'])
         formats = []
         for url, q in re.findall(
-                r'<a href="(?P<url>[^"]+)".*?>(?P<q>[^<]+)</a>', links_code):
+                r'<a\s+href="(?P<url>[^"]+)".*?>(?:<button[^>]*>)?(?P<q>[^<]+)(?:</button>)?</a>', links_code):
             format_id = compat_urllib_parse_urlparse(url).scheme + '-' + q
             formats.append({
                 'format_id': format_id,
@@ -56,7 +74,7 @@ class VuClipIE(InfoExtractor):
         self._sort_formats(formats)
 
         duration = parse_duration(self._search_regex(
-            r'\(([0-9:]+)\)</span></h1>', webpage, 'duration', fatal=False))
+            r'\(([0-9:]+)\)</span>', webpage, 'duration', fatal=False))
 
         return {
             'id': video_id,

From fdb4d278bf6c2481778e01ab0ff25c2050664f1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Sep 2014 20:11:51 +0700
Subject: [PATCH 0135/1937] [spankwire] Fix extraction and modernize

---
 youtube_dl/extractor/spankwire.py | 48 +++++++++++++++++++------------
 1 file changed, 29 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 2007a0013..21491027a 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -9,7 +9,6 @@ from ..utils import (
     compat_urllib_parse,
     unified_strdate,
     str_to_int,
-    int_or_none,
 )
 from ..aes import aes_decrypt_text
 
@@ -40,31 +39,42 @@ class SpankwireIE(InfoExtractor):
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 
-        title = self._html_search_regex(r'<h1>([^<]+)', webpage, 'title')
+        title = self._html_search_regex(
+            r'<h1>([^<]+)', webpage, 'title')
         description = self._html_search_regex(
-            r'<div\s+id="descriptionContent">([^<]+)<', webpage, 'description', fatal=False)
+            r'<div\s+id="descriptionContent">([^<]+)<',
+            webpage, 'description', fatal=False)
         thumbnail = self._html_search_regex(
-            r'flashvars\.image_url = "([^"]+)', webpage, 'thumbnail', fatal=False)
+            r'playerData\.screenShot\s*=\s*"([^"]+)"',
+            webpage, 'thumbnail', fatal=False)
 
         uploader = self._html_search_regex(
-            r'by:\s*<a [^>]*>(.+?)</a>', webpage, 'uploader', fatal=False)
+            r'by:\s*<a [^>]*>(.+?)</a>',
+            webpage, 'uploader', fatal=False)
         uploader_id = self._html_search_regex(
-            r'by:\s*<a href="/Profile\.aspx\?.*?UserId=(\d+).*?"', webpage, 'uploader id', fatal=False)
-        upload_date = self._html_search_regex(r'</a> on (.+?) at \d+:\d+', webpage, 'upload date', fatal=False)
-        if upload_date:
-            upload_date = unified_strdate(upload_date)
-        
-        view_count = self._html_search_regex(
-            r'<div id="viewsCounter"><span>([^<]+)</span> views</div>', webpage, 'view count', fatal=False)
-        if view_count:
-            view_count = str_to_int(view_count)
-        comment_count = int_or_none(self._html_search_regex(
-            r'<span id="spCommentCount">\s*(\d+)</span> Comments</div>', webpage, 'comment count', fatal=False))
+            r'by:\s*<a href="/Profile\.aspx\?.*?UserId=(\d+).*?"',
+            webpage, 'uploader id', fatal=False)
+        upload_date = unified_strdate(self._html_search_regex(
+            r'</a> on (.+?) at \d+:\d+',
+            webpage, 'upload date', fatal=False))
 
-        video_urls = list(map(compat_urllib_parse.unquote , re.findall(r'flashvars\.quality_[0-9]{3}p = "([^"]+)', webpage)))
+        view_count = str_to_int(self._html_search_regex(
+            r'<div id="viewsCounter"><span>([\d,\.]+)</span> views</div>',
+            webpage, 'view count', fatal=False))
+        comment_count = str_to_int(self._html_search_regex(
+            r'Comments<span[^>]+>\s*\(([\d,\.]+)\)</span>',
+            webpage, 'comment count', fatal=False))
+
+        video_urls = list(map(
+            compat_urllib_parse.unquote,
+            re.findall(r'playerData\.cdnPath[0-9]{3,}\s*=\s*"([^"]+)', webpage)))
         if webpage.find('flashvars\.encrypted = "true"') != -1:
-            password = self._html_search_regex(r'flashvars\.video_title = "([^"]+)', webpage, 'password').replace('+', ' ')
-            video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls))
+            password = self._html_search_regex(
+                r'flashvars\.video_title = "([^"]+)',
+                webpage, 'password').replace('+', ' ')
+            video_urls = list(map(
+                lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'),
+                video_urls))
 
         formats = []
         for video_url in video_urls:

From c816336cbdb91efa282c0ede8552157861f10e76 Mon Sep 17 00:00:00 2001
From: net <net@h-MacBook-Pro-sl-net.local>
Date: Mon, 29 Sep 2014 21:58:42 +0300
Subject: [PATCH 0136/1937] [karaoketv] Add new extractor

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/karaoketv.py | 47 +++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 youtube_dl/extractor/karaoketv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 079221567..d12e741a0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -173,6 +173,7 @@ from .jukebox import JukeboxIE
 from .justintv import JustinTVIE
 from .jpopsukitv import JpopsukiIE
 from .kankan import KankanIE
+from .karaoketv import KaraoketvIE
 from .keezmovies import KeezMoviesIE
 from .khanacademy import KhanAcademyIE
 from .kickstarter import KickStarterIE
diff --git a/youtube_dl/extractor/karaoketv.py b/youtube_dl/extractor/karaoketv.py
new file mode 100644
index 000000000..4d50308cc
--- /dev/null
+++ b/youtube_dl/extractor/karaoketv.py
@@ -0,0 +1,47 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+import sys
+
+from .common import InfoExtractor
+from ..utils import compat_urllib_parse, ExtractorError
+
+
+class KaraoketvIE(InfoExtractor):
+    _VALID_URL = r'http://karaoketv\.co\.il/\?container=songs&id=(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://karaoketv.co.il/?container=songs&id=171568',
+        'info_dict': {
+            'id': '171568',
+            'ext': 'mp4',
+            'title': 'אל העולם שלך - רותם כהן - שרים קריוקי',
+        }
+    }
+
+    def _real_extract(self, url):
+
+        # BUG: SSL23_GET_SERVER_HELLO:unknown protocol 
+        if sys.hexversion < 0x03000000:
+            raise ExtractorError("Only python 3 supported.\n")
+
+        mobj = re.match(self._VALID_URL, url)
+        
+        video_id = mobj.group('id')
+
+        webpage = self._download_webpage(url, video_id)
+
+        settings_json = compat_urllib_parse.unquote_plus(self._search_regex(r'config=(.*)', self._og_search_video_url(webpage ,video_id), ''))
+        
+        urls_info_webpage = self._download_webpage(settings_json, 'Downloading settings json')
+
+        urls_info_json = json.loads(urls_info_webpage.replace('\'', '"'))
+
+        url = urls_info_json['playlist'][0]['url']
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'url': url,
+        }
\ No newline at end of file

From a7a747d6872c4680df228f8558e31c359f5a6630 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 21:47:57 +0200
Subject: [PATCH 0137/1937] [vuclip] Remove test code

---
 youtube_dl/extractor/vuclip.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vuclip.py b/youtube_dl/extractor/vuclip.py
index c2aa497bf..ab1e7946e 100644
--- a/youtube_dl/extractor/vuclip.py
+++ b/youtube_dl/extractor/vuclip.py
@@ -28,9 +28,7 @@ class VuClipIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
-        #webpage = self._download_webpage(url, video_id)
-        import io
-        webpage = io.open('922692425_http_-_m.vuclip.com_wcid=922692425_fid=70295_z=1010_nvar_frm=index.html.dump', encoding='utf-8').read()
+        webpage = self._download_webpage(url, video_id)
         ad_m = re.search(
             r'''value="No.*?" onClick="location.href='([^"']+)'"''', webpage)
         if ad_m:

From 2d29ac4f23c6769cb17c4be83cba5a6cd6801707 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 21:48:44 +0200
Subject: [PATCH 0138/1937] [vuclip] Fix regexp

---
 youtube_dl/extractor/vuclip.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vuclip.py b/youtube_dl/extractor/vuclip.py
index ab1e7946e..ec3c010ad 100644
--- a/youtube_dl/extractor/vuclip.py
+++ b/youtube_dl/extractor/vuclip.py
@@ -53,7 +53,7 @@ class VuClipIE(InfoExtractor):
                 )
                 (.*?)
                 (?:
-                    <a\s+href="fblike'|<div\s+class="social">
+                    <a\s+href="fblike|<div\s+class="social">
                 )
             ''', webpage, 'links')
         title = self._html_search_regex(

From 7a08ad7d592a91ed12eead2cdc80ef509cf5c1cf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 22:11:24 +0200
Subject: [PATCH 0139/1937] [test/helper] Modernize

---
 test/helper.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 7f3ab8438..53494196e 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import errno
 import io
 import hashlib
@@ -40,10 +42,10 @@ def report_warning(message):
     If stderr is a tty file the 'WARNING:' will be colored
     '''
     if sys.stderr.isatty() and os.name != 'nt':
-        _msg_header = u'\033[0;33mWARNING:\033[0m'
+        _msg_header = '\033[0;33mWARNING:\033[0m'
     else:
-        _msg_header = u'WARNING:'
-    output = u'%s %s\n' % (_msg_header, message)
+        _msg_header = 'WARNING:'
+    output = '%s %s\n' % (_msg_header, message)
     if 'b' in getattr(sys.stderr, 'mode', '') or sys.version_info[0] < 3:
         output = output.encode(preferredencoding())
     sys.stderr.write(output)
@@ -103,22 +105,22 @@ def expect_info_dict(self, expected_dict, got_dict):
 
             self.assertTrue(
                 isinstance(got, compat_str),
-                u'Expected a %s object, but got %s for field %s' % (
+                'Expected a %s object, but got %s for field %s' % (
                     compat_str.__name__, type(got).__name__, info_field))
             self.assertTrue(
                 match_rex.match(got),
-                u'field %s (value: %r) should match %r' % (info_field, got, match_str))
+                'field %s (value: %r) should match %r' % (info_field, got, match_str))
         elif isinstance(expected, type):
             got = got_dict.get(info_field)
             self.assertTrue(isinstance(got, expected),
-                u'Expected type %r for field %s, but got value %r of type %r' % (expected, info_field, got, type(got)))
+                'Expected type %r for field %s, but got value %r of type %r' % (expected, info_field, got, type(got)))
         else:
             if isinstance(expected, compat_str) and expected.startswith('md5:'):
                 got = 'md5:' + md5(got_dict.get(info_field))
             else:
                 got = got_dict.get(info_field)
             self.assertEqual(expected, got,
-                u'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
+                'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
 
     # Check for the presence of mandatory fields
     if got_dict.get('_type') != 'playlist':
@@ -126,7 +128,7 @@ def expect_info_dict(self, expected_dict, got_dict):
             self.assertTrue(got_dict.get(key), 'Missing mandatory field %s' % key)
     # Check for mandatory fields that are automatically set by YoutubeDL
     for key in ['webpage_url', 'extractor', 'extractor_key']:
-        self.assertTrue(got_dict.get(key), u'Missing field: %s' % key)
+        self.assertTrue(got_dict.get(key), 'Missing field: %s' % key)
 
     # Are checkable fields missing from the test case definition?
     test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value))
@@ -134,7 +136,7 @@ def expect_info_dict(self, expected_dict, got_dict):
         if value and key in ('title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location'))
     missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys())
     if missing_keys:
-        sys.stderr.write(u'\n"info_dict": ' + json.dumps(test_info_dict, ensure_ascii=False, indent=4) + u'\n')
+        sys.stderr.write('\n"info_dict": ' + json.dumps(test_info_dict, ensure_ascii=False, indent=4) + '\n')
         self.assertFalse(
             missing_keys,
             'Missing keys in test definition: %s' % (

From c0f64ac689627bfaaea677c6edfc7ec153ccd9be Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 22:19:11 +0200
Subject: [PATCH 0140/1937] [test/helper] Improve output of missing test
 definition dictionaries

---
 test/helper.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index 53494196e..62cb3ce02 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -14,6 +14,7 @@ from youtube_dl import YoutubeDL
 from youtube_dl.utils import (
     compat_str,
     preferredencoding,
+    write_string,
 )
 
 
@@ -136,7 +137,15 @@ def expect_info_dict(self, expected_dict, got_dict):
         if value and key in ('title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location'))
     missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys())
     if missing_keys:
-        sys.stderr.write('\n"info_dict": ' + json.dumps(test_info_dict, ensure_ascii=False, indent=4) + '\n')
+        def _repr(v):
+            if isinstance(v, compat_str):
+                return "'%s'" % v.replace('\\', '\\\\').replace("'", "\\'")
+            else:
+                return repr(v)
+        info_dict_str = ''.join(
+            '    %s: %s,\n' % (_repr(k), _repr(v))
+            for k, v in test_info_dict.items())
+        write_string('\n"info_dict": {' + info_dict_str + '}\n', out=sys.stderr)
         self.assertFalse(
             missing_keys,
             'Missing keys in test definition: %s' % (

From 58ea7ec81ecc1d0166e04a5a94afc8b70e3791db Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 22:23:21 +0200
Subject: [PATCH 0141/1937] [vimeo] Fix description extraction

---
 youtube_dl/extractor/vimeo.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index a002555a9..d2c36b58a 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -91,6 +91,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
                 'uploader_id': 'openstreetmapus',
                 'uploader': 'OpenStreetMap US',
                 'title': 'Andy Allan - Putting the Carto into OpenStreetMap Cartography',
+                'description': 'md5:380943ec71b89736ff4bf27183233d09',
                 'duration': 1595,
             },
         },
@@ -105,6 +106,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
                 'uploader': 'The BLN & Business of Software',
                 'uploader_id': 'theblnbusinessofsoftware',
                 'duration': 3610,
+                'description': None,
             },
         },
         {
@@ -119,6 +121,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
                 'uploader_id': 'user18948128',
                 'uploader': 'Jaime Marquínez Ferrándiz',
                 'duration': 10,
+                'description': 'This is "youtube-dl password protected test video" by Jaime Marquínez Ferrándiz on Vimeo, the home for high quality videos and the people who love them.',
             },
             'params': {
                 'videopassword': 'youtube-dl',
@@ -205,6 +208,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
         # Extract ID from URL
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
+        orig_url = url
         if mobj.group('pro') or mobj.group('player'):
             url = 'http://player.vimeo.com/video/' + video_id
 
@@ -275,9 +279,23 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
                 _, video_thumbnail = sorted((int(width if width.isdigit() else 0), t_url) for (width, t_url) in video_thumbs.items())[-1]
 
         # Extract video description
+
         video_description = self._html_search_regex(
-            r'(?s)<div class="[^"]*description"[^>]*>(.*?)</div>',
-            webpage, 'description', fatal=False)
+            r'(?s)<div\s+class="[^"]*description[^"]*"[^>]*>(.*?)</div>',
+            webpage, 'description', default=None)
+        if not video_description:
+            video_description = self._html_search_meta(
+                'description', webpage, default=None)
+        if not video_description and mobj.group('pro'):
+            orig_webpage = self._download_webpage(
+                orig_url, video_id,
+                note='Downloading webpage for description',
+                fatal=False)
+            if orig_webpage:
+                video_description = self._html_search_meta(
+                    'description', orig_webpage, default=None)
+        if not video_description and not mobj.group('player'):
+            self._downloader.report_warning('Cannot find video description')
 
         # Extract video duration
         video_duration = int_or_none(config["video"].get("duration"))

From 4dc19c0982e4190a88a18b92afd940d6866a2c04 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 30 Sep 2014 02:26:16 +0300
Subject: [PATCH 0142/1937] [lrt] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/lrt.py      | 69 ++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+)
 create mode 100644 youtube_dl/extractor/lrt.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 89a9d8106..ce1a2b32b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -190,6 +190,7 @@ from .livestream import (
     LivestreamOriginalIE,
     LivestreamShortenerIE,
 )
+from .lrt import LRTIE
 from .lynda import (
     LyndaIE,
     LyndaCourseIE
diff --git a/youtube_dl/extractor/lrt.py b/youtube_dl/extractor/lrt.py
new file mode 100644
index 000000000..fca0bfef0
--- /dev/null
+++ b/youtube_dl/extractor/lrt.py
@@ -0,0 +1,69 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    js_to_json,
+    parse_duration,
+    remove_end,
+)
+
+
+class LRTIE(InfoExtractor):
+    IE_NAME = 'lrt.lt'
+    _VALID_URL = r'https?://(?:www\.)?lrt\.lt/mediateka/irasas/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.lrt.lt/mediateka/irasas/54391/',
+        'info_dict': {
+            'id': '54391',
+            'ext': 'mp4',
+            'title': 'Septynios Kauno dienos',
+            'description': 'Kauno miesto ir apskrities naujienos',
+            'duration': 1783,
+        },
+        'params': {
+            'skip_download': True,  # HLS download
+        },
+
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = remove_end(self._og_search_title(webpage), ' - LRT')
+        thumbnail = self._og_search_thumbnail(webpage)
+        description = self._og_search_description(webpage)
+        duration = parse_duration(self._search_regex(
+            r"'duration':\s*'([^']+)',", webpage,
+            'duration', fatal=False, default=None))
+
+        formats = []
+        for js in re.findall(r'(?s)config:\s*(\{.*?\})', webpage):
+            data = json.loads(js_to_json(js))
+            if data['provider'] == 'rtmp':
+                formats.append({
+                    'format_id': 'rtmp',
+                    'ext': determine_ext(data['file']),
+                    'url': data['streamer'],
+                    'play_path': 'mp4:%s' % data['file'],
+                    'preference': -1,
+                })
+            else:
+                formats.extend(
+                    self._extract_m3u8_formats(data['file'], video_id, 'mp4'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'description': description,
+            'duration': duration,
+        }

From b7bb0df21e46d980e95295ffdadf3f2a8307b43f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Sep 2014 19:50:14 +0700
Subject: [PATCH 0143/1937] [vgtv] Fix tests' exts

---
 youtube_dl/extractor/vgtv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 7d27d6c57..964470070 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -31,7 +31,7 @@ class VGTVIE(InfoExtractor):
             'url': 'http://www.vgtv.no/#!/live/100764/opptak-vgtv-foelger-em-kvalifiseringen',
             'info_dict': {
                 'id': '100764',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'OPPTAK: VGTV følger EM-kvalifiseringen',
                 'description': 'md5:3772d9c0dc2dff92a886b60039a7d4d3',
                 'thumbnail': 're:^https?://.*\.jpg',
@@ -50,7 +50,7 @@ class VGTVIE(InfoExtractor):
             'url': 'http://www.vgtv.no/#!/live/100015/direkte-her-kan-du-se-laksen-live-fra-suldalslaagen',
             'info_dict': {
                 'id': '100015',
-                'ext': 'mp4',
+                'ext': 'flv',
                 'title': 'DIREKTE: Her kan du se laksen live fra Suldalslågen!',
                 'description': 'md5:9a60cc23fa349f761628924e56eeec2d',
                 'thumbnail': 're:^https?://.*\.jpg',

From a3b6be104d4da99b38942a9569a953cd2dc5a6fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Sep 2014 19:55:30 +0700
Subject: [PATCH 0144/1937] [tvigle] Replace 404 test

---
 youtube_dl/extractor/tvigle.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index dc8697850..e81ade707 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -17,16 +17,16 @@ class TvigleIE(InfoExtractor):
 
     _TESTS = [
         {
-            'url': 'http://www.tvigle.ru/video/brat-2/',
-            'md5': '72cb7eab33e54314e1790da402d3c9c3',
+            'url': 'http://www.tvigle.ru/video/brat/',
+            'md5': 'ff4344a4894b0524441fb6f8218dc716',
             'info_dict': {
-                'id': '5119390',
-                'display_id': 'brat-2',
+                'id': '5118490',
+                'display_id': 'brat',
                 'ext': 'mp4',
-                'title': 'Брат 2 ',
-                'description': 'md5:5751f4fe345a58e1692585c361294bd8',
-                'duration': 7356.369,
-                'age_limit': 0,
+                'title': 'Брат',
+                'description': 'md5:d16ac7c0b47052ea51fddb92c4e413eb',
+                'duration': 5722.6,
+                'age_limit': 16,
             },
         },
         {

From e497a7f2ca46d485c2839969abc598faa16c49dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Sep 2014 20:00:21 +0700
Subject: [PATCH 0145/1937] [tvigle] Extract format file sizes

---
 youtube_dl/extractor/tvigle.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index e81ade707..27962b5fe 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -71,6 +71,7 @@ class TvigleIE(InfoExtractor):
                     'format_id': '%s-%s' % (vcodec, quality),
                     'vcodec': vcodec,
                     'height': int(quality[:-1]),
+                    'filesize': item['video_files_size'][vcodec][quality],
                 })
         self._sort_formats(formats)
 

From 4644ac5527e48a1a8c48dc790621c73913e6dbf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Sep 2014 22:27:53 +0700
Subject: [PATCH 0146/1937] [core] Decode environment variables with filesystem
 encoding (Fixes #3854, Fixes #3217, Fixes #2918)

Introduces compat versions of os.getenv and os.path.expanduser
---
 test/test_utils.py      | 13 ++++++++++
 youtube_dl/YoutubeDL.py |  3 ++-
 youtube_dl/__init__.py  |  3 ++-
 youtube_dl/cache.py     |  3 ++-
 youtube_dl/options.py   | 14 ++++++-----
 youtube_dl/utils.py     | 53 +++++++++++++++++++++++++++++++++++++----
 6 files changed, 76 insertions(+), 13 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 97551ce9c..19c9ba7f8 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -44,6 +44,9 @@ from youtube_dl.utils import (
     limit_length,
     escape_rfc3986,
     escape_url,
+    get_filesystem_encoding,
+    compat_getenv,
+    compat_expanduser,
 )
 
 
@@ -331,5 +334,15 @@ class TestUtil(unittest.TestCase):
         )
         self.assertEqual(escape_url('http://vimeo.com/56015672#at=0'), 'http://vimeo.com/56015672#at=0')
 
+    def test_compat_getenv(self):
+        test_str = 'тест'
+        os.environ['YOUTUBE-DL-TEST'] = test_str.encode(get_filesystem_encoding())
+        self.assertEqual(compat_getenv('YOUTUBE-DL-TEST'), test_str)
+
+    def test_compat_expanduser(self):
+        test_str = 'C:\Documents and Settings\тест\Application Data'
+        os.environ['HOME'] = test_str.encode(get_filesystem_encoding())
+        self.assertEqual(compat_expanduser('~'), test_str)
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4a9610355..34a1e3b5c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -24,6 +24,7 @@ if os.name == 'nt':
 
 from .utils import (
     compat_cookiejar,
+    compat_expanduser,
     compat_http_client,
     compat_str,
     compat_urllib_error,
@@ -447,7 +448,7 @@ class YoutubeDL(object):
             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 
             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
-            tmpl = os.path.expanduser(outtmpl)
+            tmpl = compat_expanduser(outtmpl)
             filename = tmpl % template_dict
             return filename
         except ValueError as err:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 7f2b4dfcc..e73bc5c37 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -94,6 +94,7 @@ from .options import (
     parseOpts,
 )
 from .utils import (
+    compat_expanduser,
     compat_getpass,
     compat_print,
     DateRange,
@@ -285,7 +286,7 @@ def _real_main(argv=None):
                      u' template'.format(outtmpl))
 
     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
-    download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
+    download_archive_fn = compat_expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 
     ydl_opts = {
         'usenetrc': opts.usenetrc,
diff --git a/youtube_dl/cache.py b/youtube_dl/cache.py
index 79ff09f78..ac5925d32 100644
--- a/youtube_dl/cache.py
+++ b/youtube_dl/cache.py
@@ -9,6 +9,7 @@ import shutil
 import traceback
 
 from .utils import (
+    compat_expanduser,
     write_json_file,
 )
 
@@ -22,7 +23,7 @@ class Cache(object):
         if res is None:
             cache_root = os.environ.get('XDG_CACHE_HOME', '~/.cache')
             res = os.path.join(cache_root, 'youtube-dl')
-        return os.path.expanduser(res)
+        return compat_expanduser(res)
 
     def _get_cache_fn(self, section, key, dtype):
         assert re.match(r'^[a-zA-Z0-9_.-]+$', section), \
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index f651337ad..e6f9f33a2 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -6,6 +6,8 @@ import shlex
 import sys
 
 from .utils import (
+    compat_expanduser,
+    compat_getenv,
     get_term_width,
     write_string,
 )
@@ -27,19 +29,19 @@ def parseOpts(overrideArguments=None):
         return res
 
     def _readUserConf():
-        xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
+        xdg_config_home = compat_getenv('XDG_CONFIG_HOME')
         if xdg_config_home:
             userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
             if not os.path.isfile(userConfFile):
                 userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
         else:
-            userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
+            userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl', 'config')
             if not os.path.isfile(userConfFile):
-                userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
+                userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl.conf')
         userConf = _readOptions(userConfFile, None)
 
         if userConf is None:
-            appdata_dir = os.environ.get('appdata')
+            appdata_dir = compat_getenv('appdata')
             if appdata_dir:
                 userConf = _readOptions(
                     os.path.join(appdata_dir, 'youtube-dl', 'config'),
@@ -51,11 +53,11 @@ def parseOpts(overrideArguments=None):
 
         if userConf is None:
             userConf = _readOptions(
-                os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'),
+                os.path.join(compat_expanduser('~'), 'youtube-dl.conf'),
                 default=None)
         if userConf is None:
             userConf = _readOptions(
-                os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'),
+                os.path.join(compat_expanduser('~'), 'youtube-dl.conf.txt'),
                 default=None)
 
         if userConf is None:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f05747097..afe32ae05 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -203,6 +203,48 @@ def compat_ord(c):
     if type(c) is int: return c
     else: return ord(c)
 
+
+# Environment variables should be decoded with filesystem encoding
+# otherwise this results in issues like #3854 #2918 #3217
+if sys.version_info >= (3, 0):
+    compat_getenv = os.getenv
+    compat_expanduser = os.path.expanduser
+else:
+    def compat_getenv(key, default=None):
+        env = os.getenv(key, default)
+        if env:
+            env = env.decode(get_filesystem_encoding())
+        return env
+
+    def compat_expanduser(path):
+        """Expand ~ and ~user constructs.
+
+        If user or $HOME is unknown, do nothing."""
+        if path[:1] != '~':
+            return path
+        i, n = 1, len(path)
+        while i < n and path[i] not in '/\\':
+            i += 1
+
+        if 'HOME' in os.environ:
+            userhome = compat_getenv('HOME')
+        elif 'USERPROFILE' in os.environ:
+            userhome = compat_getenv('USERPROFILE')
+        elif not 'HOMEPATH' in os.environ:
+            return path
+        else:
+            try:
+                drive = compat_getenv('HOMEDRIVE')
+            except KeyError:
+                drive = ''
+            userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
+
+        if i != 1:  # ~user
+            userhome = os.path.join(os.path.dirname(userhome), path[1:i])
+
+        return userhome + path[i:]
+
+
 # This is not clearly defined otherwise
 compiled_regex_type = type(re.compile(''))
 
@@ -1204,11 +1246,14 @@ class locked_file(object):
         return self.f.read(*args)
 
 
+def get_filesystem_encoding():
+    encoding = sys.getfilesystemencoding()
+    return encoding if encoding is not None else 'utf-8'
+
+
 def shell_quote(args):
     quoted_args = []
-    encoding = sys.getfilesystemencoding()
-    if encoding is None:
-        encoding = 'utf-8'
+    encoding = get_filesystem_encoding()
     for a in args:
         if isinstance(a, bytes):
             # We may get a filename encoded with 'encodeFilename'
@@ -1258,7 +1303,7 @@ def format_bytes(bytes):
 
 
 def get_term_width():
-    columns = os.environ.get('COLUMNS', None)
+    columns = compat_getenv('COLUMNS', None)
     if columns:
         return int(columns)
 

From f8fb4a7ca8a65362ff3f82b67998744f08d5570d Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 30 Sep 2014 20:01:37 +0300
Subject: [PATCH 0147/1937] [nfl] Use compatible urlparse

---
 youtube_dl/extractor/nfl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index 4832b3ce4..cc7c921c3 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -6,7 +6,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    compat_urllib_parse,
+    compat_urllib_parse_urlparse,
     int_or_none,
     remove_end,
 )
@@ -90,7 +90,7 @@ class NFLIE(InfoExtractor):
         cdn_data = video_data.get('cdnData', {})
         streams = cdn_data.get('bitrateInfo', [])
         if cdn_data.get('format') == 'EXTERNAL_HTTP_STREAM':
-            parts = compat_urllib_parse.urlparse(cdn_data.get('uri'))
+            parts = compat_urllib_parse_urlparse(cdn_data.get('uri'))
             protocol, host = parts.scheme, parts.netloc
             for stream in streams:
                 formats.append(

From 07e764439a1cdd3a3b95fbf21acc6a517c6a889e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 29 Sep 2014 22:24:35 +0200
Subject: [PATCH 0148/1937] [generic] Delete test case

The page is not available any more.
---
 youtube_dl/extractor/generic.py | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 742bc2856..14c024e48 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -225,21 +225,6 @@ class GenericIE(InfoExtractor):
                 'skip_download': 'Requires rtmpdump'
             }
         },
-        # smotri embed
-        {
-            'url': 'http://rbctv.rbc.ru/archive/news/562949990879132.shtml',
-            'md5': 'ec40048448e9284c9a1de77bb188108b',
-            'info_dict': {
-                'id': 'v27008541fad',
-                'ext': 'mp4',
-                'title': 'Крым и Севастополь вошли в состав России',
-                'description': 'md5:fae01b61f68984c7bd2fa741e11c3175',
-                'duration': 900,
-                'upload_date': '20140318',
-                'uploader': 'rbctv_2012_4',
-                'uploader_id': 'rbctv_2012_4',
-            },
-        },
         # Condé Nast embed
         {
             'url': 'http://www.wired.com/2014/04/honda-asimo/',

From 410f3e73ab268f74a455798ee39de5caba90caea Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 30 Sep 2014 07:56:24 +0200
Subject: [PATCH 0149/1937] [utils] Fix js_to_json

---
 test/test_utils.py  | 10 ++++++++++
 youtube_dl/utils.py |  6 +++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 97551ce9c..113aa44b2 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -44,6 +44,7 @@ from youtube_dl.utils import (
     limit_length,
     escape_rfc3986,
     escape_url,
+    js_to_json,
 )
 
 
@@ -331,5 +332,14 @@ class TestUtil(unittest.TestCase):
         )
         self.assertEqual(escape_url('http://vimeo.com/56015672#at=0'), 'http://vimeo.com/56015672#at=0')
 
+    def test_js_to_json(self):
+        inp = '''{
+                'clip':{'provider':'pseudo'}
+        }'''
+        self.assertEqual(js_to_json(inp), '''{
+                "clip":{"provider":"pseudo"}
+        }''')
+        json.loads(js_to_json(inp))
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f05747097..59851a8c0 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -673,6 +673,8 @@ class ExtractorError(Exception):
             expected = True
         if video_id is not None:
             msg = video_id + ': ' + msg
+        if cause:
+            msg += u' (caused by %r)' % cause
         if not expected:
             msg = msg + u'; please report this issue on https://yt-dl.org/bug . Be sure to call youtube-dl with the --verbose flag and include its complete output. Make sure you are using the latest version; type  youtube-dl -U  to update.'
         super(ExtractorError, self).__init__(msg)
@@ -1598,7 +1600,9 @@ def js_to_json(code):
             ([{,]\s*)
             ("[^"]*"|\'[^\']*\'|[a-z0-9A-Z]+)
             (:\s*)
-            ([0-9.]+|true|false|"[^"]*"|\'[^\']*\'|\[|\{)
+            ([0-9.]+|true|false|"[^"]*"|\'[^\']*\'|
+                (?=\[|\{)
+            )
         ''', fix_kv, code)
     res = re.sub(r',(\s*\])', lambda m: m.group(1), res)
     return res

From e7b6d12254702a4aa6a9f54420f80e6ea456b120 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 30 Sep 2014 11:12:59 +0200
Subject: [PATCH 0150/1937] [utils] Improve and test js_to_json

---
 test/test_utils.py             | 20 +++++++++++++++---
 youtube_dl/extractor/common.py |  6 +++++-
 youtube_dl/utils.py            | 37 +++++++++++++++-------------------
 3 files changed, 38 insertions(+), 25 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 113aa44b2..bcca0efea 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -332,14 +332,28 @@ class TestUtil(unittest.TestCase):
         )
         self.assertEqual(escape_url('http://vimeo.com/56015672#at=0'), 'http://vimeo.com/56015672#at=0')
 
-    def test_js_to_json(self):
+    def test_js_to_json_realworld(self):
         inp = '''{
-                'clip':{'provider':'pseudo'}
+            'clip':{'provider':'pseudo'}
         }'''
         self.assertEqual(js_to_json(inp), '''{
-                "clip":{"provider":"pseudo"}
+            "clip":{"provider":"pseudo"}
         }''')
         json.loads(js_to_json(inp))
 
+        inp = '''{
+            'playlist':[{'controls':{'all':null}}]
+        }'''
+        self.assertEqual(js_to_json(inp), '''{
+            "playlist":[{"controls":{"all":null}}]
+        }''')
+
+    def test_js_to_json_edgecases(self):
+        on = js_to_json("{abc_def:'1\\'\\\\2\\\\\\'3\"4'}")
+        self.assertEqual(json.loads(on), {"abc_def": "1'\\2\\'3\"4"})
+
+        on = js_to_json('{"abc": true}')
+        self.assertEqual(json.loads(on), {'abc': True})
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f43a0a569..611cf95f1 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -334,7 +334,11 @@ class InfoExtractor(object):
         try:
             return json.loads(json_string)
         except ValueError as ve:
-            raise ExtractorError('Failed to download JSON', cause=ve)
+            errmsg = '%s: Failed to parse JSON ' % video_id
+            if fatal:
+                raise ExtractorError(errmsg, cause=ve)
+            else:
+                self.report_warning(errmsg + str(ve))
 
     def report_warning(self, msg, video_id=None):
         idstr = '' if video_id is None else '%s: ' % video_id
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 59851a8c0..f8dd9c72d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1580,29 +1580,24 @@ def strip_jsonp(code):
 
 def js_to_json(code):
     def fix_kv(m):
-        key = m.group(2)
-        if key.startswith("'"):
-            assert key.endswith("'")
-            assert '"' not in key
-            key = '"%s"' % key[1:-1]
-        elif not key.startswith('"'):
-            key = '"%s"' % key
-
-        value = m.group(4)
-        if value.startswith("'"):
-            assert value.endswith("'")
-            assert '"' not in value
-            value = '"%s"' % value[1:-1]
-
-        return m.group(1) + key + m.group(3) + value
+        v = m.group(0)
+        if v in ('true', 'false', 'null'):
+            return v
+        if v.startswith('"'):
+            return v
+        if v.startswith("'"):
+            v = v[1:-1]
+            v = re.sub(r"\\\\|\\'|\"", lambda m: {
+                '\\\\': '\\\\',
+                "\\'": "'",
+                '"': '\\"',
+            }[m.group(0)], v)
+        return '"%s"' % v
 
     res = re.sub(r'''(?x)
-            ([{,]\s*)
-            ("[^"]*"|\'[^\']*\'|[a-z0-9A-Z]+)
-            (:\s*)
-            ([0-9.]+|true|false|"[^"]*"|\'[^\']*\'|
-                (?=\[|\{)
-            )
+        "(?:[^"\\]*(?:\\\\|\\")?)*"|
+        '(?:[^'\\]*(?:\\\\|\\')?)*'|
+        [a-zA-Z_][a-zA-Z_0-9]*
         ''', fix_kv, code)
     res = re.sub(r',(\s*\])', lambda m: m.group(1), res)
     return res

From fc66e4a0d59d064518c3f18d65d1f4d87de8fb8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Oct 2014 19:48:55 +0700
Subject: [PATCH 0151/1937] [utils] Add posix expanduser implementation and
 clarify the original source

---
 youtube_dl/utils.py | 84 +++++++++++++++++++++++++++++++--------------
 1 file changed, 59 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index afe32ae05..526d2cc02 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -204,45 +204,79 @@ def compat_ord(c):
     else: return ord(c)
 
 
-# Environment variables should be decoded with filesystem encoding
-# otherwise this results in issues like #3854 #2918 #3217
 if sys.version_info >= (3, 0):
     compat_getenv = os.getenv
     compat_expanduser = os.path.expanduser
 else:
+    # Environment variables should be decoded with filesystem encoding.
+    # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
+
     def compat_getenv(key, default=None):
         env = os.getenv(key, default)
         if env:
             env = env.decode(get_filesystem_encoding())
         return env
 
-    def compat_expanduser(path):
-        """Expand ~ and ~user constructs.
+    # HACK: The default implementations of os.path.expanduser from cpython do not decode
+    # environment variables with filesystem encoding. We will work around this by
+    # providing adjusted implementations.
+    # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
+    # for different platforms with correct environment variables decoding.
 
-        If user or $HOME is unknown, do nothing."""
-        if path[:1] != '~':
-            return path
-        i, n = 1, len(path)
-        while i < n and path[i] not in '/\\':
-            i += 1
+    if os.name == 'posix':
+        def compat_expanduser(path):
+            """Expand ~ and ~user constructions.  If user or $HOME is unknown,
+            do nothing."""
+            if not path.startswith('~'):
+                return path
+            i = path.find('/', 1)
+            if i < 0:
+                i = len(path)
+            if i == 1:
+                if 'HOME' not in os.environ:
+                    import pwd
+                    userhome = pwd.getpwuid(os.getuid()).pw_dir
+                else:
+                    userhome = compat_getenv('HOME')
+            else:
+                import pwd
+                try:
+                    pwent = pwd.getpwnam(path[1:i])
+                except KeyError:
+                    return path
+                userhome = pwent.pw_dir
+            userhome = userhome.rstrip('/')
+            return (userhome + path[i:]) or '/'
+    elif os.name == 'nt' or os.name == 'ce':
+        def compat_expanduser(path):
+            """Expand ~ and ~user constructs.
 
-        if 'HOME' in os.environ:
-            userhome = compat_getenv('HOME')
-        elif 'USERPROFILE' in os.environ:
-            userhome = compat_getenv('USERPROFILE')
-        elif not 'HOMEPATH' in os.environ:
-            return path
-        else:
-            try:
-                drive = compat_getenv('HOMEDRIVE')
-            except KeyError:
-                drive = ''
-            userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
+            If user or $HOME is unknown, do nothing."""
+            if path[:1] != '~':
+                return path
+            i, n = 1, len(path)
+            while i < n and path[i] not in '/\\':
+                i = i + 1
 
-        if i != 1:  # ~user
-            userhome = os.path.join(os.path.dirname(userhome), path[1:i])
+            if 'HOME' in os.environ:
+                userhome = compat_getenv('HOME')
+            elif 'USERPROFILE' in os.environ:
+                userhome = compat_getenv('USERPROFILE')
+            elif not 'HOMEPATH' in os.environ:
+                return path
+            else:
+                try:
+                    drive = compat_getenv('HOMEDRIVE')
+                except KeyError:
+                    drive = ''
+                userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
 
-        return userhome + path[i:]
+            if i != 1: #~user
+                userhome = os.path.join(os.path.dirname(userhome), path[1:i])
+
+            return userhome + path[i:]
+    else:
+        compat_expanduser = os.path.expanduser
 
 
 # This is not clearly defined otherwise

From 59d206ca2d0aa77b0f739d7262e113722d5eee16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Oct 2014 20:44:43 +0700
Subject: [PATCH 0152/1937] [sunporno] Fix duration extraction and make more
 robust

---
 youtube_dl/extractor/sunporno.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dl/extractor/sunporno.py
index 7de3c9dd5..263f09b46 100644
--- a/youtube_dl/extractor/sunporno.py
+++ b/youtube_dl/extractor/sunporno.py
@@ -39,10 +39,10 @@ class SunPornoIE(InfoExtractor):
             r'poster="([^"]+)"', webpage, 'thumbnail', fatal=False)
 
         duration = parse_duration(self._search_regex(
-            r'<span>Duration: (\d+:\d+)</span>', webpage, 'duration', fatal=False))
+            r'Duration:\s*(\d+:\d+)\s*<', webpage, 'duration', fatal=False))
 
         view_count = int_or_none(self._html_search_regex(
-            r'<span class="views">(\d+)</span>', webpage, 'view count', fatal=False))
+            r'class="views">\s*(\d+)\s*<', webpage, 'view count', fatal=False))
         comment_count = int_or_none(self._html_search_regex(
             r'(\d+)</b> Comments?', webpage, 'comment count', fatal=False))
 

From d3c9af84fcd1614aa920e669e1c951a6d94c373a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 1 Oct 2014 20:53:58 +0700
Subject: [PATCH 0153/1937] [spankwire] Fix extraction

---
 youtube_dl/extractor/spankwire.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 21491027a..94602e89e 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -45,7 +45,7 @@ class SpankwireIE(InfoExtractor):
             r'<div\s+id="descriptionContent">([^<]+)<',
             webpage, 'description', fatal=False)
         thumbnail = self._html_search_regex(
-            r'playerData\.screenShot\s*=\s*"([^"]+)"',
+            r'playerData\.screenShot\s*=\s*["\']([^"\']+)["\']',
             webpage, 'thumbnail', fatal=False)
 
         uploader = self._html_search_regex(
@@ -67,7 +67,7 @@ class SpankwireIE(InfoExtractor):
 
         video_urls = list(map(
             compat_urllib_parse.unquote,
-            re.findall(r'playerData\.cdnPath[0-9]{3,}\s*=\s*"([^"]+)', webpage)))
+            re.findall(r'playerData\.cdnPath[0-9]{3,}\s*=\s*["\']([^"\']+)["\']', webpage)))
         if webpage.find('flashvars\.encrypted = "true"') != -1:
             password = self._html_search_regex(
                 r'flashvars\.video_title = "([^"]+)',

From e80f40e5ca32f2396a034965af86a9b507fad411 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 1 Oct 2014 17:26:09 +0300
Subject: [PATCH 0154/1937] [tapely] Add new extractor (closes #3861)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tapely.py   | 97 ++++++++++++++++++++++++++++++++
 2 files changed, 98 insertions(+)
 create mode 100644 youtube_dl/extractor/tapely.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ce1a2b32b..e51ea701f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -355,6 +355,7 @@ from .swrmediathek import SWRMediathekIE
 from .syfy import SyfyIE
 from .sztvhu import SztvHuIE
 from .tagesschau import TagesschauIE
+from .tapely import TapelyIE
 from .teachertube import (
     TeacherTubeIE,
     TeacherTubeUserIE,
diff --git a/youtube_dl/extractor/tapely.py b/youtube_dl/extractor/tapely.py
new file mode 100644
index 000000000..2c1178b1b
--- /dev/null
+++ b/youtube_dl/extractor/tapely.py
@@ -0,0 +1,97 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    compat_urllib_request,
+    float_or_none,
+    parse_iso8601,
+)
+
+
+class TapelyIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?tape\.ly/(?P<id>[A-Za-z0-9\-_]+)(?:/(?P<songnr>\d+))?'
+    _API_URL = 'http://tape.ly/showtape?id={0:}'
+    _S3_SONG_URL = 'http://mytape.s3.amazonaws.com/{0:}'
+    _TESTS = [
+        {
+            'url': 'http://tape.ly/my-grief-as-told-by-water',
+            'info_dict': {
+                'id': 23952,
+                'title': 'my grief as told by water',
+                'thumbnail': 're:^https?://.*\.png$',
+                'uploader_id': 16484,
+                'timestamp': 1411848286,
+                'description': 'For Robin and Ponkers, whom the tides of life have taken out to sea.',
+            },
+            'playlist_count': 13,
+        },
+        {
+            'url': 'http://tape.ly/my-grief-as-told-by-water/1',
+            'md5': '79031f459fdec6530663b854cbc5715c',
+            'info_dict': {
+                'id': 258464,
+                'title': 'Dreaming Awake  (My Brightest Diamond)',
+                'ext': 'm4a',
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('id')
+
+        playlist_url = self._API_URL.format(display_id)
+        request = compat_urllib_request.Request(playlist_url)
+        request.add_header('X-Requested-With', 'XMLHttpRequest')
+        request.add_header('Accept', 'application/json')
+
+        playlist = self._download_json(request, display_id)
+
+        tape = playlist['tape']
+
+        entries = []
+        for s in tape['songs']:
+            song = s['song']
+            entry = {
+                'id': song['id'],
+                'duration': float_or_none(song.get('songduration'), 1000),
+                'title': song['title'],
+            }
+            if song['source'] == 'S3':
+                entry.update({
+                    'url': self._S3_SONG_URL.format(song['filename']),
+                })
+                entries.append(entry)
+            elif song['source'] == 'YT':
+                _, _, yt_id = song['filename'].split('/')
+                entry.update(self.url_result(yt_id, 'Youtube', video_id=yt_id))
+                entries.append(entry)
+            else:
+                self.report_warning('Unknown song source: %s' % song['source'])
+
+        if mobj.group('songnr'):
+            songnr = int(mobj.group('songnr')) - 1
+            try:
+                return entries[songnr]
+            except IndexError:
+                raise ExtractorError(
+                    'No song with index: %s' % mobj.group('songnr'),
+                    expected=True)
+
+        return {
+            '_type': 'playlist',
+            'id': tape['id'],
+            'display_id': display_id,
+            'title': tape['name'],
+            'entries': entries,
+            'thumbnail': tape.get('image_url'),
+            'description': clean_html(tape.get('subtext')),
+            'like_count': tape.get('likescount'),
+            'uploader_id': tape.get('user_id'),
+            'timestamp': parse_iso8601(tape.get('published_at')),
+        }

From 1414df5ce24af3b915db5d630969862847349f24 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 1 Oct 2014 21:11:38 +0300
Subject: [PATCH 0155/1937] [izlesene] Prepend scheme to thumbnails

---
 youtube_dl/extractor/izlesene.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/izlesene.py b/youtube_dl/extractor/izlesene.py
index a83dd249f..07ef682ee 100644
--- a/youtube_dl/extractor/izlesene.py
+++ b/youtube_dl/extractor/izlesene.py
@@ -63,7 +63,8 @@ class IzleseneIE(InfoExtractor):
 
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
+        thumbnail = self._proto_relative_url(
+            self._og_search_thumbnail(webpage), scheme='http:')
 
         uploader = self._html_search_regex(
             r"adduserUsername\s*=\s*'([^']+)';",

From 9e77c60c9ad83c70e0f943404507fb2d2486113a Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 1 Oct 2014 21:53:45 +0300
Subject: [PATCH 0156/1937] [tapely] Catch SoundCloud tracks

---
 youtube_dl/extractor/tapely.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tapely.py b/youtube_dl/extractor/tapely.py
index 2c1178b1b..77e056242 100644
--- a/youtube_dl/extractor/tapely.py
+++ b/youtube_dl/extractor/tapely.py
@@ -17,6 +17,7 @@ class TapelyIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?tape\.ly/(?P<id>[A-Za-z0-9\-_]+)(?:/(?P<songnr>\d+))?'
     _API_URL = 'http://tape.ly/showtape?id={0:}'
     _S3_SONG_URL = 'http://mytape.s3.amazonaws.com/{0:}'
+    _SOUNDCLOUD_SONG_URL = 'http://api.soundcloud.com{0:}'
     _TESTS = [
         {
             'url': 'http://tape.ly/my-grief-as-told-by-water',
@@ -68,9 +69,15 @@ class TapelyIE(InfoExtractor):
                 })
                 entries.append(entry)
             elif song['source'] == 'YT':
-                _, _, yt_id = song['filename'].split('/')
+                self.to_screen('YouTube video detected')
+                yt_id = song['filename'].replace('/youtube/', '')
                 entry.update(self.url_result(yt_id, 'Youtube', video_id=yt_id))
                 entries.append(entry)
+            elif song['source'] == 'SC':
+                self.to_screen('SoundCloud song detected')
+                sc_url = self._SOUNDCLOUD_SONG_URL.format(song['filename'])
+                entry.update(self.url_result(sc_url, 'Soundcloud'))
+                entries.append(entry)
             else:
                 self.report_warning('Unknown song source: %s' % song['source'])
 

From d5feab9aaa9102f4dbc24c39f504f064c2888746 Mon Sep 17 00:00:00 2001
From: Anton Larionov <diffident.cat@gmail.com>
Date: Wed, 1 Oct 2014 23:24:23 +0400
Subject: [PATCH 0157/1937] [jpopsuki] Support category links

---
 youtube_dl/extractor/jpopsukitv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/jpopsukitv.py b/youtube_dl/extractor/jpopsukitv.py
index aad782578..a48a610e5 100644
--- a/youtube_dl/extractor/jpopsukitv.py
+++ b/youtube_dl/extractor/jpopsukitv.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class JpopsukiIE(InfoExtractor):
     IE_NAME = 'jpopsuki.tv'
-    _VALID_URL = r'https?://(?:www\.)?jpopsuki\.tv/video/(.*?)/(?P<id>\S+)'
+    _VALID_URL = r'https?://(?:www\.)?jpopsuki\.tv/(category/)?video/(.*?)/(?P<id>\S+)'
 
     _TEST = {
         'url': 'http://www.jpopsuki.tv/video/ayumi-hamasaki---evolution/00be659d23b0b40508169cdee4545771',

From e4d6cca0c1bb987592b576bd3bd439e6ecc9b342 Mon Sep 17 00:00:00 2001
From: net <net@h-MacBook-Pro-sl-net.local>
Date: Wed, 1 Oct 2014 23:45:35 +0300
Subject: [PATCH 0158/1937] [walla] Add new extractor

---
 youtube_dl/extractor/walla.py | 70 +++++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)
 create mode 100644 youtube_dl/extractor/walla.py

diff --git a/youtube_dl/extractor/walla.py b/youtube_dl/extractor/walla.py
new file mode 100644
index 000000000..e687c3af0
--- /dev/null
+++ b/youtube_dl/extractor/walla.py
@@ -0,0 +1,70 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+
+import re
+
+from .common import InfoExtractor
+
+
+class WallaIE(InfoExtractor):
+    _VALID_URL = r'http://vod\.walla\.co\.il/\w+/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://vod.walla.co.il/movie/2642630/one-direction-all-for-one',
+        'info_dict': {
+            'id': '2642630',
+            'ext': 'flv',
+            'title': 'וואן דיירקשן: ההיסטריה',
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        
+        video_id = mobj.group('id')
+
+        config_url = 'http://video2.walla.co.il/?w=null/null/%s/@@/video/flv_pl' % video_id
+        
+        webpage = self._download_webpage(config_url, video_id, '')
+
+        media_id = self._html_search_regex(r'<media_id>(\d+)</media_id>', webpage, video_id, 'extract media id')
+
+        prefix = '0' if len(media_id) == 7 else ''
+
+        series =  '%s%s' % (prefix, media_id[0:2])
+        session = media_id[2:5]
+        episode = media_id[5:7]
+        
+        title = self._html_search_regex(r'<title>(.*)</title>', webpage, video_id, 'title')
+
+        default_quality = self._html_search_regex(r'<qualities defaultType="(\d+)">', webpage, video_id, 0)
+
+        quality = default_quality if default_quality else '40'
+
+        media_path = '/%s/%s/%s' % (series, session, media_id) #self._html_search_regex(r'<quality type="%s">.*<src>(.*)</src>' % default_quality ,webpage, '', flags=re.DOTALL) 
+
+        playpath = 'mp4:media/%s/%s/%s-%s' % (series, session, media_id, quality) #self._html_search_regex(r'<quality type="%s">.*<src>(.*)</src>' % default_quality ,webpage, '', flags=re.DOTALL) 
+
+        subtitles = {}
+
+        subtitle_url = self._html_search_regex(r'<subtitles.*<src>(.*)</src>.*</subtitle>', webpage, video_id, 0)
+
+        print subtitle_url
+
+        if subtitle_url:
+            subtitles_page = self._download_webpage(subtitle_url, video_id, '')
+            subtitles['heb'] = subtitles_page
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': 'rtmp://wafla.walla.co.il:1935/vod',
+            'player_url': 'http://isc.walla.co.il/w9/swf/video_swf/vod/WallaMediaPlayerAvod.swf',
+            'page_url': url,
+            'app': "vod",
+            'play_path': playpath,
+            'tc_url': 'rtmp://wafla.walla.co.il:1935/vod',
+            'rtmp_protocol': 'rtmp',
+            'ext': 'flv',
+            'subtitles': subtitles,
+        }
\ No newline at end of file

From 18937a50a4d453d48b018b9e68f40cd6d0959e31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 1 Oct 2014 23:09:25 +0200
Subject: [PATCH 0159/1937] [dropbox] Fix the video url query (fixes #3868)

Previously it would convert urls without a query into 'something.mp4&dl=1'
---
 youtube_dl/extractor/dropbox.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dropbox.py b/youtube_dl/extractor/dropbox.py
index 817a9bd61..5f24ac721 100644
--- a/youtube_dl/extractor/dropbox.py
+++ b/youtube_dl/extractor/dropbox.py
@@ -29,9 +29,8 @@ class DropboxIE(InfoExtractor):
         video_id = mobj.group('id')
         fn = compat_urllib_parse_unquote(url_basename(url))
         title = os.path.splitext(fn)[0]
-        video_url = (
-            re.sub(r'[?&]dl=0', '', url) +
-            ('?' if '?' in url else '&') + 'dl=1')
+        video_url = re.sub(r'[?&]dl=0', '', url)
+        video_url += ('?' if '?' not in video_url else '&') + 'dl=1'
 
         return {
             'id': video_id,

From 08d5230945a641af8c37456af5ca7b2cff8def9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Oct 2014 19:43:25 +0700
Subject: [PATCH 0160/1937] [jpopsukitv] Improve _VALID_URL and modernize

---
 youtube_dl/extractor/jpopsukitv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/jpopsukitv.py b/youtube_dl/extractor/jpopsukitv.py
index a48a610e5..3b8a05600 100644
--- a/youtube_dl/extractor/jpopsukitv.py
+++ b/youtube_dl/extractor/jpopsukitv.py
@@ -12,14 +12,14 @@ from ..utils import (
 
 class JpopsukiIE(InfoExtractor):
     IE_NAME = 'jpopsuki.tv'
-    _VALID_URL = r'https?://(?:www\.)?jpopsuki\.tv/(category/)?video/(.*?)/(?P<id>\S+)'
+    _VALID_URL = r'https?://(?:www\.)?jpopsuki\.tv/(?:category/)?video/[^/]+/(?P<id>\S+)'
 
     _TEST = {
         'url': 'http://www.jpopsuki.tv/video/ayumi-hamasaki---evolution/00be659d23b0b40508169cdee4545771',
         'md5': '88018c0c1a9b1387940e90ec9e7e198e',
-        'file': '00be659d23b0b40508169cdee4545771.mp4',
         'info_dict': {
             'id': '00be659d23b0b40508169cdee4545771',
+            'ext': 'mp4',
             'title': 'ayumi hamasaki - evolution',
             'description': 'Release date: 2001.01.31\r\n浜崎あゆみ - evolution',
             'thumbnail': 'http://www.jpopsuki.tv/cache/89722c74d2a2ebe58bcac65321c115b2.jpg',

From 01384d6e4b04c23329c7e0bca8d1893cc65db5c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Oct 2014 19:58:28 +0700
Subject: [PATCH 0161/1937] [jpopsuki] More modernize

---
 youtube_dl/extractor/jpopsukitv.py | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/jpopsukitv.py b/youtube_dl/extractor/jpopsukitv.py
index 3b8a05600..122e2dd8c 100644
--- a/youtube_dl/extractor/jpopsukitv.py
+++ b/youtube_dl/extractor/jpopsukitv.py
@@ -1,8 +1,6 @@
 # coding=utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -30,8 +28,7 @@ class JpopsukiIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
@@ -47,11 +44,9 @@ class JpopsukiIE(InfoExtractor):
         uploader_id = self._html_search_regex(
             r'<li>from: <a href="/user/view/user/\S*?/uid/(\d*)',
             webpage, 'video uploader_id', fatal=False)
-        upload_date = self._html_search_regex(
+        upload_date = unified_strdate(self._html_search_regex(
             r'<li>uploaded: (.*?)</li>', webpage, 'video upload_date',
-            fatal=False)
-        if upload_date is not None:
-            upload_date = unified_strdate(upload_date)
+            fatal=False))
         view_count_str = self._html_search_regex(
             r'<li>Hits: ([0-9]+?)</li>', webpage, 'video view_count',
             fatal=False)

From d9bf4652009c3b2264bca8c5cdac47c42673f493 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 2 Oct 2014 20:22:46 +0700
Subject: [PATCH 0162/1937] [bandcamp] Fix id extraction and modernize (Closes
 #3866)

---
 youtube_dl/extractor/bandcamp.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index c569aa4d2..c13446665 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -15,13 +15,23 @@ class BandcampIE(InfoExtractor):
     _VALID_URL = r'https?://.*?\.bandcamp\.com/track/(?P<title>.*)'
     _TESTS = [{
         'url': 'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song',
-        'file': '1812978515.mp3',
         'md5': 'c557841d5e50261777a6585648adf439',
         'info_dict': {
-            "title": "youtube-dl  \"'/\\\u00e4\u21ad - youtube-dl test song \"'/\\\u00e4\u21ad",
-            "duration": 9.8485,
+            'id': '1812978515',
+            'ext': 'mp3',
+            'title': "youtube-dl  \"'/\\\u00e4\u21ad - youtube-dl test song \"'/\\\u00e4\u21ad",
+            'duration': 9.8485,
         },
         '_skip': 'There is a limit of 200 free downloads / month for the test song'
+    }, {
+        'url': 'http://benprunty.bandcamp.com/track/lanius-battle',
+        'md5': '2b68e5851514c20efdff2afc5603b8b4',
+        'info_dict': {
+            'id': '2650410135',
+            'ext': 'mp3',
+            'title': 'Lanius (Battle)',
+            'uploader': 'Ben Prunty Music',
+        },
     }]
 
     def _real_extract(self, url):
@@ -59,9 +69,9 @@ class BandcampIE(InfoExtractor):
                 raise ExtractorError('No free songs found')
 
         download_link = m_download.group(1)
-        video_id = re.search(
-            r'var TralbumData = {(.*?)id: (?P<id>\d*?)$',
-            webpage, re.MULTILINE | re.DOTALL).group('id')
+        video_id = self._search_regex(
+            r'var TralbumData = {.*?id: (?P<id>\d+),?$',
+            webpage, 'video id', flags=re.MULTILINE | re.DOTALL)
 
         download_webpage = self._download_webpage(download_link, video_id, 'Downloading free downloads page')
         # We get the dictionary of the track from some javascript code

From 10606050bcb7cdad744fb619d466788f260d6a0a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 2 Oct 2014 15:30:19 +0200
Subject: [PATCH 0163/1937] release 2014.10.02

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1384b496b..b7a60ac24 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.09.29.2'
+__version__ = '2014.10.02'

From 8489578df42620a4e60fa233458520986b510105 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 2 Oct 2014 21:42:45 +0300
Subject: [PATCH 0164/1937] [generic] Support embedded Dailymotion playlists
 (fixes #3822)

---
 youtube_dl/extractor/generic.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 14c024e48..c16da70f1 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -639,6 +639,16 @@ class GenericIE(InfoExtractor):
             return _playlist_from_matches(
                 matches, lambda m: unescapeHTML(m[1]))
 
+        # Look for embedded Dailymotion playlist player (#3822)
+        m = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.[a-z]{2,3}/widget/jukebox\?.+?)\1', webpage)
+        if m:
+            playlists = re.findall(
+                r'list\[\]=/playlist/([^/]+)/', unescapeHTML(m.group('url')))
+            if playlists:
+                return _playlist_from_matches(
+                    playlists, lambda p: '//dailymotion.com/playlist/%s' % p)
+
         # Look for embedded Wistia player
         match = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)

From f78c01f68b1e16e385ec319c7f5d351b173c7ec5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Oct 2014 18:57:18 +0700
Subject: [PATCH 0165/1937] [breakcom] Cover more URLs with _VALID_URL (Closes
 #3876)

---
 youtube_dl/extractor/breakcom.py | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/breakcom.py b/youtube_dl/extractor/breakcom.py
index 1bfc9f35b..9b0aa2b1b 100644
--- a/youtube_dl/extractor/breakcom.py
+++ b/youtube_dl/extractor/breakcom.py
@@ -7,25 +7,27 @@ from .common import InfoExtractor
 
 
 class BreakIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?break\.com/video/([^/]+)'
-    _TEST = {
+    _VALID_URL = r'http://(?:www\.)?break\.com/video/(?:[^/]+/)*.+-(?P<id>\d+)'
+    _TESTS = [{
         'url': 'http://www.break.com/video/when-girls-act-like-guys-2468056',
-        'md5': 'a3513fb1547fba4fb6cfac1bffc6c46b',
+        'md5': '33aa4ff477ecd124d18d7b5d23b87ce5',
         'info_dict': {
             'id': '2468056',
             'ext': 'mp4',
             'title': 'When Girls Act Like D-Bags',
         }
-    }
+    }, {
+        'url': 'http://www.break.com/video/ugc/baby-flex-2773063',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1).split("-")[-1]
-        embed_url = 'http://www.break.com/embed/%s' % video_id
-        webpage = self._download_webpage(embed_url, video_id)
-        info_json = self._search_regex(r'var embedVars = ({.*})\s*?</script>',
-            webpage, 'info json', flags=re.DOTALL)
-        info = json.loads(info_json)
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(
+            'http://www.break.com/embed/%s' % video_id, video_id)
+        info = json.loads(self._search_regex(
+            r'var embedVars = ({.*})\s*?</script>',
+            webpage, 'info json', flags=re.DOTALL))
         video_url = info['videoUri']
         youtube_id = info.get('youtubeId')
         if youtube_id:

From 146c80e2562dab0f229f13a1fbc0e8ae116d4cb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Oct 2014 19:37:25 +0700
Subject: [PATCH 0166/1937] [utils] Add parse_age_limit

---
 youtube_dl/utils.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f8dd9c72d..2615553c1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1574,6 +1574,13 @@ US_RATINGS = {
 }
 
 
+def parse_age_limit(s):
+    if s is None:
+        return 0
+    m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
+    return int(m.group('age')) if m else US_RATINGS.get(s, 0)
+
+
 def strip_jsonp(code):
     return re.sub(r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?\s*$', r'\1', code)
 

From d397c0b3dd17d17c8235286eb88950d5fda53cee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 3 Oct 2014 19:37:47 +0700
Subject: [PATCH 0167/1937] [breakcom] Extract all formats

---
 youtube_dl/extractor/breakcom.py | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/breakcom.py b/youtube_dl/extractor/breakcom.py
index 9b0aa2b1b..2c0e5eea2 100644
--- a/youtube_dl/extractor/breakcom.py
+++ b/youtube_dl/extractor/breakcom.py
@@ -4,6 +4,10 @@ import re
 import json
 
 from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_age_limit,
+)
 
 
 class BreakIE(InfoExtractor):
@@ -28,15 +32,33 @@ class BreakIE(InfoExtractor):
         info = json.loads(self._search_regex(
             r'var embedVars = ({.*})\s*?</script>',
             webpage, 'info json', flags=re.DOTALL))
-        video_url = info['videoUri']
+
         youtube_id = info.get('youtubeId')
         if youtube_id:
             return self.url_result(youtube_id, 'Youtube')
 
-        final_url = video_url + '?' + info['AuthToken']
+        formats = [{
+            'url': media['uri'] + '?' + info['AuthToken'],
+            'tbr': media['bitRate'],
+            'width': media['width'],
+            'height': media['height'],
+        } for media in info['media']]
+
+        if not formats:
+            formats.append({
+                'url': info['videoUri']
+            })
+
+        self._sort_formats(formats)
+
+        duration = int_or_none(info.get('videoLengthInSeconds'))
+        age_limit = parse_age_limit(info.get('audienceRating'))
+
         return {
             'id': video_id,
-            'url': final_url,
             'title': info['contentName'],
             'thumbnail': info['thumbUri'],
+            'duration': duration,
+            'age_limit': age_limit,
+            'formats': formats,
         }

From fe506288bd327d3e7e9a90706ba10f3ff3d36ffe Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 3 Oct 2014 19:43:36 +0300
Subject: [PATCH 0168/1937] [planetaplay] Add new extractor (closes #3839)

---
 youtube_dl/extractor/__init__.py    |  1 +
 youtube_dl/extractor/planetaplay.py | 60 +++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 youtube_dl/extractor/planetaplay.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e51ea701f..348f5767a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -275,6 +275,7 @@ from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
 from .photobucket import PhotobucketIE
+from .planetaplay import PlanetaPlayIE
 from .played import PlayedIE
 from .playfm import PlayFMIE
 from .playvid import PlayvidIE
diff --git a/youtube_dl/extractor/planetaplay.py b/youtube_dl/extractor/planetaplay.py
new file mode 100644
index 000000000..596c621d7
--- /dev/null
+++ b/youtube_dl/extractor/planetaplay.py
@@ -0,0 +1,60 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class PlanetaPlayIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?planetaplay\.com/\?sng=(?P<id>[0-9]+)'
+    _API_URL = 'http://planetaplay.com/action/playlist/?sng={0:}'
+    _THUMBNAIL_URL = 'http://planetaplay.com/img/thumb/{thumb:}'
+    _TEST = {
+        'url': 'http://planetaplay.com/?sng=3586',
+        'md5': '9d569dceb7251a4e01355d5aea60f9db',
+        'info_dict': {
+            'id': '3586',
+            'ext': 'flv',
+            'title': 'md5:e829428ee28b1deed00de90de49d1da1',
+        }
+    }
+
+    _SONG_FORMATS = {
+        'lq': (0, 'http://www.planetaplay.com/videoplayback/{med_hash:}'),
+        'hq': (1, 'http://www.planetaplay.com/videoplayback/hi/{med_hash:}'),
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        response = self._download_json(
+            self._API_URL.format(video_id), video_id)['response']
+        try:
+            data = response.get('data')[0]
+        except IndexError:
+            raise ExtractorError(
+                '%s: failed to get the playlist' % self.IE_NAME, expected=True)
+
+        title = '{song_artists:} - {sng_name:}'.format(**data)
+        thumbnail = self._THUMBNAIL_URL.format(**data)
+
+        formats = []
+        for format_id, (quality, url_template) in self._SONG_FORMATS.items():
+            formats.append({
+                'format_id': format_id,
+                'url': url_template.format(**data),
+                'quality': quality,
+                'ext': 'flv',
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+        }

From d838b1bd4a256484820af7aeb0c2711934cb00ad Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 3 Oct 2014 20:17:10 +0200
Subject: [PATCH 0169/1937] [utils] Default age_limit to None

If we can't parse it, it means we don't have any information, not that the content is unrestricted.
---
 youtube_dl/extractor/common.py | 2 ++
 youtube_dl/utils.py            | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 611cf95f1..450c7dfd6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -138,6 +138,8 @@ class InfoExtractor(object):
 
     Unless mentioned otherwise, the fields should be Unicode strings.
 
+    Unless mentioned otherwise, None is equivalent to absence of information.
+
     Subclasses of this one should re-define the _real_initialize() and
     _real_extract() methods and define a _VALID_URL regexp.
     Probably, they should also be added to the list of extractors.
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2615553c1..2c9081733 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1576,9 +1576,9 @@ US_RATINGS = {
 
 def parse_age_limit(s):
     if s is None:
-        return 0
+        return None
     m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
-    return int(m.group('age')) if m else US_RATINGS.get(s, 0)
+    return int(m.group('age')) if m else US_RATINGS.get(s, None)
 
 
 def strip_jsonp(code):

From f47754f061abd4615ebcbcfaf3008f0b68d9e525 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 4 Oct 2014 18:56:36 +0700
Subject: [PATCH 0170/1937] [globo] Initial extractor implementation (Closes
 #3823)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/globo.py    | 398 +++++++++++++++++++++++++++++++
 2 files changed, 399 insertions(+)
 create mode 100644 youtube_dl/extractor/globo.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 348f5767a..b59fd65ca 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -134,6 +134,7 @@ from .gamestar import GameStarIE
 from .gametrailers import GametrailersIE
 from .gdcvault import GDCVaultIE
 from .generic import GenericIE
+from .globo import GloboIE
 from .godtube import GodTubeIE
 from .golem import GolemIE
 from .googleplus import GooglePlusIE
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
new file mode 100644
index 000000000..77c3ad4fc
--- /dev/null
+++ b/youtube_dl/extractor/globo.py
@@ -0,0 +1,398 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import random
+import math
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+    compat_str,
+    compat_chr,
+    compat_ord,
+)
+
+
+class GloboIE(InfoExtractor):
+    _VALID_URL = 'https?://.+?\.globo\.com/(?P<id>.+)'
+
+    _API_URL_TEMPLATE = 'http://api.globovideos.com/videos/%s/playlist'
+    _SECURITY_URL_TEMPLATE = 'http://security.video.globo.com/videos/%s/hash?player=flash&version=2.9.9.50&resource_id=%s'
+
+    _VIDEOID_REGEXES = [
+        r'\bdata-video-id="(\d+)"',
+        r'\bdata-player-videosids="(\d+)"',
+        r'<div[^>]+\bid="(\d+)"',
+    ]
+
+    _RESIGN_EXPIRATION = 86400
+
+    _TESTS = [
+        {
+            'url': 'http://globotv.globo.com/sportv/futebol-nacional/v/os-gols-de-atletico-mg-3-x-2-santos-pela-24a-rodada-do-brasileirao/3654973/',
+            'md5': '03ebf41cb7ade43581608b7d9b71fab0',
+            'info_dict': {
+                'id': '3654973',
+                'ext': 'mp4',
+                'title': 'Os gols de Atlético-MG 3 x 2 Santos pela 24ª rodada do Brasileirão',
+                'duration': 251.585,
+                'uploader': 'SporTV',
+                'uploader_id': 698,
+                'like_count': int,
+            }
+        },
+        {
+            'url': 'http://g1.globo.com/carros/autoesporte/videos/t/exclusivos-do-g1/v/mercedes-benz-gla-passa-por-teste-de-colisao-na-europa/3607726/',
+            'md5': 'b3ccc801f75cd04a914d51dadb83a78d',
+            'info_dict': {
+                'id': '3607726',
+                'ext': 'mp4',
+                'title': 'Mercedes-Benz GLA passa por teste de colisão na Europa',
+                'duration': 103.204,
+                'uploader': 'Globo.com',
+                'uploader_id': 265,
+                'like_count': int,
+            }
+        },
+        {
+            'url': 'http://g1.globo.com/jornal-nacional/noticia/2014/09/novidade-na-fiscalizacao-de-bagagem-pela-receita-provoca-discussoes.html',
+            'md5': '307fdeae4390ccfe6ba1aa198cf6e72b',
+            'info_dict': {
+                'id': '3652183',
+                'ext': 'mp4',
+                'title': 'Receita Federal explica como vai fiscalizar bagagens de quem retorna ao Brasil de avião',
+                'duration': 110.711,
+                'uploader': 'Rede Globo',
+                'uploader_id': 196,
+                'like_count': int,
+            }
+        },
+    ]
+
+    class MD5():
+        HEX_FORMAT_LOWERCASE = 0
+        HEX_FORMAT_UPPERCASE = 1
+        BASE64_PAD_CHARACTER_DEFAULT_COMPLIANCE = ''
+        BASE64_PAD_CHARACTER_RFC_COMPLIANCE = '='
+        PADDING = '=0xFF01DD'
+        hexcase = 0
+        b64pad = ''
+
+        def __init__(self):
+            pass
+
+        class JSArray(list):
+            def __getitem__(self, y):
+                try:
+                    return list.__getitem__(self, y)
+                except IndexError:
+                    return 0
+
+            def __setitem__(self, i, y):
+                try:
+                    return list.__setitem__(self, i, y)
+                except IndexError:
+                    self.extend([0] * (i - len(self) + 1))
+                    self[-1] = y
+
+        @classmethod
+        def hex_md5(cls, param1):
+            return cls.rstr2hex(cls.rstr_md5(cls.str2rstr_utf8(param1)))
+
+        @classmethod
+        def b64_md5(cls, param1, param2=None):
+            return cls.rstr2b64(cls.rstr_md5(cls.str2rstr_utf8(param1, param2)))
+
+        @classmethod
+        def any_md5(cls, param1, param2):
+            return cls.rstr2any(cls.rstr_md5(cls.str2rstr_utf8(param1)), param2)
+
+        @classmethod
+        def rstr_md5(cls, param1):
+            return cls.binl2rstr(cls.binl_md5(cls.rstr2binl(param1), len(param1) * 8))
+
+        @classmethod
+        def rstr2hex(cls, param1):
+            _loc_2 = '0123456789ABCDEF' if cls.hexcase else '0123456789abcdef'
+            _loc_3 = ''
+            for _loc_5 in range(0, len(param1)):
+                _loc_4 = compat_ord(param1[_loc_5])
+                _loc_3 += _loc_2[_loc_4 >> 4 & 15] + _loc_2[_loc_4 & 15]
+            return _loc_3
+
+        @classmethod
+        def rstr2b64(cls, param1):
+            _loc_2 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'
+            _loc_3 = ''
+            _loc_4 = len(param1)
+            for _loc_5 in range(0, _loc_4, 3):
+                _loc_6_1 = compat_ord(param1[_loc_5]) << 16
+                _loc_6_2 = compat_ord(param1[_loc_5 + 1]) << 8 if _loc_5 + 1 < _loc_4 else 0
+                _loc_6_3 = compat_ord(param1[_loc_5 + 2]) if _loc_5 + 2 < _loc_4 else 0
+                _loc_6 = _loc_6_1 | _loc_6_2 | _loc_6_3
+                for _loc_7 in range(0, 4):
+                    if _loc_5 * 8 + _loc_7 * 6 > len(param1) * 8:
+                        _loc_3 += cls.b64pad
+                    else:
+                        _loc_3 += _loc_2[_loc_6 >> 6 * (3 - _loc_7) & 63]
+            return _loc_3
+
+        @staticmethod
+        def rstr2any(param1, param2):
+            _loc_3 = len(param2)
+            _loc_4 = []
+            _loc_9 = [0] * ((len(param1) >> 2) + 1)
+            for _loc_5 in range(0, len(_loc_9)):
+                _loc_9[_loc_5] = compat_ord(param1[_loc_5 * 2]) << 8 | compat_ord(param1[_loc_5 * 2 + 1])
+
+            while len(_loc_9) > 0:
+                _loc_8 = []
+                _loc_7 = 0
+                for _loc_5 in range(0, len(_loc_9)):
+                    _loc_7 = (_loc_7 << 16) + _loc_9[_loc_5]
+                    _loc_6 = math.floor(_loc_7 / _loc_3)
+                    _loc_7 -= _loc_6 * _loc_3
+                    if len(_loc_8) > 0 or _loc_6 > 0:
+                        _loc_8[len(_loc_8)] = _loc_6
+
+                _loc_4[len(_loc_4)] = _loc_7
+                _loc_9 = _loc_8
+
+            _loc_10 = ''
+            _loc_5 = len(_loc_4) - 1
+            while _loc_5 >= 0:
+                _loc_10 += param2[_loc_4[_loc_5]]
+                _loc_5 -= 1
+
+            return _loc_10
+
+        @classmethod
+        def str2rstr_utf8(cls, param1, param2=None):
+            _loc_3 = ''
+            _loc_4 = -1
+            if not param2:
+                param2 = cls.PADDING
+            param1 = param1 + param2[1:9]
+            while True:
+                _loc_4 += 1
+                if _loc_4 >= len(param1):
+                    break
+                _loc_5 = compat_ord(param1[_loc_4])
+                _loc_6 = compat_ord(param1[_loc_4 + 1]) if _loc_4 + 1 < len(param1) else 0
+                if 55296 <= _loc_5 <= 56319 and 56320 <= _loc_6 <= 57343:
+                    _loc_5 = 65536 + ((_loc_5 & 1023) << 10) + (_loc_6 & 1023)
+                    _loc_4 += 1
+                if _loc_5 <= 127:
+                    _loc_3 += compat_chr(_loc_5)
+                    continue
+                if _loc_5 <= 2047:
+                    _loc_3 += compat_chr(192 | _loc_5 >> 6 & 31) + compat_chr(128 | _loc_5 & 63)
+                    continue
+                if _loc_5 <= 65535:
+                    _loc_3 += compat_chr(224 | _loc_5 >> 12 & 15) + compat_chr(128 | _loc_5 >> 6 & 63) + compat_chr(
+                        128 | _loc_5 & 63)
+                    continue
+                if _loc_5 <= 2097151:
+                    _loc_3 += compat_chr(240 | _loc_5 >> 18 & 7) + compat_chr(128 | _loc_5 >> 12 & 63) + compat_chr(
+                        128 | _loc_5 >> 6 & 63) + compat_chr(128 | _loc_5 & 63)
+            return _loc_3
+
+        @staticmethod
+        def rstr2binl(param1):
+            _loc_2 = [0] * ((len(param1) >> 2) + 1)
+            for _loc_3 in range(0, len(_loc_2)):
+                _loc_2[_loc_3] = 0
+            for _loc_3 in range(0, len(param1) * 8, 8):
+                _loc_2[_loc_3 >> 5] |= (compat_ord(param1[_loc_3 // 8]) & 255) << _loc_3 % 32
+            return _loc_2
+
+        @staticmethod
+        def binl2rstr(param1):
+            _loc_2 = ''
+            for _loc_3 in range(0, len(param1) * 32, 8):
+                _loc_2 += compat_chr(param1[_loc_3 >> 5] >> _loc_3 % 32 & 255)
+            return _loc_2
+
+        @classmethod
+        def binl_md5(cls, param1, param2):
+            param1 = cls.JSArray(param1)
+            param1[param2 >> 5] |= 128 << param2 % 32
+            param1[(param2 + 64 >> 9 << 4) + 14] = param2
+            _loc_3 = 1732584193
+            _loc_4 = -271733879
+            _loc_5 = -1732584194
+            _loc_6 = 271733878
+            for _loc_7 in range(0, len(param1), 16):
+                _loc_8 = _loc_3
+                _loc_9 = _loc_4
+                _loc_10 = _loc_5
+                _loc_11 = _loc_6
+                _loc_3 = cls.md5_ff(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 0], 7, -680876936)
+                _loc_6 = cls.md5_ff(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 1], 12, -389564586)
+                _loc_5 = cls.md5_ff(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 2], 17, 606105819)
+                _loc_4 = cls.md5_ff(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 3], 22, -1044525330)
+                _loc_3 = cls.md5_ff(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 4], 7, -176418897)
+                _loc_6 = cls.md5_ff(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 5], 12, 1200080426)
+                _loc_5 = cls.md5_ff(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 6], 17, -1473231341)
+                _loc_4 = cls.md5_ff(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 7], 22, -45705983)
+                _loc_3 = cls.md5_ff(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 8], 7, 1770035416)
+                _loc_6 = cls.md5_ff(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 9], 12, -1958414417)
+                _loc_5 = cls.md5_ff(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 10], 17, -42063)
+                _loc_4 = cls.md5_ff(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 11], 22, -1990404162)
+                _loc_3 = cls.md5_ff(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 12], 7, 1804603682)
+                _loc_6 = cls.md5_ff(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 13], 12, -40341101)
+                _loc_5 = cls.md5_ff(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 14], 17, -1502002290)
+                _loc_4 = cls.md5_ff(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 15], 22, 1236535329)
+                _loc_3 = cls.md5_gg(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 1], 5, -165796510)
+                _loc_6 = cls.md5_gg(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 6], 9, -1069501632)
+                _loc_5 = cls.md5_gg(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 11], 14, 643717713)
+                _loc_4 = cls.md5_gg(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 0], 20, -373897302)
+                _loc_3 = cls.md5_gg(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 5], 5, -701558691)
+                _loc_6 = cls.md5_gg(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 10], 9, 38016083)
+                _loc_5 = cls.md5_gg(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 15], 14, -660478335)
+                _loc_4 = cls.md5_gg(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 4], 20, -405537848)
+                _loc_3 = cls.md5_gg(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 9], 5, 568446438)
+                _loc_6 = cls.md5_gg(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 14], 9, -1019803690)
+                _loc_5 = cls.md5_gg(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 3], 14, -187363961)
+                _loc_4 = cls.md5_gg(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 8], 20, 1163531501)
+                _loc_3 = cls.md5_gg(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 13], 5, -1444681467)
+                _loc_6 = cls.md5_gg(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 2], 9, -51403784)
+                _loc_5 = cls.md5_gg(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 7], 14, 1735328473)
+                _loc_4 = cls.md5_gg(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 12], 20, -1926607734)
+                _loc_3 = cls.md5_hh(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 5], 4, -378558)
+                _loc_6 = cls.md5_hh(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 8], 11, -2022574463)
+                _loc_5 = cls.md5_hh(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 11], 16, 1839030562)
+                _loc_4 = cls.md5_hh(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 14], 23, -35309556)
+                _loc_3 = cls.md5_hh(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 1], 4, -1530992060)
+                _loc_6 = cls.md5_hh(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 4], 11, 1272893353)
+                _loc_5 = cls.md5_hh(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 7], 16, -155497632)
+                _loc_4 = cls.md5_hh(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 10], 23, -1094730640)
+                _loc_3 = cls.md5_hh(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 13], 4, 681279174)
+                _loc_6 = cls.md5_hh(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 0], 11, -358537222)
+                _loc_5 = cls.md5_hh(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 3], 16, -722521979)
+                _loc_4 = cls.md5_hh(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 6], 23, 76029189)
+                _loc_3 = cls.md5_hh(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 9], 4, -640364487)
+                _loc_6 = cls.md5_hh(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 12], 11, -421815835)
+                _loc_5 = cls.md5_hh(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 15], 16, 530742520)
+                _loc_4 = cls.md5_hh(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 2], 23, -995338651)
+                _loc_3 = cls.md5_ii(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 0], 6, -198630844)
+                _loc_6 = cls.md5_ii(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 7], 10, 1126891415)
+                _loc_5 = cls.md5_ii(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 14], 15, -1416354905)
+                _loc_4 = cls.md5_ii(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 5], 21, -57434055)
+                _loc_3 = cls.md5_ii(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 12], 6, 1700485571)
+                _loc_6 = cls.md5_ii(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 3], 10, -1894986606)
+                _loc_5 = cls.md5_ii(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 10], 15, -1051523)
+                _loc_4 = cls.md5_ii(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 1], 21, -2054922799)
+                _loc_3 = cls.md5_ii(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 8], 6, 1873313359)
+                _loc_6 = cls.md5_ii(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 15], 10, -30611744)
+                _loc_5 = cls.md5_ii(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 6], 15, -1560198380)
+                _loc_4 = cls.md5_ii(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 13], 21, 1309151649)
+                _loc_3 = cls.md5_ii(_loc_3, _loc_4, _loc_5, _loc_6, param1[_loc_7 + 4], 6, -145523070)
+                _loc_6 = cls.md5_ii(_loc_6, _loc_3, _loc_4, _loc_5, param1[_loc_7 + 11], 10, -1120210379)
+                _loc_5 = cls.md5_ii(_loc_5, _loc_6, _loc_3, _loc_4, param1[_loc_7 + 2], 15, 718787259)
+                _loc_4 = cls.md5_ii(_loc_4, _loc_5, _loc_6, _loc_3, param1[_loc_7 + 9], 21, -343485551)
+                _loc_3 = cls.safe_add(_loc_3, _loc_8)
+                _loc_4 = cls.safe_add(_loc_4, _loc_9)
+                _loc_5 = cls.safe_add(_loc_5, _loc_10)
+                _loc_6 = cls.safe_add(_loc_6, _loc_11)
+            return [_loc_3, _loc_4, _loc_5, _loc_6]
+
+        @classmethod
+        def md5_cmn(cls, param1, param2, param3, param4, param5, param6):
+            return cls.safe_add(
+                cls.bit_rol(cls.safe_add(cls.safe_add(param2, param1), cls.safe_add(param4, param6)), param5), param3)
+
+        @classmethod
+        def md5_ff(cls, param1, param2, param3, param4, param5, param6, param7):
+            return cls.md5_cmn(param2 & param3 | ~param2 & param4, param1, param2, param5, param6, param7)
+
+        @classmethod
+        def md5_gg(cls, param1, param2, param3, param4, param5, param6, param7):
+            return cls.md5_cmn(param2 & param4 | param3 & ~param4, param1, param2, param5, param6, param7)
+
+        @classmethod
+        def md5_hh(cls, param1, param2, param3, param4, param5, param6, param7):
+            return cls.md5_cmn(param2 ^ param3 ^ param4, param1, param2, param5, param6, param7)
+
+        @classmethod
+        def md5_ii(cls, param1, param2, param3, param4, param5, param6, param7):
+            return cls.md5_cmn(param3 ^ (param2 | ~param4), param1, param2, param5, param6, param7)
+
+        @classmethod
+        def safe_add(cls, param1, param2):
+            _loc_3 = (param1 & 65535) + (param2 & 65535)
+            _loc_4 = (param1 >> 16) + (param2 >> 16) + (_loc_3 >> 16)
+            return cls.lshift(_loc_4, 16) | _loc_3 & 65535
+
+        @classmethod
+        def bit_rol(cls, param1, param2):
+            return cls.lshift(param1, param2) | (param1 & 0xFFFFFFFF) >> (32 - param2)
+
+        @staticmethod
+        def lshift(value, count):
+            r = (0xFFFFFFFF & value) << count
+            return -(~(r - 1) & 0xFFFFFFFF) if r > 0x7FFFFFFF else r
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        video_id = self._search_regex(self._VIDEOID_REGEXES, webpage, 'video id')
+
+        video = self._download_json(
+            self._API_URL_TEMPLATE % video_id, video_id)['videos'][0]
+
+        title = video['title']
+        duration = float_or_none(video['duration'], 1000)
+        like_count = video['likes']
+        uploader = video['channel']
+        uploader_id = video['channel_id']
+
+        formats = []
+
+        for resource in video['resources']:
+            resource_id = resource.get('_id')
+            if not resource_id:
+                continue
+
+            security = self._download_json(
+                self._SECURITY_URL_TEMPLATE % (video_id, resource_id),
+                video_id, 'Downloading security hash for %s' % resource_id)
+
+            security_hash = security.get('hash')
+            if not security_hash:
+                message = security.get('message')
+                if message:
+                    raise ExtractorError(
+                        '%s returned error: %s' % (self.IE_NAME, message), expected=True)
+                continue
+
+            hash_code = security_hash[:2]
+            received_time = int(security_hash[2:12])
+            received_random = security_hash[12:22]
+            received_md5 = security_hash[22:]
+
+            sign_time = received_time + self._RESIGN_EXPIRATION
+            padding = '%010d' % random.randint(1, 10000000000)
+
+            signed_md5 = self.MD5.b64_md5(received_md5 + compat_str(sign_time) + padding)
+            signed_hash = hash_code + compat_str(received_time) + received_random + compat_str(sign_time) + padding + signed_md5
+
+            formats.append({
+                'url': '%s?h=%s&k=%s' % (resource['url'], signed_hash, 'flash'),
+                'format_id': resource_id,
+                'height': resource['height']
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'duration': duration,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'like_count': like_count,
+            'formats': formats
+        }
\ No newline at end of file

From e9be9a6acdc47e53dd8292341fa521234ce9dd78 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 4 Oct 2014 22:38:23 +0300
Subject: [PATCH 0171/1937] [utils] Add additional format to unified_strdate

---
 youtube_dl/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2c9081733..d7ae5a90a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -894,6 +894,7 @@ def unified_strdate(date_str):
         '%Y/%m/%d %H:%M:%S',
         '%d/%m/%Y %H:%M:%S',
         '%Y-%m-%d %H:%M:%S',
+        '%Y-%m-%d %H:%M:%S.%f',
         '%d.%m.%Y %H:%M',
         '%d.%m.%Y %H.%M',
         '%Y-%m-%dT%H:%M:%SZ',

From 5e69192ef74f9dd4057905d2ffc2d4afb051fb57 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 4 Oct 2014 22:40:36 +0300
Subject: [PATCH 0172/1937] [thesixtyone] Add new extractor (closes #3781)

---
 youtube_dl/extractor/__init__.py    |   1 +
 youtube_dl/extractor/thesixtyone.py | 100 ++++++++++++++++++++++++++++
 2 files changed, 101 insertions(+)
 create mode 100644 youtube_dl/extractor/thesixtyone.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b59fd65ca..d118daa68 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -371,6 +371,7 @@ from .tenplay import TenPlayIE
 from .testurl import TestURLIE
 from .tf1 import TF1IE
 from .theplatform import ThePlatformIE
+from .thesixtyone import TheSixtyOneIE
 from .thisav import ThisAVIE
 from .tinypic import TinyPicIE
 from .tlc import TlcIE, TlcDeIE
diff --git a/youtube_dl/extractor/thesixtyone.py b/youtube_dl/extractor/thesixtyone.py
new file mode 100644
index 000000000..a77c6a2fc
--- /dev/null
+++ b/youtube_dl/extractor/thesixtyone.py
@@ -0,0 +1,100 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import unified_strdate
+
+
+class TheSixtyOneIE(InfoExtractor):
+    _VALID_URL = r'''(?x)https?://(?:www\.)?thesixtyone\.com/
+        (?:.*?/)*
+        (?:
+            s|
+            song/comments/list|
+            song
+        )/(?P<id>[A-Za-z0-9]+)/?$'''
+    _SONG_URL_TEMPLATE = 'http://thesixtyone.com/s/{0:}'
+    _SONG_FILE_URL_TEMPLATE = 'http://{audio_server:}.thesixtyone.com/thesixtyone_production/audio/{0:}_stream'
+    _THUMBNAIL_URL_TEMPLATE = '{photo_base_url:}_desktop'
+    _TESTS = [
+        {
+            'url': 'http://www.thesixtyone.com/s/SrE3zD7s1jt/',
+            'md5': '821cc43b0530d3222e3e2b70bb4622ea',
+            'info_dict': {
+                'id': 'SrE3zD7s1jt',
+                'ext': 'mp3',
+                'title': 'CASIO - Unicorn War Mixtape',
+                'thumbnail': 're:^https?://.*_desktop$',
+                'upload_date': '20071217',
+                'duration': 3208,
+            }
+        },
+        {
+            'url': 'http://www.thesixtyone.com/song/comments/list/SrE3zD7s1jt',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.thesixtyone.com/s/ULoiyjuJWli#/s/SrE3zD7s1jt/',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.thesixtyone.com/#/s/SrE3zD7s1jt/',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.thesixtyone.com/song/SrE3zD7s1jt/',
+            'only_matching': True,
+        },
+    ]
+
+    _DECODE_MAP = {
+        "x": "a",
+        "m": "b",
+        "w": "c",
+        "q": "d",
+        "n": "e",
+        "p": "f",
+        "a": "0",
+        "h": "1",
+        "e": "2",
+        "u": "3",
+        "s": "4",
+        "i": "5",
+        "o": "6",
+        "y": "7",
+        "r": "8",
+        "c": "9"
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        song_id = mobj.group('id')
+
+        webpage = self._download_webpage(
+            self._SONG_URL_TEMPLATE.format(song_id), song_id)
+
+        song_data = json.loads(self._search_regex(
+            r'"%s":\s(\{.*?\})' % song_id, webpage, 'song_data'))
+        keys = [self._DECODE_MAP.get(s, s) for s in song_data['key']]
+        url = self._SONG_FILE_URL_TEMPLATE.format(
+            "".join(reversed(keys)), **song_data)
+
+        formats = [{
+            'format_id': 'sd',
+            'url': url,
+            'ext': 'mp3',
+        }]
+
+        return {
+            'id': song_id,
+            'title': '{artist:} - {name:}'.format(**song_data),
+            'formats': formats,
+            'comment_count': song_data.get('comments_count'),
+            'duration': song_data.get('play_time'),
+            'like_count': song_data.get('score'),
+            'thumbnail': self._THUMBNAIL_URL_TEMPLATE.format(**song_data),
+            'upload_date': unified_strdate(song_data.get('publish_date')),
+        }

From b81f484b60c5e06e7d30720b104d01245c466e14 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 5 Oct 2014 07:53:02 +0200
Subject: [PATCH 0173/1937] [gorillavid] Add support for movpod.in (Fixes
 #3881)

---
 youtube_dl/extractor/gorillavid.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index ca5f7c417..8c3ad8c7d 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -12,10 +12,10 @@ from ..utils import (
 
 
 class GorillaVidIE(InfoExtractor):
-    IE_DESC = 'GorillaVid.in and daclips.in'
+    IE_DESC = 'GorillaVid.in, daclips.in and movpod.in'
     _VALID_URL = r'''(?x)
         https?://(?P<host>(?:www\.)?
-            (?:daclips\.in|gorillavid\.in))/
+            (?:daclips\.in|gorillavid\.in|movpod\.in))/
         (?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)?
     '''
 
@@ -46,6 +46,9 @@ class GorillaVidIE(InfoExtractor):
             'title': 'Micro Pig piglets ready on 16th July 2009',
             'thumbnail': 're:http://.*\.jpg',
         },
+    }, {
+        'url': 'http://movpod.in/0wguyyxi1yca',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From bb0c206f592b47982753bc63d1ee49868c925cec Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 5 Oct 2014 07:53:25 +0200
Subject: [PATCH 0174/1937] release 2014.10.05

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b7a60ac24..9f48107a1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.10.02'
+__version__ = '2014.10.05'

From e4b85e35d083fca70d29a751c01071e287b5cdae Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Mon, 6 Oct 2014 01:47:22 +0700
Subject: [PATCH 0175/1937] [gorillavid] Fix title extraction and make
 thumbnail optional (Closes #3884)

---
 youtube_dl/extractor/gorillavid.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index 8c3ad8c7d..0f0012f1f 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     determine_ext,
     compat_urllib_parse,
     compat_urllib_request,
@@ -25,7 +26,7 @@ class GorillaVidIE(InfoExtractor):
         'info_dict': {
             'id': '06y9juieqpmi',
             'ext': 'flv',
-            'title': 'Rebecca Black My Moment Official Music Video Reaction',
+            'title': 'Rebecca Black My Moment Official Music Video Reaction-6GK87Rc8bzQ',
             'thumbnail': 're:http://.*\.jpg',
         },
     }, {
@@ -72,14 +73,14 @@ class GorillaVidIE(InfoExtractor):
 
             webpage = self._download_webpage(req, video_id, 'Downloading video page')
 
-        title = self._search_regex(r'style="z-index: [0-9]+;">([0-9a-zA-Z ]+)(?:-.+)?</span>', webpage, 'title')
-        thumbnail = self._search_regex(r'image:\'(http[^\']+)\',', webpage, 'thumbnail')
-        url = self._search_regex(r'file: \'(http[^\']+)\',', webpage, 'file url')
+        title = self._search_regex(r'style="z-index: [0-9]+;">([^<]+)</span>', webpage, 'title')
+        video_url = self._search_regex(r'file\s*:\s*\'(http[^\']+)\',', webpage, 'file url')
+        thumbnail = self._search_regex(r'image\s*:\s*\'(http[^\']+)\',', webpage, 'thumbnail', fatal=False)
 
         formats = [{
             'format_id': 'sd',
-            'url': url,
-            'ext': determine_ext(url),
+            'url': video_url,
+            'ext': determine_ext(video_url),
             'quality': 1,
         }]
 

From 3ae165aa10e6747173adbcce8da0e2c8d30eed33 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Mon, 6 Oct 2014 01:48:01 +0700
Subject: [PATCH 0176/1937] [gorillavid] Add check for non existing videos

---
 youtube_dl/extractor/gorillavid.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index 0f0012f1f..45cca1d24 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -20,6 +20,8 @@ class GorillaVidIE(InfoExtractor):
         (?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)?
     '''
 
+    _FILE_NOT_FOUND_REGEX = r'>(?:404 - )?File Not Found<'
+
     _TESTS = [{
         'url': 'http://gorillavid.in/06y9juieqpmi',
         'md5': '5ae4a3580620380619678ee4875893ba',
@@ -58,6 +60,9 @@ class GorillaVidIE(InfoExtractor):
 
         webpage = self._download_webpage('http://%s/%s' % (mobj.group('host'), video_id), video_id)
 
+        if re.search(self._FILE_NOT_FOUND_REGEX, webpage) is not None:
+            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
         fields = dict(re.findall(r'''(?x)<input\s+
             type="hidden"\s+
             name="([^"]+)"\s+

From b10609d98c4a4ade205fb683ad302f7bca33a3c9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 5 Oct 2014 21:59:53 +0200
Subject: [PATCH 0177/1937] [dailymotion] Alternative title search (Fixes
 #3882)

---
 youtube_dl/extractor/dailymotion.py | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 66a8f16d9..dbcf5d6a7 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -82,11 +82,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
     ]
 
     def _real_extract(self, url):
-        # Extract id and simplified title from URL
-        mobj = re.match(self._VALID_URL, url)
-
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         url = 'http://www.dailymotion.com/video/%s' % video_id
 
         # Retrieve video webpage to extract further information
@@ -147,18 +143,23 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
             self._list_available_subtitles(video_id, webpage)
             return
 
-        view_count = self._search_regex(
-            r'video_views_count[^>]+>\s+([\d\.,]+)', webpage, 'view count', fatal=False)
-        if view_count is not None:
-            view_count = str_to_int(view_count)
+        view_count = str_to_int(self._search_regex(
+            r'video_views_count[^>]+>\s+([\d\.,]+)',
+            webpage, 'view count', fatal=False))
+
+        title = self._og_search_title(webpage, default=None)
+        if title is None:
+            title = self._html_search_regex(
+                r'(?s)<span\s+id="video_title"[^>]*>(.*?)</span>', webpage,
+                'title')
 
         return {
-            'id':       video_id,
+            'id': video_id,
             'formats': formats,
             'uploader': info['owner.screenname'],
-            'upload_date':  video_upload_date,
-            'title':    self._og_search_title(webpage),
-            'subtitles':    video_subtitles,
+            'upload_date': video_upload_date,
+            'title': title,
+            'subtitles': video_subtitles,
             'thumbnail': info['thumbnail_url'],
             'age_limit': age_limit,
             'view_count': view_count,

From 4c4de296d41a1db4642df5f9381d77e7306af3c4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 5 Oct 2014 22:00:16 +0200
Subject: [PATCH 0178/1937] release 2014.10.05.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9f48107a1..ef2196960 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.10.05'
+__version__ = '2014.10.05.1'

From 642b76ac1513f7aaa3469e03078ab2fb3d3187f4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 5 Oct 2014 22:04:02 +0200
Subject: [PATCH 0179/1937] release 2014.10.05.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index ef2196960..4f0d486b9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.10.05.1'
+__version__ = '2014.10.05.2'

From 31d06400ecee79ecf3a0bd38c0702f165fd6d958 Mon Sep 17 00:00:00 2001
From: net <net@h-MacBook-Pro-sl-net.local>
Date: Mon, 6 Oct 2014 03:03:05 +0300
Subject: [PATCH 0180/1937] add missed init file

---
 youtube_dl/extractor/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 079221567..b07c0b4cc 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -427,6 +427,7 @@ from .vporn import VpornIE
 from .vube import VubeIE
 from .vuclip import VuClipIE
 from .vulture import VultureIE
+from .walla import WallaIE
 from .washingtonpost import WashingtonPostIE
 from .wat import WatIE
 from .wayofthemaster import WayOfTheMasterIE

From 3b700f8d434cbf1b097324a577e4b95325b24d91 Mon Sep 17 00:00:00 2001
From: Xu Cheng <xucheng@me.com>
Date: Mon, 6 Oct 2014 18:58:25 +0800
Subject: [PATCH 0181/1937] support zsh completion

---
 devscripts/zsh-completion.in | 28 ++++++++++++++++++++++
 devscripts/zsh-completion.py | 46 ++++++++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+)
 create mode 100644 devscripts/zsh-completion.in
 create mode 100755 devscripts/zsh-completion.py

diff --git a/devscripts/zsh-completion.in b/devscripts/zsh-completion.in
new file mode 100644
index 000000000..b394a1ae7
--- /dev/null
+++ b/devscripts/zsh-completion.in
@@ -0,0 +1,28 @@
+#compdef youtube-dl
+
+__youtube_dl() {
+    local curcontext="$curcontext" fileopts diropts cur prev
+    typeset -A opt_args
+    fileopts="{{fileopts}}"
+    diropts="{{diropts}}"
+    cur=$words[CURRENT]
+    case $cur in
+        :)
+            _arguments '*: :(::ytfavorites ::ytrecommended ::ytsubscriptions ::ytwatchlater ::ythistory)'
+        ;;
+        *)
+            prev=$words[CURRENT-1]
+            if [[ ${prev} =~ ${fileopts} ]]; then
+                _path_files
+            elif [[ ${prev} =~ ${diropts} ]]; then
+                _path_files -/
+            elif [[ ${prev} == "--recode-video" ]]; then
+                _arguments '*: :(mp4 flv ogg webm mkv)'
+            else
+                _arguments '*: :({{flags}})'
+            fi
+        ;;
+    esac
+}
+
+__youtube_dl
\ No newline at end of file
diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py
new file mode 100755
index 000000000..e8d71928a
--- /dev/null
+++ b/devscripts/zsh-completion.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+import os
+from os.path import dirname as dirn
+import sys
+
+sys.path.append(dirn(dirn((os.path.abspath(__file__)))))
+import youtube_dl
+
+ZSH_COMPLETION_FILE = "youtube-dl.zsh"
+ZSH_COMPLETION_TEMPLATE = "devscripts/zsh-completion.in"
+
+
+def build_completion(opt_parser):
+    opts = [opt for group in opt_parser.option_groups
+            for opt in group.option_list]
+    opts_file = [opt for opt in opts if opt.metavar == "FILE"]
+    opts_dir = [opt for opt in opts if opt.metavar == "DIR"]
+
+    fileopts = []
+    for opt in opts_file:
+        if opt._short_opts:
+            fileopts.extend(opt._short_opts)
+        if opt._long_opts:
+            fileopts.extend(opt._long_opts)
+
+    diropts = []
+    for opt in opts_dir:
+        if opt._short_opts:
+            diropts.extend(opt._short_opts)
+        if opt._long_opts:
+            diropts.extend(opt._long_opts)
+
+    flags = [opt.get_opt_string() for opt in opts]
+
+    with open(ZSH_COMPLETION_TEMPLATE) as f:
+        template = f.read()
+
+    template = template.replace("{{fileopts}}", "|".join(fileopts))
+    template = template.replace("{{diropts}}", "|".join(diropts))
+    template = template.replace("{{flags}}", " ".join(flags))
+
+    with open(ZSH_COMPLETION_FILE, "w") as f:
+        f.write(template)
+
+parser = youtube_dl.parseOpts()[0]
+build_completion(parser)

From 24f7fb5e1e2fb1e70f553decf07d14892f87b419 Mon Sep 17 00:00:00 2001
From: Xu Cheng <xucheng@me.com>
Date: Mon, 6 Oct 2014 19:02:11 +0800
Subject: [PATCH 0182/1937] add zsh completion support into Makefile

---
 Makefile | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index 6272b826c..2da92863f 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
-all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.fish
+all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.fish
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish
 
 cleanall: clean
 	rm -f youtube-dl youtube-dl.exe
@@ -9,6 +9,7 @@ cleanall: clean
 PREFIX ?= /usr/local
 BINDIR ?= $(PREFIX)/bin
 MANDIR ?= $(PREFIX)/man
+SHAREDIR ?= $(PREFIX)/share
 PYTHON ?= /usr/bin/env python
 
 # set SYSCONFDIR to /etc if PREFIX=/usr or PREFIX=/usr/local
@@ -22,13 +23,15 @@ else
 	endif
 endif
 
-install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
+install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
 	install -d $(DESTDIR)$(BINDIR)
 	install -m 755 youtube-dl $(DESTDIR)$(BINDIR)
 	install -d $(DESTDIR)$(MANDIR)/man1
 	install -m 644 youtube-dl.1 $(DESTDIR)$(MANDIR)/man1
 	install -d $(DESTDIR)$(SYSCONFDIR)/bash_completion.d
 	install -m 644 youtube-dl.bash-completion $(DESTDIR)$(SYSCONFDIR)/bash_completion.d/youtube-dl
+	install -d $(DESTDIR)$(SHAREDIR)/zsh/site-functions
+	install -m 644 youtube-dl.zsh $(DESTDIR)$(SHAREDIR)/zsh/site-functions/_youtube-dl
 	install -d $(DESTDIR)$(SYSCONFDIR)/fish/completions
 	install -m 644 youtube-dl.fish $(DESTDIR)$(SYSCONFDIR)/fish/completions/youtube-dl.fish
 
@@ -38,7 +41,7 @@ test:
 
 tar: youtube-dl.tar.gz
 
-.PHONY: all clean install test tar bash-completion pypi-files fish-completion
+.PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion
 
 pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 youtube-dl.fish
 
@@ -66,12 +69,17 @@ youtube-dl.bash-completion: youtube_dl/*.py youtube_dl/*/*.py devscripts/bash-co
 
 bash-completion: youtube-dl.bash-completion
 
+youtube-dl.zsh: youtube_dl/*.py youtube_dl/*/*.py devscripts/zsh-completion.in
+	python devscripts/zsh-completion.py
+
+zsh-completion: youtube-dl.zsh
+
 youtube-dl.fish: youtube_dl/*.py youtube_dl/*/*.py devscripts/fish-completion.in
 	python devscripts/fish-completion.py
 
 fish-completion: youtube-dl.fish
 
-youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.fish
+youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
 	@tar -czf youtube-dl.tar.gz --transform "s|^|youtube-dl/|" --owner 0 --group 0 \
 		--exclude '*.DS_Store' \
 		--exclude '*.kate-swp' \
@@ -86,5 +94,5 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-
 		bin devscripts test youtube_dl docs \
 		LICENSE README.md README.txt \
 		Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion \
-		youtube-dl.fish setup.py \
+		youtube-dl.zsh youtube-dl.fish setup.py \
 		youtube-dl

From 7bc8780c576505fd87a5c85ff1f50ef2e8841d88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 7 Oct 2014 22:23:05 +0700
Subject: [PATCH 0183/1937] [walla] Fix extractor and add subtitle tests

---
 test/test_subtitles.py        | 28 +++++++++++
 youtube_dl/extractor/walla.py | 95 +++++++++++++++++++++--------------
 2 files changed, 85 insertions(+), 38 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 48c302198..eb5f2f8dd 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -15,6 +15,7 @@ from youtube_dl.extractor import (
     DailymotionIE,
     TEDIE,
     VimeoIE,
+    WallaIE,
 )
 
 
@@ -279,5 +280,32 @@ class TestVimeoSubtitles(BaseTestSubtitles):
             self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
 
 
+class TestWallsSubtitles(BaseTestSubtitles):
+    url = 'http://vod.walla.co.il/movie/2705958/the-yes-men'
+    IE = WallaIE
+
+    def test_list_subtitles(self):
+        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.params['listsubtitles'] = True
+        info_dict = self.getInfoDict()
+        self.assertEqual(info_dict, None)
+
+    def test_allsubtitles(self):
+        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['heb']))
+        self.assertEqual(md5(subtitles['heb']), 'e758c5d7cb982f6bef14f377ec7a3920')
+
+    def test_nosubtitles(self):
+        self.DL.expect_warning(u'video doesn\'t have subtitles')
+        self.url = 'http://vod.walla.co.il/movie/2642630/one-direction-all-for-one'
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(len(subtitles), 0)
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/walla.py b/youtube_dl/extractor/walla.py
index e687c3af0..672bda7a7 100644
--- a/youtube_dl/extractor/walla.py
+++ b/youtube_dl/extractor/walla.py
@@ -1,70 +1,89 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-
 import re
 
-from .common import InfoExtractor
+from .subtitles import SubtitlesInfoExtractor
+from ..utils import (
+    xpath_text,
+    int_or_none,
+)
 
 
-class WallaIE(InfoExtractor):
-    _VALID_URL = r'http://vod\.walla\.co\.il/\w+/(?P<id>\d+)'
+class WallaIE(SubtitlesInfoExtractor):
+    _VALID_URL = r'http://vod\.walla\.co\.il/[^/]+/(?P<id>\d+)/(?P<display_id>.+)'
     _TEST = {
         'url': 'http://vod.walla.co.il/movie/2642630/one-direction-all-for-one',
         'info_dict': {
             'id': '2642630',
+            'display_id': 'one-direction-all-for-one',
             'ext': 'flv',
             'title': 'וואן דיירקשן: ההיסטריה',
+            'description': 'md5:de9e2512a92442574cdb0913c49bc4d8',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'duration': 3600,
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
         }
     }
 
+    _SUBTITLE_LANGS = {
+        'עברית': 'heb',
+    }
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        
         video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        config_url = 'http://video2.walla.co.il/?w=null/null/%s/@@/video/flv_pl' % video_id
-        
-        webpage = self._download_webpage(config_url, video_id, '')
+        video = self._download_xml(
+            'http://video2.walla.co.il/?w=null/null/%s/@@/video/flv_pl' % video_id,
+            display_id)
 
-        media_id = self._html_search_regex(r'<media_id>(\d+)</media_id>', webpage, video_id, 'extract media id')
+        item = video.find('./items/item')
 
-        prefix = '0' if len(media_id) == 7 else ''
-
-        series =  '%s%s' % (prefix, media_id[0:2])
-        session = media_id[2:5]
-        episode = media_id[5:7]
-        
-        title = self._html_search_regex(r'<title>(.*)</title>', webpage, video_id, 'title')
-
-        default_quality = self._html_search_regex(r'<qualities defaultType="(\d+)">', webpage, video_id, 0)
-
-        quality = default_quality if default_quality else '40'
-
-        media_path = '/%s/%s/%s' % (series, session, media_id) #self._html_search_regex(r'<quality type="%s">.*<src>(.*)</src>' % default_quality ,webpage, '', flags=re.DOTALL) 
-
-        playpath = 'mp4:media/%s/%s/%s-%s' % (series, session, media_id, quality) #self._html_search_regex(r'<quality type="%s">.*<src>(.*)</src>' % default_quality ,webpage, '', flags=re.DOTALL) 
+        title = xpath_text(item, './title', 'title')
+        description = xpath_text(item, './synopsis', 'description')
+        thumbnail = xpath_text(item, './preview_pic', 'thumbnail')
+        duration = int_or_none(xpath_text(item, './duration', 'duration'))
 
         subtitles = {}
+        for subtitle in item.findall('./subtitles/subtitle'):
+            lang = xpath_text(subtitle, './title')
+            subtitles[self._SUBTITLE_LANGS.get(lang, lang)] = xpath_text(subtitle, './src')
 
-        subtitle_url = self._html_search_regex(r'<subtitles.*<src>(.*)</src>.*</subtitle>', webpage, video_id, 0)
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, subtitles)
+            return
 
-        print subtitle_url
+        subtitles = self.extract_subtitles(video_id, subtitles)
 
-        if subtitle_url:
-            subtitles_page = self._download_webpage(subtitle_url, video_id, '')
-            subtitles['heb'] = subtitles_page
+        formats = []
+        for quality in item.findall('./qualities/quality'):
+            format_id = xpath_text(quality, './title')
+            fmt = {
+                'url': 'rtmp://wafla.walla.co.il/vod',
+                'play_path': xpath_text(quality, './src'),
+                'player_url': 'http://isc.walla.co.il/w9/swf/video_swf/vod/WallaMediaPlayerAvod.swf',
+                'page_url': url,
+                'ext': 'flv',
+                'format_id': xpath_text(quality, './title'),
+            }
+            m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
+            if m:
+                fmt['height'] = int(m.group('height'))
+            formats.append(fmt)
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
-            'url': 'rtmp://wafla.walla.co.il:1935/vod',
-            'player_url': 'http://isc.walla.co.il/w9/swf/video_swf/vod/WallaMediaPlayerAvod.swf',
-            'page_url': url,
-            'app': "vod",
-            'play_path': playpath,
-            'tc_url': 'rtmp://wafla.walla.co.il:1935/vod',
-            'rtmp_protocol': 'rtmp',
-            'ext': 'flv',
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
             'subtitles': subtitles,
-        }
\ No newline at end of file
+        }

From 842cca7d56c155f515ba0919d709a412291ef52b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 8 Oct 2014 20:08:29 +0700
Subject: [PATCH 0184/1937] [pornhd] Fix formats extraction (Closes #3898)

---
 youtube_dl/extractor/pornhd.py | 52 ++++++++++++++--------------------
 1 file changed, 22 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index 48ce6e730..bac484c67 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -4,19 +4,27 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    qualities,
+    determine_ext,
+)
 
 
 class PornHdIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?pornhd\.com/(?:[a-z]{2,4}/)?videos/(?P<id>\d+)'
+    _VALID_URL = r'http://(?:www\.)?pornhd\.com/(?:[a-z]{2,4}/)?videos/(?P<id>\d+)(?:/(?P<display_id>.+))?'
     _TEST = {
         'url': 'http://www.pornhd.com/videos/1962/sierra-day-gets-his-cum-all-over-herself-hd-porn-video',
         'md5': '956b8ca569f7f4d8ec563e2c41598441',
         'info_dict': {
             'id': '1962',
+            'display_id': 'sierra-day-gets-his-cum-all-over-herself-hd-porn-video',
             'ext': 'mp4',
             'title': 'Sierra loves doing laundry',
             'description': 'md5:8ff0523848ac2b8f9b065ba781ccf294',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'view_count': int,
             'age_limit': 18,
         }
     }
@@ -24,8 +32,9 @@ class PornHdIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, display_id or video_id)
 
         title = self._html_search_regex(
             r'<title>(.+) porn HD.+?</title>', webpage, 'title')
@@ -33,38 +42,21 @@ class PornHdIE(InfoExtractor):
             r'<div class="description">([^<]+)</div>', webpage, 'description', fatal=False)
         view_count = int_or_none(self._html_search_regex(
             r'(\d+) views\s*</span>', webpage, 'view count', fatal=False))
+        thumbnail = self._search_regex(
+            r"'poster'\s*:\s*'([^']+)'", webpage, 'thumbnail', fatal=False)
 
-        videos = re.findall(
-            r'var __video([\da-zA-Z]+?)(Low|High)StreamUrl = \'(http://.+?)\?noProxy=1\'', webpage)
-
-        mobj = re.search(r'flashVars = (?P<flashvars>{.+?});', webpage)
-        if mobj:
-            flashvars = json.loads(mobj.group('flashvars'))
-            for key, quality in [('hashlink', 'low'), ('hd', 'high')]:
-                redirect_url = flashvars.get(key)
-                if redirect_url:
-                    videos.append(('flv', quality, redirect_url))
-            thumbnail = flashvars['urlWallpaper']
-        else:
-            thumbnail = self._og_search_thumbnail(webpage)
-
-        formats = []
-        for format_, quality, redirect_url in videos:
-            format_id = '%s-%s' % (format_.lower(), quality.lower())
-            video_url = self._download_webpage(
-                redirect_url, video_id, 'Downloading %s video link' % format_id, fatal=False)
-            if not video_url:
-                continue
-            formats.append({
-                'url': video_url,
-                'ext': format_.lower(),
-                'format_id': format_id,
-                'quality': 1 if quality.lower() == 'high' else 0,
-            })
+        quality = qualities(['SD', 'HD'])
+        formats = [{
+            'url': source['file'],
+            'format_id': '%s-%s' % (source['label'], determine_ext(source['file'])),
+            'quality': quality(source['label']),
+        } for source in json.loads(js_to_json(self._search_regex(
+            r"(?s)'sources'\s*:\s*(\[.+?\])", webpage, 'sources')))]
         self._sort_formats(formats)
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,

From ced659bb4d066c782a009d370daff0fedb7b1006 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Oct 2014 19:26:23 +0700
Subject: [PATCH 0185/1937] [generic] Ignore some non-video file extensions
 during generic extraction (Closes #3900)

---
 youtube_dl/extractor/generic.py | 34 ++++++++++++++++++---------------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c16da70f1..dfc2ef4e7 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -847,47 +847,51 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'MLB')
 
+        def check_video(vurl):
+            vpath = compat_urlparse.urlparse(vurl).path
+            vext = determine_ext(vpath)
+            return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml')
+
+        def filter_video(urls):
+            return list(filter(check_video, urls))
+
         # Start with something easy: JW Player in SWFObject
-        found = re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
+        found = filter_video(re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage))
         if not found:
             # Look for gorilla-vid style embedding
-            found = re.findall(r'''(?sx)
+            found = filter_video(re.findall(r'''(?sx)
                 (?:
                     jw_plugins|
                     JWPlayerOptions|
                     jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
                 )
-                .*?file\s*:\s*["\'](.*?)["\']''', webpage)
+                .*?file\s*:\s*["\'](.*?)["\']''', webpage))
         if not found:
             # Broaden the search a little bit
-            found = re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
+            found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
         if not found:
             # Broaden the findall a little bit: JWPlayer JS loader
-            found = re.findall(r'[^A-Za-z0-9]?file["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage)
+            found = filter_video(re.findall(
+                r'[^A-Za-z0-9]?file["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
         if not found:
             # Flow player
-            found = re.findall(r'''(?xs)
+            found = filter_video(re.findall(r'''(?xs)
                 flowplayer\("[^"]+",\s*
                     \{[^}]+?\}\s*,
                     \s*{[^}]+? ["']?clip["']?\s*:\s*\{\s*
                         ["']?url["']?\s*:\s*["']([^"']+)["']
-            ''', webpage)
+            ''', webpage))
         if not found:
             # Try to find twitter cards info
-            found = re.findall(r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage)
+            found = filter_video(re.findall(
+                r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage))
         if not found:
             # We look for Open Graph info:
             # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
             m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
             # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
             if m_video_type is not None:
-                def check_video(vurl):
-                    vpath = compat_urlparse.urlparse(vurl).path
-                    vext = determine_ext(vpath)
-                    return '.' in vpath and vext not in ('swf', 'png', 'jpg')
-                found = list(filter(
-                    check_video,
-                    re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage)))
+                found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
         if not found:
             # HTML5 video
             found = re.findall(r'(?s)<video[^<]*(?:>.*?<source[^>]+)? src="([^"]+)"', webpage)

From 385009fc443eb81dc8695dd46448950bbe76cce3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Oct 2014 19:56:55 +0700
Subject: [PATCH 0186/1937] [mlb] Fix thumbnails extraction (Closes #3905)

---
 youtube_dl/extractor/mlb.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index bfdb462eb..362c2b5c9 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -88,8 +88,9 @@ class MLBIE(InfoExtractor):
         duration = parse_duration(detail.find('./duration').text)
         timestamp = parse_iso8601(detail.attrib['date'][:-5])
 
-        thumbnail = find_xpath_attr(
-            detail, './thumbnailScenarios/thumbnailScenario', 'type', '45').text
+        thumbnails = [{
+            'url': thumbnail.text,
+        } for thumbnail in detail.findall('./thumbnailScenarios/thumbnailScenario')]
 
         formats = []
         for media_url in detail.findall('./url'):
@@ -116,5 +117,5 @@ class MLBIE(InfoExtractor):
             'duration': duration,
             'timestamp': timestamp,
             'formats': formats,
-            'thumbnail': thumbnail,
+            'thumbnails': thumbnails,
         }

From b097b5f2469e9df11c120b6338a38b28f951510c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Oct 2014 20:07:34 +0700
Subject: [PATCH 0187/1937] [mlb] Remove unused import

---
 youtube_dl/extractor/mlb.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index 362c2b5c9..42aa2e227 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -6,7 +6,6 @@ from .common import InfoExtractor
 from ..utils import (
     parse_duration,
     parse_iso8601,
-    find_xpath_attr,
 )
 
 

From fc6861b175ec1ad1d7aa322438b095840c437249 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Oct 2014 21:05:39 +0700
Subject: [PATCH 0188/1937] [sportbox] Add extractor (Closes #3906)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/sportbox.py | 81 ++++++++++++++++++++++++++++++++
 2 files changed, 82 insertions(+)
 create mode 100644 youtube_dl/extractor/sportbox.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 76726305a..f96ddfd23 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -347,6 +347,7 @@ from .spiegel import SpiegelIE, SpiegelArticleIE
 from .spiegeltv import SpiegeltvIE
 from .spike import SpikeIE
 from .sport5 import Sport5IE
+from .sportbox import SportBoxIE
 from .sportdeutschland import SportDeutschlandIE
 from .stanfordoc import StanfordOpenClassroomIE
 from .steam import SteamIE
diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
new file mode 100644
index 000000000..19cc976e3
--- /dev/null
+++ b/youtube_dl/extractor/sportbox.py
@@ -0,0 +1,81 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_duration,
+    parse_iso8601,
+    int_or_none,
+)
+
+
+class SportBoxIE(InfoExtractor):
+    _VALID_URL = r'https?://news\.sportbox\.ru/Vidy_sporta/(?:[^/]+/)+spbvideo_NI\d+_(?P<display_id>.+)'
+    _TESTS = [
+        {
+            'url': 'http://news.sportbox.ru/Vidy_sporta/Avtosport/Rossijskij/spbvideo_NI483529_Gonka-2-zaezd-Obyedinenniy-2000-klassi-Turing-i-S',
+            'md5': 'ff56a598c2cf411a9a38a69709e97079',
+            'info_dict': {
+                'id': '80822',
+                'ext': 'mp4',
+                'title': 'Гонка 2  заезд ««Объединенный 2000»: классы Туринг и Супер-продакшн',
+                'description': 'md5:81715fa9c4ea3d9e7915dc8180c778ed',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'timestamp': 1411896237,
+                'upload_date': '20140928',
+                'duration': 4846,
+                'view_count': int,
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
+        }, {
+            'url': 'http://news.sportbox.ru/Vidy_sporta/billiard/spbvideo_NI486287_CHempionat-mira-po-dinamichnoy-piramide-4',
+            'only_matching': True,
+        }
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('display_id')
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._search_regex(
+            r'src="/vdl/player/media/(\d+)"', webpage, 'video id')
+
+        player = self._download_webpage(
+            'http://news.sportbox.ru/vdl/player/media/%s' % video_id,
+            display_id, 'Downloading player webpage')
+
+        hls = self._search_regex(
+            r"var\s+original_hls_file\s*=\s*'([^']+)'", player, 'hls file')
+
+        formats = self._extract_m3u8_formats(hls, display_id, 'mp4')
+
+        title = self._html_search_regex(
+            r'<h1 itemprop="name">([^<]+)</h1>', webpage, 'title')
+        description = self._html_search_regex(
+            r'(?s)<div itemprop="description">(.+?)</div>', webpage, 'description', fatal=False)
+        thumbnail = self._og_search_thumbnail(webpage)
+        timestamp = parse_iso8601(self._search_regex(
+            r'<span itemprop="uploadDate">([^<]+)</span>', webpage, 'timestamp', fatal=False))
+        duration = parse_duration(self._html_search_regex(
+            r'<meta itemprop="duration" content="PT([^"]+)">', webpage, 'duration', fatal=False))
+        view_count = int_or_none(self._html_search_regex(
+            r'<span>Просмотров: (\d+)</span>', player, 'view count', fatal=False))
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'view_count': view_count,
+            'formats': formats,
+        }

From 0ca41c3d9c7405cb79a5f11fcd6e57e7d5e07dd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 9 Oct 2014 21:10:07 +0700
Subject: [PATCH 0189/1937] [walla] Fix typo

---
 test/test_subtitles.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index eb5f2f8dd..8f4602e5f 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -280,7 +280,7 @@ class TestVimeoSubtitles(BaseTestSubtitles):
             self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
 
 
-class TestWallsSubtitles(BaseTestSubtitles):
+class TestWallaSubtitles(BaseTestSubtitles):
     url = 'http://vod.walla.co.il/movie/2705958/the-yes-men'
     IE = WallaIE
 

From 1b725173a5bf3fefcb800222e0c473d65d6cea70 Mon Sep 17 00:00:00 2001
From: George Boyle <george@thebuds.net>
Date: Thu, 9 Oct 2014 16:00:24 +0100
Subject: [PATCH 0190/1937] Fixed typo

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4a9610355..4e2a74624 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -232,7 +232,7 @@ class YoutubeDL(object):
             # On Python 3, the Unicode filesystem API will throw errors (#1474)
             self.report_warning(
                 'Assuming --restrict-filenames since file system encoding '
-                'cannot encode all charactes. '
+                'cannot encode all characters. '
                 'Set the LC_ALL environment variable to fix this.')
             self.params['restrictfilenames'] = True
 

From 53d9009bdb064a43674136d0952a73546b828cd9 Mon Sep 17 00:00:00 2001
From: George Boyle <george@thebuds.net>
Date: Fri, 10 Oct 2014 09:33:41 +0100
Subject: [PATCH 0191/1937] KeyError on initialising YoutubeDL in python3 #3910

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4e2a74624..dec0e20e7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -228,7 +228,7 @@ class YoutubeDL(object):
 
         if (sys.version_info >= (3,) and sys.platform != 'win32' and
                 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
-                and not params['restrictfilenames']):
+                and not params.get('restrictfilenames', False)):
             # On Python 3, the Unicode filesystem API will throw errors (#1474)
             self.report_warning(
                 'Assuming --restrict-filenames since file system encoding '

From 762958d5afa04735428cd39c77e78d81c6809672 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Oct 2014 19:50:29 +0700
Subject: [PATCH 0192/1937] [yahoo] Add support for regional subdomains and
 extract duration (Closes #3915)

---
 youtube_dl/extractor/yahoo.py | 37 +++++++++++++++++++++++++++++++----
 1 file changed, 33 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 221341c13..ef5ac03f0 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import itertools
@@ -15,7 +16,7 @@ from ..utils import (
 
 class YahooIE(InfoExtractor):
     IE_DESC = 'Yahoo screen and movies'
-    _VALID_URL = r'(?P<url>https?://(?:screen|movies)\.yahoo\.com/.*?-(?P<id>[0-9]+)(?:-[a-z]+)?\.html)'
+    _VALID_URL = r'(?P<url>https?://(?:.+?\.)?(?:screen|movies)\.yahoo\.com/.*?-(?P<id>[0-9]+)(?:-[a-z]+)?\.html)'
     _TESTS = [
         {
             'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
@@ -25,6 +26,7 @@ class YahooIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Julian Smith & Travis Legg Watch Julian Smith',
                 'description': 'Julian and Travis watch Julian Smith',
+                'duration': 6863,
             },
         },
         {
@@ -34,7 +36,8 @@ class YahooIE(InfoExtractor):
                 'id': 'd1dedf8c-d58c-38c3-8963-e899929ae0a9',
                 'ext': 'mp4',
                 'title': 'Codefellas - The Cougar Lies with Spanish Moss',
-                'description': 'Agent Topple\'s mustache does its dirty work, and Nicole brokers a deal for peace. But why is the NSA collecting millions of Instagram brunch photos? And if your waffles have nothing to hide, what are they so worried about?',
+                'description': 'md5:66b627ab0a282b26352136ca96ce73c1',
+                'duration': 151,
             },
         },
         {
@@ -45,6 +48,29 @@ class YahooIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': "Yahoo Saves 'Community'",
                 'description': 'md5:4d4145af2fd3de00cbb6c1d664105053',
+                'duration': 170,
+            }
+        },
+        {
+            'url': 'https://tw.screen.yahoo.com/taipei-opinion-poll/選情站報-街頭民調-台北市篇-102823042.html',
+            'md5': '92a7fdd8a08783c68a174d7aa067dde8',
+            'info_dict': {
+                'id': '7a23b569-7bea-36cb-85b9-bd5301a0a1fb',
+                'ext': 'mp4',
+                'title': '選情站報 街頭民調 台北市篇',
+                'description': '選情站報 街頭民調 台北市篇',
+                'duration': 429,
+            }
+        },
+        {
+            'url': 'https://uk.screen.yahoo.com/editor-picks/cute-raccoon-freed-drain-using-091756545.html  ',
+            'md5': '0b51660361f0e27c9789e7037ef76f4b',
+            'info_dict': {
+                'id': 'b3affa53-2e14-3590-852b-0e0db6cd1a58',
+                'ext': 'mp4',
+                'title': 'Cute Raccoon Freed From Drain\u00a0Using Angle Grinder',
+                'description': 'md5:f66c890e1490f4910a9953c941dee944',
+                'duration': 97,
             }
         },
     ]
@@ -75,9 +101,11 @@ class YahooIE(InfoExtractor):
         return self._get_info(long_id, video_id, webpage)
 
     def _get_info(self, long_id, video_id, webpage):
+        region = self._search_regex(
+            r'"region"\s*:\s*"([^"]+)"', webpage, 'region', fatal=False, default='US')
         query = ('SELECT * FROM yahoo.media.video.streams WHERE id="%s"'
-                 ' AND plrs="86Gj0vCaSzV_Iuf6hNylf2" AND region="US"'
-                 ' AND protocol="http"' % long_id)
+                 ' AND plrs="86Gj0vCaSzV_Iuf6hNylf2" AND region="%s"'
+                 ' AND protocol="http"' % (long_id, region))
         data = compat_urllib_parse.urlencode({
             'q': query,
             'env': 'prod',
@@ -118,6 +146,7 @@ class YahooIE(InfoExtractor):
             'formats': formats,
             'description': clean_html(meta['description']),
             'thumbnail': meta['thumbnail'] if meta.get('thumbnail') else self._og_search_thumbnail(webpage),
+            'duration': int_or_none(meta.get('duration')),
         }
 
 

From 3c06d3715eac87c945e75fa2f3ad41490d0da1d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Oct 2014 22:11:30 +0700
Subject: [PATCH 0193/1937] [yahoo] Generalize, support arbitrary subdomains,
 support iframe videos, capture error message (Closes #2470)

---
 youtube_dl/extractor/__init__.py |   2 +-
 youtube_dl/extractor/yahoo.py    | 116 +++++++++++++++++++++----------
 2 files changed, 81 insertions(+), 37 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f96ddfd23..5386ac553 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -463,7 +463,7 @@ from .xvideos import XVideosIE
 from .xtube import XTubeUserIE, XTubeIE
 from .yahoo import (
     YahooIE,
-    YahooNewsIE,
+    #YahooNewsIE,
     YahooSearchIE,
 )
 from .ynet import YnetIE
diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index ef5ac03f0..117f0856a 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -7,6 +7,7 @@ import re
 
 from .common import InfoExtractor, SearchInfoExtractor
 from ..utils import (
+    ExtractorError,
     compat_urllib_parse,
     compat_urlparse,
     clean_html,
@@ -16,7 +17,7 @@ from ..utils import (
 
 class YahooIE(InfoExtractor):
     IE_DESC = 'Yahoo screen and movies'
-    _VALID_URL = r'(?P<url>https?://(?:.+?\.)?(?:screen|movies)\.yahoo\.com/.*?-(?P<id>[0-9]+)(?:-[a-z]+)?\.html)'
+    _VALID_URL = r'(?P<url>(?P<host>https?://(?:[a-zA-Z]{2}\.)?[\da-zA-Z_-]+\.yahoo\.com)/(?:[^/]+/)*(?P<display_id>.+?)-(?P<id>[0-9]+)(?:-[a-z]+)?\.html)'
     _TESTS = [
         {
             'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
@@ -63,7 +64,7 @@ class YahooIE(InfoExtractor):
             }
         },
         {
-            'url': 'https://uk.screen.yahoo.com/editor-picks/cute-raccoon-freed-drain-using-091756545.html  ',
+            'url': 'https://uk.screen.yahoo.com/editor-picks/cute-raccoon-freed-drain-using-091756545.html',
             'md5': '0b51660361f0e27c9789e7037ef76f4b',
             'info_dict': {
                 'id': 'b3affa53-2e14-3590-852b-0e0db6cd1a58',
@@ -73,13 +74,70 @@ class YahooIE(InfoExtractor):
                 'duration': 97,
             }
         },
+        {
+            'url': 'https://ca.sports.yahoo.com/video/program-makes-hockey-more-affordable-013127711.html',
+            'md5': '57e06440778b1828a6079d2f744212c4',
+            'info_dict': {
+                'id': 'c9fa2a36-0d4d-3937-b8f6-cc0fb1881e73',
+                'ext': 'mp4',
+                'title': 'Program that makes hockey more affordable not offered in Manitoba',
+                'description': 'md5:c54a609f4c078d92b74ffb9bf1f496f4',
+                'duration': 121,
+            }
+        }, {
+            'url': 'https://ca.finance.yahoo.com/news/20-most-valuable-brands-world-112600775.html',
+            'md5': '3e401e4eed6325aa29d9b96125fd5b4f',
+            'info_dict': {
+                'id': 'c1b4c09c-8ed8-3b65-8b05-169c55358a83',
+                'ext': 'mp4',
+                'title': "Apple Is The World's Most Valuable Brand",
+                'description': 'md5:73eabc1a11c6f59752593b2ceefa1262',
+                'duration': 21,
+            }
+        }, {
+            'url': 'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',
+            'md5': '67010fdf3a08d290e060a4dd96baa07b',
+            'info_dict': {
+                'id': 'f885cf7f-43d4-3450-9fac-46ac30ece521',
+                'ext': 'mp4',
+                'title': 'China Moses Is Crazy About the Blues',
+                'description': 'md5:9900ab8cd5808175c7b3fe55b979bed0',
+                'duration': 128,
+            }
+        }, {
+            'url': 'https://in.lifestyle.yahoo.com/video/connect-dots-dark-side-virgo-090247395.html',
+            'md5': 'd9a083ccf1379127bf25699d67e4791b',
+            'info_dict': {
+                'id': '52aeeaa3-b3d1-30d8-9ef8-5d0cf05efb7c',
+                'ext': 'mp4',
+                'title': 'Connect the Dots: Dark Side of Virgo',
+                'description': 'md5:1428185051cfd1949807ad4ff6d3686a',
+                'duration': 201,
+            }
+        }, {
+            'url': 'https://gma.yahoo.com/pizza-delivery-man-surprised-huge-tip-college-kids-195200785.html',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
         url = mobj.group('url')
-        webpage = self._download_webpage(url, video_id)
+        host = mobj.group('host')
+        webpage = self._download_webpage(url, display_id)
+
+        # Look for iframed media first
+        iframe_m = re.search(r'<iframe[^>]+src="(/video/.+?-\d+\.html\?format=embed.*?)"', webpage)
+        if iframe_m:
+            iframepage = self._download_webpage(
+                host + iframe_m.group(1), display_id, 'Downloading iframe webpage')
+            items_json = self._search_regex(
+                r'mediaItems: (\[.+?\])$', iframepage, 'items', flags=re.MULTILINE, default=None)
+            if items_json:
+                items = json.loads(items_json)
+                video_id = items[0]['id']
+                return self._get_info(video_id, display_id, webpage)
 
         items_json = self._search_regex(
             r'mediaItems: ({.*?})$', webpage, 'items', flags=re.MULTILINE,
@@ -90,22 +148,22 @@ class YahooIE(InfoExtractor):
                 r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"',
                 r'"first_videoid"\s*:\s*"([^"]+)"',
             ]
-            long_id = self._search_regex(CONTENT_ID_REGEXES, webpage, 'content ID')
-            video_id = long_id
+            video_id = self._search_regex(CONTENT_ID_REGEXES, webpage, 'content ID')
         else:
             items = json.loads(items_json)
             info = items['mediaItems']['query']['results']['mediaObj'][0]
             # The 'meta' field is not always in the video webpage, we request it
             # from another page
-            long_id = info['id']
-        return self._get_info(long_id, video_id, webpage)
+            video_id = info['id']
+        return self._get_info(video_id, display_id, webpage)
 
-    def _get_info(self, long_id, video_id, webpage):
+    def _get_info(self, video_id, display_id, webpage):
         region = self._search_regex(
-            r'"region"\s*:\s*"([^"]+)"', webpage, 'region', fatal=False, default='US')
+            r'\\?"region\\?"\s*:\s*\\?"([^"]+?)\\?"',
+            webpage, 'region', fatal=False, default='US')
         query = ('SELECT * FROM yahoo.media.video.streams WHERE id="%s"'
                  ' AND plrs="86Gj0vCaSzV_Iuf6hNylf2" AND region="%s"'
-                 ' AND protocol="http"' % (long_id, region))
+                 ' AND protocol="http"' % (video_id, region))
         data = compat_urllib_parse.urlencode({
             'q': query,
             'env': 'prod',
@@ -113,9 +171,17 @@ class YahooIE(InfoExtractor):
         })
         query_result = self._download_json(
             'http://video.query.yahoo.com/v1/public/yql?' + data,
-            video_id, 'Downloading video info')
+            display_id, 'Downloading video info')
+
         info = query_result['query']['results']['mediaObj'][0]
-        meta = info['meta']
+        meta = info.get('meta')
+
+        if not meta:
+            msg = info['status'].get('msg')
+            if msg:
+                raise ExtractorError(
+                    '%s returned error: %s' % (self.IE_NAME, msg), expected=True)
+            raise ExtractorError('Unable to extract media object meta')
 
         formats = []
         for s in info['streams']:
@@ -142,6 +208,7 @@ class YahooIE(InfoExtractor):
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': meta['title'],
             'formats': formats,
             'description': clean_html(meta['description']),
@@ -150,29 +217,6 @@ class YahooIE(InfoExtractor):
         }
 
 
-class YahooNewsIE(YahooIE):
-    IE_NAME = 'yahoo:news'
-    _VALID_URL = r'http://news\.yahoo\.com/video/.*?-(?P<id>\d*?)\.html'
-
-    _TESTS = [{
-        'url': 'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',
-        'md5': '67010fdf3a08d290e060a4dd96baa07b',
-        'info_dict': {
-            'id': '104538833',
-            'ext': 'mp4',
-            'title': 'China Moses Is Crazy About the Blues',
-            'description': 'md5:9900ab8cd5808175c7b3fe55b979bed0',
-        },
-    }]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        webpage = self._download_webpage(url, video_id)
-        long_id = self._search_regex(r'contentId: \'(.+?)\',', webpage, 'long id')
-        return self._get_info(long_id, video_id, webpage)
-
-
 class YahooSearchIE(SearchInfoExtractor):
     IE_DESC = 'Yahoo screen search'
     _MAX_RESULTS = 1000

From 823f1e015a8880442f9b9fcf2943f647f5cf8039 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 10 Oct 2014 22:18:37 +0700
Subject: [PATCH 0194/1937] [yahoo] Wipe out yahoo news extractor

---
 youtube_dl/extractor/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5386ac553..dd770fdf1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -463,7 +463,6 @@ from .xvideos import XVideosIE
 from .xtube import XTubeUserIE, XTubeIE
 from .yahoo import (
     YahooIE,
-    #YahooNewsIE,
     YahooSearchIE,
 )
 from .ynet import YnetIE

From 3741302a109f28081a9db805a371c744f7bd2a2b Mon Sep 17 00:00:00 2001
From: Ole Ernst <olebowle@gmx.com>
Date: Fri, 10 Oct 2014 20:35:34 +0200
Subject: [PATCH 0195/1937] [ard] Add rss support

---
 youtube_dl/extractor/ard.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 8de9c11ea..cd9c1d9be 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from .generic import GenericIE
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -12,6 +13,7 @@ from ..utils import (
     parse_duration,
     unified_strdate,
     xpath_text,
+    parse_xml,
 )
 
 
@@ -54,6 +56,11 @@ class ARDMediathekIE(InfoExtractor):
         if '>Der gewünschte Beitrag ist nicht mehr verfügbar.<' in webpage:
             raise ExtractorError('Video %s is no longer available' % video_id, expected=True)
 
+        if re.search(r'rss=true', url):
+            doc = parse_xml(webpage)
+            if doc.tag == 'rss':
+                return GenericIE()._extract_rss(url, video_id, doc)
+
         title = self._html_search_regex(
             [r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
              r'<meta name="dcterms.title" content="(.*?)"/>',

From 1409704afa2d076f1d8ea05430d83834b6ffa57a Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Sat, 11 Oct 2014 12:20:13 +0530
Subject: [PATCH 0196/1937] Fix ndr.de outdated test url

---
 youtube_dl/extractor/ndr.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 94d5ba982..cf7f0309b 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -18,16 +18,16 @@ class NDRIE(InfoExtractor):
 
     _TESTS = [
         {
-            'url': 'http://www.ndr.de/fernsehen/media/dienordreportage325.html',
-            'md5': '4a4eeafd17c3058b65f0c8f091355855',
+            'url': 'http://www.ndr.de/fernsehen/sendungen/nordmagazin/Kartoffeltage-in-der-Lewitz,nordmagazin25866.html',
+            'md5': '5bc5f5b92c82c0f8b26cddca34f8bb2c',
             'note': 'Video file',
             'info_dict': {
-                'id': '325',
+                'id': '25866',
                 'ext': 'mp4',
-                'title': 'Blaue Bohnen aus Blocken',
-                'description': 'md5:190d71ba2ccddc805ed01547718963bc',
-                'duration': 1715,
-            },
+                'title': 'Kartoffeltage in der Lewitz',
+                'description': 'md5:16d4d66907f541692e8182c33270f29a',
+                'duration': 166,
+            }
         },
         {
             'url': 'http://www.ndr.de/info/audio51535.html',

From f4cf848d1d4e0eeec5fedfda79fad50f0b55ec4a Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Sat, 11 Oct 2014 15:59:42 +0530
Subject: [PATCH 0197/1937] Replace 404 url

---
 youtube_dl/extractor/howstuffworks.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index 68684b997..fccc23884 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -28,13 +28,13 @@ class HowStuffWorksIE(InfoExtractor):
             }
         },
         {
-            'url': 'http://adventure.howstuffworks.com/39516-deadliest-catch-jakes-farewell-pots-video.htm',
+            'url': 'http://adventure.howstuffworks.com/7199-survival-zone-food-and-water-in-the-savanna-video.htm',
             'info_dict': {
-                'id': '553470',
-                'display_id': 'deadliest-catch-jakes-farewell-pots',
+                'id': '453464',
+                'display_id': 'survival-zone-food-and-water-in-the-savanna',
                 'ext': 'mp4',
-                'title': 'Deadliest Catch: Jake\'s Farewell Pots',
-                'description': 'md5:9632c346d5e43ee238028c9cefd8dbbc',
+                'title': 'Survival Zone: Food and Water In the Savanna',
+                'description': 'md5:7e1c89f6411434970c15fa094170c371',
                 'thumbnail': 're:^https?://.*\.jpg$',
             },
             'params': {

From 071420e136ea2039d45a26cff0b704104edf533c Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Sat, 11 Oct 2014 21:10:53 +0530
Subject: [PATCH 0198/1937] Fix download error in GooglePlus

---
 youtube_dl/extractor/googleplus.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/googleplus.py b/youtube_dl/extractor/googleplus.py
index 07d994b44..1ab5e9260 100644
--- a/youtube_dl/extractor/googleplus.py
+++ b/youtube_dl/extractor/googleplus.py
@@ -64,7 +64,7 @@ class GooglePlusIE(InfoExtractor):
         webpage = self._download_webpage(video_page, video_id, 'Downloading video page')
 
         # Extract video links all sizes
-        pattern = r'\d+,\d+,(\d+),"(http\://redirector\.googlevideo\.com.*?)"'
+        pattern = r'\d+,\d+,(\d+),"(https\://redirector\.googlevideo\.com.*?)"'
         mobj = re.findall(pattern, webpage)
         if len(mobj) == 0:
             raise ExtractorError('Unable to extract video links')

From 4828703f1481dc7a84e49904d90fd9decd8e9e6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 12 Oct 2014 01:44:13 +0700
Subject: [PATCH 0199/1937] [googleplus] Modernize and extract all formats

---
 youtube_dl/extractor/googleplus.py | 75 +++++++++++-------------------
 1 file changed, 28 insertions(+), 47 deletions(-)

diff --git a/youtube_dl/extractor/googleplus.py b/youtube_dl/extractor/googleplus.py
index 1ab5e9260..fcefe54cd 100644
--- a/youtube_dl/extractor/googleplus.py
+++ b/youtube_dl/extractor/googleplus.py
@@ -1,13 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import datetime
 import re
+import codecs
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-)
+from ..utils import unified_strdate
 
 
 class GooglePlusIE(InfoExtractor):
@@ -19,74 +17,57 @@ class GooglePlusIE(InfoExtractor):
         'info_dict': {
             'id': 'ZButuJc6CtH',
             'ext': 'flv',
+            'title': '嘆きの天使 降臨',
             'upload_date': '20120613',
             'uploader': '井上ヨシマサ',
-            'title': '嘆きの天使 降臨',
         }
     }
 
     def _real_extract(self, url):
-        # Extract id from URL
-        mobj = re.match(self._VALID_URL, url)
-
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         # Step 1, Retrieve post webpage to extract further information
         webpage = self._download_webpage(url, video_id, 'Downloading entry webpage')
 
-        self.report_extraction(video_id)
-
-        # Extract update date
-        upload_date = self._html_search_regex(
+        title = self._og_search_description(webpage).splitlines()[0]
+        upload_date = unified_strdate(self._html_search_regex(
             r'''(?x)<a.+?class="o-U-s\s[^"]+"\s+style="display:\s*none"\s*>
                     ([0-9]{4}-[0-9]{2}-[0-9]{2})</a>''',
-            webpage, 'upload date', fatal=False, flags=re.VERBOSE)
-        if upload_date:
-            # Convert timestring to a format suitable for filename
-            upload_date = datetime.datetime.strptime(upload_date, "%Y-%m-%d")
-            upload_date = upload_date.strftime('%Y%m%d')
-
-        # Extract uploader
-        uploader = self._html_search_regex(r'rel\="author".*?>(.*?)</a>',
-            webpage, 'uploader', fatal=False)
-
-        # Extract title
-        # Get the first line for title
-        video_title = self._og_search_description(webpage).splitlines()[0]
+            webpage, 'upload date', fatal=False, flags=re.VERBOSE))
+        uploader = self._html_search_regex(
+            r'rel="author".*?>(.*?)</a>', webpage, 'uploader', fatal=False)
 
         # Step 2, Simulate clicking the image box to launch video
         DOMAIN = 'https://plus.google.com/'
-        video_page = self._search_regex(r'<a href="((?:%s)?photos/.*?)"' % re.escape(DOMAIN),
+        video_page = self._search_regex(
+            r'<a href="((?:%s)?photos/.*?)"' % re.escape(DOMAIN),
             webpage, 'video page URL')
         if not video_page.startswith(DOMAIN):
             video_page = DOMAIN + video_page
 
         webpage = self._download_webpage(video_page, video_id, 'Downloading video page')
 
+        def unicode_escape(s):
+            decoder = codecs.getdecoder('unicode_escape')
+            return re.sub(
+                r'\\u[0-9a-fA-F]{4,}',
+                lambda m: decoder(m.group(0))[0],
+                s)
+
         # Extract video links all sizes
-        pattern = r'\d+,\d+,(\d+),"(https\://redirector\.googlevideo\.com.*?)"'
-        mobj = re.findall(pattern, webpage)
-        if len(mobj) == 0:
-            raise ExtractorError('Unable to extract video links')
-
-        # Sort in resolution
-        links = sorted(mobj)
-
-        # Choose the lowest of the sort, i.e. highest resolution
-        video_url = links[-1]
-        # Only get the url. The resolution part in the tuple has no use anymore
-        video_url = video_url[-1]
-        # Treat escaped \u0026 style hex
-        try:
-            video_url = video_url.decode("unicode_escape")
-        except AttributeError: # Python 3
-            video_url = bytes(video_url, 'ascii').decode('unicode-escape')
+        formats = [{
+            'url': unicode_escape(video_url),
+            'ext': 'flv',
+            'width': int(width),
+            'height': int(height),
+        } for width, height, video_url in re.findall(
+            r'\d+,(\d+),(\d+),"(https?://redirector\.googlevideo\.com.*?)"', webpage)]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'url': video_url,
+            'title': title,
             'uploader': uploader,
             'upload_date': upload_date,
-            'title': video_title,
-            'ext': 'flv',
+            'formats': formats,
         }

From 3357110a4c26b9a8410ec47055f0d86ce30d4f3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 11 Oct 2014 22:25:30 +0200
Subject: [PATCH 0200/1937] [vimeo] Make the protocol mandatory in the url
 (fixes #3926)

If it's missing, it will be correctly handled by the generic IE.
---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index d2c36b58a..e6a86f18e 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -56,7 +56,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
 
     # _VALID_URL matches Vimeo URLs
     _VALID_URL = r'''(?x)
-        (?P<proto>(?:https?:)?//)?
+        https?://
         (?:(?:www|(?P<player>player))\.)?
         vimeo(?P<pro>pro)?\.com/
         (?!channels/[^/?#]+/?(?:$|[?#])|album/)

From b1c3a49fffb7109125a2ad215f412f1198e3dffd Mon Sep 17 00:00:00 2001
From: "Ching Yi, Chan" <chingyichan.tw@gmail.com>
Date: Sun, 12 Oct 2014 08:32:26 +0800
Subject: [PATCH 0201/1937] apply ratelimit to f4m

---
 youtube_dl/downloader/f4m.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index b3be16ff1..54dd6ac3f 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -230,6 +230,7 @@ class F4mFD(FileDownloader):
                 'continuedl': True,
                 'quiet': True,
                 'noprogress': True,
+                'ratelimit': self.params.get('ratelimit', None),
                 'test': self.params.get('test', False),
             })
 

From b4c3c8c172d42934b6481a8d62a2260dd86855f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 12 Oct 2014 13:06:31 +0200
Subject: [PATCH 0202/1937] [mixcloud] Fix metadata extraction (fixes #3930)

---
 youtube_dl/extractor/mixcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 520f27fca..a4564d3de 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -70,7 +70,7 @@ class MixcloudIE(InfoExtractor):
             raise ExtractorError('Unable to extract track url')
 
         PREFIX = (
-            r'<div class="cloudcast-play-button-container"'
+            r'<div class="cloudcast-play-button-container[^"]*?"'
             r'(?:\s+[a-zA-Z0-9-]+(?:="[^"]+")?)*?\s+')
         title = self._html_search_regex(
             PREFIX + r'm-title="([^"]+)"', webpage, 'title')

From ac20fc047a25c2a188df9060616149bc36e27a9d Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 12 Oct 2014 15:42:35 +0300
Subject: [PATCH 0203/1937] [theonion] Add new extractor (closes #3928)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/theonion.py | 71 ++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100644 youtube_dl/extractor/theonion.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index dd770fdf1..5e38d2663 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -371,6 +371,7 @@ from .telemb import TeleMBIE
 from .tenplay import TenPlayIE
 from .testurl import TestURLIE
 from .tf1 import TF1IE
+from .theonion import TheOnionIE
 from .theplatform import ThePlatformIE
 from .thesixtyone import TheSixtyOneIE
 from .thisav import ThisAVIE
diff --git a/youtube_dl/extractor/theonion.py b/youtube_dl/extractor/theonion.py
new file mode 100644
index 000000000..8e4c3414b
--- /dev/null
+++ b/youtube_dl/extractor/theonion.py
@@ -0,0 +1,71 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class TheOnionIE(InfoExtractor):
+    _VALID_URL = r'(?x)https?://(?:www\.)?theonion\.com/video/[^,]+,(?P<article_id>[0-9]+)/?'
+    _TEST = {
+        'url': 'http://www.theonion.com/video/man-wearing-mm-jacket-gods-image,36918/',
+        'md5': '19eaa9a39cf9b9804d982e654dc791ee',
+        'info_dict': {
+            'id': '2133',
+            'ext': 'mp4',
+            'title': 'Man Wearing M&M Jacket Apparently Made In God\'s Image',
+            'description': 'md5:cc12448686b5600baae9261d3e180910',
+            'thumbnail': 're:^https?://.*\.jpg\?\d+$',
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        article_id = mobj.group('article_id')
+
+        webpage = self._download_webpage(url, article_id)
+
+        video_id = self._search_regex(
+            r'"videoId":\s(\d+),', webpage, 'video ID')
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        sources = re.findall(r'<source src="([^"]+)" type="([^"]+)"', webpage)
+        if not sources:
+            raise ExtractorError(
+                'No sources found for video %s' % (self.IE_NAME, video_id),
+                expected=True)
+
+        formats = []
+        for src, type_ in sources:
+            if type_ == 'video/mp4':
+                formats.append({
+                    'format_id': 'mp4_sd',
+                    'preference': 1,
+                    'url': src,
+                })
+            elif type_ == 'video/webm':
+                formats.append({
+                    'format_id': 'webm_sd',
+                    'preference': 0,
+                    'url': src,
+                })
+            elif type_ == 'application/x-mpegURL':
+                formats.extend(
+                    self._extract_m3u8_formats(src, video_id, preference=-1))
+            else:
+                self.report_warning(
+                    'Encountered unexpected format: %s' % type_)
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'description': description,
+        }

From d18be55533a0a57ca365c3119571e0ab2566c48d Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 12 Oct 2014 15:47:31 +0300
Subject: [PATCH 0204/1937] [theonion] Fix a small mistake in string formatting

---
 youtube_dl/extractor/theonion.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/theonion.py b/youtube_dl/extractor/theonion.py
index 8e4c3414b..b65d8e03f 100644
--- a/youtube_dl/extractor/theonion.py
+++ b/youtube_dl/extractor/theonion.py
@@ -36,8 +36,7 @@ class TheOnionIE(InfoExtractor):
         sources = re.findall(r'<source src="([^"]+)" type="([^"]+)"', webpage)
         if not sources:
             raise ExtractorError(
-                'No sources found for video %s' % (self.IE_NAME, video_id),
-                expected=True)
+                'No sources found for video %s' % video_id, expected=True)
 
         formats = []
         for src, type_ in sources:

From 2c9f31188b17211c972698972ec93ec1decc77e4 Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Sun, 12 Oct 2014 20:09:12 +0530
Subject: [PATCH 0205/1937] Description changed

---
 youtube_dl/extractor/ndr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index cf7f0309b..add4b3e5d 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -25,7 +25,7 @@ class NDRIE(InfoExtractor):
                 'id': '25866',
                 'ext': 'mp4',
                 'title': 'Kartoffeltage in der Lewitz',
-                'description': 'md5:16d4d66907f541692e8182c33270f29a',
+                'description': 'md5:48c4c04dde604c8a9971b3d4e3b9eaa8',
                 'duration': 166,
             }
         },

From 23d83ad4d5d1148f9326a55ff9db8f75bd4ea044 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 12 Oct 2014 23:18:42 +0300
Subject: [PATCH 0206/1937] [niconico] Fix ignored --netrc flag

See issue #3753
---
 youtube_dl/extractor/niconico.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index c0c139b5d..7b85589b7 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -39,18 +39,17 @@ class NiconicoIE(InfoExtractor):
 
     _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/((?:[a-z]{2})?[0-9]+)'
     _NETRC_MACHINE = 'niconico'
-    # Determine whether the downloader uses authentication to download video
-    _AUTHENTICATE = False
+    # Determine whether the downloader used authentication to download video
+    _AUTHENTICATED = False
 
     def _real_initialize(self):
-        if self._downloader.params.get('username', None) is not None:
-            self._AUTHENTICATE = True
-
-        if self._AUTHENTICATE:
-            self._login()
+        self._login()
 
     def _login(self):
         (username, password) = self._get_login_info()
+        # No authentication to be performed
+        if not username:
+            return True
 
         # Log in
         login_form_strs = {
@@ -68,6 +67,8 @@ class NiconicoIE(InfoExtractor):
         if re.search(r'(?i)<h1 class="mb8p4">Log in error</h1>', login_results) is not None:
             self._downloader.report_warning('unable to log in: bad username or password')
             return False
+        # Successful login
+        self._AUTHENTICATED = True
         return True
 
     def _real_extract(self, url):
@@ -82,7 +83,7 @@ class NiconicoIE(InfoExtractor):
             'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id, video_id,
             note='Downloading video info page')
 
-        if self._AUTHENTICATE:
+        if self._AUTHENTICATED:
             # Get flv info
             flv_info_webpage = self._download_webpage(
                 'http://flapi.nicovideo.jp/api/getflv?v=' + video_id,

From 2417dc1715c0ca3f229afcd861f59e11360d54ce Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 12 Oct 2014 22:25:18 +0200
Subject: [PATCH 0207/1937] release 2014.10.12

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4f0d486b9..8b0a704ee 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.10.05.2'
+__version__ = '2014.10.12'

From b88b45e46ca960ad4da0474a5b40a45232ab544e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 12 Oct 2014 22:45:36 +0200
Subject: [PATCH 0208/1937] [options] Mention login in --username documentation
 (#3753)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index f651337ad..ad3154a7d 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -205,7 +205,7 @@ def parseOpts(overrideArguments=None):
         help='Try to download the DASH manifest on YouTube videos (experimental)')
 
     authentication.add_option('-u', '--username',
-            dest='username', metavar='USERNAME', help='account username')
+            dest='username', metavar='USERNAME', help='login with this account ID')
     authentication.add_option('-p', '--password',
             dest='password', metavar='PASSWORD', help='account password')
     authentication.add_option('-2', '--twofactor',

From 8450c15c251f65f72d659bf56fae386e877ab66a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 12 Oct 2014 23:10:11 +0200
Subject: [PATCH 0209/1937] [options] Consistent formatting and general
 niceness

---
 youtube_dl/options.py | 706 +++++++++++++++++++++++++-----------------
 1 file changed, 428 insertions(+), 278 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index ad3154a7d..a06850e1c 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -75,7 +75,8 @@ def parseOpts(overrideArguments=None):
         if len(opts) > 1:
             opts.insert(1, ', ')
 
-        if option.takes_value(): opts.append(' %s' % option.metavar)
+        if option.takes_value():
+            opts.append(' %s' % option.metavar)
 
         return "".join(opts)
 
@@ -87,68 +88,69 @@ def parseOpts(overrideArguments=None):
         for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
             try:
                 i = opts.index(private_opt)
-                opts[i+1] = 'PRIVATE'
+                opts[i + 1] = 'PRIVATE'
             except ValueError:
                 pass
         return opts
 
-    max_width = 80
-    max_help_position = 80
-
     # No need to wrap help messages if we're on a wide console
     columns = get_term_width()
-    if columns: max_width = columns
+    max_width = columns if columns else 80
+    max_help_position = 80
 
     fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
     fmt.format_option_strings = _format_option_string
 
     kw = {
-        'version'   : __version__,
-        'formatter' : fmt,
-        'usage' : '%prog [options] url [url...]',
-        'conflict_handler' : 'resolve',
+        'version': __version__,
+        'formatter': fmt,
+        'usage': '%prog [options] url [url...]',
+        'conflict_handler': 'resolve',
     }
 
     parser = optparse.OptionParser(**kw)
 
-    # option groups
-    general        = optparse.OptionGroup(parser, 'General Options')
-    selection      = optparse.OptionGroup(parser, 'Video Selection')
-    authentication = optparse.OptionGroup(parser, 'Authentication Options')
-    video_format   = optparse.OptionGroup(parser, 'Video Format Options')
-    subtitles      = optparse.OptionGroup(parser, 'Subtitle Options')
-    downloader     = optparse.OptionGroup(parser, 'Download Options')
-    postproc       = optparse.OptionGroup(parser, 'Post-processing Options')
-    filesystem     = optparse.OptionGroup(parser, 'Filesystem Options')
-    workarounds    = optparse.OptionGroup(parser, 'Workarounds')
-    verbosity      = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
-
-    general.add_option('-h', '--help',
-            action='help', help='print this help text and exit')
-    general.add_option('-v', '--version',
-            action='version', help='print program version and exit')
-    general.add_option('-U', '--update',
-            action='store_true', dest='update_self', help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
-    general.add_option('-i', '--ignore-errors',
-            action='store_true', dest='ignoreerrors', help='continue on download errors, for example to skip unavailable videos in a playlist', default=False)
-    general.add_option('--abort-on-error',
-            action='store_false', dest='ignoreerrors',
-            help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
-    general.add_option('--dump-user-agent',
-            action='store_true', dest='dump_user_agent',
-            help='display the current browser identification', default=False)
-    general.add_option('--list-extractors',
-            action='store_true', dest='list_extractors',
-            help='List all supported extractors and the URLs they would handle', default=False)
-    general.add_option('--extractor-descriptions',
-            action='store_true', dest='list_extractor_descriptions',
-            help='Output descriptions of all supported extractors', default=False)
+    general = optparse.OptionGroup(parser, 'General Options')
     general.add_option(
-        '--proxy', dest='proxy', default=None, metavar='URL',
+        '-h', '--help',
+        action='help',
+        help='print this help text and exit')
+    general.add_option(
+        '-v', '--version',
+        action='version',
+        help='print program version and exit')
+    general.add_option(
+        '-U', '--update',
+        action='store_true', dest='update_self',
+        help='update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)')
+    general.add_option(
+        '-i', '--ignore-errors',
+        action='store_true', dest='ignoreerrors', default=False,
+        help='continue on download errors, for example to skip unavailable videos in a playlist')
+    general.add_option(
+        '--abort-on-error',
+        action='store_false', dest='ignoreerrors',
+        help='Abort downloading of further videos (in the playlist or the command line) if an error occurs')
+    general.add_option(
+        '--dump-user-agent',
+        action='store_true', dest='dump_user_agent', default=False,
+        help='display the current browser identification')
+    general.add_option(
+        '--list-extractors',
+        action='store_true', dest='list_extractors', default=False,
+        help='List all supported extractors and the URLs they would handle')
+    general.add_option(
+        '--extractor-descriptions',
+        action='store_true', dest='list_extractor_descriptions', default=False,
+        help='Output descriptions of all supported extractors')
+    general.add_option(
+        '--proxy', dest='proxy',
+        default=None, metavar='URL',
         help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
     general.add_option(
-        '--socket-timeout', dest='socket_timeout',
-        type=float, default=None, help=u'Time to wait before giving up, in seconds')
+        '--socket-timeout',
+        dest='socket_timeout', type=float, default=None,
+        help='Time to wait before giving up, in seconds')
     general.add_option(
         '--default-search',
         dest='default_search', metavar='PREFIX',
@@ -158,6 +160,7 @@ def parseOpts(overrideArguments=None):
         action='store_true',
         help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
 
+    selection = optparse.OptionGroup(parser, 'Video Selection')
     selection.add_option(
         '--playlist-start',
         dest='playliststart', metavar='NUMBER', default=1, type=int,
@@ -166,245 +169,367 @@ def parseOpts(overrideArguments=None):
         '--playlist-end',
         dest='playlistend', metavar='NUMBER', default=None, type=int,
         help='playlist video to end at (default is last)')
-    selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
-    selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
-    selection.add_option('--max-downloads', metavar='NUMBER',
-                         dest='max_downloads', type=int, default=None,
-                         help='Abort after downloading NUMBER files')
-    selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None)
-    selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None)
-    selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None)
     selection.add_option(
-        '--datebefore', metavar='DATE', dest='datebefore', default=None,
+        '--match-title',
+        dest='matchtitle', metavar='REGEX',
+        help='download only matching titles (regex or caseless sub-string)')
+    selection.add_option(
+        '--reject-title',
+        dest='rejecttitle', metavar='REGEX',
+        help='skip download for matching titles (regex or caseless sub-string)')
+    selection.add_option(
+        '--max-downloads',
+        dest='max_downloads', metavar='NUMBER', type=int, default=None,
+        help='Abort after downloading NUMBER files')
+    selection.add_option(
+        '--min-filesize',
+        metavar='SIZE', dest='min_filesize', default=None,
+        help='Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)')
+    selection.add_option(
+        '--max-filesize',
+        metavar='SIZE', dest='max_filesize', default=None,
+        help='Do not download any videos larger than SIZE (e.g. 50k or 44.6m)')
+    selection.add_option(
+        '--date',
+        metavar='DATE', dest='date', default=None,
+        help='download only videos uploaded in this date')
+    selection.add_option(
+        '--datebefore',
+        metavar='DATE', dest='datebefore', default=None,
         help='download only videos uploaded on or before this date (i.e. inclusive)')
     selection.add_option(
-        '--dateafter', metavar='DATE', dest='dateafter', default=None,
+        '--dateafter',
+        metavar='DATE', dest='dateafter', default=None,
         help='download only videos uploaded on or after this date (i.e. inclusive)')
     selection.add_option(
-        '--min-views', metavar='COUNT', dest='min_views',
-        default=None, type=int,
-        help="Do not download any videos with less than COUNT views",)
+        '--min-views',
+        metavar='COUNT', dest='min_views', default=None, type=int,
+        help='Do not download any videos with less than COUNT views',)
     selection.add_option(
-        '--max-views', metavar='COUNT', dest='max_views',
-        default=None, type=int,
-        help="Do not download any videos with more than COUNT views",)
-    selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False)
-    selection.add_option('--age-limit', metavar='YEARS', dest='age_limit',
-                         help='download only videos suitable for the given age',
-                         default=None, type=int)
-    selection.add_option('--download-archive', metavar='FILE',
-                         dest='download_archive',
-                         help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
+        '--max-views',
+        metavar='COUNT', dest='max_views', default=None, type=int,
+        help='Do not download any videos with more than COUNT views')
     selection.add_option(
-        '--include-ads', dest='include_ads',
-        action='store_true',
+        '--no-playlist',
+        action='store_true', dest='noplaylist', default=False,
+        help='download only the currently playing video')
+    selection.add_option(
+        '--age-limit',
+        metavar='YEARS', dest='age_limit', default=None, type=int,
+        help='download only videos suitable for the given age')
+    selection.add_option(
+        '--download-archive', metavar='FILE',
+        dest='download_archive',
+        help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.')
+    selection.add_option(
+        '--include-ads',
+        dest='include_ads', action='store_true',
         help='Download advertisements as well (experimental)')
     selection.add_option(
-        '--youtube-include-dash-manifest', action='store_true',
-        dest='youtube_include_dash_manifest', default=False,
+        '--youtube-include-dash-manifest',
+        action='store_true', dest='youtube_include_dash_manifest', default=False,
         help='Try to download the DASH manifest on YouTube videos (experimental)')
 
-    authentication.add_option('-u', '--username',
-            dest='username', metavar='USERNAME', help='login with this account ID')
-    authentication.add_option('-p', '--password',
-            dest='password', metavar='PASSWORD', help='account password')
-    authentication.add_option('-2', '--twofactor',
-            dest='twofactor', metavar='TWOFACTOR', help='two-factor auth code')
-    authentication.add_option('-n', '--netrc',
-            action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
-    authentication.add_option('--video-password',
-            dest='videopassword', metavar='PASSWORD', help='video password (vimeo, smotri)')
+    authentication = optparse.OptionGroup(parser, 'Authentication Options')
+    authentication.add_option(
+        '-u', '--username',
+        dest='username', metavar='USERNAME',
+        help='login with this account ID')
+    authentication.add_option(
+        '-p', '--password',
+        dest='password', metavar='PASSWORD',
+        help='account password')
+    authentication.add_option(
+        '-2', '--twofactor',
+        dest='twofactor', metavar='TWOFACTOR',
+        help='two-factor auth code')
+    authentication.add_option(
+        '-n', '--netrc',
+        action='store_true', dest='usenetrc', default=False,
+        help='use .netrc authentication data')
+    authentication.add_option(
+        '--video-password',
+        dest='videopassword', metavar='PASSWORD',
+        help='video password (vimeo, smotri)')
 
+    video_format = optparse.OptionGroup(parser, 'Video Format Options')
+    video_format.add_option(
+        '-f', '--format',
+        action='store', dest='format', metavar='FORMAT', default=None,
+        help='video format code, specify the order of preference using slashes: -f 22/17/18 .  -f mp4 , -f m4a and  -f flv  are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality. Use commas to download multiple audio formats, such as  -f  136/137/mp4/bestvideo,140/m4a/bestaudio')
+    video_format.add_option(
+        '--all-formats',
+        action='store_const', dest='format', const='all',
+        help='download all available video formats')
+    video_format.add_option(
+        '--prefer-free-formats',
+        action='store_true', dest='prefer_free_formats', default=False,
+        help='prefer free video formats unless a specific one is requested')
+    video_format.add_option(
+        '--max-quality',
+        action='store', dest='format_limit', metavar='FORMAT',
+        help='highest quality format to download')
+    video_format.add_option(
+        '-F', '--list-formats',
+        action='store_true', dest='listformats',
+        help='list all available formats')
 
-    video_format.add_option('-f', '--format',
-            action='store', dest='format', metavar='FORMAT', default=None,
-            help='video format code, specify the order of preference using slashes: -f 22/17/18 .  -f mp4 , -f m4a and  -f flv  are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality. Use commas to download multiple audio formats, such as  -f  136/137/mp4/bestvideo,140/m4a/bestaudio')
-    video_format.add_option('--all-formats',
-            action='store_const', dest='format', help='download all available video formats', const='all')
-    video_format.add_option('--prefer-free-formats',
-            action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested')
-    video_format.add_option('--max-quality',
-            action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
-    video_format.add_option('-F', '--list-formats',
-            action='store_true', dest='listformats', help='list all available formats')
+    subtitles = optparse.OptionGroup(parser, 'Subtitle Options')
+    subtitles.add_option(
+        '--write-sub', '--write-srt',
+        action='store_true', dest='writesubtitles', default=False,
+        help='write subtitle file')
+    subtitles.add_option(
+        '--write-auto-sub', '--write-automatic-sub',
+        action='store_true', dest='writeautomaticsub', default=False,
+        help='write automatic subtitle file (youtube only)')
+    subtitles.add_option(
+        '--all-subs',
+        action='store_true', dest='allsubtitles', default=False,
+        help='downloads all the available subtitles of the video')
+    subtitles.add_option(
+        '--list-subs',
+        action='store_true', dest='listsubtitles', default=False,
+        help='lists all available subtitles for the video')
+    subtitles.add_option(
+        '--sub-format',
+        action='store', dest='subtitlesformat', metavar='FORMAT', default='srt',
+        help='subtitle format (default=srt) ([sbv/vtt] youtube only)')
+    subtitles.add_option(
+        '--sub-lang', '--sub-langs', '--srt-lang',
+        action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
+        default=[], callback=_comma_separated_values_options_callback,
+        help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
 
-    subtitles.add_option('--write-sub', '--write-srt',
-            action='store_true', dest='writesubtitles',
-            help='write subtitle file', default=False)
-    subtitles.add_option('--write-auto-sub', '--write-automatic-sub',
-            action='store_true', dest='writeautomaticsub',
-            help='write automatic subtitle file (youtube only)', default=False)
-    subtitles.add_option('--all-subs',
-            action='store_true', dest='allsubtitles',
-            help='downloads all the available subtitles of the video', default=False)
-    subtitles.add_option('--list-subs',
-            action='store_true', dest='listsubtitles',
-            help='lists all available subtitles for the video', default=False)
-    subtitles.add_option('--sub-format',
-            action='store', dest='subtitlesformat', metavar='FORMAT',
-            help='subtitle format (default=srt) ([sbv/vtt] youtube only)', default='srt')
-    subtitles.add_option('--sub-lang', '--sub-langs', '--srt-lang',
-            action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
-            default=[], callback=_comma_separated_values_options_callback,
-            help='languages of the subtitles to download (optional) separated by commas, use IETF language tags like \'en,pt\'')
-
-    downloader.add_option('-r', '--rate-limit',
-            dest='ratelimit', metavar='LIMIT', help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
-    downloader.add_option('-R', '--retries',
-            dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
-    downloader.add_option('--buffer-size',
-            dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16K) (default is %default)', default="1024")
-    downloader.add_option('--no-resize-buffer',
-            action='store_true', dest='noresizebuffer',
-            help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
-    downloader.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP)
+    downloader = optparse.OptionGroup(parser, 'Download Options')
+    downloader.add_option(
+        '-r', '--rate-limit',
+        dest='ratelimit', metavar='LIMIT',
+        help='maximum download rate in bytes per second (e.g. 50K or 4.2M)')
+    downloader.add_option(
+        '-R', '--retries',
+        dest='retries', metavar='RETRIES', default=10,
+        help='number of retries (default is %default)')
+    downloader.add_option(
+        '--buffer-size',
+        dest='buffersize', metavar='SIZE', default='1024',
+        help='size of download buffer (e.g. 1024 or 16K) (default is %default)')
+    downloader.add_option(
+        '--no-resize-buffer',
+        action='store_true', dest='noresizebuffer', default=False,
+        help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.')
+    downloader.add_option(
+        '--test',
+        action='store_true', dest='test', default=False,
+        help=optparse.SUPPRESS_HELP)
 
+    workarounds = optparse.OptionGroup(parser, 'Workarounds')
     workarounds.add_option(
-        '--encoding', dest='encoding', metavar='ENCODING',
+        '--encoding',
+        dest='encoding', metavar='ENCODING',
         help='Force the specified encoding (experimental)')
     workarounds.add_option(
-        '--no-check-certificate', action='store_true',
-        dest='no_check_certificate', default=False,
+        '--no-check-certificate',
+        action='store_true', dest='no_check_certificate', default=False,
         help='Suppress HTTPS certificate validation.')
     workarounds.add_option(
-        '--prefer-insecure', '--prefer-unsecure', action='store_true', dest='prefer_insecure',
+        '--prefer-insecure',
+        '--prefer-unsecure', action='store_true', dest='prefer_insecure',
         help='Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)')
     workarounds.add_option(
-        '--user-agent', metavar='UA',
-        dest='user_agent', help='specify a custom user agent')
+        '--user-agent',
+        metavar='UA', dest='user_agent',
+        help='specify a custom user agent')
     workarounds.add_option(
-        '--referer', metavar='REF',
-        dest='referer', default=None,
+        '--referer',
+        metavar='URL', dest='referer', default=None,
         help='specify a custom referer, use if the video access is restricted to one domain',
     )
     workarounds.add_option(
-        '--add-header', metavar='FIELD:VALUE',
-        dest='headers', action='append',
+        '--add-header',
+        metavar='FIELD:VALUE', dest='headers', action='append',
         help='specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
     )
     workarounds.add_option(
-        '--bidi-workaround', dest='bidi_workaround', action='store_true',
-        help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
+        '--bidi-workaround',
+        dest='bidi_workaround', action='store_true',
+        help='Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
 
-    verbosity.add_option('-q', '--quiet',
-            action='store_true', dest='quiet', help='activates quiet mode', default=False)
+    verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
+    verbosity.add_option(
+        '-q', '--quiet',
+        action='store_true', dest='quiet', default=False,
+        help='activates quiet mode')
     verbosity.add_option(
         '--no-warnings',
         dest='no_warnings', action='store_true', default=False,
         help='Ignore warnings')
-    verbosity.add_option('-s', '--simulate',
-            action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
-    verbosity.add_option('--skip-download',
-            action='store_true', dest='skip_download', help='do not download the video', default=False)
-    verbosity.add_option('-g', '--get-url',
-            action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
-    verbosity.add_option('-e', '--get-title',
-            action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
-    verbosity.add_option('--get-id',
-            action='store_true', dest='getid', help='simulate, quiet but print id', default=False)
-    verbosity.add_option('--get-thumbnail',
-            action='store_true', dest='getthumbnail',
-            help='simulate, quiet but print thumbnail URL', default=False)
-    verbosity.add_option('--get-description',
-            action='store_true', dest='getdescription',
-            help='simulate, quiet but print video description', default=False)
-    verbosity.add_option('--get-duration',
-            action='store_true', dest='getduration',
-            help='simulate, quiet but print video length', default=False)
-    verbosity.add_option('--get-filename',
-            action='store_true', dest='getfilename',
-            help='simulate, quiet but print output filename', default=False)
-    verbosity.add_option('--get-format',
-            action='store_true', dest='getformat',
-            help='simulate, quiet but print output format', default=False)
-    verbosity.add_option('-j', '--dump-json',
-            action='store_true', dest='dumpjson',
-            help='simulate, quiet but print JSON information. See --output for a description of available keys.', default=False)
-    verbosity.add_option('--newline',
-            action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
-    verbosity.add_option('--no-progress',
-            action='store_true', dest='noprogress', help='do not print progress bar', default=False)
-    verbosity.add_option('--console-title',
-            action='store_true', dest='consoletitle',
-            help='display progress in console titlebar', default=False)
-    verbosity.add_option('-v', '--verbose',
-            action='store_true', dest='verbose', help='print various debugging information', default=False)
-    verbosity.add_option('--dump-intermediate-pages',
-            action='store_true', dest='dump_intermediate_pages', default=False,
-            help='print downloaded pages to debug problems (very verbose)')
-    verbosity.add_option('--write-pages',
-            action='store_true', dest='write_pages', default=False,
-            help='Write downloaded intermediary pages to files in the current directory to debug problems')
-    verbosity.add_option('--youtube-print-sig-code',
-            action='store_true', dest='youtube_print_sig_code', default=False,
-            help=optparse.SUPPRESS_HELP)
-    verbosity.add_option('--print-traffic',
-            dest='debug_printtraffic', action='store_true', default=False,
-            help='Display sent and read HTTP traffic')
+    verbosity.add_option(
+        '-s', '--simulate',
+        action='store_true', dest='simulate', default=False,
+        help='do not download the video and do not write anything to disk',)
+    verbosity.add_option(
+        '--skip-download',
+        action='store_true', dest='skip_download', default=False,
+        help='do not download the video',)
+    verbosity.add_option(
+        '-g', '--get-url',
+        action='store_true', dest='geturl', default=False,
+        help='simulate, quiet but print URL')
+    verbosity.add_option(
+        '-e', '--get-title',
+        action='store_true', dest='gettitle', default=False,
+        help='simulate, quiet but print title')
+    verbosity.add_option(
+        '--get-id',
+        action='store_true', dest='getid', default=False,
+        help='simulate, quiet but print id')
+    verbosity.add_option(
+        '--get-thumbnail',
+        action='store_true', dest='getthumbnail', default=False,
+        help='simulate, quiet but print thumbnail URL')
+    verbosity.add_option(
+        '--get-description',
+        action='store_true', dest='getdescription', default=False,
+        help='simulate, quiet but print video description')
+    verbosity.add_option(
+        '--get-duration',
+        action='store_true', dest='getduration', default=False,
+        help='simulate, quiet but print video length')
+    verbosity.add_option(
+        '--get-filename',
+        action='store_true', dest='getfilename', default=False,
+        help='simulate, quiet but print output filename')
+    verbosity.add_option(
+        '--get-format',
+        action='store_true', dest='getformat', default=False,
+        help='simulate, quiet but print output format')
+    verbosity.add_option(
+        '-j', '--dump-json',
+        action='store_true', dest='dumpjson', default=False,
+        help='simulate, quiet but print JSON information. See --output for a description of available keys.')
+    verbosity.add_option(
+        '--newline',
+        action='store_true', dest='progress_with_newline', default=False,
+        help='output progress bar as new lines')
+    verbosity.add_option(
+        '--no-progress',
+        action='store_true', dest='noprogress', default=False,
+        help='do not print progress bar')
+    verbosity.add_option(
+        '--console-title',
+        action='store_true', dest='consoletitle', default=False,
+        help='display progress in console titlebar')
+    verbosity.add_option(
+        '-v', '--verbose',
+        action='store_true', dest='verbose', default=False,
+        help='print various debugging information')
+    verbosity.add_option(
+        '--dump-intermediate-pages',
+        action='store_true', dest='dump_intermediate_pages', default=False,
+        help='print downloaded pages to debug problems (very verbose)')
+    verbosity.add_option(
+        '--write-pages',
+        action='store_true', dest='write_pages', default=False,
+        help='Write downloaded intermediary pages to files in the current directory to debug problems')
+    verbosity.add_option(
+        '--youtube-print-sig-code',
+        action='store_true', dest='youtube_print_sig_code', default=False,
+        help=optparse.SUPPRESS_HELP)
+    verbosity.add_option(
+        '--print-traffic',
+        dest='debug_printtraffic', action='store_true', default=False,
+        help='Display sent and read HTTP traffic')
 
-
-    filesystem.add_option('-a', '--batch-file',
-            dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
-    filesystem.add_option('--id',
-            action='store_true', dest='useid', help='use only video ID in file name', default=False)
-    filesystem.add_option('-A', '--auto-number',
-            action='store_true', dest='autonumber',
-            help='number downloaded files starting from 00000', default=False)
-    filesystem.add_option('-o', '--output',
-            dest='outtmpl', metavar='TEMPLATE',
-            help=('output filename template. Use %(title)s to get the title, '
-                  '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
-                  '%(autonumber)s to get an automatically incremented number, '
-                  '%(ext)s for the filename extension, '
-                  '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
-                  '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
-                  '%(upload_date)s for the upload date (YYYYMMDD), '
-                  '%(extractor)s for the provider (youtube, metacafe, etc), '
-                  '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
-                  '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
-                  '%(height)s and %(width)s for the width and height of the video format. '
-                  '%(resolution)s for a textual description of the resolution of the video format. '
-                  'Use - to output to stdout. Can also be used to download to a different directory, '
-                  'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
-    filesystem.add_option('--autonumber-size',
-            dest='autonumber_size', metavar='NUMBER',
-            help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
-    filesystem.add_option('--restrict-filenames',
-            action='store_true', dest='restrictfilenames',
-            help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
-    filesystem.add_option('-t', '--title',
-            action='store_true', dest='usetitle', help='[deprecated] use title in file name (default)', default=False)
-    filesystem.add_option('-l', '--literal',
-            action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False)
-    filesystem.add_option('-w', '--no-overwrites',
-            action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
-    filesystem.add_option('-c', '--continue',
-            action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
-    filesystem.add_option('--no-continue',
-            action='store_false', dest='continue_dl',
-            help='do not resume partially downloaded files (restart from beginning)')
-    filesystem.add_option('--no-part',
-            action='store_true', dest='nopart', help='do not use .part files', default=False)
-    filesystem.add_option('--no-mtime',
-            action='store_false', dest='updatetime',
-            help='do not use the Last-modified header to set the file modification time', default=True)
-    filesystem.add_option('--write-description',
-            action='store_true', dest='writedescription',
-            help='write video description to a .description file', default=False)
-    filesystem.add_option('--write-info-json',
-            action='store_true', dest='writeinfojson',
-            help='write video metadata to a .info.json file', default=False)
-    filesystem.add_option('--write-annotations',
-            action='store_true', dest='writeannotations',
-            help='write video annotations to a .annotation file', default=False)
-    filesystem.add_option('--write-thumbnail',
-            action='store_true', dest='writethumbnail',
-            help='write thumbnail image to disk', default=False)
-    filesystem.add_option('--load-info',
-            dest='load_info_filename', metavar='FILE',
-            help='json file containing the video information (created with the "--write-json" option)')
-    filesystem.add_option('--cookies',
-            dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
+    filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
+    filesystem.add_option(
+        '-a', '--batch-file',
+        dest='batchfile', metavar='FILE',
+        help='file containing URLs to download (\'-\' for stdin)')
+    filesystem.add_option(
+        '--id', default=False,
+        action='store_true', dest='useid', help='use only video ID in file name')
+    filesystem.add_option(
+        '-A', '--auto-number',
+        action='store_true', dest='autonumber', default=False,
+        help='number downloaded files starting from 00000')
+    filesystem.add_option(
+        '-o', '--output',
+        dest='outtmpl', metavar='TEMPLATE',
+        help=('output filename template. Use %(title)s to get the title, '
+              '%(uploader)s for the uploader name, %(uploader_id)s for the uploader nickname if different, '
+              '%(autonumber)s to get an automatically incremented number, '
+              '%(ext)s for the filename extension, '
+              '%(format)s for the format description (like "22 - 1280x720" or "HD"), '
+              '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
+              '%(upload_date)s for the upload date (YYYYMMDD), '
+              '%(extractor)s for the provider (youtube, metacafe, etc), '
+              '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
+              '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
+              '%(height)s and %(width)s for the width and height of the video format. '
+              '%(resolution)s for a textual description of the resolution of the video format. '
+              'Use - to output to stdout. Can also be used to download to a different directory, '
+              'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
+    filesystem.add_option(
+        '--autonumber-size',
+        dest='autonumber_size', metavar='NUMBER',
+        help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
+    filesystem.add_option(
+        '--restrict-filenames',
+        action='store_true', dest='restrictfilenames', default=False,
+        help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames')
+    filesystem.add_option(
+        '-t', '--title',
+        action='store_true', dest='usetitle', default=False,
+        help='[deprecated] use title in file name (default)')
+    filesystem.add_option(
+        '-l', '--literal', default=False,
+        action='store_true', dest='usetitle',
+        help='[deprecated] alias of --title')
+    filesystem.add_option(
+        '-w', '--no-overwrites',
+        action='store_true', dest='nooverwrites', default=False,
+        help='do not overwrite files')
+    filesystem.add_option(
+        '-c', '--continue',
+        action='store_true', dest='continue_dl', default=True,
+        help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.')
+    filesystem.add_option(
+        '--no-continue',
+        action='store_false', dest='continue_dl',
+        help='do not resume partially downloaded files (restart from beginning)')
+    filesystem.add_option(
+        '--no-part',
+        action='store_true', dest='nopart', default=False,
+        help='do not use .part files - write directly into output file')
+    filesystem.add_option(
+        '--no-mtime',
+        action='store_false', dest='updatetime', default=True,
+        help='do not use the Last-modified header to set the file modification time')
+    filesystem.add_option(
+        '--write-description',
+        action='store_true', dest='writedescription', default=False,
+        help='write video description to a .description file')
+    filesystem.add_option(
+        '--write-info-json',
+        action='store_true', dest='writeinfojson', default=False,
+        help='write video metadata to a .info.json file')
+    filesystem.add_option(
+        '--write-annotations',
+        action='store_true', dest='writeannotations', default=False,
+        help='write video annotations to a .annotation file')
+    filesystem.add_option(
+        '--write-thumbnail',
+        action='store_true', dest='writethumbnail', default=False,
+        help='write thumbnail image to disk')
+    filesystem.add_option(
+        '--load-info',
+        dest='load_info_filename', metavar='FILE',
+        help='json file containing the video information (created with the "--write-json" option)')
+    filesystem.add_option(
+        '--cookies',
+        dest='cookiefile', metavar='FILE',
+        help='file to read cookies from and dump cookie jar in')
     filesystem.add_option(
         '--cache-dir', dest='cachedir', default=None, metavar='DIR',
         help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')
@@ -412,36 +537,61 @@ def parseOpts(overrideArguments=None):
         '--no-cache-dir', action='store_const', const=False, dest='cachedir',
         help='Disable filesystem caching')
     filesystem.add_option(
-        '--rm-cache-dir', action='store_true', dest='rm_cachedir',
+        '--rm-cache-dir',
+        action='store_true', dest='rm_cachedir',
         help='Delete all filesystem cache files')
 
-
-    postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False,
-            help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
-    postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
-            help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; best by default')
-    postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5',
-            help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)')
-    postproc.add_option('--recode-video', metavar='FORMAT', dest='recodevideo', default=None,
-            help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)')
-    postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
-            help='keeps the video file on disk after the post-processing; the video is erased by default')
-    postproc.add_option('--no-post-overwrites', action='store_true', dest='nopostoverwrites', default=False,
-            help='do not overwrite post-processed files; the post-processed files are overwritten by default')
-    postproc.add_option('--embed-subs', action='store_true', dest='embedsubtitles', default=False,
-            help='embed subtitles in the video (only for mp4 videos)')
-    postproc.add_option('--embed-thumbnail', action='store_true', dest='embedthumbnail', default=False,
-            help='embed thumbnail in the audio as cover art')
-    postproc.add_option('--add-metadata', action='store_true', dest='addmetadata', default=False,
-            help='write metadata to the video file')
-    postproc.add_option('--xattrs', action='store_true', dest='xattrs', default=False,
-            help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
-    postproc.add_option('--prefer-avconv', action='store_false', dest='prefer_ffmpeg',
+    postproc = optparse.OptionGroup(parser, 'Post-processing Options')
+    postproc.add_option(
+        '-x', '--extract-audio',
+        action='store_true', dest='extractaudio', default=False,
+        help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
+    postproc.add_option(
+        '--audio-format', metavar='FORMAT', dest='audioformat', default='best',
+        help='"best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "%default" by default')
+    postproc.add_option(
+        '--audio-quality', metavar='QUALITY',
+        dest='audioquality', default='5',
+        help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default %default)')
+    postproc.add_option(
+        '--recode-video',
+        metavar='FORMAT', dest='recodevideo', default=None,
+        help='Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)')
+    postproc.add_option(
+        '-k', '--keep-video',
+        action='store_true', dest='keepvideo', default=False,
+        help='keeps the video file on disk after the post-processing; the video is erased by default')
+    postproc.add_option(
+        '--no-post-overwrites',
+        action='store_true', dest='nopostoverwrites', default=False,
+        help='do not overwrite post-processed files; the post-processed files are overwritten by default')
+    postproc.add_option(
+        '--embed-subs',
+        action='store_true', dest='embedsubtitles', default=False,
+        help='embed subtitles in the video (only for mp4 videos)')
+    postproc.add_option(
+        '--embed-thumbnail',
+        action='store_true', dest='embedthumbnail', default=False,
+        help='embed thumbnail in the audio as cover art')
+    postproc.add_option(
+        '--add-metadata',
+        action='store_true', dest='addmetadata', default=False,
+        help='write metadata to the video file')
+    postproc.add_option(
+        '--xattrs',
+        action='store_true', dest='xattrs', default=False,
+        help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
+    postproc.add_option(
+        '--prefer-avconv',
+        action='store_false', dest='prefer_ffmpeg',
         help='Prefer avconv over ffmpeg for running the postprocessors (default)')
-    postproc.add_option('--prefer-ffmpeg', action='store_true', dest='prefer_ffmpeg',
+    postproc.add_option(
+        '--prefer-ffmpeg',
+        action='store_true', dest='prefer_ffmpeg',
         help='Prefer ffmpeg over avconv for running the postprocessors')
     postproc.add_option(
-        '--exec', metavar='CMD', dest='exec_cmd',
+        '--exec',
+        metavar='CMD', dest='exec_cmd',
         help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'' )
 
     parser.add_option_group(general)
@@ -458,7 +608,7 @@ def parseOpts(overrideArguments=None):
     if overrideArguments is not None:
         opts, args = parser.parse_args(overrideArguments)
         if opts.verbose:
-            write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n')
+            write_string('[debug] Override config: ' + repr(overrideArguments) + '\n')
     else:
         commandLineConf = sys.argv[1:]
         if '--ignore-config' in commandLineConf:
@@ -474,8 +624,8 @@ def parseOpts(overrideArguments=None):
 
         opts, args = parser.parse_args(argv)
         if opts.verbose:
-            write_string(u'[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
-            write_string(u'[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
-            write_string(u'[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
+            write_string('[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
+            write_string('[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
+            write_string('[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
 
     return parser, opts, args

From 4d7b03f1f28760abee5a1326f1c902b081c75976 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 12 Oct 2014 23:31:15 +0200
Subject: [PATCH 0210/1937] [zsh-completion] Ignore generated file (#3890)

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index e44977ca3..86312d4e4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,3 +30,4 @@ updates_key.pem
 *.swp
 test/testdata
 .tox
+youtube-dl.zsh

From 203fb43f3602a6363041ee9323ee718e6c71606c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 13 Oct 2014 00:03:08 +0200
Subject: [PATCH 0211/1937] [youtube] Download DASH manifest by default (Closes
 #3887)

---
 youtube_dl/extractor/youtube.py |  2 +-
 youtube_dl/options.py           | 12 ++++++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9041cfa87..98d7fb5fc 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -938,7 +938,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
 
         # Look for the DASH manifest
-        if (self._downloader.params.get('youtube_include_dash_manifest', False)):
+        if self._downloader.params.get('youtube_include_dash_manifest', True):
             try:
                 # The DASH manifest used needs to be the one from the original video_webpage.
                 # The one found in get_video_info seems to be using different signatures.
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index a06850e1c..649361bde 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -225,10 +225,6 @@ def parseOpts(overrideArguments=None):
         '--include-ads',
         dest='include_ads', action='store_true',
         help='Download advertisements as well (experimental)')
-    selection.add_option(
-        '--youtube-include-dash-manifest',
-        action='store_true', dest='youtube_include_dash_manifest', default=False,
-        help='Try to download the DASH manifest on YouTube videos (experimental)')
 
     authentication = optparse.OptionGroup(parser, 'Authentication Options')
     authentication.add_option(
@@ -273,6 +269,14 @@ def parseOpts(overrideArguments=None):
         '-F', '--list-formats',
         action='store_true', dest='listformats',
         help='list all available formats')
+    video_format.add_option(
+        '--youtube-include-dash-manifest',
+        action='store_true', dest='youtube_include_dash_manifest', default=True,
+        help=optparse.SUPPRESS_HELP)
+    video_format.add_option(
+        '--youtube-skip-dash-manifest',
+        action='store_false', dest='youtube_include_dash_manifest',
+        help='Do not download the DASH manifest on YouTube videos')
 
     subtitles = optparse.OptionGroup(parser, 'Subtitle Options')
     subtitles.add_option(

From 1cc887cbf0371a3de2154bc2b34546d1040d01a8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 13 Oct 2014 00:09:19 +0200
Subject: [PATCH 0212/1937] [youtube] Add support for format 278

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 98d7fb5fc..cfae2de89 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -286,6 +286,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
         '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
         '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
+        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'container': 'webm', 'vcodec': 'VP9'},
         '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},

From 8a2300a59703da441932f361e4e496bb64e4b502 Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Mon, 13 Oct 2014 10:03:55 +0530
Subject: [PATCH 0213/1937] [kontrtube] Fix video title extraction

---
 youtube_dl/extractor/kontrtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kontrtube.py b/youtube_dl/extractor/kontrtube.py
index 5341ac773..8a73ecfa0 100644
--- a/youtube_dl/extractor/kontrtube.py
+++ b/youtube_dl/extractor/kontrtube.py
@@ -34,7 +34,7 @@ class KontrTubeIE(InfoExtractor):
         video_url = self._html_search_regex(r"video_url: '(.+?)/?',", webpage, 'video URL')
         thumbnail = self._html_search_regex(r"preview_url: '(.+?)/?',", webpage, 'video thumbnail', fatal=False)
         title = self._html_search_regex(
-            r'<title>(.+?) - Труба зовёт - Интересный видеохостинг</title>', webpage, 'video title')
+            r'<title>(.+?)</title>', webpage, 'video title')
         description = self._html_search_meta('description', webpage, 'video description')
 
         mobj = re.search(

From 4c1ce987b03dc9cde1b9f06db117f4f12c1dd6a4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 13 Oct 2014 10:08:59 +0200
Subject: [PATCH 0214/1937] [huffpost] Modernize

---
 youtube_dl/extractor/huffpost.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/huffpost.py b/youtube_dl/extractor/huffpost.py
index 94e7cf790..4ccf6b9b8 100644
--- a/youtube_dl/extractor/huffpost.py
+++ b/youtube_dl/extractor/huffpost.py
@@ -33,8 +33,7 @@ class HuffPostIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         api_url = 'http://embed.live.huffingtonpost.com/api/segments/%s.json' % video_id
         data = self._download_json(api_url, video_id)['data']

From 2aefb886faa855518292c4078eb04c40b55d4f90 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 13 Oct 2014 10:12:43 +0200
Subject: [PATCH 0215/1937] [ffmpeg] Improve format merging (Closes #3935)

---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 8c5f7c43b..6f010a9c7 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -487,7 +487,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
 class FFmpegMergerPP(FFmpegPostProcessor):
     def run(self, info):
         filename = info['filepath']
-        args = ['-c', 'copy']
+        args = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0', '-shortest']
         self._downloader.to_screen(u'[ffmpeg] Merging formats into "%s"' % filename)
         self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args)
         return True, info

From 9b36dcbd6512fb6469b037006c30cedd2cfae3fc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 13 Oct 2014 10:12:51 +0200
Subject: [PATCH 0216/1937] release 2014.10.13

---
 README.md             | 13 +++++++------
 youtube_dl/version.py |  2 +-
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index cabc5eb9a..90ba928c3 100644
--- a/README.md
+++ b/README.md
@@ -99,8 +99,6 @@ which means you can modify it, redistribute it or use it however you like.
                                      downloaded videos in it.
     --include-ads                    Download advertisements as well
                                      (experimental)
-    --youtube-include-dash-manifest  Try to download the DASH manifest on
-                                     YouTube videos (experimental)
 
 ## Download Options:
     -r, --rate-limit LIMIT           maximum download rate in bytes per second
@@ -158,7 +156,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      downloads if possible.
     --no-continue                    do not resume partially downloaded files
                                      (restart from beginning)
-    --no-part                        do not use .part files
+    --no-part                        do not use .part files - write directly
+                                     into output file
     --no-mtime                       do not use the Last-modified header to set
                                      the file modification time
     --write-description              write video description to a .description
@@ -216,7 +215,7 @@ which means you can modify it, redistribute it or use it however you like.
                                      information about the video. (Currently
                                      supported only for YouTube)
     --user-agent UA                  specify a custom user agent
-    --referer REF                    specify a custom referer, use if the video
+    --referer URL                    specify a custom referer, use if the video
                                      access is restricted to one domain
     --add-header FIELD:VALUE         specify a custom HTTP header and its value,
                                      separated by a colon ':'. You can use this
@@ -241,6 +240,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      one is requested
     --max-quality FORMAT             highest quality format to download
     -F, --list-formats               list all available formats
+    --youtube-skip-dash-manifest     Do not download the DASH manifest on
+                                     YouTube videos
 
 ## Subtitle Options:
     --write-sub                      write subtitle file
@@ -256,7 +257,7 @@ which means you can modify it, redistribute it or use it however you like.
                                      language tags like 'en,pt'
 
 ## Authentication Options:
-    -u, --username USERNAME          account username
+    -u, --username USERNAME          login with this account ID
     -p, --password PASSWORD          account password
     -2, --twofactor TWOFACTOR        two-factor auth code
     -n, --netrc                      use .netrc authentication data
@@ -267,7 +268,7 @@ which means you can modify it, redistribute it or use it however you like.
                                      (requires ffmpeg or avconv and ffprobe or
                                      avprobe)
     --audio-format FORMAT            "best", "aac", "vorbis", "mp3", "m4a",
-                                     "opus", or "wav"; best by default
+                                     "opus", or "wav"; "best" by default
     --audio-quality QUALITY          ffmpeg/avconv audio quality specification,
                                      insert a value between 0 (better) and 9
                                      (worse) for VBR or a specific bitrate like
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8b0a704ee..416e2fd85 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.10.12'
+__version__ = '2014.10.13'

From 1419fafd36a6ab70d769411e779db63f2b4cb1ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 13 Oct 2014 19:59:35 +0700
Subject: [PATCH 0217/1937] [condenast] Add support for embedded videos (Closes
 #3929)

---
 youtube_dl/extractor/condenast.py | 2 ++
 youtube_dl/extractor/generic.py   | 7 +++++++
 2 files changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index ffbe4903b..7a7e79360 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -34,6 +34,8 @@ class CondeNastIE(InfoExtractor):
     _VALID_URL = r'http://(video|www|player)\.(?P<site>%s)\.com/(?P<type>watch|series|video|embed)/(?P<id>[^/?#]+)' % '|'.join(_SITES.keys())
     IE_DESC = 'Condé Nast media group: %s' % ', '.join(sorted(_SITES.values()))
 
+    EMBED_URL = r'(?:https?:)?//player\.(?P<site>%s)\.com/(?P<type>embed)/.+?' % '|'.join(_SITES.keys())
+
     _TEST = {
         'url': 'http://video.wired.com/watch/3d-printed-speakers-lit-with-led',
         'md5': '1921f713ed48aabd715691f774c451f7',
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index dfc2ef4e7..122763a23 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -28,6 +28,7 @@ from .brightcove import BrightcoveIE
 from .ooyala import OoyalaIE
 from .rutv import RUTVIE
 from .smotri import SmotriIE
+from .condenast import CondeNastIE
 
 
 class GenericIE(InfoExtractor):
@@ -847,6 +848,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'MLB')
 
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
+            webpage)
+        if mobj is not None:
+            return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')
+
         def check_video(vurl):
             vpath = compat_urlparse.urlparse(vurl).path
             vext = determine_ext(vpath)

From bd4e40df1aaef96dcde64dd64793b5152ae66055 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 13 Oct 2014 16:26:53 +0200
Subject: [PATCH 0218/1937] [brightcove] Add a test for playlists

---
 youtube_dl/extractor/brightcove.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 294670386..ad22cbafd 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -87,6 +87,15 @@ class BrightcoveIE(InfoExtractor):
                 'description': 'UCI MTB World Cup 2014: Fort William, UK - Downhill Finals',
             },
         },
+        {
+            # playlist test
+            # from http://support.brightcove.com/en/video-cloud/docs/playlist-support-single-video-players
+            'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=3550052898001&playerKey=AQ%7E%7E%2CAAABmA9XpXk%7E%2C-Kp7jNgisre1fG5OdqpAFUTcs0lP_ZoL',
+            'info_dict': {
+                'title': 'Sealife',
+            },
+            'playlist_mincount': 7,
+        },
     ]
 
     @classmethod

From a86c73cf80130e55b676dcf5f5979801f28acaec Mon Sep 17 00:00:00 2001
From: Damon Timm <damontimm@gmail.com>
Date: Mon, 13 Oct 2014 14:08:29 -0700
Subject: [PATCH 0219/1937] [glide] Add new extractor

Added an extractor for glide.me shared messages.  Glide is a movile video messaging services.  You can share the link to the messages easily enough and this would allow you to download and save the actual video.
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/glide.py    | 33 ++++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)
 create mode 100644 youtube_dl/extractor/glide.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5e38d2663..c0b26c5b3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -134,6 +134,7 @@ from .gamestar import GameStarIE
 from .gametrailers import GametrailersIE
 from .gdcvault import GDCVaultIE
 from .generic import GenericIE
+from .glide import GlideIE
 from .globo import GloboIE
 from .godtube import GodTubeIE
 from .golem import GolemIE
diff --git a/youtube_dl/extractor/glide.py b/youtube_dl/extractor/glide.py
new file mode 100644
index 000000000..175d85197
--- /dev/null
+++ b/youtube_dl/extractor/glide.py
@@ -0,0 +1,33 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class GlideIE(InfoExtractor):
+    IE_DESC = 'Glide mobile video messages (glide.me)'
+    _VALID_URL = r'https?://share\.glide\.me/(?P<id>[A-Za-z0-9\-=_+]+)'
+    _TEST = {
+        'url': 'http://share.glide.me/UZF8zlmuQbe4mr+7dCiQ0w==',
+        'md5': '4466372687352851af2d131cfaa8a4c7',
+        'info_dict': {
+            'id': 'UZF8zlmuQbe4mr+7dCiQ0w==',
+            'ext': 'mp4',
+            'title': 'Damon Timm\'s Glide message',
+            'thumbnail' : 'http://dk608k4lm7m9j.cloudfront.net/3ee7da5af87065a1eeb8c6c9a864ba5b_2.jpg'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
+        video_url = self._search_regex(r'<source src="(.*?)" type="video/mp4">', webpage, 'video_url')
+        thumbnail_url = self._search_regex(r'<img id="video-thumbnail" src="(.*?)" alt="Video thumbnail">', webpage, 'thumbnail_url')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url' : 'http:' + video_url,
+            'thumbnail' : 'http:' + thumbnail_url
+        }

From 5f58165def551921741b44772c9bf31a237a4ac8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 14 Oct 2014 21:43:48 +0700
Subject: [PATCH 0220/1937] [extractor/common] Fix dumping requests with long
 file abspath on Windows

---
 youtube_dl/extractor/common.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 450c7dfd6..e8366f7f9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -281,6 +281,12 @@ class InfoExtractor(object):
             raw_filename = basen + '.dump'
             filename = sanitize_filename(raw_filename, restricted=True)
             self.to_screen('Saving request to ' + filename)
+            # Working around MAX_PATH limitation on Windows (see
+            # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
+            if os.name == 'nt':
+                absfilepath = os.path.abspath(filename)
+                if len(absfilepath) > 259:
+                    filename = '\\\\?\\' + absfilepath
             with open(filename, 'wb') as outf:
                 outf.write(webpage_bytes)
 

From 62a164e71372f126d6d70997adf74fcc4e086b58 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 15 Oct 2014 00:53:54 +0200
Subject: [PATCH 0221/1937] [mixcloud] Output downloading progress

---
 youtube_dl/extractor/mixcloud.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index a4564d3de..bb8937c4d 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -33,22 +33,22 @@ class MixcloudIE(InfoExtractor):
         },
     }
 
-    def check_urls(self, url_list):
-        """Returns 1st active url from list"""
-        for url in url_list:
+    def _get_url(self, track_id, template_url):
+        server_count = 30
+        for i in range(server_count):
+            url = template_url % i
             try:
                 # We only want to know if the request succeed
                 # don't download the whole file
-                self._request_webpage(HEADRequest(url), None, False)
+                self._request_webpage(
+                    HEADRequest(url), track_id,
+                    'Checking URL %d/%d ...' % (i + 1, server_count + 1))
                 return url
             except ExtractorError:
-                url = None
+                pass
 
         return None
 
-    def _get_url(self, template_url):
-        return self.check_urls(template_url % i for i in range(30))
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         uploader = mobj.group(1)
@@ -61,11 +61,11 @@ class MixcloudIE(InfoExtractor):
             r'\s(?:data-preview-url|m-preview)="(.+?)"', webpage, 'preview url')
         song_url = preview_url.replace('/previews/', '/c/originals/')
         template_url = re.sub(r'(stream\d*)', 'stream%d', song_url)
-        final_song_url = self._get_url(template_url)
+        final_song_url = self._get_url(track_id, template_url)
         if final_song_url is None:
             self.to_screen('Trying with m4a extension')
             template_url = template_url.replace('.mp3', '.m4a').replace('originals/', 'm4a/64/')
-            final_song_url = self._get_url(template_url)
+            final_song_url = self._get_url(track_id, template_url)
         if final_song_url is None:
             raise ExtractorError('Unable to extract track url')
 

From 7b61ac3ddf0bfe660266e70fc7c10f048758698f Mon Sep 17 00:00:00 2001
From: net <net@h-MacBook-Pro-sl-net.local>
Date: Wed, 15 Oct 2014 06:46:47 +0300
Subject: [PATCH 0222/1937] Fix #2310. Play by the 8tracks rules

---
 youtube_dl/extractor/eighttracks.py | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index c1b4c729e..101a09699 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -4,10 +4,12 @@ from __future__ import unicode_literals
 import json
 import random
 import re
+import time
 
 from .common import InfoExtractor
 from ..utils import (
     compat_str,
+    ExtractorError,
 )
 
 
@@ -112,14 +114,30 @@ class EightTracksIE(InfoExtractor):
         session = str(random.randint(0, 1000000000))
         mix_id = data['id']
         track_count = data['tracks_count']
+        duration = data['duration']
+        avg_song_duration = duration / track_count
         first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id)
         next_url = first_url
         entries = []
+
         for i in range(track_count):
-            api_json = self._download_webpage(
-                next_url, playlist_id,
-                note='Downloading song information %d/%d' % (i + 1, track_count),
-                errnote='Failed to download song information')
+            
+            api_json = None
+            download_tries = 0
+
+            while api_json is None:
+                try:
+                    api_json = self._download_webpage(
+                        next_url, playlist_id,
+                        note='Downloading song information %d/%d' % (i + 1, track_count),
+                        errnote='Failed to download song information')
+                except ExtractorError:
+                    if download_tries > 3:
+                        raise
+                    else:
+                        ++download_tries
+                        time.sleep(avg_song_duration)
+
             api_data = json.loads(api_json)
             track_data = api_data['set']['track']
             info = {
@@ -131,6 +149,7 @@ class EightTracksIE(InfoExtractor):
                 'ext': 'm4a',
             }
             entries.append(info)
+
             next_url = 'http://8tracks.com/sets/%s/next?player=sm&mix_id=%s&format=jsonh&track_id=%s' % (
                 session, mix_id, track_data['id'])
         return {

From a72cbfacf067fc5539b93152d2917f30f44b88c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 15 Oct 2014 12:24:11 +0200
Subject: [PATCH 0223/1937] [ted] Add support for external videos (fixes #3948)

---
 youtube_dl/extractor/ted.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index d5e28efad..cd4af96fd 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -65,6 +65,22 @@ class TEDIE(SubtitlesInfoExtractor):
             'title': 'Who are the hackers?',
         },
         'playlist_mincount': 6,
+    }, {
+        # contains a youtube video
+        'url': 'https://www.ted.com/talks/douglas_adams_parrots_the_universe_and_everything',
+        'add_ie': ['Youtube'],
+        'info_dict': {
+            'id': '_ZG8HBuDjgc',
+            'ext': 'mp4',
+            'title': 'Douglas Adams: Parrots the Universe and Everything',
+            'description': 'md5:01ad1e199c49ac640cb1196c0e9016af',
+            'uploader': 'University of California Television (UCTV)',
+            'uploader_id': 'UCtelevision',
+            'upload_date': '20080522',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     _NATIVE_FORMATS = {
@@ -114,6 +130,13 @@ class TEDIE(SubtitlesInfoExtractor):
 
         talk_info = self._extract_info(webpage)['talks'][0]
 
+        if talk_info.get('external') is not None:
+            self.to_screen('Found video from %s' % talk_info['external']['service'])
+            return {
+                '_type': 'url',
+                'url': talk_info['external']['uri'],
+            }
+
         formats = [{
             'url': format_url,
             'format_id': format_id,

From df928d500fa2bcbee5a18fae9454e22e0c223624 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 15 Oct 2014 12:39:30 +0200
Subject: [PATCH 0224/1937] release 2014.10.15

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 416e2fd85..f17c86405 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.10.13'
+__version__ = '2014.10.15'

From 09a42738fc5a060208cad5fa5579688225e53464 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 15 Oct 2014 13:50:53 +0200
Subject: [PATCH 0225/1937] [generic] Correct handling of embedded vimeo
 players (#3955)

---
 youtube_dl/extractor/generic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 122763a23..158dc95a5 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -609,13 +609,13 @@ class GenericIE(InfoExtractor):
         if mobj:
             player_url = unescapeHTML(mobj.group('url'))
             surl = smuggle_url(player_url, {'Referer': url})
-            return self.url_result(surl, 'Vimeo')
+            return self.url_result(surl)
 
         # Look for embedded (swf embed) Vimeo player
         mobj = re.search(
-            r'<embed[^>]+?src="(https?://(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
+            r'<embed[^>]+?src="((?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
         if mobj:
-            return self.url_result(mobj.group(1), 'Vimeo')
+            return self.url_result(mobj.group(1))
 
         # Look for embedded YouTube player
         matches = re.findall(r'''(?x)

From 474086450808541954e402dd3d58bad38b75b1d5 Mon Sep 17 00:00:00 2001
From: CkuT <s3cu@hotmail.fr>
Date: Wed, 15 Oct 2014 18:24:32 +0200
Subject: [PATCH 0226/1937] [SexyKarma] Add new extractor

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/sexykarma.py | 43 +++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+)
 create mode 100644 youtube_dl/extractor/sexykarma.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5e38d2663..92f38dcd0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -316,6 +316,7 @@ from .sbs import SBSIE
 from .scivee import SciVeeIE
 from .screencast import ScreencastIE
 from .servingsys import ServingSysIE
+from .sexykarma import SexyKarmaIE
 from .shared import SharedIE
 from .sharesix import ShareSixIE
 from .sina import SinaIE
diff --git a/youtube_dl/extractor/sexykarma.py b/youtube_dl/extractor/sexykarma.py
new file mode 100644
index 000000000..cd2668872
--- /dev/null
+++ b/youtube_dl/extractor/sexykarma.py
@@ -0,0 +1,43 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class SexyKarmaIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?sexykarma\.com/gonewild/video/(?P<id>[a-zA-Z0-9\-]+)(.html)'
+    _TESTS = [{
+        'url': 'http://www.sexykarma.com/gonewild/video/taking-a-quick-pee-yHI70cOyIHt.html',
+        'md5': 'b9798e7d1ef1765116a8f516c8091dbd',
+        'info_dict': {
+            'id': 'taking-a-quick-pee-yHI70cOyIHt',
+            'ext': 'mp4',
+            'title': 'Taking a quick pee.',
+            'uploader': 'wildginger7',
+        }
+    }, {
+        'url': 'http://www.sexykarma.com/gonewild/video/pot-pixie-tribute-8Id6EZPbuHf.html',
+        'md5': 'dd216c68d29b49b12842b9babe762a5d',
+        'info_dict': {
+            'id': 'pot-pixie-tribute-8Id6EZPbuHf',
+            'ext': 'mp4',
+            'title': 'pot_pixie tribute',
+            'uploader': 'banffite',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        # TODO more code goes here, for example ...
+        webpage = self._download_webpage(url, video_id)
+        title = self._html_search_regex(r'<h2 class="he2"><span>(.*?)</span>', webpage, 'title')
+        uploader = self._html_search_regex(r'class="aupa">\n*(.*?)</a>', webpage, 'uploader')
+        url = self._html_search_regex(r'<p><a href="(.*?)" ?\n*target="_blank"><font color', webpage, 'url')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'uploader': uploader,
+            'url': url
+        }

From 1723edb1a589190b8a91af12bd7dadaa50e26549 Mon Sep 17 00:00:00 2001
From: CkuT <s3cu@hotmail.fr>
Date: Wed, 15 Oct 2014 20:17:07 +0200
Subject: [PATCH 0227/1937] Few improvements

---
 youtube_dl/extractor/sexykarma.py | 51 +++++++++++++++++++++++++------
 1 file changed, 41 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/sexykarma.py b/youtube_dl/extractor/sexykarma.py
index cd2668872..ac7745e74 100644
--- a/youtube_dl/extractor/sexykarma.py
+++ b/youtube_dl/extractor/sexykarma.py
@@ -2,42 +2,73 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-
+import re
+import datetime
 
 class SexyKarmaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?sexykarma\.com/gonewild/video/(?P<id>[a-zA-Z0-9\-]+)(.html)'
+    _VALID_URL = r'https?://(?:www\.)?sexykarma\.com/gonewild/video/.+\-(?P<id>[a-zA-Z0-9\-]+)(.html)'
     _TESTS = [{
         'url': 'http://www.sexykarma.com/gonewild/video/taking-a-quick-pee-yHI70cOyIHt.html',
         'md5': 'b9798e7d1ef1765116a8f516c8091dbd',
         'info_dict': {
-            'id': 'taking-a-quick-pee-yHI70cOyIHt',
+            'id': 'yHI70cOyIHt',
             'ext': 'mp4',
             'title': 'Taking a quick pee.',
-            'uploader': 'wildginger7',
+            'uploader_id': 'wildginger7',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': int,
+            'view_count': int,
+            'upload_date': '20141007',
         }
     }, {
         'url': 'http://www.sexykarma.com/gonewild/video/pot-pixie-tribute-8Id6EZPbuHf.html',
         'md5': 'dd216c68d29b49b12842b9babe762a5d',
         'info_dict': {
-            'id': 'pot-pixie-tribute-8Id6EZPbuHf',
+            'id': '8Id6EZPbuHf',
             'ext': 'mp4',
             'title': 'pot_pixie tribute',
-            'uploader': 'banffite',
+            'uploader_id': 'banffite',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': int,
+            'view_count': int,
+            'upload_date': '20141013',
         }
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        # TODO more code goes here, for example ...
         webpage = self._download_webpage(url, video_id)
+              
         title = self._html_search_regex(r'<h2 class="he2"><span>(.*?)</span>', webpage, 'title')
-        uploader = self._html_search_regex(r'class="aupa">\n*(.*?)</a>', webpage, 'uploader')
+        uploader_id = self._html_search_regex(r'class="aupa">\n*(.*?)</a>', webpage, 'uploader')
         url = self._html_search_regex(r'<p><a href="(.*?)" ?\n*target="_blank"><font color', webpage, 'url')
+        thumbnail = self._html_search_regex(r'<div id="player" style="z-index:1;"> <span id="edge"></span> <span id="container"><img[\n ]*src="(.+?)"', webpage, 'thumbnail')
+        
+        str_duration = self._html_search_regex(r'<tr>[\n\s]*<td>Time: </td>[\n\s]*<td align="right"><span>(.+)\n*', webpage, 'duration')
+        duration = self._to_seconds(str_duration)
+
+        str_views = self._html_search_regex(r'<tr>[\n\s]*<td>Views: </td>[\n\s]*<td align="right"><span>(.+)</span>', webpage, 'view_count')
+        view_count = int(str_views)
+        # print view_count
+
+        date = self._html_search_regex(r'class="aup">Added: <strong>(.*?)</strong>', webpage, 'date')
+        d = datetime.datetime.strptime(date, '%B %d, %Y')
+        upload_date = d.strftime('%Y%m%d')
 
         return {
             'id': video_id,
             'title': title,
-            'uploader': uploader,
-            'url': url
+            'uploader_id': uploader_id,
+            'url': url,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'view_count': view_count,
+            'upload_date': upload_date,
         }
+
+    def _to_seconds(self, timestr):
+        seconds= 0
+        for part in timestr.split(':'):
+            seconds= seconds*60 + int(part)
+        return seconds

From 7da224c907627b5b0233c7eb4eea733f5f97ae90 Mon Sep 17 00:00:00 2001
From: CkuT <s3cu@hotmail.fr>
Date: Wed, 15 Oct 2014 22:34:35 +0200
Subject: [PATCH 0228/1937] Add categories

---
 youtube_dl/extractor/sexykarma.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/sexykarma.py b/youtube_dl/extractor/sexykarma.py
index ac7745e74..629499e72 100644
--- a/youtube_dl/extractor/sexykarma.py
+++ b/youtube_dl/extractor/sexykarma.py
@@ -56,6 +56,8 @@ class SexyKarmaIE(InfoExtractor):
         d = datetime.datetime.strptime(date, '%B %d, %Y')
         upload_date = d.strftime('%Y%m%d')
 
+        categories = re.findall(r'http://www.sexykarma.com/gonewild/search/video/(?:.+?)"><span>(.*?)</span>', webpage)
+
         return {
             'id': video_id,
             'title': title,
@@ -65,6 +67,7 @@ class SexyKarmaIE(InfoExtractor):
             'duration': duration,
             'view_count': view_count,
             'upload_date': upload_date,
+            'categories': categories,
         }
 
     def _to_seconds(self, timestr):

From 355d074ff955f7f8b24321e51be44f4431572fe0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 16 Oct 2014 22:23:35 +0700
Subject: [PATCH 0229/1937] [twitch] Adapt to new API (Fixes #3946, Fixes
 #3949, Fixes #3965)

 Work in progress
---
 youtube_dl/extractor/justintv.py | 77 ++++++++++++++++++++++++++++++--
 1 file changed, 73 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/justintv.py b/youtube_dl/extractor/justintv.py
index 27017e89f..411167d7d 100644
--- a/youtube_dl/extractor/justintv.py
+++ b/youtube_dl/extractor/justintv.py
@@ -10,6 +10,7 @@ from ..utils import (
     compat_str,
     ExtractorError,
     formatSeconds,
+    parse_iso8601,
 )
 
 
@@ -43,6 +44,8 @@ class JustinTVIE(InfoExtractor):
         }
     }
 
+    _API_BASE = 'https://api.twitch.tv'
+
     # Return count of items, list of *valid* items
     def _parse_page(self, url, video_id, counter):
         info_json = self._download_webpage(
@@ -74,18 +77,84 @@ class JustinTVIE(InfoExtractor):
                 })
         return (len(response), info)
 
+    def _handle_error(self, response):
+        if not isinstance(response, dict):
+            return
+        error = response.get('error')
+        if error:
+            raise ExtractorError(
+                '%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
+                expected=True)
+
+    def _download_json(self, url, video_id, note='Downloading JSON metadata'):
+        response = super(JustinTVIE, self)._download_json(url, video_id, note)
+        self._handle_error(response)
+        return response
+
+    def _extract_media(self, item, item_id):
+
+        ITEMS = {
+            'a': 'video',
+            'c': 'chapter',
+        }
+
+        info = self._extract_info(self._download_json(
+            '%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
+            'Downloading %s info JSON' % ITEMS[item]))
+
+        response = self._download_json(
+            '%s/api/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
+            'Downloading %s playlist JSON' % ITEMS[item])
+
+        entries = []
+        chunks = response['chunks']
+        qualities = list(chunks.keys())
+        for num, fragment in enumerate(zip(*chunks.values()), start=1):
+            formats = []
+            for fmt_num, fragment_fmt in enumerate(fragment):
+                format_id = qualities[fmt_num]
+                fmt = {
+                    'url': fragment_fmt['url'],
+                    'format_id': format_id,
+                    'quality': 1 if format_id == 'live' else 0,
+                }
+                m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
+                if m:
+                    fmt['height'] = int(m.group('height'))
+                formats.append(fmt)
+            self._sort_formats(formats)
+            entry = dict(info)
+            entry['title'] = '%s part %d' % (entry['title'], num)
+            entry['formats'] = formats
+            entries.append(entry)
+        return entries
+
+    def _extract_info(self, info):
+        return {
+            'id': info['_id'],
+            'title': info['title'],
+            'description': info['description'],
+            'duration': info['length'],
+            'thumbnail': info['preview'],
+            'uploader': info['channel']['display_name'],
+            'uploader_id': info['channel']['name'],
+            'timestamp': parse_iso8601(info['recorded_at']),
+            'view_count': info['views'],
+        }
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
 
-        api_base = 'http://api.justin.tv'
+        api_base = 'http://api.twitch.tv'
         paged = False
         if mobj.group('channelid'):
             paged = True
             video_id = mobj.group('channelid')
             api = api_base + '/channel/archives/%s.json' % video_id
         elif mobj.group('chapterid'):
-            chapter_id = mobj.group('chapterid')
+            return self._extract_media('c', mobj.group('chapterid'))
 
+            """
             webpage = self._download_webpage(url, chapter_id)
             m = re.search(r'PP\.archive_id = "([0-9]+)";', webpage)
             if not m:
@@ -133,9 +202,9 @@ class JustinTVIE(InfoExtractor):
                 'uploader_id': chapter_info['channel']['name'],
             }
             return info
+            """
         else:
-            video_id = mobj.group('videoid')
-            api = api_base + '/broadcast/by_archive/%s.json' % video_id
+            return self._extract_media('a', mobj.group('videoid'))
 
         entries = []
         offset = 0

From 4698f0d85839f340fd33a387247efe39828384af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 17 Oct 2014 15:49:16 +0200
Subject: [PATCH 0230/1937] [vimeo] Improve regex for the config js dict (fixes
 #3955, fixes #3974)

The javascript code contains assignments of empty dicts with the same variable name
---
 youtube_dl/extractor/vimeo.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index e6a86f18e..07959d3fe 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -157,6 +157,18 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
                 'duration': 62,
             }
         },
+        {
+            # from https://www.ouya.tv/game/Pier-Solar-and-the-Great-Architects/
+            'url': 'https://player.vimeo.com/video/98044508',
+            'note': 'The js code contains assignments to the same variable as the config',
+            'info_dict': {
+                'id': '98044508',
+                'ext': 'mp4',
+                'title': 'Pier Solar OUYA Official Trailer',
+                'uploader': 'Tulio Gonçalves',
+                'uploader_id': 'user28849593',
+            },
+        },
     ]
 
     def _verify_video_password(self, url, video_id, webpage):
@@ -244,7 +256,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
                 # We try to find out to which variable is assigned the config dic
                 m_variable_name = re.search('(\w)\.video\.id', webpage)
                 if m_variable_name is not None:
-                    config_re = r'%s=({.+?});' % re.escape(m_variable_name.group(1))
+                    config_re = r'%s=({[^}].+?});' % re.escape(m_variable_name.group(1))
                 else:
                     config_re = [r' = {config:({.+?}),assets:', r'(?:[abc])=({.+?});']
                 config = self._search_regex(config_re, webpage, 'info section',

From 46fd0dd5a5a8d98b76830636a3dcd241ac7bcc7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Oct 2014 22:58:18 +0700
Subject: [PATCH 0231/1937] [twitch] Rename extractor and support channel
 videos

---
 youtube_dl/extractor/__init__.py              |   2 +-
 .../extractor/{justintv.py => twitch.py}      | 102 +++++-------------
 2 files changed, 28 insertions(+), 76 deletions(-)
 rename youtube_dl/extractor/{justintv.py => twitch.py} (69%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5e38d2663..c29c29c10 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -173,7 +173,6 @@ from .jadorecettepub import JadoreCettePubIE
 from .jeuxvideo import JeuxVideoIE
 from .jove import JoveIE
 from .jukebox import JukeboxIE
-from .justintv import JustinTVIE
 from .jpopsukitv import JpopsukiIE
 from .kankan import KankanIE
 from .keezmovies import KeezMoviesIE
@@ -395,6 +394,7 @@ from .tutv import TutvIE
 from .tvigle import TvigleIE
 from .tvp import TvpIE
 from .tvplay import TVPlayIE
+from .twitch import TwitchIE
 from .ubu import UbuIE
 from .udemy import (
     UdemyIE,
diff --git a/youtube_dl/extractor/justintv.py b/youtube_dl/extractor/twitch.py
similarity index 69%
rename from youtube_dl/extractor/justintv.py
rename to youtube_dl/extractor/twitch.py
index 411167d7d..42d38b224 100644
--- a/youtube_dl/extractor/justintv.py
+++ b/youtube_dl/extractor/twitch.py
@@ -1,26 +1,20 @@
 from __future__ import unicode_literals
 
 import itertools
-import json
-import os
 import re
 
 from .common import InfoExtractor
 from ..utils import (
-    compat_str,
     ExtractorError,
-    formatSeconds,
     parse_iso8601,
 )
 
 
-class JustinTVIE(InfoExtractor):
-    """Information extractor for justin.tv and twitch.tv"""
+class TwitchIE(InfoExtractor):
     # TODO: One broadcast may be split into multiple videos. The key
     # 'broadcast_id' is the same for all parts, and 'broadcast_part'
     # starts at 1 and increases. Can we treat all parts as one video?
-
-    _VALID_URL = r"""(?x)^(?:http://)?(?:www\.)?(?:twitch|justin)\.tv/
+    _VALID_URL = r"""(?x)^(?:http://)?(?:www\.)?twitch\.tv/
         (?:
             (?P<channelid>[^/]+)|
             (?:(?:[^/]+)/b/(?P<videoid>[^/]+))|
@@ -28,9 +22,8 @@ class JustinTVIE(InfoExtractor):
         )
         /?(?:\#.*)?$
         """
-    _JUSTIN_PAGE_LIMIT = 100
-    IE_NAME = 'justin.tv'
-    IE_DESC = 'justin.tv and twitch.tv'
+    _PAGE_LIMIT = 100
+    _API_BASE = 'https://api.twitch.tv'
     _TEST = {
         'url': 'http://www.twitch.tv/thegamedevhub/b/296128360',
         'md5': 'ecaa8a790c22a40770901460af191c9a',
@@ -44,39 +37,6 @@ class JustinTVIE(InfoExtractor):
         }
     }
 
-    _API_BASE = 'https://api.twitch.tv'
-
-    # Return count of items, list of *valid* items
-    def _parse_page(self, url, video_id, counter):
-        info_json = self._download_webpage(
-            url, video_id,
-            'Downloading video info JSON on page %d' % counter,
-            'Unable to download video info JSON %d' % counter)
-
-        response = json.loads(info_json)
-        if type(response) != list:
-            error_text = response.get('error', 'unknown error')
-            raise ExtractorError('Justin.tv API: %s' % error_text)
-        info = []
-        for clip in response:
-            video_url = clip['video_file_url']
-            if video_url:
-                video_extension = os.path.splitext(video_url)[1][1:]
-                video_date = re.sub('-', '', clip['start_time'][:10])
-                video_uploader_id = clip.get('user_id', clip.get('channel_id'))
-                video_id = clip['id']
-                video_title = clip.get('title', video_id)
-                info.append({
-                    'id': compat_str(video_id),
-                    'url': video_url,
-                    'title': video_title,
-                    'uploader': clip.get('channel_name', video_uploader_id),
-                    'uploader_id': video_uploader_id,
-                    'upload_date': video_date,
-                    'ext': video_extension,
-                })
-        return (len(response), info)
-
     def _handle_error(self, response):
         if not isinstance(response, dict):
             return
@@ -87,25 +47,21 @@ class JustinTVIE(InfoExtractor):
                 expected=True)
 
     def _download_json(self, url, video_id, note='Downloading JSON metadata'):
-        response = super(JustinTVIE, self)._download_json(url, video_id, note)
+        response = super(TwitchIE, self)._download_json(url, video_id, note)
         self._handle_error(response)
         return response
 
     def _extract_media(self, item, item_id):
-
         ITEMS = {
             'a': 'video',
             'c': 'chapter',
         }
-
         info = self._extract_info(self._download_json(
             '%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
             'Downloading %s info JSON' % ITEMS[item]))
-
         response = self._download_json(
             '%s/api/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
             'Downloading %s playlist JSON' % ITEMS[item])
-
         entries = []
         chunks = response['chunks']
         qualities = list(chunks.keys())
@@ -144,14 +100,7 @@ class JustinTVIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-
-        api_base = 'http://api.twitch.tv'
-        paged = False
-        if mobj.group('channelid'):
-            paged = True
-            video_id = mobj.group('channelid')
-            api = api_base + '/channel/archives/%s.json' % video_id
-        elif mobj.group('chapterid'):
+        if mobj.group('chapterid'):
             return self._extract_media('c', mobj.group('chapterid'))
 
             """
@@ -203,22 +152,25 @@ class JustinTVIE(InfoExtractor):
             }
             return info
             """
-        else:
+        elif mobj.group('videoid'):
             return self._extract_media('a', mobj.group('videoid'))
-
-        entries = []
-        offset = 0
-        limit = self._JUSTIN_PAGE_LIMIT
-        for counter in itertools.count(1):
-            page_url = api + ('?offset=%d&limit=%d' % (offset, limit))
-            page_count, page_info = self._parse_page(
-                page_url, video_id, counter)
-            entries.extend(page_info)
-            if not paged or page_count != limit:
-                break
-            offset += limit
-        return {
-            '_type': 'playlist',
-            'id': video_id,
-            'entries': entries,
-        }
+        elif mobj.group('channelid'):
+            channel_id = mobj.group('channelid')
+            info = self._download_json(
+                '%s/kraken/channels/%s' % (self._API_BASE, channel_id),
+                channel_id, 'Downloading channel info JSON')
+            channel_name = info.get('display_name') or info.get('name')
+            entries = []
+            offset = 0
+            limit = self._PAGE_LIMIT
+            for counter in itertools.count(1):
+                response = self._download_json(
+                    '%s/kraken/channels/%s/videos/?offset=%d&limit=%d'
+                    % (self._API_BASE, channel_id, offset, limit),
+                    channel_id, 'Downloading channel videos JSON page %d' % counter)
+                videos = response['videos']
+                if not videos:
+                    break
+                entries.extend([self.url_result(video['url'], 'Twitch') for video in videos])
+                offset += limit
+            return self.playlist_result(entries, channel_id, channel_name)

From 013bfdd84c09049cffff365fb81a60fdef51f125 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Oct 2014 23:46:53 +0700
Subject: [PATCH 0232/1937] [twitch] Update tests and minor improvements

---
 youtube_dl/extractor/twitch.py | 35 ++++++++++++++++++++++------------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 42d38b224..247fc2f64 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -24,18 +24,28 @@ class TwitchIE(InfoExtractor):
         """
     _PAGE_LIMIT = 100
     _API_BASE = 'https://api.twitch.tv'
-    _TEST = {
-        'url': 'http://www.twitch.tv/thegamedevhub/b/296128360',
-        'md5': 'ecaa8a790c22a40770901460af191c9a',
+    _TESTS = [{
+        'url': 'http://www.twitch.tv/riotgames/b/577357806',
         'info_dict': {
-            'id': '296128360',
-            'ext': 'flv',
-            'upload_date': '20110927',
-            'uploader_id': 25114803,
-            'uploader': 'thegamedevhub',
-            'title': 'Beginner Series - Scripting With Python Pt.1'
-        }
-    }
+            'id': 'a577357806',
+            'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
+        },
+        'playlist_mincount': 12,
+    }, {
+        'url': 'http://www.twitch.tv/acracingleague/c/5285812',
+        'info_dict': {
+            'id': 'c5285812',
+            'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
+        },
+        'playlist_mincount': 3,
+    }, {
+        'url': 'http://www.twitch.tv/vanillatv',
+        'info_dict': {
+            'id': 'vanillatv',
+            'title': 'VanillaTV',
+        },
+        'playlist_mincount': 412,
+    }]
 
     def _handle_error(self, response):
         if not isinstance(response, dict):
@@ -80,10 +90,11 @@ class TwitchIE(InfoExtractor):
                 formats.append(fmt)
             self._sort_formats(formats)
             entry = dict(info)
+            entry['id'] = '%s_%d' % (entry['id'], num),
             entry['title'] = '%s part %d' % (entry['title'], num)
             entry['formats'] = formats
             entries.append(entry)
-        return entries
+        return self.playlist_result(entries, info['id'], info['title'])
 
     def _extract_info(self, info):
         return {

From 9471c4440599b02bf648091560097eaab690db50 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 18 Oct 2014 01:52:55 +0300
Subject: [PATCH 0233/1937] [generic] Make sure Wistia embed URLs contain the
 protocol (Closes #3977)

Also, improve detection (Addresses #3662)
---
 youtube_dl/extractor/generic.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 158dc95a5..9057a6beb 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -654,15 +654,17 @@ class GenericIE(InfoExtractor):
         match = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
         if match:
+            embed_url = self._proto_relative_url(
+                unescapeHTML(match.group('url')))
             return {
                 '_type': 'url_transparent',
-                'url': unescapeHTML(match.group('url')),
+                'url': embed_url,
                 'ie_key': 'Wistia',
                 'uploader': video_uploader,
                 'title': video_title,
                 'id': video_id,
             }
-        match = re.search(r'(?:id=["\']wistia_|data-wistiaid=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
+        match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
         if match:
             return {
                 '_type': 'url_transparent',

From f9befee1f5e96e8b9b7032f33e70290ffd7769f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 18 Oct 2014 18:14:49 +0700
Subject: [PATCH 0234/1937] [arte.tv:+7] Append media type to format_id (Closes
 #3967)

---
 youtube_dl/extractor/arte.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index c3d02f85e..3a34d1ecc 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -160,6 +160,9 @@ class ArteTVPlus7IE(InfoExtractor):
                 format_id = '%s-%s' % (quality, format_info['versionCode'])
             else:
                 format_id = quality
+            media_type = format_info.get('mediaType')
+            if media_type is not None:
+                format_id += '-%s' % media_type
             info = {
                 'format_id': format_id,
                 'format_note': format_info.get('versionLibelle'),

From 159444a6688172696185404cbfea02945982e968 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sat, 18 Oct 2014 21:57:24 +0700
Subject: [PATCH 0235/1937] [twitch] Remove superfluous comma

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 247fc2f64..36aa1ad6e 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -90,7 +90,7 @@ class TwitchIE(InfoExtractor):
                 formats.append(fmt)
             self._sort_formats(formats)
             entry = dict(info)
-            entry['id'] = '%s_%d' % (entry['id'], num),
+            entry['id'] = '%s_%d' % (entry['id'], num)
             entry['title'] = '%s part %d' % (entry['title'], num)
             entry['formats'] = formats
             entries.append(entry)

From 95fa5fb569c9a29d1dfb7da744022373d279fcb7 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 19 Oct 2014 00:48:05 +0700
Subject: [PATCH 0236/1937] [sexykarma] Improve and simplify

---
 youtube_dl/extractor/sexykarma.py | 99 ++++++++++++++++++++-----------
 1 file changed, 64 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/sexykarma.py b/youtube_dl/extractor/sexykarma.py
index 629499e72..4a6345758 100644
--- a/youtube_dl/extractor/sexykarma.py
+++ b/youtube_dl/extractor/sexykarma.py
@@ -1,77 +1,106 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
 import re
-import datetime
+
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate,
+    parse_duration,
+    int_or_none,
+)
+
 
 class SexyKarmaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?sexykarma\.com/gonewild/video/.+\-(?P<id>[a-zA-Z0-9\-]+)(.html)'
+    _VALID_URL = r'https?://(?:www\.)?sexykarma\.com/gonewild/video/(?P<display_id>[^/]+)-(?P<id>[a-zA-Z0-9]+)\.html'
     _TESTS = [{
         'url': 'http://www.sexykarma.com/gonewild/video/taking-a-quick-pee-yHI70cOyIHt.html',
         'md5': 'b9798e7d1ef1765116a8f516c8091dbd',
         'info_dict': {
             'id': 'yHI70cOyIHt',
+            'display_id': 'taking-a-quick-pee',
             'ext': 'mp4',
             'title': 'Taking a quick pee.',
-            'uploader_id': 'wildginger7',
+            'description': '',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'duration': int,
-            'view_count': int,
+            'uploader': 'wildginger7',
             'upload_date': '20141007',
+            'duration': 81,
+            'view_count': int,
+            'comment_count': int,
+            'categories': list,
         }
     }, {
         'url': 'http://www.sexykarma.com/gonewild/video/pot-pixie-tribute-8Id6EZPbuHf.html',
         'md5': 'dd216c68d29b49b12842b9babe762a5d',
         'info_dict': {
             'id': '8Id6EZPbuHf',
+            'display_id': 'pot-pixie-tribute',
             'ext': 'mp4',
             'title': 'pot_pixie tribute',
-            'uploader_id': 'banffite',
+            'description': 'tribute',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'duration': int,
-            'view_count': int,
+            'uploader': 'banffite',
             'upload_date': '20141013',
+            'duration': 16,
+            'view_count': int,
+            'comment_count': int,
+            'categories': list,
         }
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        webpage = self._download_webpage(url, video_id)
-              
-        title = self._html_search_regex(r'<h2 class="he2"><span>(.*?)</span>', webpage, 'title')
-        uploader_id = self._html_search_regex(r'class="aupa">\n*(.*?)</a>', webpage, 'uploader')
-        url = self._html_search_regex(r'<p><a href="(.*?)" ?\n*target="_blank"><font color', webpage, 'url')
-        thumbnail = self._html_search_regex(r'<div id="player" style="z-index:1;"> <span id="edge"></span> <span id="container"><img[\n ]*src="(.+?)"', webpage, 'thumbnail')
-        
-        str_duration = self._html_search_regex(r'<tr>[\n\s]*<td>Time: </td>[\n\s]*<td align="right"><span>(.+)\n*', webpage, 'duration')
-        duration = self._to_seconds(str_duration)
+        webpage = self._download_webpage(url, display_id)
 
-        str_views = self._html_search_regex(r'<tr>[\n\s]*<td>Views: </td>[\n\s]*<td align="right"><span>(.+)</span>', webpage, 'view_count')
-        view_count = int(str_views)
-        # print view_count
+        video_url = self._html_search_regex(
+            r'<p>Save this video to your computer: </p><p><a href="([^"]+)"',
+            webpage, 'url')
 
-        date = self._html_search_regex(r'class="aup">Added: <strong>(.*?)</strong>', webpage, 'date')
-        d = datetime.datetime.strptime(date, '%B %d, %Y')
-        upload_date = d.strftime('%Y%m%d')
+        title = self._html_search_regex(
+            r'<h2 class="he2"><span>(.*?)</span>',
+            webpage, 'title')
+        description = self._html_search_meta(
+            'description', webpage, 'description', fatal=False, default='')
+        thumbnail = self._html_search_regex(
+            r'<span id="container"><img\s+src="([^"]+)"',
+            webpage, 'thumbnail', fatal=False)
 
-        categories = re.findall(r'http://www.sexykarma.com/gonewild/search/video/(?:.+?)"><span>(.*?)</span>', webpage)
+        uploader = self._html_search_regex(
+            r'class="aupa">\s*(.*?)</a>',
+            webpage, 'uploader')
+        upload_date = unified_strdate(self._html_search_regex(
+            r'Added: <strong>(.+?)</strong>', webpage, 'upload date', fatal=False))
+
+        duration = parse_duration(self._search_regex(
+            r'<td>Time:\s*</td>\s*<td align="right"><span>\s*(.+?)\s*</span>',
+            webpage, 'duration', fatal=False))
+
+        view_count = int_or_none(self._search_regex(
+            r'<td>Views:\s*</td>\s*<td align="right"><span>\s*(\d+)\s*</span>',
+            webpage, 'view count', fatal=False))
+        comment_count = int_or_none(self._search_regex(
+            r'<td>Comments:\s*</td>\s*<td align="right"><span>\s*(\d+)\s*</span>',
+            webpage, 'comment count', fatal=False))
+
+        categories = self._html_search_meta(
+            'keywords', webpage, 'categories',
+            fatal=False, default='').split(',')
 
         return {
             'id': video_id,
+            'display_id': display_id,
+            'url': video_url,
             'title': title,
-            'uploader_id': uploader_id,
-            'url': url,
+            'description': description,
             'thumbnail': thumbnail,
+            'uploader': uploader,
+            'upload_date': upload_date,
             'duration': duration,
             'view_count': view_count,
-            'upload_date': upload_date,
+            'comment_count': comment_count,
             'categories': categories,
         }
-
-    def _to_seconds(self, timestr):
-        seconds= 0
-        for part in timestr.split(':'):
-            seconds= seconds*60 + int(part)
-        return seconds

From 0f175a932fbc44b404d464e5485460054c697bb0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 18 Oct 2014 20:22:23 +0200
Subject: [PATCH 0237/1937] release 2014.10.18

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f17c86405..e7f6adef1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.10.15'
+__version__ = '2014.10.18'

From 4510d14f0ad7e1a7c108bc55526d8b4704420347 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 19 Oct 2014 01:35:53 +0700
Subject: [PATCH 0238/1937] [twitch] Update tests

---
 test/test_all_urls.py | 24 ++++++++++--------------
 1 file changed, 10 insertions(+), 14 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index 84b05da39..965e5d8a5 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -14,7 +14,7 @@ from test.helper import gettestcases
 from youtube_dl.extractor import (
     FacebookIE,
     gen_extractors,
-    JustinTVIE,
+    TwitchIE,
     YoutubeIE,
 )
 
@@ -72,21 +72,17 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertMatch('http://www.youtube.com/results?search_query=making+mustard', ['youtube:search_url'])
         self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url'])
 
-    def test_justin_tv_channelid_matching(self):
-        self.assertTrue(JustinTVIE.suitable('justin.tv/vanillatv'))
-        self.assertTrue(JustinTVIE.suitable('twitch.tv/vanillatv'))
-        self.assertTrue(JustinTVIE.suitable('www.justin.tv/vanillatv'))
-        self.assertTrue(JustinTVIE.suitable('www.twitch.tv/vanillatv'))
-        self.assertTrue(JustinTVIE.suitable('http://www.justin.tv/vanillatv'))
-        self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/vanillatv'))
-        self.assertTrue(JustinTVIE.suitable('http://www.justin.tv/vanillatv/'))
-        self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/vanillatv/'))
+    def test_twitch_channelid_matching(self):
+        self.assertTrue(TwitchIE.suitable('twitch.tv/vanillatv'))
+        self.assertTrue(TwitchIE.suitable('www.twitch.tv/vanillatv'))
+        self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv'))
+        self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv/'))
 
-    def test_justintv_videoid_matching(self):
-        self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/vanillatv/b/328087483'))
+    def test_twitch_videoid_matching(self):
+        self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv/b/328087483'))
 
-    def test_justin_tv_chapterid_matching(self):
-        self.assertTrue(JustinTVIE.suitable('http://www.twitch.tv/tsm_theoddone/c/2349361'))
+    def test_twitch_chapterid_matching(self):
+        self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/tsm_theoddone/c/2349361'))
 
     def test_youtube_extract(self):
         assertExtractId = lambda url, id: self.assertEqual(YoutubeIE.extract_id(url), id)

From 16efb3695f4e3a07e492a4206cb2901c05ac7c65 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 19 Oct 2014 01:53:15 +0700
Subject: [PATCH 0239/1937] [sexykarma] Add support for watchindianporn

---
 youtube_dl/extractor/sexykarma.py | 35 ++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/sexykarma.py b/youtube_dl/extractor/sexykarma.py
index 4a6345758..c833fc8ee 100644
--- a/youtube_dl/extractor/sexykarma.py
+++ b/youtube_dl/extractor/sexykarma.py
@@ -12,7 +12,8 @@ from ..utils import (
 
 
 class SexyKarmaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?sexykarma\.com/gonewild/video/(?P<display_id>[^/]+)-(?P<id>[a-zA-Z0-9]+)\.html'
+    IE_DESC = 'Sexy Karma and Watch Indian Porn'
+    _VALID_URL = r'https?://(?:www\.)?(?:sexykarma\.com|watchindianporn\.net)/(?:[^/]+/)*video/(?P<display_id>[^/]+)-(?P<id>[a-zA-Z0-9]+)\.html'
     _TESTS = [{
         'url': 'http://www.sexykarma.com/gonewild/video/taking-a-quick-pee-yHI70cOyIHt.html',
         'md5': 'b9798e7d1ef1765116a8f516c8091dbd',
@@ -21,11 +22,10 @@ class SexyKarmaIE(InfoExtractor):
             'display_id': 'taking-a-quick-pee',
             'ext': 'mp4',
             'title': 'Taking a quick pee.',
-            'description': '',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': 'wildginger7',
             'upload_date': '20141007',
-            'duration': 81,
+            'duration': 22,
             'view_count': int,
             'comment_count': int,
             'categories': list,
@@ -38,7 +38,6 @@ class SexyKarmaIE(InfoExtractor):
             'display_id': 'pot-pixie-tribute',
             'ext': 'mp4',
             'title': 'pot_pixie tribute',
-            'description': 'tribute',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': 'banffite',
             'upload_date': '20141013',
@@ -47,6 +46,22 @@ class SexyKarmaIE(InfoExtractor):
             'comment_count': int,
             'categories': list,
         }
+    }, {
+        'url': 'http://www.watchindianporn.net/video/desi-dancer-namrata-stripping-completely-nude-and-dancing-on-a-hot-number-dW2mtctxJfs.html',
+        'md5': '9afb80675550406ed9a63ac2819ef69d',
+        'info_dict': {
+            'id': 'dW2mtctxJfs',
+            'display_id': 'desi-dancer-namrata-stripping-completely-nude-and-dancing-on-a-hot-number',
+            'ext': 'mp4',
+            'title': 'Desi dancer namrata stripping completely nude and dancing on a hot number',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'Don',
+            'upload_date': '20140213',
+            'duration': 83,
+            'view_count': int,
+            'comment_count': int,
+            'categories': list,
+        }
     }]
 
     def _real_extract(self, url):
@@ -57,14 +72,11 @@ class SexyKarmaIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         video_url = self._html_search_regex(
-            r'<p>Save this video to your computer: </p><p><a href="([^"]+)"',
-            webpage, 'url')
+            r"url: escape\('([^']+)'\)", webpage, 'url')
 
         title = self._html_search_regex(
             r'<h2 class="he2"><span>(.*?)</span>',
             webpage, 'title')
-        description = self._html_search_meta(
-            'description', webpage, 'description', fatal=False, default='')
         thumbnail = self._html_search_regex(
             r'<span id="container"><img\s+src="([^"]+)"',
             webpage, 'thumbnail', fatal=False)
@@ -86,16 +98,15 @@ class SexyKarmaIE(InfoExtractor):
             r'<td>Comments:\s*</td>\s*<td align="right"><span>\s*(\d+)\s*</span>',
             webpage, 'comment count', fatal=False))
 
-        categories = self._html_search_meta(
-            'keywords', webpage, 'categories',
-            fatal=False, default='').split(',')
+        categories = re.findall(
+            r'<a href="[^"]+/search/video/desi"><span>([^<]+)</span></a>',
+            webpage)
 
         return {
             'id': video_id,
             'display_id': display_id,
             'url': video_url,
             'title': title,
-            'description': description,
             'thumbnail': thumbnail,
             'uploader': uploader,
             'upload_date': upload_date,

From ce519b747e1e404f6d15ae68e1b1607a27beff1c Mon Sep 17 00:00:00 2001
From: scheiba <dennis.scheiba@gmx.de>
Date: Sat, 18 Oct 2014 22:15:47 +0200
Subject: [PATCH 0240/1937] added "bild.de" as extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/bild.py     | 46 ++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 youtube_dl/extractor/bild.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 070f9ff19..e0957987c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -26,6 +26,7 @@ from .bandcamp import BandcampIE, BandcampAlbumIE
 from .bbccouk import BBCCoUkIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
+from .bild import BildIE
 from .bilibili import BiliBiliIE
 from .blinkx import BlinkxIE
 from .bliptv import BlipTVIE, BlipTVUserIE
diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py
new file mode 100644
index 000000000..3a822a5c0
--- /dev/null
+++ b/youtube_dl/extractor/bild.py
@@ -0,0 +1,46 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class BildIE(InfoExtractor):
+    IE_NAME = 'bild'
+    _TEST = {
+        'url': 'http://www.bild.de/video/clip/apple-ipad-air/das-koennen-die-neuen-ipads-38184146.bild.html',
+        'info_dict': {
+            'id': '38184146',
+            'title': 'BILD hat sie getestet',
+            'thumbnail': 'http://bilder.bild.de/fotos/stand-das-koennen-die-neuen-ipads-38184138/Bild/1.bild.jpg',
+            'duration': 196,
+        }
+    }
+    
+    #http://www.bild.de/video/clip/apple-ipad-air/das-koennen-die-neuen-ipads-38184146.bild.html
+    _VALID_URL = r'http?://(?:www\.)?bild\.de/(?:[^/]+/)+(?P<display_id>[^/]+)-(?P<id>\d+)(?:,auto=true)?\.bild\.html'
+    
+    def _real_extract(self, url):
+        m = re.match(self._VALID_URL, url)
+        video_id = m.group('id')
+        
+        #webpage = self._download_webpage(url, video_id)
+        
+        xml_url = url.split(".bild.html")[0]+",view=xml.bild.xml"
+        
+        doc = self._download_xml(xml_url, video_id)
+        
+        video_url = doc.attrib['src']
+        title = doc.attrib['ueberschrift']
+        description = doc.attrib['text']
+        thumbnail = doc.attrib['img']
+        duration = int(doc.attrib['duration'])/1000
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'url': video_url,
+            'thumbnail': thumbnail,
+            'duration': duration,
+        }

From cc98a3f09651423f4405933265c6dcf2ae51011f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 19 Oct 2014 15:14:37 +0200
Subject: [PATCH 0241/1937] [cnn] Fix for urls ending in '.cnn-ap' (fixes
 #3985)

---
 youtube_dl/extractor/cnn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index dae40c136..78877b1cf 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class CNNIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://((edition|www)\.)?cnn\.com/video/(data/.+?|\?)/
-        (?P<path>.+?/(?P<title>[^/]+?)(?:\.cnn|(?=&)))'''
+        (?P<path>.+?/(?P<title>[^/]+?)(?:\.cnn(-ap)?|(?=&)))'''
 
     _TESTS = [{
         'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',

From 8230018c20595a22e636b834ebb522a6a85d0d8b Mon Sep 17 00:00:00 2001
From: Gabriel Schubiner <gabeos@cs.washington.edu>
Date: Sun, 19 Oct 2014 22:47:05 -0700
Subject: [PATCH 0242/1937] Added extractor for crunchyroll 'playlists' i.e.
 series. so that one can, e.g. download all episodes of a series

---
 youtube_dl/extractor/__init__.py    |  5 ++++-
 youtube_dl/extractor/crunchyroll.py | 35 +++++++++++++++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 070f9ff19..0dd763006 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -60,7 +60,10 @@ from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
 from .criterion import CriterionIE
-from .crunchyroll import CrunchyrollIE
+from .crunchyroll import (
+    CrunchyrollIE,
+    CrunchyrollShowPlaylistIE
+)
 from .cspan import CSpanIE
 from .d8 import D8IE
 from .dailymotion import (
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index f99888ecc..414c46b0d 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -24,6 +24,7 @@ from ..aes import (
     aes_cbc_decrypt,
     inc,
 )
+from .common import InfoExtractor
 
 
 class CrunchyrollIE(SubtitlesInfoExtractor):
@@ -285,3 +286,37 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             'subtitles':   subtitles,
             'formats':     formats,
         }
+
+
+class CrunchyrollShowPlaylistIE(InfoExtractor):
+    IE_NAME = "crunchyroll:playlist"
+    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<show>[\w\-]+))/?$'
+    _TITLE_EXTR = r'<span\s+itemprop="name">\s*(?P<showtitle>[\w\s]+)'
+
+    _TESTS = [{
+        'url' : 'http://www.crunchyroll.com/attack-on-titan',
+        'info_dict' : {
+            'title' : 'Attack on Titan'
+        },
+        'playlist_count' : 15
+    }]
+
+    def _extract_title_entries(self,id,webpage):
+        _EPISODE_ID_EXTR = r'id="showview_videos_media_(?P<vidid>\d+)".*?href="/{0}/(?P<vidurl>[\w\-]+-(?P=vidid))"'.format(id)
+        title = self._html_search_regex(self._TITLE_EXTR,webpage,"title",flags=re.UNICODE|re.MULTILINE)
+        episode_urls = [self.url_result('http://www.crunchyroll.com/{0}/{1}'.format(id, showmatch[1])) for
+                    showmatch in re.findall(_EPISODE_ID_EXTR, webpage,re.UNICODE|re.MULTILINE|re.DOTALL)]
+        return title, episode_urls
+
+
+    def _real_extract(self, url):
+        url_match = re.match(self._VALID_URL,url)
+        show_id = url_match.group('show')
+        webpage = self._download_webpage(url,show_id)
+        (title,entries) = self._extract_title_entries(show_id,webpage)
+        return {
+            '_type' : 'playlist',
+            'id' : show_id,
+            'title' : title,
+            'entries' : entries
+        }
\ No newline at end of file

From d24a2b20b4908b01f9b5bfa88cd293c189fe6475 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 20 Oct 2014 20:27:59 +0700
Subject: [PATCH 0243/1937] [arte.tv:+7] Use original format ids to avoid
 duplicates

---
 youtube_dl/extractor/arte.py | 30 +++++++++++-------------------
 1 file changed, 11 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 3a34d1ecc..a42ebff8e 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -12,6 +12,7 @@ from ..utils import (
     get_element_by_id,
     compat_str,
     get_element_by_attribute,
+    int_or_none,
 )
 
 # There are different sources of video in arte.tv, the extraction process 
@@ -98,7 +99,11 @@ class ArteTVPlus7IE(InfoExtractor):
             'thumbnail': player_info.get('programImage') or player_info.get('VTU', {}).get('IUR'),
         }
 
-        all_formats = player_info['VSR'].values()
+        all_formats = []
+        for format_id, format_dict in player_info['VSR'].items():
+            fmt = dict(format_dict)
+            fmt['format_id'] = format_id
+            all_formats.append(fmt)
         # Some formats use the m3u8 protocol
         all_formats = list(filter(lambda f: f.get('videoFormat') != 'M3U8', all_formats))
         def _match_lang(f):
@@ -149,25 +154,12 @@ class ArteTVPlus7IE(InfoExtractor):
                 )
         formats = sorted(formats, key=sort_key)
         def _format(format_info):
-            quality = ''
-            height = format_info.get('height')
-            if height is not None:
-                quality = compat_str(height)
-            bitrate = format_info.get('bitrate')
-            if bitrate is not None:
-                quality += '-%d' % bitrate
-            if format_info.get('versionCode') is not None:
-                format_id = '%s-%s' % (quality, format_info['versionCode'])
-            else:
-                format_id = quality
-            media_type = format_info.get('mediaType')
-            if media_type is not None:
-                format_id += '-%s' % media_type
             info = {
-                'format_id': format_id,
-                'format_note': format_info.get('versionLibelle'),
-                'width': format_info.get('width'),
-                'height': height,
+                'format_id': format_info['format_id'],
+                'format_note': '%s, %s' % (format_info.get('versionCode'), format_info.get('versionLibelle')),
+                'width': int_or_none(format_info.get('width')),
+                'height': int_or_none(format_info.get('height')),
+                'tbr': int_or_none(format_info.get('bitrate')),
             }
             if format_info['mediaType'] == 'rtmp':
                 info['url'] = format_info['streamer']

From 4231235cdaa18d050f3119a80ac409138fb8e8bd Mon Sep 17 00:00:00 2001
From: h-collector <githcoll@gmail.com>
Date: Sun, 6 Jul 2014 01:42:41 +0200
Subject: [PATCH 0244/1937] Fix issues with fc2

Fix issues #2912 and #3171
---
 youtube_dl/extractor/fc2.py | 49 ++++++++++++++++++++++++++++++++++---
 1 file changed, 46 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index c663a0f81..ecfb233f9 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -7,14 +7,16 @@ import hashlib
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    compat_urllib_parse,
     compat_urllib_request,
     compat_urlparse,
 )
 
 
 class FC2IE(InfoExtractor):
-    _VALID_URL = r'^http://video\.fc2\.com/((?P<lang>[^/]+)/)?content/(?P<id>[^/]+)'
+    _VALID_URL = r'^http://video\.fc2\.com/((?P<lang>[^/]+)/)?(a/)?content/(?P<id>[^/]+)'
     IE_NAME = 'fc2'
+    _NETRC_MACHINE = 'fc2'
     _TEST = {
         'url': 'http://video.fc2.com/en/content/20121103kUan1KHs',
         'md5': 'a6ebe8ebe0396518689d963774a54eb7',
@@ -25,17 +27,53 @@ class FC2IE(InfoExtractor):
         },
     }
 
+    #def _real_initialize(self):
+    #    self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if (username is None) or (password is None):
+           self._downloader.report_warning('unable to log in: will be downloading in non authorized mode')
+           return False
+
+        # Log in
+        login_form_strs = {
+            'email':    username,
+            'password': password,
+            'done':     'video',
+            'Submit':   ' Login ',
+        }
+
+        # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
+        # chokes on unicode
+        login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in login_form_strs.items())
+        login_data = compat_urllib_parse.urlencode(login_form).encode('utf-8')
+        request    = compat_urllib_request.Request(
+            'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
+
+        login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
+        if 'mode=redirect&login=done' not in login_results:
+            self._downloader.report_warning('unable to log in: bad username or password')
+            return False
+        
+        # this is also needed
+        login_redir = compat_urllib_request.Request('http://id.fc2.com/?mode=redirect&login=done')
+        redir_res   = self._download_webpage(login_redir, None, note='Login redirect', errnote='Something is not right')
+
+        return True
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
         webpage = self._download_webpage(url, video_id)
         self._downloader.cookiejar.clear_session_cookies()  # must clear
+        self._login()
 
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
-        refer = url.replace('/content/', '/a/content/')
 
+        refer = (url if '/a/content/' in url else url.replace('/content/', '/a/content/'));
         mimi = hashlib.md5((video_id + '_gGddgPfeaf_gzyr').encode('utf-8')).hexdigest()
 
         info_url = (
@@ -47,7 +85,12 @@ class FC2IE(InfoExtractor):
         info = compat_urlparse.parse_qs(info_webpage)
 
         if 'err_code' in info:
-            raise ExtractorError('Error code: %s' % info['err_code'][0])
+            #raise ExtractorError('Error code: %s' % info['err_code'][0])
+            # most of the time we can still download wideo even if err_code is 403 or 602
+            print 'Error code was: %s... but still trying' % info['err_code'][0]
+            
+        if 'filepath' not in info:
+            raise ExtractorError('No file path for download. Maybe not logged?')
 
         video_url = info['filepath'][0] + '?mid=' + info['mid'][0]
         title_info = info.get('title')

From 40b1cbafacea338e51d43fa78438eaf21a1bcbcd Mon Sep 17 00:00:00 2001
From: h-collector <githcoll@gmail.com>
Date: Sun, 6 Jul 2014 01:48:07 +0200
Subject: [PATCH 0245/1937] Update fc2.py

---
 youtube_dl/extractor/fc2.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index ecfb233f9..0933485e0 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -87,10 +87,10 @@ class FC2IE(InfoExtractor):
         if 'err_code' in info:
             #raise ExtractorError('Error code: %s' % info['err_code'][0])
             # most of the time we can still download wideo even if err_code is 403 or 602
-            print 'Error code was: %s... but still trying' % info['err_code'][0]
-            
+            self._downloader.report_warning('Error code was: %s... but still trying' % info['err_code'][0])
+
         if 'filepath' not in info:
-            raise ExtractorError('No file path for download. Maybe not logged?')
+            raise ExtractorError('Cannot download file. Are you logged?')
 
         video_url = info['filepath'][0] + '?mid=' + info['mid'][0]
         title_info = info.get('title')

From 5a000b45b339f6516f2a5a3bdfd2869713e8438a Mon Sep 17 00:00:00 2001
From: h-collector <githcoll@gmail.com>
Date: Mon, 7 Jul 2014 22:47:28 +0200
Subject: [PATCH 0246/1937] Don't use report_warning for reporting warnings

In tests warning is converted to error
---
 youtube_dl/extractor/fc2.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 0933485e0..3073ab0d4 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -33,7 +33,7 @@ class FC2IE(InfoExtractor):
     def _login(self):
         (username, password) = self._get_login_info()
         if (username is None) or (password is None):
-           self._downloader.report_warning('unable to log in: will be downloading in non authorized mode')
+           self.to_screen('unable to log in: will be downloading in non authorized mode') # report_warning
            return False
 
         # Log in
@@ -53,7 +53,7 @@ class FC2IE(InfoExtractor):
 
         login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
         if 'mode=redirect&login=done' not in login_results:
-            self._downloader.report_warning('unable to log in: bad username or password')
+            self.to_screen('unable to log in: bad username or password') # report_warning
             return False
         
         # this is also needed
@@ -87,7 +87,7 @@ class FC2IE(InfoExtractor):
         if 'err_code' in info:
             #raise ExtractorError('Error code: %s' % info['err_code'][0])
             # most of the time we can still download wideo even if err_code is 403 or 602
-            self._downloader.report_warning('Error code was: %s... but still trying' % info['err_code'][0])
+            self.to_screen('Error code was: %s... but still trying' % info['err_code'][0]) # report_warning
 
         if 'filepath' not in info:
             raise ExtractorError('Cannot download file. Are you logged?')

From 1b10a011ec7544f49159ed60642128720333b8aa Mon Sep 17 00:00:00 2001
From: Gabriel Schubiner <gabeos@cs.washington.edu>
Date: Mon, 20 Oct 2014 18:38:42 -0700
Subject: [PATCH 0247/1937] Forgot to reverse extracted video urls so they are
 in correct order for video selection args

---
 youtube_dl/extractor/crunchyroll.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 414c46b0d..9ac86c2be 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -306,6 +306,7 @@ class CrunchyrollShowPlaylistIE(InfoExtractor):
         title = self._html_search_regex(self._TITLE_EXTR,webpage,"title",flags=re.UNICODE|re.MULTILINE)
         episode_urls = [self.url_result('http://www.crunchyroll.com/{0}/{1}'.format(id, showmatch[1])) for
                     showmatch in re.findall(_EPISODE_ID_EXTR, webpage,re.UNICODE|re.MULTILINE|re.DOTALL)]
+        episode_urls.reverse()
         return title, episode_urls
 
 

From 74214d35c592976ab48b3c89577df586705059e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Oct 2014 20:08:20 +0700
Subject: [PATCH 0248/1937] [arte.tv:+7] Improve title extraction (Closes
 #3995)

---
 youtube_dl/extractor/arte.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index a42ebff8e..844a8f563 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -91,9 +91,14 @@ class ArteTVPlus7IE(InfoExtractor):
         if not upload_date_str:
             upload_date_str = player_info.get('VDA', '').split(' ')[0]
 
+        title = player_info['VTI'].strip()
+        subtitle = player_info.get('VSU', '').strip()
+        if subtitle:
+            title += ' - %s' % subtitle
+
         info_dict = {
             'id': player_info['VID'],
-            'title': player_info['VTI'],
+            'title': title,
             'description': player_info.get('VDE'),
             'upload_date': unified_strdate(upload_date_str),
             'thumbnail': player_info.get('programImage') or player_info.get('VTU', {}).get('IUR'),

From 40bca5f92780586f0de085f718bb2aa9ee94b6fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 21 Oct 2014 21:06:24 +0700
Subject: [PATCH 0249/1937] [arte.tv] Remove unused import

---
 youtube_dl/extractor/arte.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 844a8f563..b9a9440c0 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -10,7 +10,6 @@ from ..utils import (
     unified_strdate,
     determine_ext,
     get_element_by_id,
-    compat_str,
     get_element_by_attribute,
     int_or_none,
 )

From 6bf6962062ff674013d8651e190998b506c681e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 22 Oct 2014 01:24:04 +0700
Subject: [PATCH 0250/1937] [francetv] Force m3u8 formats extension to mp4
 (Closes #3997)

---
 youtube_dl/extractor/francetv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 0b3374d97..07165e330 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -46,7 +46,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
                         f4m_format['preference'] = 1
                     formats.extend(f4m_formats)
             elif video_url.endswith('.m3u8'):
-                formats.extend(self._extract_m3u8_formats(video_url, video_id))
+                formats.extend(self._extract_m3u8_formats(video_url, video_id, 'mp4'))
             elif video_url.startswith('rtmp'):
                 formats.append({
                     'url': video_url,

From bfd91588f3fab0d5a14726810c186e5ec2fdd0f4 Mon Sep 17 00:00:00 2001
From: Ole Ernst <olebowle@gmx.com>
Date: Wed, 22 Oct 2014 14:24:53 +0200
Subject: [PATCH 0251/1937] [ard] make rss match more universal

---
 youtube_dl/extractor/ard.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index cd9c1d9be..630b1faa9 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -56,7 +56,7 @@ class ARDMediathekIE(InfoExtractor):
         if '>Der gewünschte Beitrag ist nicht mehr verfügbar.<' in webpage:
             raise ExtractorError('Video %s is no longer available' % video_id, expected=True)
 
-        if re.search(r'rss=true', url):
+        if re.search(r'[\?&]rss($|[=&])', url):
             doc = parse_xml(webpage)
             if doc.tag == 'rss':
                 return GenericIE()._extract_rss(url, video_id, doc)

From 6b445558ffe292c11327862ab510b1abf4e36616 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 23 Oct 2014 00:44:21 +0200
Subject: [PATCH 0252/1937] [youtube] Only set language when necessary

We only need to set the language when we are using a login; otherwise it will always be English.
---
 youtube_dl/extractor/youtube.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index cfae2de89..4ab56e0ac 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -191,8 +191,9 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     def _real_initialize(self):
         if self._downloader is None:
             return
-        if not self._set_language():
-            return
+        if self._get_login_info()[0] is not None:
+            if not self._set_language():
+                return
         if not self._login():
             return
         self._confirm_age()

From 9bbec55255bdf7189f782e42c1acaeb596eb9bf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Oct 2014 20:41:45 +0700
Subject: [PATCH 0253/1937] [pbs] Add support for tabbed frontline videos
 (Closes #4006)

---
 youtube_dl/extractor/pbs.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 8f140d626..6118ed5c2 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -80,8 +80,14 @@ class PBSIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.jpg$',
                 'upload_date': '20140122',
             }
+        },
+        {
+            'url': 'http://www.pbs.org/wgbh/pages/frontline/united-states-of-secrets/',
+            'info_dict': {
+                'id': 'united-states-of-secrets',
+            },
+            'playlist_count': 2,
         }
-
     ]
 
     def _extract_webpage(self, url):
@@ -96,6 +102,12 @@ class PBSIE(InfoExtractor):
                 r'<input type="hidden" id="air_date_[0-9]+" value="([^"]+)"',
                 webpage, 'upload date', default=None))
 
+            # tabbed frontline videos
+            tabbed_videos = re.findall(
+                r'<div[^>]+class="videotab[^"]*"[^>]+vid="(\d+)"', webpage)
+            if tabbed_videos:
+                return tabbed_videos, presumptive_id, upload_date
+
             MEDIA_ID_REGEXES = [
                 r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'",  # frontline video embed
                 r'class="coveplayerid">([^<]+)<',                       # coveplayer
@@ -130,6 +142,12 @@ class PBSIE(InfoExtractor):
     def _real_extract(self, url):
         video_id, display_id, upload_date = self._extract_webpage(url)
 
+        if isinstance(video_id, list):
+            entries = [self.url_result(
+                'http://video.pbs.org/video/%s' % vid_id, 'PBS', vid_id)
+                for vid_id in video_id]
+            return self.playlist_result(entries, display_id)
+
         info_url = 'http://video.pbs.org/videoInfo/%s?format=json' % video_id
         info = self._download_json(info_url, display_id)
 

From 8f3b5397a761d68122bc1bd66d049fbbe31289a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Oct 2014 21:35:51 +0700
Subject: [PATCH 0254/1937] [cinemassacre] Fix extraction (Closes #4008)

---
 youtube_dl/extractor/cinemassacre.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index 496271be4..d064a28f9 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -42,7 +42,7 @@ class CinemassacreIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
         video_date = mobj.group('date_Y') + mobj.group('date_m') + mobj.group('date_d')
-        mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?id=(?:Cinemassacre-)?(?P<video_id>.+?))"', webpage)
+        mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?:Cinemassacre-)?(?P<video_id>.+?))"', webpage)
         if not mobj:
             raise ExtractorError('Can\'t extract embed url and video id')
         playerdata_url = mobj.group('embed_url')
@@ -53,17 +53,22 @@ class CinemassacreIE(InfoExtractor):
         video_description = self._html_search_regex(
             r'<div class="entry-content">(?P<description>.+?)</div>',
             webpage, 'description', flags=re.DOTALL, fatal=False)
+        video_thumbnail = self._og_search_thumbnail(webpage)
 
         playerdata = self._download_webpage(playerdata_url, video_id, 'Downloading player webpage')
-        video_thumbnail = self._search_regex(
-            r'image: \'(?P<thumbnail>[^\']+)\'', playerdata, 'thumbnail', fatal=False)
-        sd_url = self._search_regex(r'file: \'([^\']+)\', label: \'SD\'', playerdata, 'sd_file')
-        videolist_url = self._search_regex(r'file: \'([^\']+\.smil)\'}', playerdata, 'videolist_url')
 
+        vidurl = self._search_regex(
+            r'\'vidurl\'\s*:\s*"([^\']+)"', playerdata, 'vidurl').replace('\\/', '/')
+        vidid = self._search_regex(
+            r'\'vidid\'\s*:\s*"([^\']+)"', playerdata, 'vidid')
+        videoserver = self._html_search_regex(
+            r"'videoserver'\s*:\s*'([^']+)'", playerdata, 'videoserver')
+
+        videolist_url = 'http://%s/vod/smil:%s.smil/jwplayer.smil' % (videoserver, vidid)
         videolist = self._download_xml(videolist_url, video_id, 'Downloading videolist XML')
 
         formats = []
-        baseurl = sd_url[:sd_url.rfind('/')+1]
+        baseurl = vidurl[:vidurl.rfind('/')+1]
         for video in videolist.findall('.//video'):
             src = video.get('src')
             if not src:

From 52cffcb186432f9f1b30fba159414aff6a923f59 Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Thu, 23 Oct 2014 20:27:13 +0530
Subject: [PATCH 0255/1937] [thoughtworks] wistia support added

---
 youtube_dl/extractor/generic.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9057a6beb..877447381 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -380,6 +380,16 @@ class GenericIE(InfoExtractor):
                 'uploader': 'education-portal.com',
             },
         },
+        {
+            'url': 'http://thoughtworks.wistia.com/medias/uxjb0lwrcz',
+            'md5': 'baf49c2baa8a7de5f3fc145a8506dcd4',
+            'info_dict': {
+                'id': 'uxjb0lwrcz',
+                'ext': 'mp4',
+                'title': 'Conversation about Hexagonal Rails Part 1',
+                'duration': 1715.0,
+            },   
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -651,6 +661,10 @@ class GenericIE(InfoExtractor):
                     playlists, lambda p: '//dailymotion.com/playlist/%s' % p)
 
         # Look for embedded Wistia player
+        mobj = re.search(r'<meta content=(["\'])(?P<url>https?://fast\.wistia\.net/embed/iframe/(?P<id>[a-zA-Z0-9_]+)(["\']))', webpage)
+        if mobj:
+            return self.url_result(mobj.group('url'), 'Wistia')
+        
         match = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
         if match:
@@ -664,6 +678,7 @@ class GenericIE(InfoExtractor):
                 'title': video_title,
                 'id': video_id,
             }
+            
         match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
         if match:
             return {

From 2399535fd1f2ef3125fcf6c1eabf76de53b9b8ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Oct 2014 22:30:34 +0700
Subject: [PATCH 0256/1937] [francetv] Lower preference of direct links (Closes
 #4010, closes #3947)

Direct links fail with 403, fallback on f4m for now until further investigations.
---
 youtube_dl/extractor/francetv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 07165e330..566e20d76 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -58,7 +58,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
                 formats.append({
                     'url': video_url,
                     'format_id': format_id,
-                    'preference': 2,
+                    'preference': -1,
                 })
         self._sort_formats(formats)
 

From 85d7b7658617895703a3645f14e76a65a1a2f7e6 Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Thu, 23 Oct 2014 21:23:56 +0530
Subject: [PATCH 0257/1937] [thoughtworks] wistia regex modified

---
 youtube_dl/extractor/generic.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 877447381..bf957fa3f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -386,8 +386,9 @@ class GenericIE(InfoExtractor):
             'info_dict': {
                 'id': 'uxjb0lwrcz',
                 'ext': 'mp4',
-                'title': 'Conversation about Hexagonal Rails Part 1',
+                'title': 'Conversation about Hexagonal Rails Part 1 - ThoughtWorks',
                 'duration': 1715.0,
+                'uploader': 'thoughtworks.wistia.com',
             },   
         },
     ]
@@ -661,12 +662,8 @@ class GenericIE(InfoExtractor):
                     playlists, lambda p: '//dailymotion.com/playlist/%s' % p)
 
         # Look for embedded Wistia player
-        mobj = re.search(r'<meta content=(["\'])(?P<url>https?://fast\.wistia\.net/embed/iframe/(?P<id>[a-zA-Z0-9_]+)(["\']))', webpage)
-        if mobj:
-            return self.url_result(mobj.group('url'), 'Wistia')
-        
         match = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
+            r'(?:<meta content|<iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
         if match:
             embed_url = self._proto_relative_url(
                 unescapeHTML(match.group('url')))

From 281d3f1d6877a022d51d8d654595c4958dd83d7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Oct 2014 23:03:07 +0700
Subject: [PATCH 0258/1937] [generic/wistia] Improve regex

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index bf957fa3f..e13816c66 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -663,7 +663,7 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded Wistia player
         match = re.search(
-            r'(?:<meta content|<iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
+            r'<(?:meta[^>]+?content|iframe[^>]+?src)=(["\'])(?P<url>(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage)
         if match:
             embed_url = self._proto_relative_url(
                 unescapeHTML(match.group('url')))

From 75da98e9e1f55f8fc27200f2b3d56386ead3b56d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 23 Oct 2014 23:07:58 +0700
Subject: [PATCH 0259/1937] [funnyordie] Fix extraction (Closes #4011)

---
 youtube_dl/extractor/funnyordie.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index d966e8403..ec6d96ada 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -37,7 +37,7 @@ class FunnyOrDieIE(InfoExtractor):
         video_id = mobj.group('id')
         webpage = self._download_webpage(url, video_id)
 
-        links = re.findall(r'<source src="([^"]+/v)\d+\.([^"]+)" type=\'video', webpage)
+        links = re.findall(r'<source src="([^"]+/v)[^"]+\.([^"]+)" type=\'video', webpage)
         if not links:
             raise ExtractorError('No media links available for %s' % video_id)
 

From ad5f53ac72d32803819d221b3e9a68331c3002a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 23 Oct 2014 20:08:55 +0200
Subject: [PATCH 0260/1937] [telecinco] Add extractor (closes #4005)

It uses the same extraction process as mitele.es, but with a few small differences.
---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/mitele.py    | 12 ++++++++++--
 youtube_dl/extractor/telecinco.py | 18 ++++++++++++++++++
 3 files changed, 29 insertions(+), 2 deletions(-)
 create mode 100644 youtube_dl/extractor/telecinco.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 070f9ff19..0de0b4587 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -367,6 +367,7 @@ from .teachingchannel import TeachingChannelIE
 from .teamcoco import TeamcocoIE
 from .techtalks import TechTalksIE
 from .ted import TEDIE
+from .telecinco import TelecincoIE
 from .telemb import TeleMBIE
 from .tenplay import TenPlayIE
 from .testurl import TestURLIE
diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 979f3d692..df7fde87c 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -39,13 +39,21 @@ class MiTeleIE(InfoExtractor):
         ).replace('\'', '"')
         embed_data = json.loads(embed_data_json)
 
-        info_url = embed_data['flashvars']['host']
+        domain = embed_data['mediaUrl']
+        if not domain.startswith('http'):
+            # only happens in telecinco.es videos
+            domain = 'http://' + domain
+        info_url = compat_urllib_parse.urljoin(
+            domain,
+            compat_urllib_parse.unquote(embed_data['flashvars']['host'])
+        )
         info_el = self._download_xml(info_url, episode).find('./video/info')
 
         video_link = info_el.find('videoUrl/link').text
         token_query = compat_urllib_parse.urlencode({'id': video_link})
         token_info = self._download_json(
-            'http://token.mitele.es/?' + token_query, episode,
+            embed_data['flashvars']['ov_tk'] + '?' + token_query,
+            episode,
             transform_source=strip_jsonp
         )
 
diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
new file mode 100644
index 000000000..161656aa6
--- /dev/null
+++ b/youtube_dl/extractor/telecinco.py
@@ -0,0 +1,18 @@
+from __future__ import unicode_literals
+
+from .mitele import MiTeleIE
+
+
+class TelecincoIE(MiTeleIE):
+    IE_NAME = 'telecinco.es'
+    _VALID_URL = r'https?://www\.telecinco\.es/[^/]+/[^/]+/[^/]+/(?P<episode>.*?)\.html'
+
+    _TEST = {
+        'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html',
+        'info_dict': {
+            'id': 'MDSVID20141015_0058',
+            'ext': 'mp4',
+            'title': 'Con Martín Berasategui, hacer un bacalao al ...',
+            'duration': 662,
+        },
+    }

From f3cd403c2b0b0819504236946895e8ead9814f1f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 23 Oct 2014 20:21:27 +0200
Subject: [PATCH 0261/1937] [telecino] Add coding declaration

---
 youtube_dl/extractor/telecinco.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index 161656aa6..db9788c18 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -1,3 +1,4 @@
+#coding: utf-8
 from __future__ import unicode_literals
 
 from .mitele import MiTeleIE

From 4661e243f8db0befd0180090874a33dfc375b33c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 23 Oct 2014 20:21:38 +0200
Subject: [PATCH 0262/1937] release 2014.10.23

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e7f6adef1..75da03df1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.10.18'
+__version__ = '2014.10.23'

From f2f2c0c2c642aecc1a8954ad3d8639425036e066 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 23 Oct 2014 21:13:45 +0200
Subject: [PATCH 0263/1937] [generic] Allow --default-search without colon

---
 youtube_dl/__init__.py          | 2 --
 youtube_dl/extractor/generic.py | 3 ++-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 7f2b4dfcc..a53f6a621 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -255,8 +255,6 @@ def _real_main(argv=None):
         date = DateRange.day(opts.date)
     else:
         date = DateRange(opts.dateafter, opts.datebefore)
-    if opts.default_search not in ('auto', 'auto_warning', 'error', 'fixup_error', None) and ':' not in opts.default_search:
-        parser.error(u'--default-search invalid; did you forget a colon (:) at the end?')
 
     # Do not download videos when there are audio-only formats
     if opts.extractaudio and not opts.keepvideo and opts.format is None:
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index e13816c66..9b6498894 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -487,7 +487,8 @@ class GenericIE(InfoExtractor):
                      'Set --default-search "ytsearch" (or run  youtube-dl "ytsearch:%s" ) to search YouTube'
                     ) % (url, url), expected=True)
             else:
-                assert ':' in default_search
+                if ':' not in default_search:
+                    default_search += ':'
                 return self.url_result(default_search + url)
 
         url, smuggled_data = unsmuggle_url(url)

From 2c63ccec78faec092394d4461cb9428504e8c830 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 23 Oct 2014 21:26:48 +0200
Subject: [PATCH 0264/1937] [mitele] Fix on python 2.x

---
 youtube_dl/extractor/mitele.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index df7fde87c..6691521e5 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -6,6 +6,7 @@ import json
 from .common import InfoExtractor
 from ..utils import (
     compat_urllib_parse,
+    compat_urlparse,
     get_element_by_attribute,
     parse_duration,
     strip_jsonp,
@@ -43,7 +44,7 @@ class MiTeleIE(InfoExtractor):
         if not domain.startswith('http'):
             # only happens in telecinco.es videos
             domain = 'http://' + domain
-        info_url = compat_urllib_parse.urljoin(
+        info_url = compat_urlparse.urljoin(
             domain,
             compat_urllib_parse.unquote(embed_data['flashvars']['host'])
         )

From 34e48bed3b562ef32c3bfa14ba59101786316464 Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Fri, 24 Oct 2014 02:41:10 +0530
Subject: [PATCH 0265/1937] [extractor/common] Added support for f4m manifest
 Version 2.0

---
 youtube_dl/extractor/common.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e8366f7f9..290df1175 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -648,6 +648,8 @@ class InfoExtractor(object):
 
         formats = []
         media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
+        if not media_nodes:
+            media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             format_id = 'f4m-%d' % (i if tbr is None else tbr)

From b1edd7a48ace472f89cda7503f104a98fc67ee30 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 23 Oct 2014 23:25:02 +0200
Subject: [PATCH 0266/1937] [crunchyroll] Correct parsing (Fixes #4014)

---
 youtube_dl/extractor/crunchyroll.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index f99888ecc..e3057d900 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -39,6 +39,7 @@ class CrunchyrollIE(SubtitlesInfoExtractor):
             'thumbnail': 'http://img1.ak.crunchyroll.com/i/spire1-tmb/20c6b5e10f1a47b10516877d3c039cae1380951166_full.jpg',
             'uploader': 'Yomiuri Telecasting Corporation (YTV)',
             'upload_date': '20131013',
+            'url': 're:(?!.*&amp)',
         },
         'params': {
             # rtmp
@@ -237,12 +238,14 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             streamdata_req.data = 'req=RpcApiVideoEncode%5FGetStreamInfo&video%5Fencode%5Fquality='+stream_quality+'&media%5Fid='+stream_id+'&video%5Fformat='+stream_format
             streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
             streamdata_req.add_header('Content-Length', str(len(streamdata_req.data)))
-            streamdata = self._download_webpage(streamdata_req, video_id, note='Downloading media info for '+video_format)
-            video_url = self._search_regex(r'<host>([^<]+)', streamdata, 'video_url')
-            video_play_path = self._search_regex(r'<file>([^<]+)', streamdata, 'video_play_path')
+            streamdata = self._download_xml(
+                streamdata_req, video_id,
+                note='Downloading media info for %s' % video_format)
+            video_url = streamdata.find('.//host').text
+            video_play_path = streamdata.find('.//file').text
             formats.append({
                 'url': video_url,
-                'play_path':   video_play_path,
+                'play_path': video_play_path,
                 'ext': 'flv',
                 'format': video_format,
                 'format_id': video_format,

From 67500bf939d7db66cdfb9f742fdedca1b83b8309 Mon Sep 17 00:00:00 2001
From: xavier <xavier.beynon@gmail.com>
Date: Thu, 23 Oct 2014 16:55:39 -0500
Subject: [PATCH 0267/1937] Initial version of audiomack.py

---
 youtube_dl/extractor/audiomack.py | 43 +++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 youtube_dl/extractor/audiomack.py

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
new file mode 100644
index 000000000..c5214f401
--- /dev/null
+++ b/youtube_dl/extractor/audiomack.py
@@ -0,0 +1,43 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+import datetime
+import time
+import urllib.request
+import json
+
+
+class AudiomackIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?audiomack\.com/song/(?P<id>[\w/-]+)'
+    _TEST = {
+        'url': 'https://www.audiomack.com/song/crewneckkramer/story-i-tell',
+        'info_dict': {
+            'id': 'story-i-tell',
+            'ext': 'mp3',
+            'title': 'story-i-tell'
+        }
+    }
+
+    def _real_extract(self, url):
+        # TODO more code goes here, for example ...
+        #webpage = self._download_webpage(url, video_id)
+        #title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
+	
+        assert("/song/" in url)
+        songurl = url[url.index("/song/")+5:]
+        title = songurl[songurl.rindex("/")+1:]
+        video_id = title
+        t = int(time.mktime(datetime.datetime.now().timetuple()))
+        s = "http://www.audiomack.com/api/music/url/song"+songurl+"?_="+str(t)
+        f = urllib.request.urlopen(s)
+        j = f.read(1000).decode("utf-8")
+        data = json.loads(j)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url' : data["url"],
+            'ext' : 'mp3'
+            # TODO more properties (see youtube_dl/extractor/common.py)
+        }   

From 5c565ac9e7fed4f6a7123332cbaf35fc44a5a921 Mon Sep 17 00:00:00 2001
From: xavier <xavier.beynon@gmail.com>
Date: Thu, 23 Oct 2014 16:58:11 -0500
Subject: [PATCH 0268/1937] Added init.py initializer

---
 youtube_dl/extractor/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 070f9ff19..29f32cdef 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -20,6 +20,7 @@ from .arte import (
     ArteTVDDCIE,
     ArteTVEmbedIE,
 )
+from .audiomack import AudiomackIE
 from .auengine import AUEngineIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE

From 7a47d07c6dbc8b6bb3be321af886c1774d68e0d6 Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Fri, 24 Oct 2014 09:47:39 +0530
Subject: [PATCH 0269/1937] [extractor/common] href attribute added

---
 youtube_dl/extractor/common.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 290df1175..a8674dd85 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -647,10 +647,14 @@ class InfoExtractor(object):
             'Unable to download f4m manifest')
 
         formats = []
+        manifest_version = '1.0'
         media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
         if not media_nodes:
+            manifest_version = '2.0'
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
+            if manifest_version == '2.0':
+                manifest_url = '/'.join(manifest_url.split('/')[:-1]) + '/' + media_el.attrib.get('href')
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             format_id = 'f4m-%d' % (i if tbr is None else tbr)
             formats.append({

From 25fadd06d078d54e30e187063d8b45fae31bed81 Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Fri, 24 Oct 2014 09:49:01 +0530
Subject: [PATCH 0270/1937] [Rte] New extractor added

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/rte.py      | 53 ++++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)
 create mode 100644 youtube_dl/extractor/rte.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0de0b4587..7918f9ab7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -297,6 +297,7 @@ from .ro220 import Ro220IE
 from .rottentomatoes import RottenTomatoesIE
 from .roxwel import RoxwelIE
 from .rtbf import RTBFIE
+from .rte import RteIE
 from .rtlnl import RtlXlIE
 from .rtlnow import RTLnowIE
 from .rts import RTSIE
diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
new file mode 100644
index 000000000..ab12bb5fd
--- /dev/null
+++ b/youtube_dl/extractor/rte.py
@@ -0,0 +1,53 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import (
+    float_or_none,
+)
+
+
+class RteIE(InfoExtractor):
+    _VALID_URL = r'http?://(?:www\.)?rte\.ie/player/in/show/(?P<id>[0-9]+)/'
+    _TEST = {
+        'url': 'http://www.rte.ie/player/in/show/10336191/',
+        'info_dict': {
+            'id': '10336191',
+            'ext': 'mp4',
+            'title': 'Nine News',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'The One O\'Clock News followed by Weather.',
+            'duration': 1622963.0,
+        }
+    }
+    
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        title = self._og_search_title(webpage)
+        
+        description = self._search_regex(r'<meta name="description" content="(.*?)" />', webpage, 'description')
+        duration = float_or_none(self._html_search_meta('duration', webpage, 'duration'))
+        
+        thumbnail_id = self._search_regex(r'<meta name="thumbnail" content="uri:irus:(.*?)" />', webpage, 'thumbnail')
+        thumbnail = 'http://img.rasset.ie/' + thumbnail_id + '.jpg' 
+        
+        feeds_url = self._html_search_meta("feeds-prefix", webpage, 'feeds url') + video_id
+        json_string = self._download_json(feeds_url, video_id)
+        
+        # f4m_url = server + relative_url
+        f4m_url = json_string['shows'][0]['media:group'][0]['rte:server'] + json_string['shows'][0]['media:group'][0]['url']
+        f4m_formats = self._extract_f4m_formats(f4m_url, video_id)
+        for f in f4m_formats:
+            del f['tbr']
+        
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': f4m_formats,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+        }
\ No newline at end of file

From 9e9bc793f3abddc4824cfcb13f569163fb0a4ba7 Mon Sep 17 00:00:00 2001
From: xavier <xavier.beynon@gmail.com>
Date: Thu, 23 Oct 2014 23:54:59 -0500
Subject: [PATCH 0271/1937] Finished audiomack extractor

---
 youtube_dl/extractor/audiomack.py | 82 ++++++++++++++++++++-----------
 1 file changed, 53 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index c5214f401..2ececa998 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -1,43 +1,67 @@
+# Xavier Beynon 2014
 # coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from .soundcloud import SoundcloudIE
 import datetime
 import time
-import urllib.request
-import json
 
 
 class AudiomackIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?audiomack\.com/song/(?P<id>[\w/-]+)'
-    _TEST = {
-        'url': 'https://www.audiomack.com/song/crewneckkramer/story-i-tell',
-        'info_dict': {
-            'id': 'story-i-tell',
-            'ext': 'mp3',
-            'title': 'story-i-tell'
+    IE_NAME = 'audiomack'
+    _TESTS = [
+        #hosted on audiomack
+        {
+            'url': 'http://www.audiomack.com/song/roosh-williams/extraordinary',
+            'file': 'Roosh Williams - Extraordinary.mp3',
+            'info_dict':
+            {
+                'ext': 'mp3',
+                'title': 'Roosh Williams - Extraordinary'
+            }
+        },
+        #hosted on soundcloud via audiomack
+        {
+            'url': 'http://www.audiomack.com/song/xclusiveszone/take-kare',
+            'file': '172419696.mp3',
+            'info_dict':
+            {
+                'ext': 'mp3',
+                'title': 'Young Thug ft Lil Wayne - Take Kare',
+                "upload_date": "20141016",
+                "description": "New track produced by London On Da Track called “Take Kare\"\n\nhttp://instagram.com/theyoungthugworld\nhttps://www.facebook.com/ThuggerThuggerCashMoney\n",
+                "uploader": "Young Thug World"
+            }
         }
-    }
+    ]
 
     def _real_extract(self, url):
-        # TODO more code goes here, for example ...
-        #webpage = self._download_webpage(url, video_id)
-        #title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
-	
-        assert("/song/" in url)
-        songurl = url[url.index("/song/")+5:]
-        title = songurl[songurl.rindex("/")+1:]
-        video_id = title
-        t = int(time.mktime(datetime.datetime.now().timetuple()))
-        s = "http://www.audiomack.com/api/music/url/song"+songurl+"?_="+str(t)
-        f = urllib.request.urlopen(s)
-        j = f.read(1000).decode("utf-8")
-        data = json.loads(j)
+        #id is what follows /song/ in url, usually the uploader name + title
+        id = url[url.index("/song/")+5:]
 
-        return {
-            'id': video_id,
-            'title': title,
-            'url' : data["url"],
-            'ext' : 'mp3'
-            # TODO more properties (see youtube_dl/extractor/common.py)
-        }   
+        #Call the api, which gives us a json doc with the real url inside
+        rightnow = int(time.mktime(datetime.datetime.now().timetuple()))
+        apiresponse = self._download_json("http://www.audiomack.com/api/music/url/song"+id+"?_="+str(rightnow), id)
+        if not url in apiresponse:
+            raise Exception("Unable to deduce api url of song")
+        realurl = apiresponse["url"]
+
+        #Audiomack wraps a lot of soundcloud tracks in their branded wrapper
+        # - if so, pass the work off to the soundcloud extractor
+        if SoundcloudIE.suitable(realurl):
+            sc = SoundcloudIE(downloader=self._downloader)
+            return sc._real_extract(realurl)
+        else:
+            #Pull out metadata
+            page = self._download_webpage(url, id)
+            artist = self._html_search_regex(r'<span class="artist">(.*)</span>', page, "artist")
+            songtitle = self._html_search_regex(r'<h1 class="profile-title song-title"><span class="artist">.*</span>(.*)</h1>', page, "title")
+            title = artist+" - "+songtitle
+            return {
+                'id': title,  # ignore id, which is not useful in song name
+                'title': title,
+                'url': realurl,
+                'ext': 'mp3'
+            }

From 057a5206cc602f2b6b4a622530df5244f1b41782 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 24 Oct 2014 14:48:12 +0200
Subject: [PATCH 0272/1937] Add --flat-playlist option (Closes #4003)

---
 youtube_dl/YoutubeDL.py | 11 +++++++++--
 youtube_dl/__init__.py  |  1 +
 youtube_dl/options.py   |  5 +++++
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index dec0e20e7..623f9d6fe 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -165,6 +165,8 @@ class YoutubeDL(object):
                        'auto' for elaborate guessing
     encoding:          Use this encoding instead of the system-specified.
     extract_flat:      Do not resolve URLs, return the immediate result.
+                       Pass in 'in_playlist' to only show this behavior for
+                       playlist items.
 
     The following parameters are not used by YoutubeDL itself, they are used by
     the FileDownloader:
@@ -568,8 +570,13 @@ class YoutubeDL(object):
 
         result_type = ie_result.get('_type', 'video')
 
-        if self.params.get('extract_flat', False):
-            if result_type in ('url', 'url_transparent'):
+        if result_type in ('url', 'url_transparent'):
+            extract_flat = self.params.get('extract_flat', False)
+            if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
+                    extract_flat is True):
+                self.add_extra_info(ie_result, extra_info)
+                if self.params.get('forcejson', False):
+                    self.to_stdout(json.dumps(ie_result))
                 return ie_result
 
         if result_type == 'video':
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index a53f6a621..c4e1d32db 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -367,6 +367,7 @@ def _real_main(argv=None):
         'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
         'encoding': opts.encoding,
         'exec_cmd': opts.exec_cmd,
+        'extract_flat': opts.extract_flat,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 649361bde..2b1cd7438 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -159,6 +159,11 @@ def parseOpts(overrideArguments=None):
         '--ignore-config',
         action='store_true',
         help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
+    general.add_option(
+        '--flat-playlist',
+        action='store_const', dest='extract_flat', const='in_playlist',
+        default=False,
+        help='Do not extract the videos of a playlist, only list them.')
 
     selection = optparse.OptionGroup(parser, 'Video Selection')
     selection.add_option(

From 36f1c9049722fc80154d8ee571b1335286829111 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 24 Oct 2014 14:48:19 +0200
Subject: [PATCH 0273/1937] release 2014.10.24

---
 README.md             | 2 ++
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 90ba928c3..e772fc22a 100644
--- a/README.md
+++ b/README.md
@@ -69,6 +69,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      configuration in ~/.config/youtube-dl.conf
                                      (%APPDATA%/youtube-dl/config.txt on
                                      Windows)
+    --flat-playlist                  Do not extract the videos of a playlist,
+                                     only list them.
 
 ## Video Selection:
     --playlist-start NUMBER          playlist video to start at (default is 1)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 75da03df1..59cb3b1a1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.10.23'
+__version__ = '2014.10.24'

From b5a14350b9909cb9da7e82120fcaf1bf4aa61e43 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 24 Oct 2014 15:09:43 +0200
Subject: [PATCH 0274/1937] [bild] Simplify (#3983)

---
 youtube_dl/extractor/bild.py | 85 +++++++++++++++++-------------------
 1 file changed, 39 insertions(+), 46 deletions(-)

diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py
index 3a822a5c0..0269d1174 100644
--- a/youtube_dl/extractor/bild.py
+++ b/youtube_dl/extractor/bild.py
@@ -1,46 +1,39 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-
-
-class BildIE(InfoExtractor):
-    IE_NAME = 'bild'
-    _TEST = {
-        'url': 'http://www.bild.de/video/clip/apple-ipad-air/das-koennen-die-neuen-ipads-38184146.bild.html',
-        'info_dict': {
-            'id': '38184146',
-            'title': 'BILD hat sie getestet',
-            'thumbnail': 'http://bilder.bild.de/fotos/stand-das-koennen-die-neuen-ipads-38184138/Bild/1.bild.jpg',
-            'duration': 196,
-        }
-    }
-    
-    #http://www.bild.de/video/clip/apple-ipad-air/das-koennen-die-neuen-ipads-38184146.bild.html
-    _VALID_URL = r'http?://(?:www\.)?bild\.de/(?:[^/]+/)+(?P<display_id>[^/]+)-(?P<id>\d+)(?:,auto=true)?\.bild\.html'
-    
-    def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('id')
-        
-        #webpage = self._download_webpage(url, video_id)
-        
-        xml_url = url.split(".bild.html")[0]+",view=xml.bild.xml"
-        
-        doc = self._download_xml(xml_url, video_id)
-        
-        video_url = doc.attrib['src']
-        title = doc.attrib['ueberschrift']
-        description = doc.attrib['text']
-        thumbnail = doc.attrib['img']
-        duration = int(doc.attrib['duration'])/1000
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'url': video_url,
-            'thumbnail': thumbnail,
-            'duration': duration,
-        }
+#coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class BildIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?bild\.de/(?:[^/]+/)+(?P<display_id>[^/]+)-(?P<id>\d+)(?:,auto=true)?\.bild\.html'
+    IE_DESC = 'Bild.de'
+    _TEST = {
+        'url': 'http://www.bild.de/video/clip/apple-ipad-air/das-koennen-die-neuen-ipads-38184146.bild.html',
+        'md5': 'dd495cbd99f2413502a1713a1156ac8a',
+        'info_dict': {
+            'id': '38184146',
+            'ext': 'mp4',
+            'title': 'BILD hat sie getestet',
+            'thumbnail': 'http://bilder.bild.de/fotos/stand-das-koennen-die-neuen-ipads-38184138/Bild/1.bild.jpg',
+            'duration': 196,
+            'description': 'Mit dem iPad Air 2 und dem iPad Mini 3 hat Apple zwei neue Tablet-Modelle präsentiert. BILD-Reporter Sven Stein durfte die Geräte bereits testen. ',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        xml_url = url.split(".bild.html")[0] + ",view=xml.bild.xml"
+        doc = self._download_xml(xml_url, video_id)
+
+        duration = int_or_none(doc.attrib.get('duration'), scale=1000)
+
+        return {
+            'id': video_id,
+            'title': doc.attrib['ueberschrift'],
+            'description': doc.attrib.get('text'),
+            'url': doc.attrib['src'],
+            'thumbnail': doc.attrib.get('img'),
+            'duration': duration,
+        }

From 2acfe95f58e3dc52864a29f1d5b30e44d75d7b38 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 24 Oct 2014 15:12:29 +0200
Subject: [PATCH 0275/1937] Credit @capital-G for bild.de (#3983)

---
 youtube_dl/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index c4e1d32db..a1079e596 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -79,6 +79,7 @@ __authors__  = (
     'Carlos Ramos',
     '5moufl',
     'lenaten',
+    'Dennis Scheiba',
 )
 
 __license__ = 'Public Domain'

From 95ee84421e57982f208ec00c314abfabaa3418e8 Mon Sep 17 00:00:00 2001
From: winwon <wish4read@gmail.com>
Date: Fri, 17 Oct 2014 17:38:16 +0100
Subject: [PATCH 0276/1937] vidzi Add new extractor

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/vidzi.py    | 30 ++++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/vidzi.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3023c3095..b52f8c7d1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -493,7 +493,7 @@ from .youtube import (
 )
 
 from .zdf import ZDFIE
-
+from .vidzi import VidziIE
 
 _ALL_CLASSES = [
     klass
diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
new file mode 100644
index 000000000..7d93dd301
--- /dev/null
+++ b/youtube_dl/extractor/vidzi.py
@@ -0,0 +1,30 @@
+import re
+from .common import InfoExtractor
+
+class VidziIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?vidzi\.tv/(?P<id>\w+)'
+    _TEST = {
+        'url': 'http://vidzi.tv/m1chxrwq7bx9',
+        'md5': '5c4c4a8ca2281a199c8eefe8f411d630',
+        'info_dict': {
+            'id': 'm1chxrwq7bx9',
+            'ext': 'mp4',
+            'title': 'Watch Cadbury Dream Factory S01E04 HDTV x264 FiHTV mp4',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        
+        webpage = self._download_webpage('http://vidzi.tv/' + video_id, video_id)
+        video_url = self._html_search_regex(r'{\s*file\s*:\s*"([^"]+)"\s*}', webpage, u'vidzi url')
+        title = self._html_search_regex(r'<Title>([^<]+)<\/Title>', webpage, u'vidzi title')
+        
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+            'ext': 'mp4',
+        }
+        
\ No newline at end of file

From e65e06fbe28365bf630608bb1d6101074c475e21 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 24 Oct 2014 15:15:54 +0200
Subject: [PATCH 0277/1937] [vidzi] Correct order in imports (#3989)

---
 youtube_dl/extractor/__init__.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b52f8c7d1..81b816974 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -423,6 +423,7 @@ from .videopremium import VideoPremiumIE
 from .videott import VideoTtIE
 from .videoweed import VideoWeedIE
 from .vidme import VidmeIE
+from .vidzi import VidziIE
 from .vimeo import (
     VimeoIE,
     VimeoAlbumIE,
@@ -491,9 +492,7 @@ from .youtube import (
     YoutubeUserIE,
     YoutubeWatchLaterIE,
 )
-
 from .zdf import ZDFIE
-from .vidzi import VidziIE
 
 _ALL_CLASSES = [
     klass

From 018e835594f961dfd625abc932451ef3c2c85038 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 24 Oct 2014 15:17:17 +0200
Subject: [PATCH 0278/1937] [vidzi] Simplify

---
 youtube_dl/extractor/vidzi.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index 7d93dd301..82d2179e9 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -1,6 +1,8 @@
-import re
+from __future__ import unicode_literals
+
 from .common import InfoExtractor
 
+
 class VidziIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?vidzi\.tv/(?P<id>\w+)'
     _TEST = {
@@ -14,12 +16,13 @@ class VidziIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         
-        webpage = self._download_webpage('http://vidzi.tv/' + video_id, video_id)
-        video_url = self._html_search_regex(r'{\s*file\s*:\s*"([^"]+)"\s*}', webpage, u'vidzi url')
-        title = self._html_search_regex(r'<Title>([^<]+)<\/Title>', webpage, u'vidzi title')
+        webpage = self._download_webpage(url, video_id)
+        video_url = self._html_search_regex(
+            r'{\s*file\s*:\s*"([^"]+)"\s*}', webpage, 'video url')
+        title = self._html_search_regex(
+            r'<Title>([^<]+)<\/Title>', webpage, 'title')
         
         return {
             'id': video_id,

From 2c26df763c42433fea30ef46736156c689a74fd9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 24 Oct 2014 15:27:02 +0200
Subject: [PATCH 0279/1937] [vidzi] Use proper test case and simplify (#3989)

---
 youtube_dl/extractor/vidzi.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index 82d2179e9..669979e13 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -1,3 +1,4 @@
+#coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
@@ -6,12 +7,12 @@ from .common import InfoExtractor
 class VidziIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?vidzi\.tv/(?P<id>\w+)'
     _TEST = {
-        'url': 'http://vidzi.tv/m1chxrwq7bx9',
-        'md5': '5c4c4a8ca2281a199c8eefe8f411d630',
+        'url': 'http://vidzi.tv/cghql9yq6emu.html',
+        'md5': '4f16c71ca0c8c8635ab6932b5f3f1660',
         'info_dict': {
-            'id': 'm1chxrwq7bx9',
+            'id': 'cghql9yq6emu',
             'ext': 'mp4',
-            'title': 'Watch Cadbury Dream Factory S01E04 HDTV x264 FiHTV mp4',
+            'title': 'youtube-dl test video  1\\\\2\'3/4<5\\\\6ä7↭',
         },
     }
 
@@ -22,12 +23,11 @@ class VidziIE(InfoExtractor):
         video_url = self._html_search_regex(
             r'{\s*file\s*:\s*"([^"]+)"\s*}', webpage, 'video url')
         title = self._html_search_regex(
-            r'<Title>([^<]+)<\/Title>', webpage, 'title')
+            r'(?s)<h2 class="video-title">(.*?)</h2>', webpage, 'title')
         
         return {
             'id': video_id,
             'title': title,
             'url': video_url,
-            'ext': 'mp4',
         }
         
\ No newline at end of file

From 964ae0a1228c9a0853e2b9142452d66e7333bf8c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 24 Oct 2014 15:29:44 +0200
Subject: [PATCH 0280/1937] Credit @thornomad for glide (#3944)

---
 youtube_dl/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index a1079e596..6b0efe059 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -80,6 +80,7 @@ __authors__  = (
     '5moufl',
     'lenaten',
     'Dennis Scheiba',
+    'Damon Timm',
 )
 
 __license__ = 'Public Domain'

From 1ede5b2481a271fcfd0e179a494ef84cca2576b7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 24 Oct 2014 15:34:19 +0200
Subject: [PATCH 0281/1937] [glide] Simplify

---
 youtube_dl/extractor/common.py |  2 +-
 youtube_dl/extractor/glide.py  | 19 +++++++++++++------
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e8366f7f9..8849208de 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -618,7 +618,7 @@ class InfoExtractor(object):
         formats.sort(key=_formats_key)
 
     def http_scheme(self):
-        """ Either "https:" or "https:", depending on the user's preferences """
+        """ Either "http:" or "https:", depending on the user's preferences """
         return (
             'http:'
             if self._downloader.params.get('prefer_insecure', False)
diff --git a/youtube_dl/extractor/glide.py b/youtube_dl/extractor/glide.py
index 175d85197..9561ed5fb 100644
--- a/youtube_dl/extractor/glide.py
+++ b/youtube_dl/extractor/glide.py
@@ -14,20 +14,27 @@ class GlideIE(InfoExtractor):
             'id': 'UZF8zlmuQbe4mr+7dCiQ0w==',
             'ext': 'mp4',
             'title': 'Damon Timm\'s Glide message',
-            'thumbnail' : 'http://dk608k4lm7m9j.cloudfront.net/3ee7da5af87065a1eeb8c6c9a864ba5b_2.jpg'
+            'thumbnail': 're:^https?://.*?\.cloudfront\.net/.*\.jpg$',
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
-        video_url = self._search_regex(r'<source src="(.*?)" type="video/mp4">', webpage, 'video_url')
-        thumbnail_url = self._search_regex(r'<img id="video-thumbnail" src="(.*?)" alt="Video thumbnail">', webpage, 'thumbnail_url')
+        title = self._html_search_regex(
+            r'<title>(.*?)</title>', webpage, 'title')
+        video_url = self.http_scheme() + self._search_regex(
+            r'<source src="(.*?)" type="video/mp4">', webpage, 'video URL')
+        thumbnail_url = self._search_regex(
+            r'<img id="video-thumbnail" src="(.*?)"',
+            webpage, 'thumbnail url', fatal=False)
+        thumbnail = (
+            thumbnail_url if thumbnail_url is None
+            else self.http_scheme() + thumbnail_url)
 
         return {
             'id': video_id,
             'title': title,
-            'url' : 'http:' + video_url,
-            'thumbnail' : 'http:' + thumbnail_url
+            'url': video_url,
+            'thumbnail': thumbnail,
         }

From e82c1e9a6e709502acc683fb90864f7611d96d07 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 24 Oct 2014 16:13:45 +0200
Subject: [PATCH 0282/1937] [YoutubeDL] Do not apply playlist info to videos
 when extract_flat is set (#4003)

---
 youtube_dl/YoutubeDL.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 623f9d6fe..6aa1fd105 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -574,7 +574,6 @@ class YoutubeDL(object):
             extract_flat = self.params.get('extract_flat', False)
             if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
                     extract_flat is True):
-                self.add_extra_info(ie_result, extra_info)
                 if self.params.get('forcejson', False):
                     self.to_stdout(json.dumps(ie_result))
                 return ie_result

From 5823eda139d851e3ff3cfd1e3fd6b109e1f7c19e Mon Sep 17 00:00:00 2001
From: Filip B <b_filip@zoho.com>
Date: Fri, 24 Oct 2014 21:26:32 +0400
Subject: [PATCH 0283/1937] Fix motherless test 0

Apparently motherless no longer serves flv for that video (or maybe even
other videos). So I changed expected extension from flv to mp4 and
changed expected md5 accordingly.
---
 youtube_dl/extractor/motherless.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 6229b2173..55f282966 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -15,10 +15,10 @@ class MotherlessIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://motherless.com/AC3FFE1',
-            'md5': '5527fef81d2e529215dad3c2d744a7d9',
+            'md5': '310f62e325a9fafe64f68c0bccb6e75f',
             'info_dict': {
                 'id': 'AC3FFE1',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Fucked in the ass while playing PS3',
                 'categories': ['Gaming', 'anal', 'reluctant', 'rough', 'Wife'],
                 'upload_date': '20100913',

From 7608815cc24e96d3f4895f7c056ee8d769b678d2 Mon Sep 17 00:00:00 2001
From: Filip B <b_filip@zoho.com>
Date: Fri, 24 Oct 2014 21:09:55 +0400
Subject: [PATCH 0284/1937] Add another motherless test

This test is for videos posted in groups.
URL looks like this:
motherless.com/g/group_name/ID
which is basically the same as
motherless.com/ID
So far this test fails because I haven't improved the extractor yet.
---
 youtube_dl/extractor/motherless.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 55f282966..9ce080a30 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -40,6 +40,20 @@ class MotherlessIE(InfoExtractor):
                 'thumbnail': 're:http://.*\.jpg',
                 'age_limit': 18,
             }
+        },
+        {
+            'url': 'http://motherless.com/g/cosplay/633979F',
+            'md5': '0b2a43f447a49c3e649c93ad1fafa4a0',
+            'info_dict': {
+                'id': '633979F',
+                'ext': 'mp4',
+                'title': 'Turtlette',
+                'categories': ['superheroine heroine  superher'],
+                'upload_date': '20140827',
+                'uploader_id': 'shade0230',
+                'thumbnail': 're:http://.*\.jpg',
+                'age_limit': 18,
+            }
         }
     ]
 

From 7f9ced64cbd33e597287de2f67b7862af123bb7b Mon Sep 17 00:00:00 2001
From: Filip B <b_filip@zoho.com>
Date: Fri, 24 Oct 2014 21:44:21 +0400
Subject: [PATCH 0285/1937] Add group videos support for motherless

Modified motherless regular expression.
Previously it matched only URLs like this:
motherless.com/ID
Now it also matches this:
motherless.com/g/group_name/ID
All tests including the newly added one passed.
---
 youtube_dl/extractor/motherless.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 9ce080a30..50083bd4a 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class MotherlessIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?motherless\.com/(?P<id>[A-Z0-9]+)'
+    _VALID_URL = r'http://(?:www\.)?motherless\.com/(?:g/[a-z0-9_]+/)?(?P<id>[A-Z0-9]+)'
     _TESTS = [
         {
             'url': 'http://motherless.com/AC3FFE1',

From 8efd06aa428df6feed925396f4170a9ad8ca2b4d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 24 Oct 2014 19:53:48 +0200
Subject: [PATCH 0286/1937] [motherless] Simplify

---
 youtube_dl/extractor/motherless.py | 36 +++++++++++++++++-------------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 50083bd4a..3621ff99e 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -5,7 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    int_or_none,
+    str_to_int,
     unified_strdate,
 )
 
@@ -57,30 +57,34 @@ class MotherlessIE(InfoExtractor):
         }
     ]
 
-    def _real_extract(self,url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'id="view-upload-title">\s+([^<]+)<', webpage, 'title')
-        
-        video_url = self._html_search_regex(r'setup\(\{\s+"file".+: "([^"]+)",', webpage, 'video_url')
+        title = self._html_search_regex(
+            r'id="view-upload-title">\s+([^<]+)<', webpage, 'title')
+        video_url = self._html_search_regex(
+            r'setup\(\{\s+"file".+: "([^"]+)",', webpage, 'video URL')
         age_limit = self._rta_search(webpage)
-
-        view_count = self._html_search_regex(r'<strong>Views</strong>\s+([^<]+)<', webpage, 'view_count')
+        view_count = str_to_int(self._html_search_regex(
+            r'<strong>Views</strong>\s+([^<]+)<',
+            webpage, 'view count', fatal=False))
+        like_count = str_to_int(self._html_search_regex(
+            r'<strong>Favorited</strong>\s+([^<]+)<',
+            webpage, 'like count', fatal=False))
  
-        upload_date = self._html_search_regex(r'<strong>Uploaded</strong>\s+([^<]+)<', webpage, 'upload_date')
+        upload_date = self._html_search_regex(
+            r'<strong>Uploaded</strong>\s+([^<]+)<', webpage, 'upload date')
         if 'Ago' in upload_date:
             days = int(re.search(r'([0-9]+)', upload_date).group(1))
             upload_date = (datetime.datetime.now() - datetime.timedelta(days=days)).strftime('%Y%m%d')
         else:
             upload_date = unified_strdate(upload_date)
 
-        like_count = self._html_search_regex(r'<strong>Favorited</strong>\s+([^<]+)<', webpage, 'like_count')
-
         comment_count = webpage.count('class="media-comment-contents"')
-        uploader_id = self._html_search_regex(r'"thumb-member-username">\s+<a href="/m/([^"]+)"', webpage, 'uploader_id')
+        uploader_id = self._html_search_regex(
+            r'"thumb-member-username">\s+<a href="/m/([^"]+)"',
+            webpage, 'uploader_id')
 
         categories = self._html_search_meta('keywords', webpage)
         if categories:
@@ -93,8 +97,8 @@ class MotherlessIE(InfoExtractor):
             'uploader_id': uploader_id,
             'thumbnail': self._og_search_thumbnail(webpage),
             'categories': categories,
-            'view_count': int_or_none(view_count.replace(',', '')),
-            'like_count': int_or_none(like_count.replace(',', '')),
+            'view_count': view_count,
+            'like_count': like_count,
             'comment_count': comment_count,
             'age_limit': age_limit,
             'url': video_url,

From cdc5cb7c2b2a30a6bac849248849189624339d5f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 24 Oct 2014 22:31:55 +0200
Subject: [PATCH 0287/1937] [hark] Modernize

---
 youtube_dl/extractor/hark.py | 48 +++++++++++++++++-------------------
 1 file changed, 22 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/hark.py b/youtube_dl/extractor/hark.py
index 5bdd08afa..b6cc15b6f 100644
--- a/youtube_dl/extractor/hark.py
+++ b/youtube_dl/extractor/hark.py
@@ -1,37 +1,33 @@
 # -*- coding: utf-8 -*-
-
-import re
-import json
+from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import determine_ext
+
 
 class HarkIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.hark\.com/clips/(.+?)-.+'
+    _VALID_URL = r'https?://www\.hark\.com/clips/(?P<id>.+?)-.+'
     _TEST = {
-        u'url': u'http://www.hark.com/clips/mmbzyhkgny-obama-beyond-the-afghan-theater-we-only-target-al-qaeda-on-may-23-2013',
-        u'file': u'mmbzyhkgny.mp3',
-        u'md5': u'6783a58491b47b92c7c1af5a77d4cbee',
-        u'info_dict': {
-            u'title': u"Obama: 'Beyond The Afghan Theater, We Only Target Al Qaeda' on May 23, 2013",
-            u'description': u'President Barack Obama addressed the nation live on May 23, 2013 in a speech aimed at addressing counter-terrorism policies including the use of drone strikes, detainees at Guantanamo Bay prison facility, and American citizens who are terrorists.',
-            u'duration': 11,
+        'url': 'http://www.hark.com/clips/mmbzyhkgny-obama-beyond-the-afghan-theater-we-only-target-al-qaeda-on-may-23-2013',
+        'md5': '6783a58491b47b92c7c1af5a77d4cbee',
+        'info_dict': {
+            'id': 'mmbzyhkgny',
+            'ext': 'mp3',
+            'title': 'Obama: \'Beyond The Afghan Theater, We Only Target Al Qaeda\' on May 23, 2013',
+            'description': 'President Barack Obama addressed the nation live on May 23, 2013 in a speech aimed at addressing counter-terrorism policies including the use of drone strikes, detainees at Guantanamo Bay prison facility, and American citizens who are terrorists.',
+            'duration': 11,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
-        json_url = "http://www.hark.com/clips/%s.json" %(video_id)
-        info_json = self._download_webpage(json_url, video_id)
-        info = json.loads(info_json)
-        final_url = info['url']
+        video_id = self._match_id(url)
+        data = self._download_json(
+            'http://www.hark.com/clips/%s.json' % video_id, video_id)
 
-        return {'id': video_id,
-                'url' : final_url,
-                'title': info['name'],
-                'ext': determine_ext(final_url),
-                'description': info['description'],
-                'thumbnail': info['image_original'],
-                'duration': info['duration'],
-                }
+        return {
+            'id': video_id,
+            'url': data['url'],
+            'title': data['name'],
+            'description': data.get('description'),
+            'thumbnail': data.get('image_original'),
+            'duration': data.get('duration'),
+        }

From c64ed2a310542a46e79e8969df32d14a5403e53d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 25 Oct 2014 00:10:11 +0200
Subject: [PATCH 0288/1937] [viddler] Use API

---
 test/helper.py                  |   2 +-
 youtube_dl/extractor/common.py  |   5 ++
 youtube_dl/extractor/viddler.py | 100 +++++++++++++++++++++-----------
 3 files changed, 71 insertions(+), 36 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 62cb3ce02..2fa45631a 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -145,7 +145,7 @@ def expect_info_dict(self, expected_dict, got_dict):
         info_dict_str = ''.join(
             '    %s: %s,\n' % (_repr(k), _repr(v))
             for k, v in test_info_dict.items())
-        write_string('\n"info_dict": {' + info_dict_str + '}\n', out=sys.stderr)
+        write_string('\n"info_dict": {\n' + info_dict_str + '}\n', out=sys.stderr)
         self.assertFalse(
             missing_keys,
             'Missing keys in test definition: %s' % (
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8849208de..cf3781cd6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -89,6 +89,10 @@ class InfoExtractor(object):
                                  format, irrespective of the file format.
                                  -1 for default (order by other properties),
                                  -2 or smaller for less than default.
+                    * source_preference  Order number for this video source
+                                  (quality takes higher priority)
+                                 -1 for default (order by other properties),
+                                 -2 or smaller for less than default.
                     * http_referer  HTTP Referer header value to set.
                     * http_method  HTTP method to use for the download.
                     * http_headers  A dictionary of additional HTTP headers
@@ -613,6 +617,7 @@ class InfoExtractor(object):
                 audio_ext_preference,
                 f.get('filesize') if f.get('filesize') is not None else -1,
                 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
+                f.get('source_preference') if f.get('source_preference') is not None else -1,
                 f.get('format_id'),
             )
         formats.sort(key=_formats_key)
diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index 9328ef4a2..0faa729c6 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -1,55 +1,85 @@
-import json
-import re
+from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    int_or_none,
+)
 
 
 class ViddlerIE(InfoExtractor):
-    _VALID_URL = r'(?P<domain>https?://(?:www\.)?viddler\.com)/(?:v|embed|player)/(?P<id>[a-z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?viddler\.com/(?:v|embed|player)/(?P<id>[a-z0-9]+)'
     _TEST = {
-        u"url": u"http://www.viddler.com/v/43903784",
-        u'file': u'43903784.mp4',
-        u'md5': u'fbbaedf7813e514eb7ca30410f439ac9',
-        u'info_dict': {
-            u"title": u"Video Made Easy",
-            u"uploader": u"viddler",
-            u"duration": 100.89,
+        "url": "http://www.viddler.com/v/43903784",
+        'md5': 'ae43ad7cb59431ce043f0ff7fa13cbf4',
+        'info_dict': {
+            'id': '43903784',
+            'ext': 'mp4',
+            "title": "Video Made Easy",
+            'description': 'You don\'t need to be a professional to make high-quality video content. Viddler provides some quick and easy tips on how to produce great video content with limited resources. ',
+            "uploader": "viddler",
+            'timestamp': 1335371429,
+            'upload_date': '20120425',
+            "duration": 100.89,
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'view_count': int,
+            'categories': ['video content', 'high quality video', 'video made easy', 'how to produce video with limited resources', 'viddler'],
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
-        embed_url = mobj.group('domain') + u'/embed/' + video_id
-        webpage = self._download_webpage(embed_url, video_id)
+        json_url = (
+            'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?video_id=%s&key=v0vhrt7bg2xq1vyxhkct' %
+            video_id)
+        data = self._download_json(json_url, video_id)['video']
 
-        video_sources_code = self._search_regex(
-            r"(?ms)sources\s*:\s*(\{.*?\})", webpage, u'video URLs')
-        video_sources = json.loads(video_sources_code.replace("'", '"'))
+        formats = []
+        for filed in data['files']:
+            if filed.get('status', 'ready') != 'ready':
+                continue
+            f = {
+                'format_id': filed['profile_id'],
+                'format_note': filed['profile_name'],
+                'url': self._proto_relative_url(filed['url']),
+                'width': int_or_none(filed.get('width')),
+                'height': int_or_none(filed.get('height')),
+                'filesize': int_or_none(filed.get('size')),
+                'ext': filed.get('ext'),
+                'source_preference': -1,
+            }
+            formats.append(f)
 
-        formats = [{
-            'url': video_url,
-            'format': format_id,
-        } for video_url, format_id in video_sources.items()]
+            if filed.get('cdn_url'):
+                f = f.copy()
+                f['url'] = self._proto_relative_url(filed['cdn_url'])
+                f['format_id'] = filed['profile_id'] + '-cdn'
+                f['source_preference'] = 1
+                formats.append(f)
 
-        title = self._html_search_regex(
-            r"title\s*:\s*'([^']*)'", webpage, u'title')
-        uploader = self._html_search_regex(
-            r"authorName\s*:\s*'([^']*)'", webpage, u'uploader', fatal=False)
-        duration_s = self._html_search_regex(
-            r"duration\s*:\s*([0-9.]*)", webpage, u'duration', fatal=False)
-        duration = float(duration_s) if duration_s else None
-        thumbnail = self._html_search_regex(
-            r"thumbnail\s*:\s*'([^']*)'",
-            webpage, u'thumbnail', fatal=False)
+            if filed.get('html5_video_source'):
+                f = f.copy()
+                f['url'] = self._proto_relative_url(
+                    filed['html5_video_source'])
+                f['format_id'] = filed['profile_id'] + '-html5'
+                f['source_preference'] = 0
+                formats.append(f)
+        self._sort_formats(formats)
+
+        categories = [
+            t.get('text') for t in data.get('tags', []) if 'text' in t]
 
         return {
             '_type': 'video',
             'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'uploader': uploader,
-            'duration': duration,
+            'title': data['title'],
             'formats': formats,
+            'description': data.get('description'),
+            'timestamp': int_or_none(data.get('upload_time')),
+            'thumbnail': self._proto_relative_url(data.get('thumbnail_url')),
+            'uploader': data.get('author'),
+            'duration': float_or_none(data.get('length')),
+            'view_count': int_or_none(data.get('view_count')),
+            'categories': categories,
         }

From 63e0be341571450a2016a29f917e521b70d36020 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 25 Oct 2014 00:30:57 +0200
Subject: [PATCH 0289/1937] New option --dump-single-json (#4003)

---
 youtube_dl/YoutubeDL.py | 9 ++++++++-
 youtube_dl/__init__.py  | 3 ++-
 youtube_dl/options.py   | 4 ++++
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6aa1fd105..75461f19d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -107,6 +107,8 @@ class YoutubeDL(object):
     forcefilename:     Force printing final filename.
     forceduration:     Force printing duration.
     forcejson:         Force printing info_dict as JSON.
+    dump_single_json:  Force printing the info_dict of the whole playlist
+                       (or video) as a single JSON line.
     simulate:          Do not download the video files.
     format:            Video format code.
     format_limit:      Highest quality format to try.
@@ -903,6 +905,8 @@ class YoutubeDL(object):
         if self.params.get('forcejson', False):
             info_dict['_filename'] = filename
             self.to_stdout(json.dumps(info_dict))
+        if self.params.get('dump_single_json', False):
+            info_dict['_filename'] = filename
 
         # Do nothing else if in simulate mode
         if self.params.get('simulate', False):
@@ -1070,12 +1074,15 @@ class YoutubeDL(object):
         for url in url_list:
             try:
                 #It also downloads the videos
-                self.extract_info(url)
+                res = self.extract_info(url)
             except UnavailableVideoError:
                 self.report_error('unable to download video')
             except MaxDownloadsReached:
                 self.to_screen('[info] Maximum number of downloaded files reached.')
                 raise
+            else:
+                if self.params.get('dump_single_json', False):
+                    self.to_stdout(json.dumps(res))
 
         return self._download_retcode
 
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 6b0efe059..c8bcc6e46 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -284,7 +284,7 @@ def _real_main(argv=None):
                      u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
                      u' template'.format(outtmpl))
 
-    any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson
+    any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
     download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 
     ydl_opts = {
@@ -304,6 +304,7 @@ def _real_main(argv=None):
         'forcefilename': opts.getfilename,
         'forceformat': opts.getformat,
         'forcejson': opts.dumpjson,
+        'dump_single_json': opts.dump_single_json,
         'simulate': opts.simulate,
         'skip_download': (opts.skip_download or opts.simulate or any_printing),
         'format': opts.format,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 2b1cd7438..2ccc63fc5 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -417,6 +417,10 @@ def parseOpts(overrideArguments=None):
         '-j', '--dump-json',
         action='store_true', dest='dumpjson', default=False,
         help='simulate, quiet but print JSON information. See --output for a description of available keys.')
+    verbosity.add_option(
+        '-J', '--dump-single-json',
+        action='store_true', dest='dump_single_json', default=False,
+        help='simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist information in a single line.')
     verbosity.add_option(
         '--newline',
         action='store_true', dest='progress_with_newline', default=False,

From 724d03189324920fe5ea4bc9d39cc3e170d376d0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 25 Oct 2014 00:41:06 +0200
Subject: [PATCH 0290/1937] release 2014.10.25

---
 README.md             | 4 ++++
 youtube_dl/version.py | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e772fc22a..d62546433 100644
--- a/README.md
+++ b/README.md
@@ -199,6 +199,10 @@ which means you can modify it, redistribute it or use it however you like.
     -j, --dump-json                  simulate, quiet but print JSON information.
                                      See --output for a description of available
                                      keys.
+    -J, --dump-single-json           simulate, quiet but print JSON information
+                                     for each command-line argument. If the URL
+                                     refers to a playlist, dump the whole
+                                     playlist information in a single line.
     --newline                        output progress bar as new lines
     --no-progress                    do not print progress bar
     --console-title                  display progress in console titlebar
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 59cb3b1a1..d822ae330 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.10.24'
+__version__ = '2014.10.25'

From fdfefa1b9c86ef58d943a26a157dc234c2df14d9 Mon Sep 17 00:00:00 2001
From: xavier <xavier.beynon@gmail.com>
Date: Fri, 24 Oct 2014 21:07:01 -0500
Subject: [PATCH 0291/1937] Made changes per phihag

---
 youtube_dl/__init__.py            |  1 +
 youtube_dl/extractor/audiomack.py | 17 +++++++++--------
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 7f2b4dfcc..78cdf14df 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -79,6 +79,7 @@ __authors__  = (
     'Carlos Ramos',
     '5moufl',
     'lenaten',
+    'Xavier Beynon'
 )
 
 __license__ = 'Public Domain'
diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 2ececa998..bdcc51235 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -1,9 +1,9 @@
-# Xavier Beynon 2014
 # coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from .soundcloud import SoundcloudIE
+from ..utils import ExtractorError
 import datetime
 import time
 
@@ -15,9 +15,9 @@ class AudiomackIE(InfoExtractor):
         #hosted on audiomack
         {
             'url': 'http://www.audiomack.com/song/roosh-williams/extraordinary',
-            'file': 'Roosh Williams - Extraordinary.mp3',
             'info_dict':
             {
+                'id' : 'roosh-williams/extraordinary',
                 'ext': 'mp3',
                 'title': 'Roosh Williams - Extraordinary'
             }
@@ -39,13 +39,14 @@ class AudiomackIE(InfoExtractor):
 
     def _real_extract(self, url):
         #id is what follows /song/ in url, usually the uploader name + title
-        id = url[url.index("/song/")+5:]
+        id = self._match_id(url)
 
         #Call the api, which gives us a json doc with the real url inside
-        rightnow = int(time.mktime(datetime.datetime.now().timetuple()))
-        apiresponse = self._download_json("http://www.audiomack.com/api/music/url/song"+id+"?_="+str(rightnow), id)
-        if not url in apiresponse:
-            raise Exception("Unable to deduce api url of song")
+        rightnow = int(time.time())
+        apiresponse = self._download_json("http://www.audiomack.com/api/music/url/song/"+id+"?_="+str(rightnow), id)
+
+        if "url" not in apiresponse:
+            raise ExtractorError("Unable to deduce api url of song")
         realurl = apiresponse["url"]
 
         #Audiomack wraps a lot of soundcloud tracks in their branded wrapper
@@ -60,7 +61,7 @@ class AudiomackIE(InfoExtractor):
             songtitle = self._html_search_regex(r'<h1 class="profile-title song-title"><span class="artist">.*</span>(.*)</h1>', page, "title")
             title = artist+" - "+songtitle
             return {
-                'id': title,  # ignore id, which is not useful in song name
+                'id': id,  # ignore id, which is not useful in song name
                 'title': title,
                 'url': realurl,
                 'ext': 'mp3'

From d36cae46d877ebcc656f23a41b53e31731d2b77e Mon Sep 17 00:00:00 2001
From: xavier <xavier.beynon@gmail.com>
Date: Fri, 24 Oct 2014 21:11:46 -0500
Subject: [PATCH 0292/1937] Not directly calling soundcloud extractor anymore

---
 youtube_dl/extractor/audiomack.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index bdcc51235..2f32253af 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -52,8 +52,7 @@ class AudiomackIE(InfoExtractor):
         #Audiomack wraps a lot of soundcloud tracks in their branded wrapper
         # - if so, pass the work off to the soundcloud extractor
         if SoundcloudIE.suitable(realurl):
-            sc = SoundcloudIE(downloader=self._downloader)
-            return sc._real_extract(realurl)
+            return {'_type': 'url', 'url': realurl, 'ie_key': 'Soundcloud'}
         else:
             #Pull out metadata
             page = self._download_webpage(url, id)

From ec9c978481eee1035ac478926a73f729232be204 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 25 Oct 2014 08:42:44 +0200
Subject: [PATCH 0293/1937] Credit @winwon for vidzi (#3989)

---
 youtube_dl/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index c8bcc6e46..59b851056 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -81,6 +81,7 @@ __authors__  = (
     'lenaten',
     'Dennis Scheiba',
     'Damon Timm',
+    'winwon',
 )
 
 __license__ = 'Public Domain'

From 43d9718fb93ddaafd35e552158ec5bfe61b4c83f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 25 Oct 2014 13:56:21 +0700
Subject: [PATCH 0294/1937] [nhl] Improve video URL extraction (Closes #4013)

---
 youtube_dl/extractor/nhl.py | 34 ++++++++++++++++++++++++----------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index 072d9cf8e..d66c2c6f8 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -22,21 +22,23 @@ class NHLBaseInfoExtractor(InfoExtractor):
         self.report_extraction(video_id)
 
         initial_video_url = info['publishPoint']
-        data = compat_urllib_parse.urlencode({
-            'type': 'fvod',
-            'path': initial_video_url.replace('.mp4', '_sd.mp4'),
-        })
-        path_url = 'http://video.nhl.com/videocenter/servlets/encryptvideopath?' + data
-        path_doc = self._download_xml(
-            path_url, video_id, 'Downloading final video url')
-        video_url = path_doc.find('path').text
+        if info['formats'] == '1':
+            data = compat_urllib_parse.urlencode({
+                'type': 'fvod',
+                'path': initial_video_url.replace('.mp4', '_sd.mp4'),
+            })
+            path_url = 'http://video.nhl.com/videocenter/servlets/encryptvideopath?' + data
+            path_doc = self._download_xml(
+                path_url, video_id, 'Downloading final video url')
+            video_url = path_doc.find('path').text
+        else:
+           video_url = initial_video_url
 
         join = compat_urlparse.urljoin
         return {
             'id': video_id,
             'title': info['name'],
             'url': video_url,
-            'ext': determine_ext(video_url),
             'description': info['description'],
             'duration': int(info['duration']),
             'thumbnail': join(join(video_url, '/u/'), info['bigImage']),
@@ -46,10 +48,11 @@ class NHLBaseInfoExtractor(InfoExtractor):
 
 class NHLIE(NHLBaseInfoExtractor):
     IE_NAME = 'nhl.com'
-    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/console(?:\?(?:.*?[?&])?)id=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/console(?:\?(?:.*?[?&])?)id=(?P<id>[0-9a-z-]+)'
 
     _TESTS = [{
         'url': 'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614',
+        'md5': 'db704a4ea09e8d3988c85e36cc892d09',
         'info_dict': {
             'id': '453614',
             'ext': 'mp4',
@@ -58,6 +61,17 @@ class NHLIE(NHLBaseInfoExtractor):
             'duration': 18,
             'upload_date': '20131006',
         },
+    }, {
+        'url': 'http://video.nhl.com/videocenter/console?id=2014020024-628-h',
+        'md5': 'd22e82bc592f52d37d24b03531ee9696',
+        'info_dict': {
+            'id': '2014020024-628-h',
+            'ext': 'mp4',
+            'title': 'Alex Galchenyuk Goal on Ray Emery (14:40/3rd)',
+            'description': 'Home broadcast - Montreal Canadiens at Philadelphia Flyers - October 11, 2014',
+            'duration': 0,
+            'upload_date': '20141011',
+        },
     }, {
         'url': 'http://video.flames.nhl.com/videocenter/console?id=630616',
         'only_matching': True,

From d3c72db8945cb979a2505099319071717a525816 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 25 Oct 2014 08:58:03 +0200
Subject: [PATCH 0295/1937] [audiomack] Simplify

---
 youtube_dl/extractor/audiomack.py | 40 ++++++++++++++++---------------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 2f32253af..57446fddd 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -38,30 +38,32 @@ class AudiomackIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        #id is what follows /song/ in url, usually the uploader name + title
-        id = self._match_id(url)
+        video_id = self._match_id(url)
 
-        #Call the api, which gives us a json doc with the real url inside
-        rightnow = int(time.time())
-        apiresponse = self._download_json("http://www.audiomack.com/api/music/url/song/"+id+"?_="+str(rightnow), id)
+        api_response = self._download_json(
+            "http://www.audiomack.com/api/music/url/song/%s?_=%d" % (
+                video_id, time.time()),
+            video_id)
 
-        if "url" not in apiresponse:
+        if "url" not in api_response:
             raise ExtractorError("Unable to deduce api url of song")
-        realurl = apiresponse["url"]
+        realurl = api_response["url"]
 
         #Audiomack wraps a lot of soundcloud tracks in their branded wrapper
         # - if so, pass the work off to the soundcloud extractor
         if SoundcloudIE.suitable(realurl):
             return {'_type': 'url', 'url': realurl, 'ie_key': 'Soundcloud'}
-        else:
-            #Pull out metadata
-            page = self._download_webpage(url, id)
-            artist = self._html_search_regex(r'<span class="artist">(.*)</span>', page, "artist")
-            songtitle = self._html_search_regex(r'<h1 class="profile-title song-title"><span class="artist">.*</span>(.*)</h1>', page, "title")
-            title = artist+" - "+songtitle
-            return {
-                'id': id,  # ignore id, which is not useful in song name
-                'title': title,
-                'url': realurl,
-                'ext': 'mp3'
-            }
+
+        webpage = self._download_webpage(url, video_id)
+        artist = self._html_search_regex(
+            r'<span class="artist">(.*?)</span>', webpage, "artist")
+        songtitle = self._html_search_regex(
+            r'<h1 class="profile-title song-title"><span class="artist">.*?</span>(.*?)</h1>',
+            webpage, "title")
+        title = artist + " - " + songtitle
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': realurl,
+        }

From 9a76f416ce67eaaf9c1cdd9058a4e509950950ef Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 25 Oct 2014 13:24:46 +0300
Subject: [PATCH 0296/1937] [lrt] Updated test

---
 youtube_dl/extractor/lrt.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lrt.py b/youtube_dl/extractor/lrt.py
index fca0bfef0..db5df4078 100644
--- a/youtube_dl/extractor/lrt.py
+++ b/youtube_dl/extractor/lrt.py
@@ -22,7 +22,7 @@ class LRTIE(InfoExtractor):
             'id': '54391',
             'ext': 'mp4',
             'title': 'Septynios Kauno dienos',
-            'description': 'Kauno miesto ir apskrities naujienos',
+            'description': 'md5:24d84534c7dc76581e59f5689462411a',
             'duration': 1783,
         },
         'params': {

From 0eb9fb9f24b3eaf1e1eaa5e4576a06ad7794d944 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 25 Oct 2014 22:32:01 +0200
Subject: [PATCH 0297/1937] [soundcloud] Modernize and fix tests

---
 youtube_dl/extractor/soundcloud.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 4719ba45c..c77671fd3 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -40,14 +40,15 @@ class SoundcloudIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy',
-            'file': '62986583.mp3',
             'md5': 'ebef0a451b909710ed1d7787dddbf0d7',
             'info_dict': {
-                "upload_date": "20121011",
-                "description": "No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o'd",
-                "uploader": "E.T. ExTerrestrial Music",
-                "title": "Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1",
-                "duration": 143,
+                'id': '62986583',
+                'ext': 'mp3',
+                'upload_date': '20121011',
+                'description': 'No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o\'d',
+                'uploader': 'E.T. ExTerrestrial Music',
+                'title': 'Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1',
+                'duration': 143,
             }
         },
         # not streamable song
@@ -103,7 +104,7 @@ class SoundcloudIE(InfoExtractor):
                 'id': '128590877',
                 'ext': 'mp3',
                 'title': 'Bus Brakes',
-                'description': 'md5:0170be75dd395c96025d210d261c784e',
+                'description': 'md5:0053ca6396e8d2fd7b7e1595ef12ab66',
                 'uploader': 'oddsamples',
                 'upload_date': '20140109',
                 'duration': 17,

From 681b9caa9cec6105707381412b90bfe998aa3e9d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 25 Oct 2014 22:42:56 +0200
Subject: [PATCH 0298/1937] [tumblr] Fix extraction (fixes #4029)

---
 youtube_dl/extractor/tumblr.py | 35 +++++++++++++---------------------
 1 file changed, 13 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 306fe8974..40c53ff17 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -4,9 +4,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-)
 
 
 class TumblrIE(InfoExtractor):
@@ -18,7 +15,7 @@ class TumblrIE(InfoExtractor):
             'id': '54196191430',
             'ext': 'mp4',
             'title': 'tatiana maslany news, Orphan Black || DVD extra - behind the scenes ↳...',
-            'description': 'md5:dfac39636969fe6bf1caa2d50405f069',
+            'description': 'md5:37db8211e40b50c7c44e95da14f630b7',
             'thumbnail': 're:http://.*\.jpg',
         }
     }, {
@@ -27,7 +24,7 @@ class TumblrIE(InfoExtractor):
         'info_dict': {
             'id': '90208453769',
             'ext': 'mp4',
-            'title': '5SOS STRUM ;)',
+            'title': '5SOS STRUM ;]',
             'description': 'md5:dba62ac8639482759c8eb10ce474586a',
             'thumbnail': 're:http://.*\.jpg',
         }
@@ -41,18 +38,12 @@ class TumblrIE(InfoExtractor):
         url = 'http://%s.tumblr.com/post/%s/' % (blog, video_id)
         webpage = self._download_webpage(url, video_id)
 
-        re_video = r'src=\\x22(?P<video_url>http://%s\.tumblr\.com/video_file/%s/(.*?))\\x22 type=\\x22video/(?P<ext>.*?)\\x22' % (blog, video_id)
-        video = re.search(re_video, webpage)
-        if video is None:
-            raise ExtractorError('Unable to extract video')
-        video_url = video.group('video_url')
-        ext = video.group('ext')
-
-        video_thumbnail = self._search_regex(
-            r'posters.*?\[\\x22(.*?)\\x22',
-            webpage, 'thumbnail', fatal=False)  # We pick the first poster
-        if video_thumbnail:
-            video_thumbnail = video_thumbnail.replace('\\\\/', '/')
+        iframe_url = self._search_regex(
+            r'src=\'(https?://www\.tumblr\.com/video/[^\']+)\'',
+            webpage, 'iframe url')
+        iframe = self._download_webpage(iframe_url, video_id)
+        video_url = self._search_regex(r'<source src="([^"]+)"',
+            iframe, 'video url')
 
         # The only place where you can get a title, it's not complete,
         # but searching in other places doesn't work for all videos
@@ -62,9 +53,9 @@ class TumblrIE(InfoExtractor):
 
         return {
             'id': video_id,
-             'url': video_url,
-             'title': video_title,
-             'description': self._html_search_meta('description', webpage),
-             'thumbnail': video_thumbnail,
-             'ext': ext,
+            'url': video_url,
+            'ext': 'mp4',
+            'title': video_title,
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
         }

From f64f8a46623d4940e6f53c3bf4b23050b054b0f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Oct 2014 08:00:01 +0700
Subject: [PATCH 0299/1937] [sportbox] Remove view count

---
 youtube_dl/extractor/sportbox.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index 19cc976e3..b9cd35109 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -26,7 +26,6 @@ class SportBoxIE(InfoExtractor):
                 'timestamp': 1411896237,
                 'upload_date': '20140928',
                 'duration': 4846,
-                'view_count': int,
             },
             'params': {
                 # m3u8 download
@@ -65,8 +64,6 @@ class SportBoxIE(InfoExtractor):
             r'<span itemprop="uploadDate">([^<]+)</span>', webpage, 'timestamp', fatal=False))
         duration = parse_duration(self._html_search_regex(
             r'<meta itemprop="duration" content="PT([^"]+)">', webpage, 'duration', fatal=False))
-        view_count = int_or_none(self._html_search_regex(
-            r'<span>Просмотров: (\d+)</span>', player, 'view count', fatal=False))
 
         return {
             'id': video_id,
@@ -76,6 +73,5 @@ class SportBoxIE(InfoExtractor):
             'thumbnail': thumbnail,
             'timestamp': timestamp,
             'duration': duration,
-            'view_count': view_count,
             'formats': formats,
         }

From b13ccb1b87199dcfd10b12ed5c3002ff6f951769 Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Sun, 26 Oct 2014 13:35:00 +0530
Subject: [PATCH 0300/1937] [BelgianNational] New extractor added

---
 youtube_dl/extractor/__init__.py        |  1 +
 youtube_dl/extractor/belgiannational.py | 64 +++++++++++++++++++++++++
 2 files changed, 65 insertions(+)
 create mode 100644 youtube_dl/extractor/belgiannational.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 691fef5ca..ec299b72f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -27,6 +27,7 @@ from .bandcamp import BandcampIE, BandcampAlbumIE
 from .bbccouk import BBCCoUkIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
+from .belgiannational import BelgianNationalIE
 from .bild import BildIE
 from .bilibili import BiliBiliIE
 from .blinkx import BlinkxIE
diff --git a/youtube_dl/extractor/belgiannational.py b/youtube_dl/extractor/belgiannational.py
new file mode 100644
index 000000000..2367d1f2b
--- /dev/null
+++ b/youtube_dl/extractor/belgiannational.py
@@ -0,0 +1,64 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+class BelgianNationalIE(InfoExtractor):    
+    _VALID_URL = r'http://(?:deredactie|sporza|cobra)\.be/cm/(.*)/(?P<video_id>[^\']+)'
+    _TESTS = [
+    # deredactie.be
+    {
+        'url': 'http://deredactie.be/cm/vrtnieuws/videozone/programmas/journaal/EP_141025_JOL',
+        'md5': '4cebde1eb60a53782d4f3992cbd46ec8',
+        'info_dict': {
+            'id': 'EP_141025_JOL',
+            'title': 'Het journaal L - 25/10/14',
+            'ext': 'mp4',
+            'duration': 929,
+        }
+    },
+    # sporza.be
+    {
+        'url': 'http://sporza.be/cm/sporza/videozone/programmas/extratime/EP_141020_Extra_time',
+        'md5': '11f53088da9bf8e7cfc42456697953ff',
+        'info_dict': {
+            'id': 'EP_141020_Extra_time',
+            'title': 'Bekijk Extra Time van 20 oktober',
+            'ext': 'mp4',
+            'duration': 3238,
+        }
+        
+    },
+    # cobra.be
+    {
+        'url': 'http://cobra.be/cm/cobra/videozone/rubriek/film-videozone/141022-mv-ellis-cafecorsari',
+        'md5': '78a2b060a5083c4f055449a72477409d',
+        'info_dict': {
+            'id': '141022-mv-ellis-cafecorsari',
+            'title': 'Bret Easton Ellis in Café Corsari',
+            'ext': 'mp4',
+            'duration': 661,
+        }
+    }
+    ]
+    
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('video_id')
+        
+        webpage = self._download_webpage(url, video_id)
+        title = self._og_search_title(webpage)
+        
+        video_url = self._search_regex(r'data-video-src="(.*?)"', webpage, 'Video url') + '/manifest.f4m'
+        duration = int_or_none(self._search_regex(r'data-video-sitestat-duration="(.*?)"', webpage, 'Duration'))
+        
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+            'ext': 'mp4',
+            'duration': duration,
+        }
\ No newline at end of file

From 03936f6e6ddf222a9391193f15b3b6e0b75983cd Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Sun, 26 Oct 2014 13:47:41 +0530
Subject: [PATCH 0301/1937] [BelgianNational] corrected indentation

---
 youtube_dl/extractor/belgiannational.py | 67 ++++++++++++-------------
 1 file changed, 33 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/belgiannational.py b/youtube_dl/extractor/belgiannational.py
index 2367d1f2b..cabff50f2 100644
--- a/youtube_dl/extractor/belgiannational.py
+++ b/youtube_dl/extractor/belgiannational.py
@@ -9,40 +9,39 @@ from ..utils import int_or_none
 class BelgianNationalIE(InfoExtractor):    
     _VALID_URL = r'http://(?:deredactie|sporza|cobra)\.be/cm/(.*)/(?P<video_id>[^\']+)'
     _TESTS = [
-    # deredactie.be
-    {
-        'url': 'http://deredactie.be/cm/vrtnieuws/videozone/programmas/journaal/EP_141025_JOL',
-        'md5': '4cebde1eb60a53782d4f3992cbd46ec8',
-        'info_dict': {
-            'id': 'EP_141025_JOL',
-            'title': 'Het journaal L - 25/10/14',
-            'ext': 'mp4',
-            'duration': 929,
-        }
-    },
-    # sporza.be
-    {
-        'url': 'http://sporza.be/cm/sporza/videozone/programmas/extratime/EP_141020_Extra_time',
-        'md5': '11f53088da9bf8e7cfc42456697953ff',
-        'info_dict': {
-            'id': 'EP_141020_Extra_time',
-            'title': 'Bekijk Extra Time van 20 oktober',
-            'ext': 'mp4',
-            'duration': 3238,
-        }
-        
-    },
-    # cobra.be
-    {
-        'url': 'http://cobra.be/cm/cobra/videozone/rubriek/film-videozone/141022-mv-ellis-cafecorsari',
-        'md5': '78a2b060a5083c4f055449a72477409d',
-        'info_dict': {
-            'id': '141022-mv-ellis-cafecorsari',
-            'title': 'Bret Easton Ellis in Café Corsari',
-            'ext': 'mp4',
-            'duration': 661,
-        }
-    }
+        # deredactie.be
+        {
+            'url': 'http://deredactie.be/cm/vrtnieuws/videozone/programmas/journaal/EP_141025_JOL',
+            'md5': '4cebde1eb60a53782d4f3992cbd46ec8',
+            'info_dict': {
+                'id': 'EP_141025_JOL',
+                'title': 'Het journaal L - 25/10/14',
+                'ext': 'mp4',
+                'duration': 929,
+            }
+        },
+        # sporza.be
+        {
+            'url': 'http://sporza.be/cm/sporza/videozone/programmas/extratime/EP_141020_Extra_time',
+            'md5': '11f53088da9bf8e7cfc42456697953ff',
+            'info_dict': {
+                'id': 'EP_141020_Extra_time',
+                'title': 'Bekijk Extra Time van 20 oktober',
+                'ext': 'mp4',
+                'duration': 3238,
+            }  
+        },
+        # cobra.be
+        {
+            'url': 'http://cobra.be/cm/cobra/videozone/rubriek/film-videozone/141022-mv-ellis-cafecorsari',
+            'md5': '78a2b060a5083c4f055449a72477409d',
+            'info_dict': {
+                'id': '141022-mv-ellis-cafecorsari',
+                'title': 'Bret Easton Ellis in Café Corsari',
+                'ext': 'mp4',
+                'duration': 661,
+            }
+        },
     ]
     
     def _real_extract(self, url):

From 911344e5aca14d207731148f4862913827c8c871 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 26 Oct 2014 16:48:11 +0700
Subject: [PATCH 0302/1937] [vrt] Improve extractor

---
 youtube_dl/extractor/__init__.py        |  2 +-
 youtube_dl/extractor/belgiannational.py | 63 ----------------
 youtube_dl/extractor/vrt.py             | 95 +++++++++++++++++++++++++
 3 files changed, 96 insertions(+), 64 deletions(-)
 delete mode 100644 youtube_dl/extractor/belgiannational.py
 create mode 100644 youtube_dl/extractor/vrt.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ec299b72f..5ec9b4745 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -27,7 +27,6 @@ from .bandcamp import BandcampIE, BandcampAlbumIE
 from .bbccouk import BBCCoUkIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
-from .belgiannational import BelgianNationalIE
 from .bild import BildIE
 from .bilibili import BiliBiliIE
 from .blinkx import BlinkxIE
@@ -446,6 +445,7 @@ from .viki import VikiIE
 from .vk import VKIE
 from .vodlocker import VodlockerIE
 from .vporn import VpornIE
+from .vrt import VRTIE
 from .vube import VubeIE
 from .vuclip import VuClipIE
 from .vulture import VultureIE
diff --git a/youtube_dl/extractor/belgiannational.py b/youtube_dl/extractor/belgiannational.py
deleted file mode 100644
index cabff50f2..000000000
--- a/youtube_dl/extractor/belgiannational.py
+++ /dev/null
@@ -1,63 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import int_or_none
-
-class BelgianNationalIE(InfoExtractor):    
-    _VALID_URL = r'http://(?:deredactie|sporza|cobra)\.be/cm/(.*)/(?P<video_id>[^\']+)'
-    _TESTS = [
-        # deredactie.be
-        {
-            'url': 'http://deredactie.be/cm/vrtnieuws/videozone/programmas/journaal/EP_141025_JOL',
-            'md5': '4cebde1eb60a53782d4f3992cbd46ec8',
-            'info_dict': {
-                'id': 'EP_141025_JOL',
-                'title': 'Het journaal L - 25/10/14',
-                'ext': 'mp4',
-                'duration': 929,
-            }
-        },
-        # sporza.be
-        {
-            'url': 'http://sporza.be/cm/sporza/videozone/programmas/extratime/EP_141020_Extra_time',
-            'md5': '11f53088da9bf8e7cfc42456697953ff',
-            'info_dict': {
-                'id': 'EP_141020_Extra_time',
-                'title': 'Bekijk Extra Time van 20 oktober',
-                'ext': 'mp4',
-                'duration': 3238,
-            }  
-        },
-        # cobra.be
-        {
-            'url': 'http://cobra.be/cm/cobra/videozone/rubriek/film-videozone/141022-mv-ellis-cafecorsari',
-            'md5': '78a2b060a5083c4f055449a72477409d',
-            'info_dict': {
-                'id': '141022-mv-ellis-cafecorsari',
-                'title': 'Bret Easton Ellis in Café Corsari',
-                'ext': 'mp4',
-                'duration': 661,
-            }
-        },
-    ]
-    
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
-        
-        webpage = self._download_webpage(url, video_id)
-        title = self._og_search_title(webpage)
-        
-        video_url = self._search_regex(r'data-video-src="(.*?)"', webpage, 'Video url') + '/manifest.f4m'
-        duration = int_or_none(self._search_regex(r'data-video-sitestat-duration="(.*?)"', webpage, 'Duration'))
-        
-        return {
-            'id': video_id,
-            'title': title,
-            'url': video_url,
-            'ext': 'mp4',
-            'duration': duration,
-        }
\ No newline at end of file
diff --git a/youtube_dl/extractor/vrt.py b/youtube_dl/extractor/vrt.py
new file mode 100644
index 000000000..57ef8dc30
--- /dev/null
+++ b/youtube_dl/extractor/vrt.py
@@ -0,0 +1,95 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import float_or_none
+
+
+class VRTIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:deredactie|sporza|cobra)\.be/cm/(?:[^/]+/)+(?P<id>[^/]+)/*'
+    _TESTS = [
+        # deredactie.be
+        {
+            'url': 'http://deredactie.be/cm/vrtnieuws/videozone/programmas/journaal/EP_141025_JOL',
+            'md5': '4cebde1eb60a53782d4f3992cbd46ec8',
+            'info_dict': {
+                'id': '2129880',
+                'ext': 'flv',
+                'title': 'Het journaal L - 25/10/14',
+                'description': None,
+                'timestamp': 1414271750.949,
+                'upload_date': '20141025',
+                'duration': 929,
+            }
+        },
+        # sporza.be
+        {
+            'url': 'http://sporza.be/cm/sporza/videozone/programmas/extratime/EP_141020_Extra_time',
+            'md5': '11f53088da9bf8e7cfc42456697953ff',
+            'info_dict': {
+                'id': '2124639',
+                'ext': 'flv',
+                'title': 'Bekijk Extra Time van 20 oktober',
+                'description': 'md5:83ac5415a4f1816c6a93f8138aef2426',
+                'timestamp': 1413835980.560,
+                'upload_date': '20141020',
+                'duration': 3238,
+            }  
+        },
+        # cobra.be
+        {
+            'url': 'http://cobra.be/cm/cobra/videozone/rubriek/film-videozone/141022-mv-ellis-cafecorsari',
+            'md5': '78a2b060a5083c4f055449a72477409d',
+            'info_dict': {
+                'id': '2126050',
+                'ext': 'flv',
+                'title': 'Bret Easton Ellis in Café Corsari',
+                'description': 'md5:f699986e823f32fd6036c1855a724ee9',
+                'timestamp': 1413967500.494,
+                'upload_date': '20141022',
+                'duration': 661,
+            }
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_id = self._search_regex(
+            r'data-video-id="([^"]+)_[^"]+"', webpage, 'video id', fatal=False)
+
+        formats = []
+        mobj = re.search(
+            r'data-video-iphone-server="(?P<server>[^"]+)"\s+data-video-iphone-path="(?P<path>[^"]+)"',
+            webpage)
+        if mobj:
+            formats.extend(self._extract_m3u8_formats(
+                '%s/%s' % (mobj.group('server'), mobj.group('path')),
+                video_id, 'mp4'))
+        mobj = re.search(r'data-video-src="(?P<src>[^"]+)"', webpage)
+        if mobj:
+            formats.extend(self._extract_f4m_formats(
+                '%s/manifest.f4m' % mobj.group('src'), video_id))
+        self._sort_formats(formats)
+
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage, default=None)
+        thumbnail = self._og_search_thumbnail(webpage)
+        timestamp = float_or_none(self._search_regex(
+            r'data-video-sitestat-pubdate="(\d+)"', webpage, 'timestamp', fatal=False), 1000)
+        duration = float_or_none(self._search_regex(
+            r'data-video-duration="(\d+)"', webpage, 'duration', fatal=False), 1000)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'formats': formats,
+        }
\ No newline at end of file

From 1bdeb7be2e5bef703d54d8786fbc6f3c0c23faef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 26 Oct 2014 13:01:57 +0100
Subject: [PATCH 0303/1937] Set '--simulate' if any of the printing options is
 given (fixes #3036)

That's what the help messages say.
Previously it would only set '--skip-download', which would write thumbnail,
descriptions or subtitles if they were requested (for example you may have set
'--write-thumbnail' in the config file).
---
 youtube_dl/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 4f5ce604f..578ecc4ab 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -307,8 +307,8 @@ def _real_main(argv=None):
         'forceformat': opts.getformat,
         'forcejson': opts.dumpjson,
         'dump_single_json': opts.dump_single_json,
-        'simulate': opts.simulate,
-        'skip_download': (opts.skip_download or opts.simulate or any_printing),
+        'simulate': opts.simulate or any_printing,
+        'skip_download': opts.skip_download,
         'format': opts.format,
         'format_limit': opts.format_limit,
         'listformats': opts.listformats,

From a7e97f6db17240ae33719550d62df0816ab410ad Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 14:15:48 +0100
Subject: [PATCH 0304/1937] [generic] Allow new SWFObject()-style imports

This embed style is used on http://www.bitburger-open.de/ , but that is not included as a test case since the format is likely to be temporary.
---
 youtube_dl/extractor/generic.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 9b6498894..524215408 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -635,7 +635,8 @@ class GenericIE(InfoExtractor):
                 <iframe[^>]+?src=|
                 data-video-url=|
                 <embed[^>]+?src=|
-                embedSWF\(?:\s*
+                embedSWF\(?:\s*|
+                new\s+SWFObject\(
             )
             (["\'])
                 (?P<url>(?:https?:)?//(?:www\.)?youtube(?:-nocookie)?\.com/

From d28b5171541aaef568372a78384d6a5c862b2369 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 16:31:52 +0100
Subject: [PATCH 0305/1937] [YoutubeDL] Output avconv/ffmpeg versions if -v is
 given

---
 youtube_dl/YoutubeDL.py              | 16 ++++++++++---
 youtube_dl/postprocessor/__init__.py | 12 ++++++----
 youtube_dl/postprocessor/ffmpeg.py   | 34 +++++++++++++++++++++++-----
 3 files changed, 48 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 242affb5b..ecf426e37 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -62,7 +62,7 @@ from .utils import (
 from .cache import Cache
 from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader
-from .postprocessor import FFmpegMergerPP
+from .postprocessor import FFmpegMergerPP, FFmpegPostProcessor
 from .version import __version__
 
 
@@ -1311,8 +1311,18 @@ class YoutubeDL(object):
                 sys.exc_clear()
             except:
                 pass
-        self._write_string('[debug] Python version %s - %s' %
-                     (platform.python_version(), platform_name()) + '\n')
+        self._write_string('[debug] Python version %s - %s\n' % (
+            platform.python_version(), platform_name()))
+
+        exe_versions = FFmpegPostProcessor.get_versions()
+        exe_str = ', '.join(
+            '%s %s' % (exe, v)
+            for exe, v in sorted(exe_versions.items())
+            if v
+        )
+        if not exe_str:
+            exe_str = 'none'
+        self._write_string('[debug] exe versions: %s\n' % exe_str)
 
         proxy_map = {}
         for handler in self._opener.handlers:
diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index 15aa0daa9..6ac67cbae 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -1,24 +1,26 @@
 
 from .atomicparsley import AtomicParsleyPP
 from .ffmpeg import (
+    FFmpegPostProcessor,
     FFmpegAudioFixPP,
+    FFmpegEmbedSubtitlePP,
+    FFmpegExtractAudioPP,
     FFmpegMergerPP,
     FFmpegMetadataPP,
     FFmpegVideoConvertor,
-    FFmpegExtractAudioPP,
-    FFmpegEmbedSubtitlePP,
 )
 from .xattrpp import XAttrMetadataPP
 from .execafterdownload import ExecAfterDownloadPP
 
 __all__ = [
     'AtomicParsleyPP',
+    'ExecAfterDownloadPP',
     'FFmpegAudioFixPP',
+    'FFmpegEmbedSubtitlePP',
+    'FFmpegExtractAudioPP',
     'FFmpegMergerPP',
     'FFmpegMetadataPP',
+    'FFmpegPostProcessor',
     'FFmpegVideoConvertor',
-    'FFmpegExtractAudioPP',
-    'FFmpegEmbedSubtitlePP',
     'XAttrMetadataPP',
-    'ExecAfterDownloadPP',
 ]
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 6f010a9c7..867e75d80 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -1,4 +1,5 @@
 import os
+import re
 import subprocess
 import sys
 import time
@@ -18,6 +19,23 @@ from ..utils import (
 )
 
 
+def get_version(executable):
+    """ Returns the version of the specified executable,
+    or False if the executable is not present """
+    try:
+        out, err = subprocess.Popen(
+            [executable, '-version'],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+    except OSError:
+        return False
+    firstline = out.partition(b'\n')[0].decode('ascii', 'ignore')
+    m = re.search(r'version\s+([0-9._-a-zA-Z]+)', firstline)
+    if not m:
+        return u'present'
+    else:
+        return m.group(1)
+
+
 class FFmpegPostProcessorError(PostProcessingError):
     pass
 
@@ -25,22 +43,26 @@ class FFmpegPostProcessorError(PostProcessingError):
 class FFmpegPostProcessor(PostProcessor):
     def __init__(self, downloader=None, deletetempfiles=False):
         PostProcessor.__init__(self, downloader)
-        self._exes = self.detect_executables()
+        self._versions = self.get_versions()
         self._deletetempfiles = deletetempfiles
 
     @staticmethod
-    def detect_executables():
+    def get_versions():
         programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
-        return dict((program, check_executable(program, ['-version'])) for program in programs)
+        return dict((program, get_version(program)) for program in programs)
 
     def _get_executable(self):
         if self._downloader.params.get('prefer_ffmpeg', False):
-            return self._exes['ffmpeg'] or self._exes['avconv']
+            prefs = ('ffmpeg', 'avconv')
         else:
-            return self._exes['avconv'] or self._exes['ffmpeg']
+            prefs = ('avconv', 'ffmpeg')
+        for p in prefs:
+            if self._versions[p]:
+                return p
+        return None
 
     def _uses_avconv(self):
-        return self._get_executable() == self._exes['avconv']
+        return self._get_executable() == 'avconv'
 
     def run_ffmpeg_multiple_files(self, input_paths, out_path, opts):
         if not self._get_executable():

From 488447455d3d90e1d83a7ebc2f9ce552e031e0d8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 16:46:34 +0100
Subject: [PATCH 0306/1937] [ffmpeg] Warn if ffmpeg/avconv version is too old
 (Fixes #4026)

---
 youtube_dl/YoutubeDL.py            |  2 +-
 youtube_dl/postprocessor/ffmpeg.py | 28 ++++++++++++++++++++--------
 youtube_dl/utils.py                | 13 +++++++++++++
 3 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ecf426e37..28dcc0195 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1026,7 +1026,7 @@ class YoutubeDL(object):
                         downloaded = []
                         success = True
                         merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
-                        if not merger._get_executable():
+                        if not merger._executable:
                             postprocessors = []
                             self.report_warning('You have requested multiple '
                                 'formats but ffmpeg or avconv are not installed.'
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 867e75d80..e13deab4b 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -12,6 +12,7 @@ from ..utils import (
     compat_subprocess_get_DEVNULL,
     encodeArgument,
     encodeFilename,
+    is_outdated_version,
     PostProcessingError,
     prepend_extension,
     shell_quote,
@@ -41,17 +42,29 @@ class FFmpegPostProcessorError(PostProcessingError):
 
 
 class FFmpegPostProcessor(PostProcessor):
-    def __init__(self, downloader=None, deletetempfiles=False):
+    def __init__(self, downloader, deletetempfiles=False):
         PostProcessor.__init__(self, downloader)
         self._versions = self.get_versions()
         self._deletetempfiles = deletetempfiles
 
+    def check_version(self):
+        if not self._executable:
+            raise FFmpegPostProcessorError(u'ffmpeg or avconv not found. Please install one.')
+
+        REQUIRED_VERSION = '1.0'
+        if is_outdated_version(
+                self._versions[self._executable], REQUIRED_VERSION):
+            warning = u'Your copy of %s is outdated, update %s to version %s or newer if you encounter any errors.' % (
+                self._executable, self._executable, REQUIRED_VERSION)
+            self._downloader.report_warning(warning)
+
     @staticmethod
     def get_versions():
         programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
         return dict((program, get_version(program)) for program in programs)
 
-    def _get_executable(self):
+    @property
+    def _executable(self):
         if self._downloader.params.get('prefer_ffmpeg', False):
             prefs = ('ffmpeg', 'avconv')
         else:
@@ -62,16 +75,15 @@ class FFmpegPostProcessor(PostProcessor):
         return None
 
     def _uses_avconv(self):
-        return self._get_executable() == 'avconv'
+        return self._executable == 'avconv'
 
     def run_ffmpeg_multiple_files(self, input_paths, out_path, opts):
-        if not self._get_executable():
-            raise FFmpegPostProcessorError(u'ffmpeg or avconv not found. Please install one.')
+        self.check_version()
 
         files_cmd = []
         for path in input_paths:
             files_cmd.extend(['-i', encodeFilename(path, True)])
-        cmd = ([self._get_executable(), '-y'] + files_cmd
+        cmd = ([self._executable, '-y'] + files_cmd
                + [encodeArgument(o) for o in opts] +
                [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])
 
@@ -204,14 +216,14 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
             if self._nopostoverwrites and os.path.exists(encodeFilename(new_path)):
                 self._downloader.to_screen(u'[youtube] Post-process file %s exists, skipping' % new_path)
             else:
-                self._downloader.to_screen(u'[' + self._get_executable() + '] Destination: ' + new_path)
+                self._downloader.to_screen(u'[' + self._executable + '] Destination: ' + new_path)
                 self.run_ffmpeg(path, new_path, acodec, more_opts)
         except:
             etype,e,tb = sys.exc_info()
             if isinstance(e, AudioConversionError):
                 msg = u'audio conversion failed: ' + e.msg
             else:
-                msg = u'error running ' + self._get_executable()
+                msg = u'error running ' + self._executable
             raise PostProcessingError(msg)
 
         # Try to update the date time for extracted audio file.
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9287edd8d..6c0c39ca5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1723,3 +1723,16 @@ def limit_length(s, length):
     if len(s) > length:
         return s[:length - len(ELLIPSES)] + ELLIPSES
     return s
+
+
+def version_tuple(v):
+    return [int(e) for e in v.split('.')]
+
+
+def is_outdated_version(version, limit, assume_new=True):
+    if not version:
+        return not assume_new
+    try:
+        return version_tuple(version) < version_tuple(limit)
+    except ValueError:
+        return not assume_new

From 23be51d8ce132dbb967f460e1225fdaaa43dff39 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 17:05:44 +0100
Subject: [PATCH 0307/1937] [generic] Handle audio streams that do not
 implement HEAD (Fixes #4032)

---
 youtube_dl/extractor/common.py  |  7 +++--
 youtube_dl/extractor/generic.py | 53 +++++++++++++++++----------------
 2 files changed, 32 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index cf3781cd6..e1bd6bb49 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -242,7 +242,6 @@ class InfoExtractor(object):
 
     def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
         """ Returns a tuple (page content as string, URL handle) """
-
         # Strip hashes from the URL (#1038)
         if isinstance(url_or_request, (compat_str, str)):
             url_or_request = url_or_request.partition('#')[0]
@@ -251,6 +250,10 @@ class InfoExtractor(object):
         if urlh is False:
             assert not fatal
             return False
+        content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal)
+        return (content, urlh)
+
+    def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True):
         content_type = urlh.headers.get('Content-Type', '')
         webpage_bytes = urlh.read()
         m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
@@ -309,7 +312,7 @@ class InfoExtractor(object):
                 msg += ' Visit %s for more details' % blocked_iframe
             raise ExtractorError(msg, expected=True)
 
-        return (content, urlh)
+        return content
 
     def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
         """ Returns the data of the page as a string """
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 524215408..51dbbc8db 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -503,14 +503,14 @@ class GenericIE(InfoExtractor):
         self.to_screen('%s: Requesting header' % video_id)
 
         head_req = HEADRequest(url)
-        response = self._request_webpage(
+        head_response = self._request_webpage(
             head_req, video_id,
             note=False, errnote='Could not send HEAD request to %s' % url,
             fatal=False)
 
-        if response is not False:
+        if head_response is not False:
             # Check for redirect
-            new_url = response.geturl()
+            new_url = head_response.geturl()
             if url != new_url:
                 self.report_following_redirect(new_url)
                 if force_videoid:
@@ -518,34 +518,35 @@ class GenericIE(InfoExtractor):
                         new_url, {'force_videoid': force_videoid})
                 return self.url_result(new_url)
 
-            # Check for direct link to a video
-            content_type = response.headers.get('Content-Type', '')
-            m = re.match(r'^(?P<type>audio|video|application(?=/ogg$))/(?P<format_id>.+)$', content_type)
-            if m:
-                upload_date = response.headers.get('Last-Modified')
-                if upload_date:
-                    upload_date = unified_strdate(upload_date)
-                return {
-                    'id': video_id,
-                    'title': os.path.splitext(url_basename(url))[0],
-                    'formats': [{
-                        'format_id': m.group('format_id'),
-                        'url': url,
-                        'vcodec': 'none' if m.group('type') == 'audio' else None
-                    }],
-                    'upload_date': upload_date,
-                }
+        full_response = None
+        if head_response is False:
+            full_response = self._request_webpage(url, video_id)
+            head_response = full_response
+
+        # Check for direct link to a video
+        content_type = head_response.headers.get('Content-Type', '')
+        m = re.match(r'^(?P<type>audio|video|application(?=/ogg$))/(?P<format_id>.+)$', content_type)
+        if m:
+            upload_date = unified_strdate(
+                head_response.headers.get('Last-Modified'))
+            return {
+                'id': video_id,
+                'title': os.path.splitext(url_basename(url))[0],
+                'formats': [{
+                    'format_id': m.group('format_id'),
+                    'url': url,
+                    'vcodec': 'none' if m.group('type') == 'audio' else None
+                }],
+                'upload_date': upload_date,
+            }
 
         if not self._downloader.params.get('test', False) and not is_intentional:
             self._downloader.report_warning('Falling back on generic information extractor.')
 
-        try:
+        if full_response:
+            webpage = _webpage_read_content(url, video_id)
+        else:
             webpage = self._download_webpage(url, video_id)
-        except ValueError:
-            # since this is the last-resort InfoExtractor, if
-            # this error is thrown, it'll be thrown here
-            raise ExtractorError('Failed to download URL: %s' % url)
-
         self.report_extraction(video_id)
 
         # Is it an RSS feed?

From 09e5d6a6e564ecaa44a174456882bd1998eacbf8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 17:28:09 +0100
Subject: [PATCH 0308/1937] [crunchyroll:playlist] Simplify (#3988)

---
 youtube_dl/extractor/crunchyroll.py | 50 +++++++++++++++--------------
 1 file changed, 26 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 2dca52660..05b21e872 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -293,34 +293,36 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
 class CrunchyrollShowPlaylistIE(InfoExtractor):
     IE_NAME = "crunchyroll:playlist"
-    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<show>[\w\-]+))/?$'
-    _TITLE_EXTR = r'<span\s+itemprop="name">\s*(?P<showtitle>[\w\s]+)'
+    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?!(?:news|anime-news|library|forum|launchcalendar|lineup|store|comics|freetrial|login))(?P<id>[\w\-]+))/?$'
 
     _TESTS = [{
-        'url' : 'http://www.crunchyroll.com/attack-on-titan',
-        'info_dict' : {
-            'title' : 'Attack on Titan'
+        'url': 'http://www.crunchyroll.com/a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi',
+        'info_dict': {
+            'id': 'a-bridge-to-the-starry-skies-hoshizora-e-kakaru-hashi',
+            'title': 'A Bridge to the Starry Skies - Hoshizora e Kakaru Hashi'
         },
-        'playlist_count' : 15
+        'playlist_count': 13,
     }]
 
-    def _extract_title_entries(self,id,webpage):
-        _EPISODE_ID_EXTR = r'id="showview_videos_media_(?P<vidid>\d+)".*?href="/{0}/(?P<vidurl>[\w\-]+-(?P=vidid))"'.format(id)
-        title = self._html_search_regex(self._TITLE_EXTR,webpage,"title",flags=re.UNICODE|re.MULTILINE)
-        episode_urls = [self.url_result('http://www.crunchyroll.com/{0}/{1}'.format(id, showmatch[1])) for
-                    showmatch in re.findall(_EPISODE_ID_EXTR, webpage,re.UNICODE|re.MULTILINE|re.DOTALL)]
-        episode_urls.reverse()
-        return title, episode_urls
-
-
     def _real_extract(self, url):
-        url_match = re.match(self._VALID_URL,url)
-        show_id = url_match.group('show')
-        webpage = self._download_webpage(url,show_id)
-        (title,entries) = self._extract_title_entries(show_id,webpage)
+        show_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, show_id)
+        title = self._html_search_regex(
+            r'(?s)<h1[^>]*>\s*<span itemprop="name">(.*?)</span>',
+            webpage, 'title')
+        episode_paths = re.findall(
+            r'(?s)<li id="showview_videos_media_[0-9]+"[^>]+>.*?<a href="([^"]+)"',
+            webpage)
+        entries = [
+            self.url_result('http://www.crunchyroll.com' + ep, 'Crunchyroll')
+            for ep in episode_paths
+        ]
+        entries.reverse()
+
         return {
-            '_type' : 'playlist',
-            'id' : show_id,
-            'title' : title,
-            'entries' : entries
-        }
\ No newline at end of file
+            '_type': 'playlist',
+            'id': show_id,
+            'title': title,
+            'entries': entries,
+        }

From 5894a4f4eee4dd9684b3f575f4a7d030c7084eb2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 17:28:50 +0100
Subject: [PATCH 0309/1937] Credit @gabeos for crunchyroll:playlist (#3988)

---
 youtube_dl/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index cb4f2e41c..e2c6f38a9 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -82,7 +82,8 @@ __authors__  = (
     'Dennis Scheiba',
     'Damon Timm',
     'winwon',
-    'Xavier Beynon'
+    'Xavier Beynon',
+    'Gabriel Schubiner',
 )
 
 __license__ = 'Public Domain'

From d481699a7a9bc8769deb9f7b9cdb5754a1fec5ba Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 17:29:27 +0100
Subject: [PATCH 0310/1937] release 2014.10.26

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d822ae330..f5f60cdc3 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.10.25'
+__version__ = '2014.10.26'

From 64269e4d0117a04096696532bccd90ee6b176f25 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 18:00:50 +0100
Subject: [PATCH 0311/1937] Move AUTHORS to root (closes #2985)

---
 AUTHORS                | 82 ++++++++++++++++++++++++++++++++++++++++
 youtube_dl/__init__.py | 85 ------------------------------------------
 2 files changed, 82 insertions(+), 85 deletions(-)
 create mode 100644 AUTHORS

diff --git a/AUTHORS b/AUTHORS
new file mode 100644
index 000000000..a574949b3
--- /dev/null
+++ b/AUTHORS
@@ -0,0 +1,82 @@
+Ricardo Garcia Gonzalez
+Danny Colligan
+Benjamin Johnson
+Vasyl' Vavrychuk
+Witold Baryluk
+Paweł Paprota
+Gergely Imreh
+Rogério Brito
+Philipp Hagemeister
+Sören Schulze
+Kevin Ngo
+Ori Avtalion
+shizeeg
+Filippo Valsorda
+Christian Albrecht
+Dave Vasilevsky
+Jaime Marquínez Ferrándiz
+Jeff Crouse
+Osama Khalid
+Michael Walter
+M. Yasoob Ullah Khalid
+Julien Fraichard
+Johny Mo Swag
+Axel Noack
+Albert Kim
+Pierre Rudloff
+Huarong Huo
+Ismael Mejía
+Steffan 'Ruirize' James
+Andras Elso
+Jelle van der Waa
+Marcin Cieślak
+Anton Larionov
+Takuya Tsuchida
+Sergey M.
+Michael Orlitzky
+Chris Gahan
+Saimadhav Heblikar
+Mike Col
+Oleg Prutz
+pulpe
+Andreas Schmitz
+Michael Kaiser
+Niklas Laxström
+David Triendl
+Anthony Weems
+David Wagner
+Juan C. Olivares
+Mattias Harrysson
+phaer
+Sainyam Kapoor
+Nicolas Évrard
+Jason Normore
+Hoje Lee
+Adam Thalhammer
+Georg Jähnig
+Ralf Haring
+Koki Takahashi
+Ariset Llerena
+Adam Malcontenti-Wilson
+Tobias Bell
+Naglis Jonaitis
+Charles Chen
+Hassaan Ali
+Dobrosław Żybort
+David Fabijan
+Sebastian Haas
+Alexander Kirk
+Erik Johnson
+Keith Beckman
+Ole Ernst
+Aaron McDaniel (mcd1992)
+Magnus Kolstad
+Hari Padmanaban
+Carlos Ramos
+5moufl
+lenaten
+Dennis Scheiba
+Damon Timm
+winwon
+Xavier Beynon
+Gabriel Schubiner
\ No newline at end of file
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index e2c6f38a9..7dc971884 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -1,91 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-__authors__  = (
-    'Ricardo Garcia Gonzalez',
-    'Danny Colligan',
-    'Benjamin Johnson',
-    'Vasyl\' Vavrychuk',
-    'Witold Baryluk',
-    'Paweł Paprota',
-    'Gergely Imreh',
-    'Rogério Brito',
-    'Philipp Hagemeister',
-    'Sören Schulze',
-    'Kevin Ngo',
-    'Ori Avtalion',
-    'shizeeg',
-    'Filippo Valsorda',
-    'Christian Albrecht',
-    'Dave Vasilevsky',
-    'Jaime Marquínez Ferrándiz',
-    'Jeff Crouse',
-    'Osama Khalid',
-    'Michael Walter',
-    'M. Yasoob Ullah Khalid',
-    'Julien Fraichard',
-    'Johny Mo Swag',
-    'Axel Noack',
-    'Albert Kim',
-    'Pierre Rudloff',
-    'Huarong Huo',
-    'Ismael Mejía',
-    'Steffan \'Ruirize\' James',
-    'Andras Elso',
-    'Jelle van der Waa',
-    'Marcin Cieślak',
-    'Anton Larionov',
-    'Takuya Tsuchida',
-    'Sergey M.',
-    'Michael Orlitzky',
-    'Chris Gahan',
-    'Saimadhav Heblikar',
-    'Mike Col',
-    'Oleg Prutz',
-    'pulpe',
-    'Andreas Schmitz',
-    'Michael Kaiser',
-    'Niklas Laxström',
-    'David Triendl',
-    'Anthony Weems',
-    'David Wagner',
-    'Juan C. Olivares',
-    'Mattias Harrysson',
-    'phaer',
-    'Sainyam Kapoor',
-    'Nicolas Évrard',
-    'Jason Normore',
-    'Hoje Lee',
-    'Adam Thalhammer',
-    'Georg Jähnig',
-    'Ralf Haring',
-    'Koki Takahashi',
-    'Ariset Llerena',
-    'Adam Malcontenti-Wilson',
-    'Tobias Bell',
-    'Naglis Jonaitis',
-    'Charles Chen',
-    'Hassaan Ali',
-    'Dobrosław Żybort',
-    'David Fabijan',
-    'Sebastian Haas',
-    'Alexander Kirk',
-    'Erik Johnson',
-    'Keith Beckman',
-    'Ole Ernst',
-    'Aaron McDaniel (mcd1992)',
-    'Magnus Kolstad',
-    'Hari Padmanaban',
-    'Carlos Ramos',
-    '5moufl',
-    'lenaten',
-    'Dennis Scheiba',
-    'Damon Timm',
-    'winwon',
-    'Xavier Beynon',
-    'Gabriel Schubiner',
-)
-
 __license__ = 'Public Domain'
 
 import codecs

From 5f82b129e085eb122b6129df36e45925f5b4546a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 26 Oct 2014 18:11:31 +0100
Subject: [PATCH 0312/1937] [ffmpeg] Also look into stderr for extracting the
 version

At least with avconv 11, it will print 'avconv version 11, ..' to stderr, not stdout.
---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index e13deab4b..19fc135dc 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -26,7 +26,7 @@ def get_version(executable):
     try:
         out, err = subprocess.Popen(
             [executable, '-version'],
-            stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+            stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
     except OSError:
         return False
     firstline = out.partition(b'\n')[0].decode('ascii', 'ignore')

From 579657ad8726ab0cdefd98fbbb28f09fbcf94e96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 26 Oct 2014 19:08:36 +0100
Subject: [PATCH 0313/1937] [soundcloud] Set the 'webpage_url' field for each
 track

For playlists, YoutubeDL would set it to the playlist url.
---
 youtube_dl/extractor/soundcloud.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index c77671fd3..54256e1a2 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -141,6 +141,7 @@ class SoundcloudIE(InfoExtractor):
             'description': info['description'],
             'thumbnail': thumbnail,
             'duration': int_or_none(info.get('duration'), 1000),
+            'webpage_url': info.get('permalink_url'),
         }
         formats = []
         if info.get('downloadable', False):

From 0551a02b82b4209422e0734150e0186210cd0723 Mon Sep 17 00:00:00 2001
From: dinesh <dsdiscoverdinesh@gmail.com>
Date: Mon, 27 Oct 2014 01:08:51 +0530
Subject: [PATCH 0314/1937] [Rte] Improve extractor

---
 youtube_dl/extractor/rte.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index ab12bb5fd..8d4e94e55 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -18,7 +18,7 @@ class RteIE(InfoExtractor):
             'title': 'Nine News',
             'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'The One O\'Clock News followed by Weather.',
-            'duration': 1622963.0,
+            'duration': 1622.963,
         }
     }
     
@@ -29,7 +29,7 @@ class RteIE(InfoExtractor):
         title = self._og_search_title(webpage)
         
         description = self._search_regex(r'<meta name="description" content="(.*?)" />', webpage, 'description')
-        duration = float_or_none(self._html_search_meta('duration', webpage, 'duration'))
+        duration = float_or_none(self._html_search_meta('duration', webpage, 'duration'), 1000)
         
         thumbnail_id = self._search_regex(r'<meta name="thumbnail" content="uri:irus:(.*?)" />', webpage, 'thumbnail')
         thumbnail = 'http://img.rasset.ie/' + thumbnail_id + '.jpg' 
@@ -40,8 +40,13 @@ class RteIE(InfoExtractor):
         # f4m_url = server + relative_url
         f4m_url = json_string['shows'][0]['media:group'][0]['rte:server'] + json_string['shows'][0]['media:group'][0]['url']
         f4m_formats = self._extract_f4m_formats(f4m_url, video_id)
-        for f in f4m_formats:
-            del f['tbr']
+        f4m_formats = [{
+            'format_id': f['format_id'],
+            'url': f['url'],
+            'ext': 'mp4',
+            'width': f['width'],
+            'height': f['height'],
+        } for f in f4m_formats ]
         
         return {
             'id': video_id,

From 70b7e3fbb62e9bec12328bb9d506327fe77a85c0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 20:49:51 +0100
Subject: [PATCH 0315/1937] [generic] Add a test case for direct links with
 broken HEAD (#4032)

---
 test/helper.py                  | 10 ++++++++++
 test/test_download.py           |  2 ++
 youtube_dl/extractor/generic.py | 17 ++++++++++++++++-
 3 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index 2fa45631a..fb8618120 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -171,3 +171,13 @@ def assertGreaterEqual(self, got, expected, msg=None):
         if msg is None:
             msg = '%r not greater than or equal to %r' % (got, expected)
         self.assertTrue(got >= expected, msg)
+
+
+def expect_warnings(ydl, warnings_re):
+    real_warning = ydl.report_warning
+
+    def _report_warning(w):
+        if not any(re.search(w_re, w) for w_re in warnings_re):
+            real_warning(w)
+
+    ydl.report_warning = _report_warning
diff --git a/test/test_download.py b/test/test_download.py
index 8178015ea..608e4528e 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -8,6 +8,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from test.helper import (
     assertGreaterEqual,
+    expect_warnings,
     get_params,
     gettestcases,
     expect_info_dict,
@@ -100,6 +101,7 @@ def generator(test_case):
             if status['status'] == 'finished':
                 finished_hook_called.add(status['filename'])
         ydl.add_progress_hook(_hook)
+        expect_warnings(ydl, test_case.get('expected_warnings', []))
 
         def get_tc_filename(tc):
             return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 51dbbc8db..52f286ac6 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -389,8 +389,23 @@ class GenericIE(InfoExtractor):
                 'title': 'Conversation about Hexagonal Rails Part 1 - ThoughtWorks',
                 'duration': 1715.0,
                 'uploader': 'thoughtworks.wistia.com',
-            },   
+            },
         },
+        # Direct download with broken HEAD
+        {
+            'url': 'http://ai-radio.org:8000/radio.opus',
+            'info_dict': {
+                'id': 'radio',
+                'ext': 'opus',
+                'title': 'radio',
+            },
+            'params': {
+                'skip_download': True,  # infinite live stream
+            },
+            'expected_warnings': [
+                r'501.*Not Implemented'
+            ],
+        }
     ]
 
     def report_following_redirect(self, new_url):

From 37d66e7f1e5d325802781c69042571968afa1da0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 20:58:09 +0100
Subject: [PATCH 0316/1937] [generic] Correct call to
 _webpage_read_full_content

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 52f286ac6..42284f3d5 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -559,7 +559,7 @@ class GenericIE(InfoExtractor):
             self._downloader.report_warning('Falling back on generic information extractor.')
 
         if full_response:
-            webpage = _webpage_read_content(url, video_id)
+            webpage = self._webpage_read_content(full_response, url, video_id)
         else:
             webpage = self._download_webpage(url, video_id)
         self.report_extraction(video_id)

From 6194bb1419842e908a5694b8f9a666ba97d3ac9c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 21:00:42 +0100
Subject: [PATCH 0317/1937] [ffmpeg] Make downloader optional (Fixes #4039)

---
 youtube_dl/postprocessor/ffmpeg.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 19fc135dc..4b93556af 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -42,7 +42,7 @@ class FFmpegPostProcessorError(PostProcessingError):
 
 
 class FFmpegPostProcessor(PostProcessor):
-    def __init__(self, downloader, deletetempfiles=False):
+    def __init__(self, downloader=None, deletetempfiles=False):
         PostProcessor.__init__(self, downloader)
         self._versions = self.get_versions()
         self._deletetempfiles = deletetempfiles
@@ -56,7 +56,8 @@ class FFmpegPostProcessor(PostProcessor):
                 self._versions[self._executable], REQUIRED_VERSION):
             warning = u'Your copy of %s is outdated, update %s to version %s or newer if you encounter any errors.' % (
                 self._executable, self._executable, REQUIRED_VERSION)
-            self._downloader.report_warning(warning)
+            if self._downloader:
+                self._downloader.report_warning(warning)
 
     @staticmethod
     def get_versions():

From 1a253e134ce1f30dd640ba47ffb8fb8a74fe8bed Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 21:03:16 +0100
Subject: [PATCH 0318/1937] [ffmpeg] Fix call to ffprobe (Fixes #4041)

---
 youtube_dl/postprocessor/ffmpeg.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 4b93556af..bd2d749f9 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -75,6 +75,17 @@ class FFmpegPostProcessor(PostProcessor):
                 return p
         return None
 
+    @property
+    def _probe_executable(self):
+        if self._downloader.params.get('prefer_ffmpeg', False):
+            prefs = ('ffproe', 'avprobe')
+        else:
+            prefs = ('avprobe', 'ffprobe')
+        for p in prefs:
+            if self._versions[p]:
+                return p
+        return None
+
     def _uses_avconv(self):
         return self._executable == 'avconv'
 
@@ -120,11 +131,12 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
         self._nopostoverwrites = nopostoverwrites
 
     def get_audio_codec(self, path):
-        if not self._exes['ffprobe'] and not self._exes['avprobe']:
+
+        if not self._probe_executable:
             raise PostProcessingError(u'ffprobe or avprobe not found. Please install one.')
         try:
             cmd = [
-                self._exes['avprobe'] or self._exes['ffprobe'],
+                self._probe_executable,
                 '-show_streams',
                 encodeFilename(self._ffmpeg_filename_argument(path), True)]
             handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE)

From 3d6eed9b5224cbcd0f57044f8e09a52a4a7eabfd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 21:03:38 +0100
Subject: [PATCH 0319/1937] release 2014.10.26.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f5f60cdc3..8eb60c1e2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.10.26'
+__version__ = '2014.10.26.1'

From 50b51830fbeea2e9b2a3235e0c8b6b146f28eb3a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 21:31:51 +0100
Subject: [PATCH 0320/1937] [ffmpeg] Fix typo

---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index bd2d749f9..4963ffc92 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -78,7 +78,7 @@ class FFmpegPostProcessor(PostProcessor):
     @property
     def _probe_executable(self):
         if self._downloader.params.get('prefer_ffmpeg', False):
-            prefs = ('ffproe', 'avprobe')
+            prefs = ('ffprobe', 'avprobe')
         else:
             prefs = ('avprobe', 'ffprobe')
         for p in prefs:

From 83855f3a1fdf9d344e0b25c757e3eccd9246c2bf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 21:44:23 +0100
Subject: [PATCH 0321/1937] [livestream:original] Fix RTMP parameters (Fixes
 #4040)

---
 youtube_dl/extractor/livestream.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 516147417..363a12ad0 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -190,7 +190,8 @@ class LivestreamOriginalIE(InfoExtractor):
             'id': video_id,
             'title': item.find('title').text,
             'url': 'rtmp://extondemand.livestream.com/ondemand',
-            'play_path': 'mp4:trans/dv15/mogulus-{0}.mp4'.format(path),
+            'play_path': 'trans/dv15/mogulus-{0}'.format(path),
+            'player_url': 'http://static.livestream.com/chromelessPlayer/v21/playerapi.swf?hash=5uetk&v=0803&classid=D27CDB6E-AE6D-11cf-96B8-444553540000&jsEnabled=false&wmode=opaque',
             'ext': 'flv',
             'thumbnail': thumbnail_url,
         }

From bfc2bedcfcbfd531b6f452cae3b7af654ceaab1a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 21:49:29 +0100
Subject: [PATCH 0322/1937] [youtube] Make confirm_age non-fatal (#4042)

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4ab56e0ac..382d20c5a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -185,8 +185,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
         self._download_webpage(
             req, None,
-            note='Confirming age', errnote='Unable to confirm age')
-        return True
+            note='Confirming age', errnote='Unable to confirm age',
+            fatal=False)
 
     def _real_initialize(self):
         if self._downloader is None:

From 77ec444d9a20fb0e3aa1be2941979189d53ebdb3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 21:49:52 +0100
Subject: [PATCH 0323/1937] release 2014.10.26.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8eb60c1e2..8e5a642a8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.10.26.1'
+__version__ = '2014.10.26.2'

From 526b276fd7199414d199f2913bc794677d0933ac Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 23:11:15 +0100
Subject: [PATCH 0324/1937] [faz] Modernize

---
 youtube_dl/extractor/faz.py | 36 +++++++++++++++++++-----------------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/faz.py b/youtube_dl/extractor/faz.py
index c6ab6952e..f3d45b2be 100644
--- a/youtube_dl/extractor/faz.py
+++ b/youtube_dl/extractor/faz.py
@@ -1,5 +1,5 @@
 # encoding: utf-8
-import re
+from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
@@ -8,42 +8,44 @@ from ..utils import (
 
 
 class FazIE(InfoExtractor):
-    IE_NAME = u'faz.net'
+    IE_NAME = 'faz.net'
     _VALID_URL = r'https?://www\.faz\.net/multimedia/videos/.*?-(?P<id>\d+)\.html'
 
     _TEST = {
-        u'url': u'http://www.faz.net/multimedia/videos/stockholm-chemie-nobelpreis-fuer-drei-amerikanische-forscher-12610585.html',
-        u'file': u'12610585.mp4',
-        u'info_dict': {
-            u'title': u'Stockholm: Chemie-Nobelpreis für drei amerikanische Forscher',
-            u'description': u'md5:1453fbf9a0d041d985a47306192ea253',
+        'url': 'http://www.faz.net/multimedia/videos/stockholm-chemie-nobelpreis-fuer-drei-amerikanische-forscher-12610585.html',
+        'info_dict': {
+            'id': '12610585',
+            'ext': 'mp4',
+            'title': 'Stockholm: Chemie-Nobelpreis für drei amerikanische Forscher',
+            'description': 'md5:1453fbf9a0d041d985a47306192ea253',
         },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        self.to_screen(video_id)
+        video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
-        config_xml_url = self._search_regex(r'writeFLV\(\'(.+?)\',', webpage,
-            u'config xml url')
-        config = self._download_xml(config_xml_url, video_id,
-            u'Downloading config xml')
+        config_xml_url = self._search_regex(
+            r'writeFLV\(\'(.+?)\',', webpage, 'config xml url')
+        config = self._download_xml(
+            config_xml_url, video_id, 'Downloading config xml')
 
         encodings = config.find('ENCODINGS')
         formats = []
-        for code in ['LOW', 'HIGH', 'HQ']:
+        for pref, code in enumerate(['LOW', 'HIGH', 'HQ']):
             encoding = encodings.find(code)
             if encoding is None:
                 continue
             encoding_url = encoding.find('FILENAME').text
             formats.append({
                 'url': encoding_url,
-                'ext': determine_ext(encoding_url),
                 'format_id': code.lower(),
+                'quality': pref,
             })
+        self._sort_formats(formats)
 
-        descr = self._html_search_regex(r'<p class="Content Copy">(.*?)</p>', webpage, u'description')
+        descr = self._html_search_regex(
+            r'<p class="Content Copy">(.*?)</p>', webpage, 'description', fatal=False)
         return {
             'id': video_id,
             'title': self._og_search_title(webpage),

From c9f08154a3dbf7272b1373c8e6aeea4a8d92b190 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 23:13:42 +0100
Subject: [PATCH 0325/1937] Remove unused imports

---
 youtube_dl/extractor/audiomack.py  | 2 +-
 youtube_dl/extractor/br.py         | 2 --
 youtube_dl/extractor/cliphunter.py | 5 +----
 youtube_dl/extractor/faz.py        | 3 ---
 youtube_dl/extractor/nhl.py        | 1 -
 youtube_dl/extractor/sportbox.py   | 1 -
 youtube_dl/extractor/vimeo.py      | 2 --
 youtube_dl/postprocessor/ffmpeg.py | 1 -
 8 files changed, 2 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 57446fddd..6232d2cd0 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from .soundcloud import SoundcloudIE
 from ..utils import ExtractorError
-import datetime
+
 import time
 
 
diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py
index 2e277c8c3..45ba51732 100644
--- a/youtube_dl/extractor/br.py
+++ b/youtube_dl/extractor/br.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
diff --git a/youtube_dl/extractor/cliphunter.py b/youtube_dl/extractor/cliphunter.py
index d4227e6eb..2edab90a3 100644
--- a/youtube_dl/extractor/cliphunter.py
+++ b/youtube_dl/extractor/cliphunter.py
@@ -4,7 +4,6 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..utils import int_or_none
 
 
 _translation_table = {
@@ -39,9 +38,7 @@ class CliphunterIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         video_title = self._search_regex(
diff --git a/youtube_dl/extractor/faz.py b/youtube_dl/extractor/faz.py
index f3d45b2be..3c39ca451 100644
--- a/youtube_dl/extractor/faz.py
+++ b/youtube_dl/extractor/faz.py
@@ -2,9 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
-    determine_ext,
-)
 
 
 class FazIE(InfoExtractor):
diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index d66c2c6f8..82af6e330 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -7,7 +7,6 @@ from .common import InfoExtractor
 from ..utils import (
     compat_urlparse,
     compat_urllib_parse,
-    determine_ext,
     unified_strdate,
 )
 
diff --git a/youtube_dl/extractor/sportbox.py b/youtube_dl/extractor/sportbox.py
index b9cd35109..becdf658f 100644
--- a/youtube_dl/extractor/sportbox.py
+++ b/youtube_dl/extractor/sportbox.py
@@ -7,7 +7,6 @@ from .common import InfoExtractor
 from ..utils import (
     parse_duration,
     parse_iso8601,
-    int_or_none,
 )
 
 
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 07959d3fe..5839c550d 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -8,13 +8,11 @@ import itertools
 from .common import InfoExtractor
 from .subtitles import SubtitlesInfoExtractor
 from ..utils import (
-    clean_html,
     compat_HTTPError,
     compat_urllib_parse,
     compat_urllib_request,
     compat_urlparse,
     ExtractorError,
-    get_element_by_attribute,
     InAdvancePagedList,
     int_or_none,
     RegexNotFoundError,
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 4963ffc92..083c79592 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -8,7 +8,6 @@ import time
 from .common import AudioConversionError, PostProcessor
 
 from ..utils import (
-    check_executable,
     compat_subprocess_get_DEVNULL,
     encodeArgument,
     encodeFilename,

From 2bcae58d46b88200a2247a7e2bee999f459f75c4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 23:23:10 +0100
Subject: [PATCH 0326/1937] [srmediathek] New extractor

---
 youtube_dl/extractor/__init__.py    |  1 +
 youtube_dl/extractor/srmediathek.py | 43 +++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+)
 create mode 100644 youtube_dl/extractor/srmediathek.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8e31de93d..17ab49283 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -355,6 +355,7 @@ from .spike import SpikeIE
 from .sport5 import Sport5IE
 from .sportbox import SportBoxIE
 from .sportdeutschland import SportDeutschlandIE
+from .srmediathek import SRMediathekIE
 from .stanfordoc import StanfordOpenClassroomIE
 from .steam import SteamIE
 from .streamcloud import StreamcloudIE
diff --git a/youtube_dl/extractor/srmediathek.py b/youtube_dl/extractor/srmediathek.py
new file mode 100644
index 000000000..26bf9e34c
--- /dev/null
+++ b/youtube_dl/extractor/srmediathek.py
@@ -0,0 +1,43 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..utils import js_to_json
+
+
+class SRMediathekIE(InfoExtractor):
+    IE_NAME = 'Süddeutscher Rundfunk'
+    _VALID_URL = r'https?://sr-mediathek\.sr-online\.de/index\.php\?.*?&id=(?P<id>[0-9]+)'
+
+    _TEST = {
+        'url': 'http://sr-mediathek.sr-online.de/index.php?seite=7&id=28455',
+        'info_dict': {
+            'id': '28455',
+            'ext': 'mp4',
+            'title': 'sportarena (26.10.2014)',
+            'description': 'Ringen: KSV Köllerbach gegen Aachen-Walheim; Frauen-Fußball: 1. FC Saarbrücken gegen Sindelfingen; Motorsport: Rallye in Losheim; dazu: Interview mit Timo Bernhard; Turnen: TG Saar; Reitsport: Deutscher Voltigier-Pokal; Badminton: Interview mit Michael Fuchs ',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        urls = json.loads(js_to_json(self._search_regex(
+            r'var mediaURLs\s*=\s*(.*?);\n', webpage, 'video URLs')))
+        formats = [{'url': url} for url in urls]
+        self._sort_formats(formats)
+
+        title = json.loads(js_to_json(self._search_regex(
+            r'var mediaTitles\s*=\s*(.*?);\n', webpage, 'title')))[0]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+        }

From cf501a23d29e13e8bcfac5f8ce10ac96d097f6e0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 26 Oct 2014 23:23:53 +0100
Subject: [PATCH 0327/1937] [srmediathek] Correct IE_NAME/IE_DESC

---
 youtube_dl/extractor/srmediathek.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/srmediathek.py b/youtube_dl/extractor/srmediathek.py
index 26bf9e34c..d92d14d65 100644
--- a/youtube_dl/extractor/srmediathek.py
+++ b/youtube_dl/extractor/srmediathek.py
@@ -8,7 +8,7 @@ from ..utils import js_to_json
 
 
 class SRMediathekIE(InfoExtractor):
-    IE_NAME = 'Süddeutscher Rundfunk'
+    IE_DESC = 'Süddeutscher Rundfunk'
     _VALID_URL = r'https?://sr-mediathek\.sr-online\.de/index\.php\?.*?&id=(?P<id>[0-9]+)'
 
     _TEST = {

From 8d11b59bbb2cc4cdf10b2659c04589fd81b9c6f9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 00:05:34 +0100
Subject: [PATCH 0328/1937] [ynet] Remove test md5sums

These fluctuate regularly.
---
 youtube_dl/extractor/ynet.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/ynet.py b/youtube_dl/extractor/ynet.py
index 944d7da38..9cd7989cc 100644
--- a/youtube_dl/extractor/ynet.py
+++ b/youtube_dl/extractor/ynet.py
@@ -13,7 +13,6 @@ class YnetIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://hot.ynet.co.il/home/0,7340,L-11659-99244,00.html',
-            'md5': '4b29cb57c3dddd57642b3f051f535b07',
             'info_dict': {
                 'id': 'L-11659-99244',
                 'ext': 'flv',
@@ -22,7 +21,6 @@ class YnetIE(InfoExtractor):
             }
         }, {
             'url': 'http://hot.ynet.co.il/home/0,7340,L-8859-84418,00.html',
-            'md5': '8194c2ea221e9a639cac96b6b0753dc5',
             'info_dict': {
                 'id': 'L-8859-84418',
                 'ext': 'flv',

From f46a8702ccfcb221b3e01ca95553be92af3f8f4e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 00:06:47 +0100
Subject: [PATCH 0329/1937] [youtube:playlist] Fix test title

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 382d20c5a..40fe4662a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1057,7 +1057,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'note': 'issue #673',
         'url': 'PLBB231211A4F62143',
         'info_dict': {
-            'title': 'Team Fortress 2 (Class-based LP)',
+            'title': '[OLD]Team Fortress 2 (Class-based LP)',
         },
         'playlist_mincount': 26,
     }, {

From 3adba6fa2ae1af3c90e9c6623161b223697cc874 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 00:08:37 +0100
Subject: [PATCH 0330/1937] [xtube] Fix test description

---
 youtube_dl/extractor/xtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 273d93d9e..c3bb9b2cf 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -20,7 +20,7 @@ class XTubeIE(InfoExtractor):
             'id': 'kVTUy_G222_',
             'ext': 'mp4',
             'title': 'strange erotica',
-            'description': 'surreal gay themed erotica...almost an ET kind of thing',
+            'description': 'http://www.xtube.com an ET kind of thing',
             'uploader': 'greenshowers',
             'duration': 450,
             'age_limit': 18,

From 7706927370f7b2ec9b759961ce5bf34b24ef5ce5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 00:11:34 +0100
Subject: [PATCH 0331/1937] [vine:user] Adapt test to changed list size

---
 youtube_dl/extractor/vine.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index e7754158d..42995226e 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -70,7 +70,7 @@ class VineUserIE(InfoExtractor):
         'info_dict': {
             'id': 'Visa',
         },
-        'playlist_mincount': 47,
+        'playlist_mincount': 46,
     }
 
     def _real_extract(self, url):

From 9dec99303d589bdf78493ee124f3ae230d8cd606 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 00:13:40 +0100
Subject: [PATCH 0332/1937] [vimeo:review] Fix test title

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 5839c550d..d9cad0ea5 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -512,7 +512,7 @@ class VimeoReviewIE(InfoExtractor):
         'info_dict': {
             'id': '91613211',
             'ext': 'mp4',
-            'title': 'Death by dogma versus assembling agile - Sander Hoogendoorn',
+            'title': 're:(?i)^Death by dogma versus assembling agile . Sander Hoogendoorn',
             'uploader': 'DevWeek Events',
             'duration': 2773,
             'thumbnail': 're:^https?://.*\.jpg$',

From 3d6047113ceade1c3588b46fa6a65ee360643b02 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 00:14:52 +0100
Subject: [PATCH 0333/1937] [vgtv] Simplify

---
 youtube_dl/extractor/vgtv.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 964470070..bb6cc83b5 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -67,9 +67,7 @@ class VGTVIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         data = self._download_json(
             'http://svp.vg.no/svp/api/v1/vgtv/assets/%s?appName=vgtv-website' % video_id,
             video_id, 'Downloading media JSON')

From 6009b69f81ce7320a69baf1172a61e7e36ea8096 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 00:15:42 +0100
Subject: [PATCH 0334/1937] [vgtv] Fix test title

---
 youtube_dl/extractor/vgtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index bb6cc83b5..d3fa70e0e 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -17,7 +17,7 @@ class VGTVIE(InfoExtractor):
             'info_dict': {
                 'id': '84196',
                 'ext': 'mp4',
-                'title': 'Hevnen er søt episode 10: Abu',
+                'title': 'Hevnen er søt episode 1:10 - Abu',
                 'description': 'md5:e25e4badb5f544b04341e14abdc72234',
                 'thumbnail': 're:^https?://.*\.jpg',
                 'duration': 648.000,

From 446a03bd96d99d99b36f1dbfe0617033327f463f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 00:18:10 +0100
Subject: [PATCH 0335/1937] [ustream:channel] Change test playlist size (Seems
 to have been limited that way on the website as well)

---
 youtube_dl/extractor/ustream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 994b60a76..cee1ea8f6 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -72,7 +72,7 @@ class UstreamChannelIE(InfoExtractor):
         'info_dict': {
             'id': '10874166',
         },
-        'playlist_mincount': 54,
+        'playlist_mincount': 17,
     }
 
     def _real_extract(self, url):

From 94f052cbf458ba816e95c1f73cae7bcd63c9e549 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 00:19:10 +0100
Subject: [PATCH 0336/1937] [syfy] Remove test checksum

We have the minsize test now.
---
 youtube_dl/extractor/syfy.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/syfy.py b/youtube_dl/extractor/syfy.py
index f76b6e2b2..5ca079f88 100644
--- a/youtube_dl/extractor/syfy.py
+++ b/youtube_dl/extractor/syfy.py
@@ -10,7 +10,6 @@ class SyfyIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.syfy.com/videos/Robot%20Combat%20League/Behind%20the%20Scenes/vid:2631458',
-        'md5': 'e07de1d52c7278adbb9b9b1c93a66849',
         'info_dict': {
             'id': 'NmqMrGnXvmO1',
             'ext': 'flv',

From 8d32abff9e3d56bf74eb6eec8284623a94f4551f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 00:20:54 +0100
Subject: [PATCH 0337/1937] [ruhd] Simplify

---
 youtube_dl/extractor/ruhd.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/ruhd.py b/youtube_dl/extractor/ruhd.py
index 55b58e5e6..0e470e73f 100644
--- a/youtube_dl/extractor/ruhd.py
+++ b/youtube_dl/extractor/ruhd.py
@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -21,19 +19,20 @@ class RUHDIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         video_url = self._html_search_regex(
             r'<param name="src" value="([^"]+)"', webpage, 'video url')
         title = self._html_search_regex(
-            r'<title>([^<]+)&nbsp;&nbsp; RUHD.ru - Видео Высокого качества №1 в России!</title>', webpage, 'title')
+            r'<title>([^<]+)&nbsp;&nbsp; RUHD.ru - Видео Высокого качества №1 в России!</title>',
+            webpage, 'title')
         description = self._html_search_regex(
-            r'(?s)<div id="longdesc">(.+?)<span id="showlink">', webpage, 'description', fatal=False)
+            r'(?s)<div id="longdesc">(.+?)<span id="showlink">',
+            webpage, 'description', fatal=False)
         thumbnail = self._html_search_regex(
-            r'<param name="previewImage" value="([^"]+)"', webpage, 'thumbnail', fatal=False)
+            r'<param name="previewImage" value="([^"]+)"',
+            webpage, 'thumbnail', fatal=False)
         if thumbnail:
             thumbnail = 'http://www.ruhd.ru' + thumbnail
 

From 249efaf44bbdedf24f7da7c9d27afc397251d106 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 00:33:35 +0100
Subject: [PATCH 0338/1937] [pornhub] Modernize and fix test definition

---
 youtube_dl/extractor/pornhub.py | 13 ++++++-------
 youtube_dl/extractor/rtlnow.py  |  2 +-
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 4118ee956..618e8f5dd 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -16,13 +16,14 @@ from ..aes import (
 
 
 class PornHubIE(InfoExtractor):
-    _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>pornhub\.com/view_video\.php\?viewkey=(?P<videoid>[0-9a-f]+))'
+    _VALID_URL = r'^https?://(?:www\.)?pornhub\.com/view_video\.php\?viewkey=(?P<id>[0-9a-f]+)'
     _TEST = {
         'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
-        'file': '648719015.mp4',
         'md5': '882f488fa1f0026f023f33576004a2ed',
         'info_dict': {
-            "uploader": "BABES-COM",
+            'id': '648719015',
+            'ext': 'mp4',
+            "uploader": "Babes",
             "title": "Seductive Indian beauty strips down and fingers her pink pussy",
             "age_limit": 18
         }
@@ -35,9 +36,7 @@ class PornHubIE(InfoExtractor):
         return count
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
-        url = 'http://www.' + mobj.group('url')
+        video_id = self._match_id(url)
 
         req = compat_urllib_request.Request(url)
         req.add_header('Cookie', 'age_verified=1')
@@ -45,7 +44,7 @@ class PornHubIE(InfoExtractor):
 
         video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, 'title')
         video_uploader = self._html_search_regex(
-            r'(?s)From:&nbsp;.+?<(?:a href="/users/|<span class="username)[^>]+>(.+?)<',
+            r'(?s)From:&nbsp;.+?<(?:a href="/users/|a href="/channels/|<span class="username)[^>]+>(.+?)<',
             webpage, 'uploader', fatal=False)
         thumbnail = self._html_search_regex(r'"image_url":"([^"]+)', webpage, 'thumbnail', fatal=False)
         if thumbnail:
diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py
index a45884b25..1a41cbe40 100644
--- a/youtube_dl/extractor/rtlnow.py
+++ b/youtube_dl/extractor/rtlnow.py
@@ -81,7 +81,7 @@ class RTLnowIE(InfoExtractor):
                 'id': '99205',
                 'ext': 'flv',
                 'title': 'Medicopter 117 - Angst!',
-                'description': 'md5:895b1df01639b5f61a04fc305a5cb94d',
+                'description': 're:^Im Therapiezentrum \'Sonnalm\' kommen durch eine Unachtsamkeit die für die B.handlung mit Phobikern gehaltenen Voglespinnen frei\. Eine Ausreißerin',
                 'thumbnail': 'http://autoimg.static-fra.de/superrtlnow/287529/1500x1500/image2.jpg',
                 'upload_date': '20080928',
                 'duration': 2691,

From f744c0f3984cb8bb09f16bc4ef1d3c9c356b1b33 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 00:39:39 +0100
Subject: [PATCH 0339/1937] [test_download] Improve error message

---
 test/test_download.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_download.py b/test/test_download.py
index 608e4528e..75e0bb289 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -185,7 +185,9 @@ def generator(test_case):
                         md5_for_file = _file_md5(tc_filename)
                         self.assertEqual(md5_for_file, tc['md5'])
                 info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'
-                self.assertTrue(os.path.exists(info_json_fn))
+                self.assertTrue(
+                    os.path.exists(info_json_fn),
+                    'Missing info file %s' % info_json_fn)
                 with io.open(info_json_fn, encoding='utf-8') as infof:
                     info_dict = json.load(infof)
 

From 640743233389714dda8a3ccb01cf7335e7b43c37 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 00:40:07 +0100
Subject: [PATCH 0340/1937] [Makefile] remove temporary files in clean target

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 2da92863f..3e1debc7e 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part
 
 cleanall: clean
 	rm -f youtube-dl youtube-dl.exe

From 2e9ff8f36284c43d9f98e19fa58e10ce5bd5eece Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 00:44:27 +0100
Subject: [PATCH 0341/1937] [gorillavid] Fix test title

---
 youtube_dl/extractor/gorillavid.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index 45cca1d24..e21e57510 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -46,9 +46,9 @@ class GorillaVidIE(InfoExtractor):
         'info_dict': {
             'id': '3rso4kdn6f9m',
             'ext': 'mp4',
-            'title': 'Micro Pig piglets ready on 16th July 2009',
+            'title': 'Micro Pig piglets ready on 16th July 2009-bG0PdrCdxUc',
             'thumbnail': 're:http://.*\.jpg',
-        },
+        }
     }, {
         'url': 'http://movpod.in/0wguyyxi1yca',
         'only_matching': True,

From d6d9186f0d12e226ab1b908c66d67e14adb88aea Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 00:45:15 +0100
Subject: [PATCH 0342/1937] [generic] Fix test title

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 42284f3d5..35a7664b2 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -325,7 +325,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'age_limit': 18,
                 'uploader': 'www.handjobhub.com',
-                'title': 'Busty Blonde Siri Tit Fuck While Wank at Handjob Hub',
+                'title': 'Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com',
             }
         },
         # RSS feed

From 586f7082ef279049bf8a774836556b0a72be7a62 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 00:46:34 +0100
Subject: [PATCH 0343/1937] [francetv] Remove changing md5sum

---
 youtube_dl/extractor/francetv.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 566e20d76..35d7d15e1 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -93,7 +93,6 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.francetvinfo.fr/replay-jt/france-3/soir-3/jt-grand-soir-3-lundi-26-aout-2013_393427.html',
-        'md5': '9cecf35f99c4079c199e9817882a9a1c',
         'info_dict': {
             'id': '84981923',
             'ext': 'flv',

From 15956b5aa1867e9511d5edeebc5d4e9e70ed39c0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 00:50:22 +0100
Subject: [PATCH 0344/1937] [promptfile] Fix check for deleted videos

---
 youtube_dl/extractor/promptfile.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/promptfile.py b/youtube_dl/extractor/promptfile.py
index 463e85501..7fcde086c 100644
--- a/youtube_dl/extractor/promptfile.py
+++ b/youtube_dl/extractor/promptfile.py
@@ -14,7 +14,6 @@ from ..utils import (
 
 class PromptFileIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?promptfile\.com/l/(?P<id>[0-9A-Z\-]+)'
-    _FILE_NOT_FOUND_REGEX = r'<div.+id="not_found_msg".+>.+</div>[^-]'
     _TEST = {
         'url': 'http://www.promptfile.com/l/D21B4746E9-F01462F0FF',
         'md5': 'd1451b6302da7215485837aaea882c4c',
@@ -27,11 +26,10 @@ class PromptFileIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        if re.search(self._FILE_NOT_FOUND_REGEX, webpage) is not None:
+        if re.search(r'<div.+id="not_found_msg".+>(?!We are).+</div>[^-]', webpage) is not None:
             raise ExtractorError('Video %s does not exist' % video_id,
                                  expected=True)
 

From 10e3d734723def2d91bdd3d02b85a149c056dcfb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 01:14:17 +0100
Subject: [PATCH 0345/1937] [nbc] Fix ThePlatform embedded videos

---
 youtube_dl/extractor/nbc.py         |  7 +++---
 youtube_dl/extractor/theplatform.py | 34 +++++++++++++++--------------
 2 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index e75ab7c39..7b5449031 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -26,8 +26,7 @@ class NBCIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         theplatform_url = self._search_regex('class="video-player video-player-full" data-mpx-url="(.*?)"', webpage, 'theplatform url')
         if theplatform_url.startswith('//'):
@@ -57,7 +56,7 @@ class NBCNewsIE(InfoExtractor):
             'md5': 'b2421750c9f260783721d898f4c42063',
             'info_dict': {
                 'id': 'I1wpAI_zmhsQ',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'How Twitter Reacted To The Snowden Interview',
                 'description': 'md5:65a0bd5d76fe114f3c2727aa3a81fe64',
             },
@@ -97,6 +96,8 @@ class NBCNewsIE(InfoExtractor):
             ]
 
             for base_url in base_urls:
+                if not base_url:
+                    continue
                 playlist_url = base_url + '?form=MPXNBCNewsAPI'
                 all_videos = self._download_json(playlist_url, title)['videos']
 
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 0be793b1c..a04925633 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -6,6 +6,7 @@ import json
 from .common import InfoExtractor
 from ..utils import (
     compat_str,
+    determine_ext,
     ExtractorError,
     xpath_with_ns,
 )
@@ -34,10 +35,21 @@ class ThePlatformIE(InfoExtractor):
             'skip_download': True,
         },
     }
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        if mobj.group('config'):
+            config_url = url+ '&form=json'
+            config_url = config_url.replace('swf/', 'config/')
+            config_url = config_url.replace('onsite/', 'onsite/config/')
+            config = self._download_json(config_url, video_id, 'Downloading config')
+            smil_url = config['releaseUrl'] + '&format=SMIL&formats=MPEG4&manifest=f4m'
+        else:
+            smil_url = ('http://link.theplatform.com/s/dJ5BDC/{0}/meta.smil?'
+                'format=smil&mbr=true'.format(video_id))
+
 
-    def _get_info(self, video_id, smil_url):
         meta = self._download_xml(smil_url, video_id)
-
         try:
             error_msg = next(
                 n.attrib['abstract']
@@ -89,10 +101,14 @@ class ThePlatformIE(InfoExtractor):
                 for f in switch.findall(_x('smil:video')):
                     attr = f.attrib
                     vbr = int(attr['system-bitrate']) // 1000
+                    ext = determine_ext(attr['src'])
+                    if ext == 'once':
+                        ext = 'mp4'
                     formats.append({
                         'format_id': compat_str(vbr),
                         'url': attr['src'],
                         'vbr': vbr,
+                        'ext': ext,
                     })
             self._sort_formats(formats)
 
@@ -104,17 +120,3 @@ class ThePlatformIE(InfoExtractor):
             'thumbnail': info['defaultThumbnailUrl'],
             'duration': info['duration']//1000,
         }
-        
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        if mobj.group('config'):
-            config_url = url+ '&form=json'
-            config_url = config_url.replace('swf/', 'config/')
-            config_url = config_url.replace('onsite/', 'onsite/config/')
-            config = self._download_json(config_url, video_id, 'Downloading config')
-            smil_url = config['releaseUrl'] + '&format=SMIL&formats=MPEG4&manifest=f4m'
-        else:
-            smil_url = ('http://link.theplatform.com/s/dJ5BDC/{0}/meta.smil?'
-                'format=smil&mbr=true'.format(video_id))
-        return self._get_info(video_id, smil_url)

From 4a6c94288ae42402f4a61a924318cedc5d1bb511 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 01:16:18 +0100
Subject: [PATCH 0346/1937] [kickstarter] Simplify and fix test case

---
 youtube_dl/extractor/kickstarter.py | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/kickstarter.py b/youtube_dl/extractor/kickstarter.py
index 56a76380c..827091e60 100644
--- a/youtube_dl/extractor/kickstarter.py
+++ b/youtube_dl/extractor/kickstarter.py
@@ -1,8 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -21,22 +19,17 @@ class KickStarterIE(InfoExtractor):
     }, {
         'note': 'Embedded video (not using the native kickstarter video service)',
         'url': 'https://www.kickstarter.com/projects/597507018/pebble-e-paper-watch-for-iphone-and-android/posts/659178',
-        'playlist': [
-            {
-                'info_dict': {
-                    'id': '78704821',
-                    'ext': 'mp4',
-                    'uploader_id': 'pebble',
-                    'uploader': 'Pebble Technology',
-                    'title': 'Pebble iOS Notifications',
-                }
-            }
-        ],
+        'info_dict': {
+            'id': '78704821',
+            'ext': 'mp4',
+            'uploader_id': 'pebble',
+            'uploader': 'Pebble Technology',
+            'title': 'Pebble iOS Notifications',
+        }
     }]
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(

From ecfe623422f61df886b83e6eae98a0e4a297802e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 01:33:49 +0100
Subject: [PATCH 0347/1937] [heise] Fix extraction

Now they use an XML format instead of JSON.
---
 youtube_dl/extractor/heise.py | 48 +++++++++++++++--------------------
 1 file changed, 21 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/heise.py b/youtube_dl/extractor/heise.py
index f97b1e085..d41c0413f 100644
--- a/youtube_dl/extractor/heise.py
+++ b/youtube_dl/extractor/heise.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     get_meta_content,
+    int_or_none,
     parse_iso8601,
 )
 
@@ -28,20 +29,26 @@ class HeiseIE(InfoExtractor):
             'timestamp': 1411812600,
             'upload_date': '20140927',
             'description': 'In uplink-Episode 3.3 geht es darum, wie man sich von Cloud-Anbietern emanzipieren kann, worauf man beim Kauf einer Tastatur achten sollte und was Smartphones über uns verraten.',
+            'thumbnail': 're:https?://.*\.jpg$',
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         webpage = self._download_webpage(url, video_id)
-        json_url = self._search_regex(
-            r'json_url:\s*"([^"]+)"', webpage, 'json URL')
-        config = self._download_json(json_url, video_id)
+
+        container_id = self._search_regex(
+            r'<div class="videoplayerjw".*?data-container="([0-9]+)"',
+            webpage, 'container ID')
+        sequenz_id = self._search_regex(
+            r'<div class="videoplayerjw".*?data-sequenz="([0-9]+)"',
+            webpage, 'sequenz ID')
+        data_url = 'http://www.heise.de/videout/feed?container=%s&sequenz=%s' % (container_id, sequenz_id)
+        doc = self._download_xml(data_url, video_id)
 
         info = {
             'id': video_id,
-            'thumbnail': config.get('poster'),
+            'thumbnail': self._og_search_thumbnail(webpage),
             'timestamp': parse_iso8601(get_meta_content('date', webpage)),
             'description': self._og_search_description(webpage),
         }
@@ -49,32 +56,19 @@ class HeiseIE(InfoExtractor):
         title = get_meta_content('fulltitle', webpage)
         if title:
             info['title'] = title
-        elif config.get('title'):
-            info['title'] = config['title']
         else:
             info['title'] = self._og_search_title(webpage)
 
         formats = []
-        for t, rs in config['formats'].items():
-            if not rs or not hasattr(rs, 'items'):
-                self._downloader.report_warning(
-                    'formats: {0}: no resolutions'.format(t))
-                continue
-
-            for height_str, obj in rs.items():
-                format_id = '{0}_{1}'.format(t, height_str)
-
-                if not obj or not obj.get('url'):
-                    self._downloader.report_warning(
-                        'formats: {0}: no url'.format(format_id))
-                    continue
-
-                formats.append({
-                    'url': obj['url'],
-                    'format_id': format_id,
-                    'height': self._int(height_str, 'height'),
-                })
-
+        for source_node in doc.findall('.//{http://rss.jwpcdn.com/}source'):
+            label = source_node.attrib['label']
+            height = int_or_none(self._search_regex(
+                r'^(.*?_)?([0-9]+)p$', label, 'height', default=None))
+            formats.append({
+                'url': source_node.attrib['file'],
+                'format_note': label,
+                'height': height,
+            })
         self._sort_formats(formats)
         info['formats'] = formats
 

From aea856621fa24cbe7125ba8f38ed1dfce73cada6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 02:14:07 +0100
Subject: [PATCH 0348/1937] [zdf] Simplify

---
 youtube_dl/extractor/zdf.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 3b1ac4e9f..52b066ce3 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class ZDFIE(InfoExtractor):
-    _VALID_URL = r'^https?://www\.zdf\.de/ZDFmediathek(?P<hash>#)?/(.*beitrag/(?:video/)?)(?P<video_id>[0-9]+)(?:/[^/?]+)?(?:\?.*)?'
+    _VALID_URL = r'^https?://www\.zdf\.de/ZDFmediathek(?P<hash>#)?/(.*beitrag/(?:video/)?)(?P<id>[0-9]+)(?:/[^/?]+)?(?:\?.*)?'
 
     _TEST = {
         'url': 'http://www.zdf.de/ZDFmediathek/beitrag/video/2037704/ZDFspezial---Ende-des-Machtpokers--?bc=sts;stt',
@@ -29,8 +29,7 @@ class ZDFIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
+        video_id = self._match_id(url)
 
         xml_url = 'http://www.zdf.de/ZDFmediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id
         doc = self._download_xml(

From bf7aa6301b30156d3badbd67a5a12bdb500a8366 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 02:26:05 +0100
Subject: [PATCH 0349/1937] [fktv] Modernize

---
 youtube_dl/extractor/fktv.py | 42 +++++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index d7048c8c1..990bee8ba 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import re
 import random
 import json
@@ -11,15 +13,16 @@ from ..utils import (
 
 
 class FKTVIE(InfoExtractor):
-    IE_NAME = u'fernsehkritik.tv'
-    _VALID_URL = r'(?:http://)?(?:www\.)?fernsehkritik\.tv/folge-(?P<ep>[0-9]+)(?:/.*)?'
+    IE_NAME = 'fernsehkritik.tv'
+    _VALID_URL = r'http://(?:www\.)?fernsehkritik\.tv/folge-(?P<ep>[0-9]+)(?:/.*)?'
 
     _TEST = {
-        u'url': u'http://fernsehkritik.tv/folge-1',
-        u'file': u'00011.flv',
-        u'info_dict': {
-            u'title': u'Folge 1 vom 10. April 2007',
-            u'description': u'md5:fb4818139c7cfe6907d4b83412a6864f',
+        'url': 'http://fernsehkritik.tv/folge-1',
+        'info_dict': {
+            'id': '00011',
+            'ext': 'flv',
+            'title': 'Folge 1 vom 10. April 2007',
+            'description': 'md5:fb4818139c7cfe6907d4b83412a6864f',
         },
     }
 
@@ -32,7 +35,7 @@ class FKTVIE(InfoExtractor):
         start_webpage = self._download_webpage('http://fernsehkritik.tv/folge-%d/Start' % episode,
             episode)
         playlist = self._search_regex(r'playlist = (\[.*?\]);', start_webpage,
-            u'playlist', flags=re.DOTALL)
+            'playlist', flags=re.DOTALL)
         files = json.loads(re.sub('{[^{}]*?}', '{}', playlist))
         # TODO: return a single multipart video
         videos = []
@@ -42,7 +45,6 @@ class FKTVIE(InfoExtractor):
             videos.append({
                 'id': video_id,
                 'url': video_url,
-                'ext': determine_ext(video_url),
                 'title': clean_html(get_element_by_id('eptitle', start_webpage)),
                 'description': clean_html(get_element_by_id('contentlist', start_webpage)),
                 'thumbnail': video_thumbnail
@@ -51,14 +53,15 @@ class FKTVIE(InfoExtractor):
 
 
 class FKTVPosteckeIE(InfoExtractor):
-    IE_NAME = u'fernsehkritik.tv:postecke'
-    _VALID_URL = r'(?:http://)?(?:www\.)?fernsehkritik\.tv/inline-video/postecke\.php\?(.*&)?ep=(?P<ep>[0-9]+)(&|$)'
+    IE_NAME = 'fernsehkritik.tv:postecke'
+    _VALID_URL = r'http://(?:www\.)?fernsehkritik\.tv/inline-video/postecke\.php\?(.*&)?ep=(?P<ep>[0-9]+)(&|$)'
     _TEST = {
-        u'url': u'http://fernsehkritik.tv/inline-video/postecke.php?iframe=true&width=625&height=440&ep=120',
-        u'file': u'0120.flv',
-        u'md5': u'262f0adbac80317412f7e57b4808e5c4',
-        u'info_dict': {
-            u"title": u"Postecke 120"
+        'url': 'http://fernsehkritik.tv/inline-video/postecke.php?iframe=true&width=625&height=440&ep=120',
+        'md5': '262f0adbac80317412f7e57b4808e5c4',
+        'info_dict': {
+            'id': '0120',
+            'ext': 'flv',
+            'title': 'Postecke 120',
         }
     }
 
@@ -71,8 +74,7 @@ class FKTVPosteckeIE(InfoExtractor):
         video_url = 'http://dl%d.fernsehkritik.tv/postecke/postecke%d.flv' % (server, episode)
         video_title = 'Postecke %d' % episode
         return {
-            'id':       video_id,
-            'url':      video_url,
-            'ext':      determine_ext(video_url),
-            'title':    video_title,
+            'id': video_id,
+            'url': video_url,
+            'title': video_title,
         }

From 8112d4b28429a4319bc3d9d111b69fa62e514de3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 02:27:49 +0100
Subject: [PATCH 0350/1937] [lrt] Modernize

---
 youtube_dl/extractor/lrt.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/lrt.py b/youtube_dl/extractor/lrt.py
index db5df4078..d72d470aa 100644
--- a/youtube_dl/extractor/lrt.py
+++ b/youtube_dl/extractor/lrt.py
@@ -32,9 +32,7 @@ class LRTIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         title = remove_end(self._og_search_title(webpage), ' - LRT')

From 81515ad9f674597e0d41be6acfcb8592f2230f32 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 02:28:37 +0100
Subject: [PATCH 0351/1937] [extractor/common] Improve m3u8 output

---
 youtube_dl/extractor/common.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e1bd6bb49..b77db52f7 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -689,7 +689,10 @@ class InfoExtractor(object):
             if re.match(r'^https?://', u)
             else compat_urlparse.urljoin(m3u8_url, u))
 
-        m3u8_doc = self._download_webpage(m3u8_url, video_id)
+        m3u8_doc = self._download_webpage(
+            m3u8_url, video_id,
+            note='Downloading m3u8 information',
+            errnote='Failed to download m3u8 information')
         last_info = None
         kv_rex = re.compile(
             r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')

From b0fb6d4db1ed19dfa3a38ee5fd1d38e6227c3d93 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 02:32:44 +0100
Subject: [PATCH 0352/1937] [ku6] Modernize

---
 youtube_dl/extractor/ku6.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/ku6.py b/youtube_dl/extractor/ku6.py
index 484239b19..89013e522 100644
--- a/youtube_dl/extractor/ku6.py
+++ b/youtube_dl/extractor/ku6.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -18,11 +16,11 @@ class Ku6IE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        title = self._search_regex(r'<h1 title=.*>(.*?)</h1>', webpage, 'title')
+
+        title = self._html_search_regex(
+            r'<h1 title=.*>(.*?)</h1>', webpage, 'title')
         dataUrl = 'http://v.ku6.com/fetchVideo4Player/%s.html' % video_id
         jsonData = self._download_json(dataUrl, video_id)
         downloadUrl = jsonData['data']['f']

From 8cc3eba79ae19cf5ec4780356b75ccb9813916f0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 02:43:59 +0100
Subject: [PATCH 0353/1937] [phoenix] Add new extractor (Fixes #4036)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/phoenix.py  |  31 +++++++
 youtube_dl/extractor/zdf.py      | 151 ++++++++++++++++---------------
 3 files changed, 109 insertions(+), 74 deletions(-)
 create mode 100644 youtube_dl/extractor/phoenix.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 17ab49283..3979b8270 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -280,6 +280,7 @@ from .orf import (
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
+from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
 from .planetaplay import PlanetaPlayIE
 from .played import PlayedIE
diff --git a/youtube_dl/extractor/phoenix.py b/youtube_dl/extractor/phoenix.py
new file mode 100644
index 000000000..a20672c0c
--- /dev/null
+++ b/youtube_dl/extractor/phoenix.py
@@ -0,0 +1,31 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .zdf import extract_from_xml_url
+
+
+class PhoenixIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?phoenix\.de/content/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.phoenix.de/content/884301',
+        'md5': 'ed249f045256150c92e72dbb70eadec6',
+        'info_dict': {
+            'id': '884301',
+            'ext': 'mp4',
+            'title': 'Michael Krons mit Hans-Werner Sinn',
+            'description': 'Im Dialog - Sa. 25.10.14, 00.00 - 00.35 Uhr',
+            'upload_date': '20141025',
+            'uploader': 'Im Dialog',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        internal_id = self._search_regex(
+            r'<div class="phx_vod" id="phx_vod_([0-9]+)"',
+            webpage, 'internal video ID')
+
+        api_url = 'http://www.phoenix.de/php/zdfplayer-v1.3/data/beitragsDetails.php?ak=web&id=%s' % internal_id
+        return extract_from_xml_url(self, video_id, api_url)
diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 52b066ce3..9ff00e26c 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -10,6 +10,82 @@ from ..utils import (
 )
 
 
+def extract_from_xml_url(ie, video_id, xml_url):
+    doc = ie._download_xml(
+        xml_url, video_id,
+        note='Downloading video info',
+        errnote='Failed to download video info')
+
+    title = doc.find('.//information/title').text
+    description = doc.find('.//information/detail').text
+    duration = int(doc.find('.//details/lengthSec').text)
+    uploader_node = doc.find('.//details/originChannelTitle')
+    uploader = None if uploader_node is None else uploader_node.text
+    uploader_id_node = doc.find('.//details/originChannelId')
+    uploader_id = None if uploader_id_node is None else uploader_id_node.text
+    upload_date = unified_strdate(doc.find('.//details/airtime').text)
+
+    def xml_to_format(fnode):
+        video_url = fnode.find('url').text
+        is_available = 'http://www.metafilegenerator' not in video_url
+
+        format_id = fnode.attrib['basetype']
+        format_m = re.match(r'''(?x)
+            (?P<vcodec>[^_]+)_(?P<acodec>[^_]+)_(?P<container>[^_]+)_
+            (?P<proto>[^_]+)_(?P<index>[^_]+)_(?P<indexproto>[^_]+)
+        ''', format_id)
+
+        ext = format_m.group('container')
+        proto = format_m.group('proto').lower()
+
+        quality = fnode.find('./quality').text
+        abr = int(fnode.find('./audioBitrate').text) // 1000
+        vbr_node = fnode.find('./videoBitrate')
+        vbr = None if vbr_node is None else int(vbr_node.text) // 1000
+
+        width_node = fnode.find('./width')
+        width = None if width_node is None else int_or_none(width_node.text)
+        height_node = fnode.find('./height')
+        height = None if height_node is None else int_or_none(height_node.text)
+
+        format_note = ''
+        if not format_note:
+            format_note = None
+
+        return {
+            'format_id': format_id + '-' + quality,
+            'url': video_url,
+            'ext': ext,
+            'acodec': format_m.group('acodec'),
+            'vcodec': format_m.group('vcodec'),
+            'abr': abr,
+            'vbr': vbr,
+            'width': width,
+            'height': height,
+            'filesize': int_or_none(fnode.find('./filesize').text),
+            'format_note': format_note,
+            'protocol': proto,
+            '_available': is_available,
+        }
+
+    format_nodes = doc.findall('.//formitaeten/formitaet')
+    formats = list(filter(
+        lambda f: f['_available'],
+        map(xml_to_format, format_nodes)))
+    ie._sort_formats(formats)
+
+    return {
+        'id': video_id,
+        'title': title,
+        'description': description,
+        'duration': duration,
+        'uploader': uploader,
+        'uploader_id': uploader_id,
+        'upload_date': upload_date,
+        'formats': formats,
+    }
+
+
 class ZDFIE(InfoExtractor):
     _VALID_URL = r'^https?://www\.zdf\.de/ZDFmediathek(?P<hash>#)?/(.*beitrag/(?:video/)?)(?P<id>[0-9]+)(?:/[^/?]+)?(?:\?.*)?'
 
@@ -32,77 +108,4 @@ class ZDFIE(InfoExtractor):
         video_id = self._match_id(url)
 
         xml_url = 'http://www.zdf.de/ZDFmediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id
-        doc = self._download_xml(
-            xml_url, video_id,
-            note='Downloading video info',
-            errnote='Failed to download video info')
-
-        title = doc.find('.//information/title').text
-        description = doc.find('.//information/detail').text
-        duration = int(doc.find('.//details/lengthSec').text)
-        uploader_node = doc.find('.//details/originChannelTitle')
-        uploader = None if uploader_node is None else uploader_node.text
-        uploader_id_node = doc.find('.//details/originChannelId')
-        uploader_id = None if uploader_id_node is None else uploader_id_node.text
-        upload_date = unified_strdate(doc.find('.//details/airtime').text)
-
-        def xml_to_format(fnode):
-            video_url = fnode.find('url').text
-            is_available = 'http://www.metafilegenerator' not in video_url
-
-            format_id = fnode.attrib['basetype']
-            format_m = re.match(r'''(?x)
-                (?P<vcodec>[^_]+)_(?P<acodec>[^_]+)_(?P<container>[^_]+)_
-                (?P<proto>[^_]+)_(?P<index>[^_]+)_(?P<indexproto>[^_]+)
-            ''', format_id)
-
-            ext = format_m.group('container')
-            proto = format_m.group('proto').lower()
-
-            quality = fnode.find('./quality').text
-            abr = int(fnode.find('./audioBitrate').text) // 1000
-            vbr_node = fnode.find('./videoBitrate')
-            vbr = None if vbr_node is None else int(vbr_node.text) // 1000
-
-            width_node = fnode.find('./width')
-            width = None if width_node is None else int_or_none(width_node.text)
-            height_node = fnode.find('./height')
-            height = None if height_node is None else int_or_none(height_node.text)
-
-            format_note = ''
-            if not format_note:
-                format_note = None
-
-            return {
-                'format_id': format_id + '-' + quality,
-                'url': video_url,
-                'ext': ext,
-                'acodec': format_m.group('acodec'),
-                'vcodec': format_m.group('vcodec'),
-                'abr': abr,
-                'vbr': vbr,
-                'width': width,
-                'height': height,
-                'filesize': int_or_none(fnode.find('./filesize').text),
-                'format_note': format_note,
-                'protocol': proto,
-                '_available': is_available,
-            }
-
-        format_nodes = doc.findall('.//formitaeten/formitaet')
-        formats = list(filter(
-            lambda f: f['_available'],
-            map(xml_to_format, format_nodes)))
-
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'duration': duration,
-            'uploader': uploader,
-            'uploader_id': uploader_id,
-            'upload_date': upload_date,
-            'formats': formats,
-        }
\ No newline at end of file
+        return extract_from_xml_url(self, video_id, xml_url)

From b2e6a1c14c142eab299266641a72101f7d679443 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 27 Oct 2014 02:44:07 +0100
Subject: [PATCH 0354/1937] release 2014.10.27

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8e5a642a8..b1f0316ee 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.10.26.2'
+__version__ = '2014.10.27'

From aaa399d2f6cf8efff6e63f05be947c6ee4de4d34 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 27 Oct 2014 18:49:48 +0200
Subject: [PATCH 0355/1937] [sphinx] Fix version import

---
 docs/conf.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 4a04ad779..594ca61a6 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -44,8 +44,8 @@ copyright = u'2014, Ricardo Garcia Gonzalez'
 # built documents.
 #
 # The short X.Y version.
-import youtube_dl
-version = youtube_dl.__version__
+from youtube_dl.version import __version__
+version = __version__
 # The full version, including alpha/beta/rc tags.
 release = version
 

From 27c542c06f816100a64a2e3dbc5396df033aa025 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 28 Oct 2014 09:12:28 +0100
Subject: [PATCH 0356/1937] [iconosquare] Simplify

---
 youtube_dl/extractor/iconosquare.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/iconosquare.py b/youtube_dl/extractor/iconosquare.py
index 1d5a10a3b..370e86e5a 100644
--- a/youtube_dl/extractor/iconosquare.py
+++ b/youtube_dl/extractor/iconosquare.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -20,13 +18,11 @@ class IconosquareIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        html_title = self._html_search_regex(
-            r'<title>(.+?)</title>',
+        title = self._html_search_regex(
+            r'<title>(.+?)(?: *\(Videos?\))? \| (?:Iconosquare|Statigram)</title>',
             webpage, 'title')
-        title = re.sub(r'(?: *\(Videos?\))? \| (?:Iconosquare|Statigram)$', '', html_title)
         uploader_id = self._html_search_regex(
             r'@([^ ]+)', title, 'uploader name', fatal=False)
 

From 48a24ab7465a41822a0fd40bba249520b3a85124 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 28 Oct 2014 10:41:24 +0100
Subject: [PATCH 0357/1937] [generic] Fix HTML5 video regexp

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 35a7664b2..47580328a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -933,7 +933,7 @@ class GenericIE(InfoExtractor):
                 found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
         if not found:
             # HTML5 video
-            found = re.findall(r'(?s)<video[^<]*(?:>.*?<source[^>]+)? src="([^"]+)"', webpage)
+            found = re.findall(r'(?s)<video[^<]*(?:>.*?<source[^>]*)?\s+src="([^"]+)"', webpage)
         if not found:
             found = re.search(
                 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'

From 9ef55c5bbc1d3429218950ad380094623f499c6f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 28 Oct 2014 10:41:37 +0100
Subject: [PATCH 0358/1937] [quickvid] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/quickvid.py | 51 ++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 youtube_dl/extractor/quickvid.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3979b8270..c3799da67 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -294,6 +294,7 @@ from .pornoxo import PornoXOIE
 from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
 from .pyvideo import PyvideoIE
+from .quickvid import QuickVidIE
 from .radiofrance import RadioFranceIE
 from .rai import RaiIE
 from .rbmaradio import RBMARadioIE
diff --git a/youtube_dl/extractor/quickvid.py b/youtube_dl/extractor/quickvid.py
new file mode 100644
index 000000000..3bc78060d
--- /dev/null
+++ b/youtube_dl/extractor/quickvid.py
@@ -0,0 +1,51 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    compat_urlparse,
+    determine_ext,
+    int_or_none,
+)
+
+
+class QuickVidIE(InfoExtractor):
+    _VALID_URL = r'https?://(www\.)?quickvid\.org/watch\.php\?v=(?P<id>[a-zA-Z_0-9-]+)'
+    _TEST = {
+        'url': 'http://quickvid.org/watch.php?v=sUQT3RCG8dx',
+        'md5': 'c0c72dd473f260c06c808a05d19acdc5',
+        'info_dict': {
+            'id': 'sUQT3RCG8dx',
+            'ext': 'mp4',
+            'title': 'Nick Offerman\'s Summer Reading Recap',
+            'thumbnail': 're:^https?://.*\.(?:png|jpg|gif)$',
+            'view_count': int,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(r'<h2>(.*?)</h2>', webpage, 'title')
+        view_count = int_or_none(self._html_search_regex(
+            r'(?s)<div id="views">(.*?)</div>',
+            webpage, 'view count', fatal=False))
+        video_code = self._search_regex(
+            r'(?s)<video id="video"[^>]*>(.*?)</video>', webpage, 'video code')
+        formats = [
+            {
+                'url': compat_urlparse.urljoin(url, src),
+                'format_id': determine_ext(src, None),
+            } for src in re.findall('<source\s+src="([^"]+)"', video_code)
+        ]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'view_count': view_count,
+        }

From bbf1092ad0e444d766f30c960e11c2e21a77f605 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 28 Oct 2014 10:44:17 +0100
Subject: [PATCH 0359/1937] [fktv] Remove unused import

---
 youtube_dl/extractor/fktv.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index 990bee8ba..21b89142c 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -6,7 +6,6 @@ import json
 
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
     get_element_by_id,
     clean_html,
 )

From 9fdece5d340f24ab3c16770f71fd26943f4996c2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 28 Oct 2014 10:44:47 +0100
Subject: [PATCH 0360/1937] [srmediathek] Choose variable name more wisely

---
 youtube_dl/extractor/srmediathek.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/srmediathek.py b/youtube_dl/extractor/srmediathek.py
index d92d14d65..666a7dcc8 100644
--- a/youtube_dl/extractor/srmediathek.py
+++ b/youtube_dl/extractor/srmediathek.py
@@ -26,9 +26,9 @@ class SRMediathekIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        urls = json.loads(js_to_json(self._search_regex(
+        murls = json.loads(js_to_json(self._search_regex(
             r'var mediaURLs\s*=\s*(.*?);\n', webpage, 'video URLs')))
-        formats = [{'url': url} for url in urls]
+        formats = [{'url': murl} for murl in murls]
         self._sort_formats(formats)
 
         title = json.loads(js_to_json(self._search_regex(

From 3511266bc39976a926a9f33a0762aa40212431ee Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 28 Oct 2014 12:54:29 +0100
Subject: [PATCH 0361/1937] [YoutubeDL] Simplify API of YoutubeDL

Calling add_default_extractors twice should be harmless since the first set of extractors will match.
---
 youtube_dl/YoutubeDL.py | 6 +++++-
 youtube_dl/__init__.py  | 3 ---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 28dcc0195..fd775b9fd 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -189,7 +189,7 @@ class YoutubeDL(object):
     _num_downloads = None
     _screen_file = None
 
-    def __init__(self, params=None):
+    def __init__(self, params=None, auto_init=True):
         """Create a FileDownloader object with the given options."""
         if params is None:
             params = {}
@@ -246,6 +246,10 @@ class YoutubeDL(object):
 
         self._setup_opener()
 
+        if auto_init:
+            self.print_debug_header()
+            self.add_default_info_extractors()
+
     def add_info_extractor(self, ie):
         """Add an InfoExtractor object to the end of the list."""
         self._ies.append(ie)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 7dc971884..3c968082c 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -293,9 +293,6 @@ def _real_main(argv=None):
     }
 
     with YoutubeDL(ydl_opts) as ydl:
-        ydl.print_debug_header()
-        ydl.add_default_info_extractors()
-
         # PostProcessors
         # Add the metadata pp first, the other pps will copy it
         if opts.addmetadata:

From 03df7baa6a8dc5cab41f5ce2f0be8cd439a6195d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 28 Oct 2014 12:54:39 +0100
Subject: [PATCH 0362/1937] Start documentation on how to embed youtube-dl

---
 README.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/README.md b/README.md
index d62546433..230c12a88 100644
--- a/README.md
+++ b/README.md
@@ -511,6 +511,20 @@ If you want to add support for a new site, you can follow this quick list (assum
 
 In any case, thank you very much for your contributions!
 
+# EMBEDDING YOUTUBE-DL
+
+youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new).
+
+From a Python program, you can embed youtube-dl in a more powerful fashion, like this:
+
+    import youtube_dl
+
+    ydl_opts = {}
+    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
+        ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
+
+Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
+
 # BUGS
 
 Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues> . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email.

From 632256d9ecd86f17b370b2cbe3f50acdf961d6cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 28 Oct 2014 20:35:02 +0700
Subject: [PATCH 0363/1937] [wimp] Update video URL regex

---
 youtube_dl/extractor/wimp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py
index c27dda944..3377a543e 100644
--- a/youtube_dl/extractor/wimp.py
+++ b/youtube_dl/extractor/wimp.py
@@ -37,7 +37,7 @@ class WimpIE(InfoExtractor):
         video_id = mobj.group(1)
         webpage = self._download_webpage(url, video_id)
         video_url = self._search_regex(
-            r's1\.addVariable\("file",\s*"([^"]+)"\);', webpage, 'video URL')
+            r"'file'\s*:\s*'([^']+)'", webpage, 'video URL')
         if YoutubeIE.suitable(video_url):
             self.to_screen('Found YouTube video')
             return {

From 9334f8f17a0e6a2b97d4e20b12a9ede52e4c0b9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 28 Oct 2014 21:06:07 +0700
Subject: [PATCH 0364/1937] [vk] Handle deleted videos

---
 youtube_dl/extractor/vk.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 918bd1098..837c3167e 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -142,6 +142,10 @@ class VKIE(InfoExtractor):
             raise ExtractorError('This video is only available for registered users, '
                 'use --username and --password options to provide account credentials.', expected=True)
 
+        if '<!>Unknown error' in info_page:
+            raise ExtractorError(
+                'Video %s does not exist' % video_id, expected=True)
+
         m_yt = re.search(r'src="(http://www.youtube.com/.*?)"', info_page)
         if m_yt is not None:
             self.to_screen('Youtube video detected')

From e0c51cdadc838a67870f5c22c48bb0b9f1951afc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 28 Oct 2014 21:35:25 +0700
Subject: [PATCH 0365/1937] [vk] Generalize errors

---
 youtube_dl/extractor/vk.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 837c3167e..36cd7e52e 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -138,13 +138,19 @@ class VKIE(InfoExtractor):
         info_url = 'http://vk.com/al_video.php?act=show&al=1&video=%s' % video_id
         info_page = self._download_webpage(info_url, video_id)
 
-        if re.search(r'<!>Please log in or <', info_page):
-            raise ExtractorError('This video is only available for registered users, '
-                'use --username and --password options to provide account credentials.', expected=True)
+        ERRORS = {
+            r'>Видеозапись .*? была изъята из публичного доступа в связи с обращением правообладателя.<':
+                'Video %s has been removed from public access due to rightholder complaint.',
+            r'<!>Please log in or <':
+                'Video %s is only available for registered users, '
+                'use --username and --password options to provide account credentials.',
+            '<!>Unknown error':
+                'Video %s does not exist.'
+        }
 
-        if '<!>Unknown error' in info_page:
-            raise ExtractorError(
-                'Video %s does not exist' % video_id, expected=True)
+        for error_re, error_msg in ERRORS.items():
+            if re.search(error_re, info_page):
+                raise ExtractorError(error_msg % video_id, expected=True)
 
         m_yt = re.search(r'src="(http://www.youtube.com/.*?)"', info_page)
         if m_yt is not None:

From 01d663bca35e138cb95f5d156bcf396dcae54f73 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 28 Oct 2014 15:51:15 +0100
Subject: [PATCH 0366/1937] [auengine] Simplify

---
 youtube_dl/extractor/auengine.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/auengine.py b/youtube_dl/extractor/auengine.py
index 20bf12550..1c765532a 100644
--- a/youtube_dl/extractor/auengine.py
+++ b/youtube_dl/extractor/auengine.py
@@ -24,8 +24,7 @@ class AUEngineIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
         title = self._html_search_regex(r'<title>(?P<title>.+?)</title>', webpage, 'title')

From e5193599ecca2d4437d066e9fed95db74b8562c7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 28 Oct 2014 16:51:32 +0100
Subject: [PATCH 0367/1937] [laola1tv] Add new extractor

The extractor works fine, but the f4m downloader cannot handle the resulting bootstrap information.
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/laola1tv.py | 75 ++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)
 create mode 100644 youtube_dl/extractor/laola1tv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c3799da67..615018c09 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -189,6 +189,7 @@ from .kontrtube import KontrTubeIE
 from .krasview import KrasViewIE
 from .ku6 import Ku6IE
 from .la7 import LA7IE
+from .laola1tv import Laola1TvIE
 from .lifenews import LifeNewsIE
 from .liveleak import LiveLeakIE
 from .livestream import (
diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
new file mode 100644
index 000000000..45ab86b03
--- /dev/null
+++ b/youtube_dl/extractor/laola1tv.py
@@ -0,0 +1,75 @@
+from __future__ import unicode_literals
+
+import random
+import re
+
+from .common import InfoExtractor
+
+
+class Laola1TvIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P<lang>[a-z]+)-(?P<portal>[a-z]+)/.*?/(?P<id>[0-9]+)\.html'
+    _TEST = {
+        'url': 'http://www.laola1.tv/de-de/live/bwf-bitburger-open-grand-prix-gold-court-1/250019.html',
+        'info_dict': {
+            'id': '250019',
+            'ext': 'mp4',
+            'title': 'Bitburger Open Grand Prix Gold - Court 1',
+            'categories': ['Badminton'],
+            'uploader': 'BWF - Badminton World Federation',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        lang = mobj.group('lang')
+        portal = mobj.group('portal')
+
+        webpage = self._download_webpage(url, video_id)
+        iframe_url = self._search_regex(
+            r'<iframe[^>]*?class="main_tv_player"[^>]*?src="([^"]+)"',
+            webpage, 'iframe URL')
+
+        iframe = self._download_webpage(
+            iframe_url, video_id, note='Downloading iframe')
+        flashvars_m = re.findall(
+            r'flashvars\.([_a-zA-Z0-9]+)\s*=\s*"([^"]*)";', iframe)
+        flashvars = dict((m[0], m[1]) for m in flashvars_m)
+
+        xml_url = ('http://www.laola1.tv/server/hd_video.php?' +
+                   'play=%s&partner=1&portal=%s&v5ident=&lang=%s' % (
+                       video_id, portal, lang))
+        hd_doc = self._download_xml(xml_url, video_id)
+
+        title = hd_doc.find('.//video/title').text
+        flash_url = hd_doc.find('.//video/url').text
+        categories = hd_doc.find('.//video/meta_sports').text.split(',')
+        uploader = hd_doc.find('.//video/meta_organistation').text
+
+        ident = random.randint(10000000, 99999999)
+        token_url = '%s&ident=%s&klub=0&unikey=0&timestamp=%s&auth=%s' % (
+            flash_url, ident, flashvars['timestamp'], flashvars['auth'])
+
+        token_doc = self._download_xml(
+            token_url, video_id, note='Downloading token')
+        token_attrib = token_doc.find('.//token').attrib
+        if token_attrib.get('auth') == 'blocked':
+            raise ExtractorError('Token error: ' % token_attrib.get('comment'))
+
+        video_url = '%s?hdnea=%s&hdcore=3.2.0' % (
+            token_attrib['url'], token_attrib['auth'])
+
+        return {
+            'id': video_id,
+            'is_live': True,
+            'title': title,
+            'url': video_url,
+            'uploader': uploader,
+            'categories': categories,
+            'ext': 'mp4',
+        }
+

From ee966928af5f35123c4c7417bb052ae5aea345c9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 28 Oct 2014 17:27:41 +0100
Subject: [PATCH 0368/1937] [f4m] Support bootstrap URLs

---
 youtube_dl/downloader/f4m.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index b3be16ff1..b607f6485 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -244,9 +244,16 @@ class F4mFD(FileDownloader):
                 lambda f: int(f[0]) == requested_bitrate, formats))[0]
 
         base_url = compat_urlparse.urljoin(man_url, media.attrib['url'])
-        bootstrap = base64.b64decode(doc.find(_add_ns('bootstrapInfo')).text)
+        bootstrap_node = doc.find(_add_ns('bootstrapInfo'))
+        if bootstrap_node.text is None:
+            bootstrap_url = compat_urlparse.urljoin(
+                base_url, bootstrap_node.attrib['url'])
+            bootstrap = self.ydl.urlopen(bootstrap_url).read()
+        else:
+            bootstrap = base64.b64decode(bootstrap_node.text)
         metadata = base64.b64decode(media.find(_add_ns('metadata')).text)
         boot_info = read_bootstrap_info(bootstrap)
+
         fragments_list = build_fragments_list(boot_info)
         if self.params.get('test', False):
             # We only download the first fragment

From 00edd4f9bef7cf0708d1c4da128477d76115fdec Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 28 Oct 2014 17:29:25 +0100
Subject: [PATCH 0369/1937] [laola1tv] Mark as broken

When the f4m downloader gets live stream support, I expect this to work magically or with very minor changes.
---
 youtube_dl/extractor/laola1tv.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index 45ab86b03..263f68773 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -23,6 +23,8 @@ class Laola1TvIE(InfoExtractor):
         }
     }
 
+    _BROKEN = True  # Not really - extractor works fine, but f4m downloader does not support live streams yet.
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')

From a9bad429b309e614b4b8905c085ef425350ceeb2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 29 Oct 2014 11:04:48 +0100
Subject: [PATCH 0370/1937] [niconico] Add extractor for playlists (closes
 #4043)

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/niconico.py | 34 ++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 615018c09..32236f0fa 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -252,7 +252,7 @@ from .newstube import NewstubeIE
 from .nfb import NFBIE
 from .nfl import NFLIE
 from .nhl import NHLIE, NHLVideocenterIE
-from .niconico import NiconicoIE
+from .niconico import NiconicoIE, NiconicoPlaylistIE
 from .ninegag import NineGagIE
 from .noco import NocoIE
 from .normalboots import NormalbootsIE
diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 7b85589b7..62d5707fe 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import re
+import json
 
 from .common import InfoExtractor
 from ..utils import (
@@ -146,3 +147,36 @@ class NiconicoIE(InfoExtractor):
             'duration': duration,
             'webpage_url': webpage_url,
         }
+
+
+class NiconicoPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.nicovideo\.jp/mylist/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://www.nicovideo.jp/mylist/27411728',
+        'info_dict': {
+            'id': '27411728',
+            'title': 'AKB48のオールナイトニッポン',
+        },
+        'playlist_mincount': 225,
+    }
+
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+        webpage = self._download_webpage(url, list_id)
+
+        entries_json = self._search_regex(r'Mylist\.preload\(\d+, (\[.*\])\);',
+            webpage, 'entries')
+        entries = json.loads(entries_json)
+        entries = [{
+            '_type': 'url',
+            'ie_key': NiconicoIE.ie_key(),
+            'url': 'http://www.nicovideo.jp/watch/%s' % entry['item_id'],
+        } for entry in entries]
+
+        return {
+            '_type': 'playlist',
+            'title': self._search_regex(r'\s+name: "(.*?)"', webpage, 'title'),
+            'id': list_id,
+            'entries': entries,
+        }

From 8abec2c8bb2ff1051af63420711706497faf6de4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 29 Oct 2014 11:13:34 +0100
Subject: [PATCH 0371/1937] [test_utils] Fix compat_getenv and
 compat_expanduser tests on python 3.x

---
 test/test_utils.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 19f9fce20..febba411e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -360,12 +360,14 @@ class TestUtil(unittest.TestCase):
 
     def test_compat_getenv(self):
         test_str = 'тест'
-        os.environ['YOUTUBE-DL-TEST'] = test_str.encode(get_filesystem_encoding())
+        os.environ['YOUTUBE-DL-TEST'] = (test_str if sys.version_info >= (3, 0)
+            else test_str.encode(get_filesystem_encoding()))
         self.assertEqual(compat_getenv('YOUTUBE-DL-TEST'), test_str)
 
     def test_compat_expanduser(self):
         test_str = 'C:\Documents and Settings\тест\Application Data'
-        os.environ['HOME'] = test_str.encode(get_filesystem_encoding())
+        os.environ['HOME'] = (test_str if sys.version_info >= (3, 0)
+            else test_str.encode(get_filesystem_encoding()))
         self.assertEqual(compat_expanduser('~'), test_str)
 
 if __name__ == '__main__':

From c451d4f5533de9bacba5ad7c186b7bbb3fb0b6cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Oct 2014 21:16:10 +0700
Subject: [PATCH 0372/1937] [trutube] Fix extraction

---
 youtube_dl/extractor/trutube.py | 38 +++++++++++++++------------------
 1 file changed, 17 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/trutube.py b/youtube_dl/extractor/trutube.py
index 57f956683..a73f3c43a 100644
--- a/youtube_dl/extractor/trutube.py
+++ b/youtube_dl/extractor/trutube.py
@@ -1,13 +1,12 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import xpath_text
 
 
 class TruTubeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?trutube\.tv/video/(?P<id>[0-9]+)/.*'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?trutube\.tv/(?:video/|nuevo/player/embed\.php\?v=)(?P<id>[0-9]+)'
+    _TESTS = [{
         'url': 'http://trutube.tv/video/14880/Ramses-II-Proven-To-Be-A-Red-Headed-Caucasoid-',
         'md5': 'c5b6e301b0a2040b074746cbeaa26ca1',
         'info_dict': {
@@ -16,29 +15,26 @@ class TruTubeIE(InfoExtractor):
             'title': 'Ramses II - Proven To Be A Red Headed Caucasoid',
             'thumbnail': 're:^http:.*\.jpg$',
         }
-    }
+    }, {
+        'url': 'https://trutube.tv/nuevo/player/embed.php?v=14880',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
-        video_title = self._og_search_title(webpage).strip()
-        thumbnail = self._search_regex(
-            r"var splash_img = '([^']+)';", webpage, 'thumbnail', fatal=False)
+        config = self._download_xml(
+            'https://trutube.tv/nuevo/player/config.php?v=%s' % video_id,
+            video_id, transform_source=lambda s: s.strip())
 
-        all_formats = re.finditer(
-            r"var (?P<key>[a-z]+)_video_file\s*=\s*'(?P<url>[^']+)';", webpage)
-        formats = [{
-            'format_id': m.group('key'),
-            'quality': -i,
-            'url': m.group('url'),
-        } for i, m in enumerate(all_formats)]
-        self._sort_formats(formats)
+        # filehd is always 404
+        video_url = xpath_text(config, './file', 'video URL', fatal=True)
+        title = xpath_text(config, './title', 'title')
+        thumbnail = xpath_text(config, './image', ' thumbnail')
 
         return {
             'id': video_id,
-            'title': video_title,
-            'formats': formats,
+            'url': video_url,
+            'title': title,
             'thumbnail': thumbnail,
         }

From dbd1283d31aa1df43bf9b3255dc27a1c8bfed4ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 29 Oct 2014 21:50:37 +0700
Subject: [PATCH 0373/1937] [naver] Capture and output error message (#4057)

---
 youtube_dl/extractor/naver.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index c0231c197..cb1af9ecc 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..utils import (
     compat_urllib_parse,
     ExtractorError,
+    clean_html,
 )
 
 
@@ -31,6 +32,11 @@ class NaverIE(InfoExtractor):
         m_id = re.search(r'var rmcPlayer = new nhn.rmcnmv.RMCVideoPlayer\("(.+?)", "(.+?)"',
             webpage)
         if m_id is None:
+            m_error = re.search(
+                r'(?s)<div class="nation_error">\s*(?:<!--.*?-->)?\s*<p class="[^"]+">(?P<msg>.+?)</p>\s*</div>',
+                webpage)
+            if m_error:
+                raise ExtractorError(clean_html(m_error.group('msg')), expected=True)
             raise ExtractorError('couldn\'t extract vid and key')
         vid = m_id.group(1)
         key = m_id.group(2)

From 6ad4013d40e839211e2896129eed05ccd40ee963 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 29 Oct 2014 20:10:00 +0100
Subject: [PATCH 0374/1937] [drtv] Allow fractional timestamps (Fixes #4059)

---
 test/test_utils.py           | 1 +
 youtube_dl/extractor/drtv.py | 7 ++-----
 youtube_dl/utils.py          | 4 ++--
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 19f9fce20..1c2b57609 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -289,6 +289,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_iso8601('2014-03-23T23:04:26+0100'), 1395612266)
         self.assertEqual(parse_iso8601('2014-03-23T22:04:26+0000'), 1395612266)
         self.assertEqual(parse_iso8601('2014-03-23T22:04:26Z'), 1395612266)
+        self.assertEqual(parse_iso8601('2014-03-23T22:04:26.1234Z'), 1395612266)
 
     def test_strip_jsonp(self):
         stripped = strip_jsonp('cb ([ {"id":"532cb",\n\n\n"x":\n3}\n]\n);')
diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index 9d6ce1f48..93b3c9f36 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .subtitles import SubtitlesInfoExtractor
 from .common import ExtractorError
 from ..utils import parse_iso8601
@@ -25,8 +23,7 @@ class DRTVIE(SubtitlesInfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         programcard = self._download_json(
             'http://www.dr.dk/mu/programcard/expanded/%s' % video_id, video_id, 'Downloading video JSON')
@@ -35,7 +32,7 @@ class DRTVIE(SubtitlesInfoExtractor):
 
         title = data['Title']
         description = data['Description']
-        timestamp = parse_iso8601(data['CreatedTime'][:-5])
+        timestamp = parse_iso8601(data['CreatedTime'])
 
         thumbnail = None
         duration = None
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 6c0c39ca5..2864e5142 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -925,7 +925,7 @@ def parse_iso8601(date_str, delimiter='T'):
         return None
 
     m = re.search(
-        r'Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$',
+        r'(\.[0-9]+)?(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
         date_str)
     if not m:
         timezone = datetime.timedelta()
@@ -938,7 +938,7 @@ def parse_iso8601(date_str, delimiter='T'):
             timezone = datetime.timedelta(
                 hours=sign * int(m.group('hours')),
                 minutes=sign * int(m.group('minutes')))
-    date_format =  '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
+    date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
     dt = datetime.datetime.strptime(date_str, date_format) - timezone
     return calendar.timegm(dt.timetuple())
 

From ac645ac7d0ce240e17f5e26f525e22e2aa71407a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 29 Oct 2014 20:27:58 +0100
Subject: [PATCH 0375/1937] [generic] Allow soundcloud embeds with additional
 attributes

---
 youtube_dl/extractor/generic.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 47580328a..a27ec342e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -405,6 +405,18 @@ class GenericIE(InfoExtractor):
             'expected_warnings': [
                 r'501.*Not Implemented'
             ],
+        },
+        # Soundcloud embed
+        {
+            'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
+            'info_dict': {
+                'id': '174391317',
+                'ext': 'mp3',
+                'description': 'md5:ff867d6b555488ad3c52572bb33d432c',
+                'uploader': 'Sophos Security',
+                'title': 'Chet Chat 171 - Oct 29, 2014',
+                'upload_date': '20141029',
+            }
         }
     ]
 
@@ -838,7 +850,7 @@ class GenericIE(InfoExtractor):
 
         # Look for embeded soundcloud player
         mobj = re.search(
-            r'<iframe src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
+            r'<iframe\s+(?:[a-zA-Z0-9_-]+="[^"]+"\s+)*src="(?P<url>https?://(?:w\.)?soundcloud\.com/player[^"]+)"',
             webpage)
         if mobj is not None:
             url = unescapeHTML(mobj.group('url'))

From d65d6286139fb6e4836113924648e35892f48de3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 29 Oct 2014 21:19:20 +0100
Subject: [PATCH 0376/1937] [crunchycroll] Fix building of ass subtitles
 (reported in #4019)

Parse the xml document instead of using regexes, otherwise unicode characters are left unescaped.
---
 youtube_dl/extractor/crunchyroll.py | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 05b21e872..cc612d08e 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -109,19 +109,17 @@ class CrunchyrollIE(SubtitlesInfoExtractor):
         decrypted_data = intlist_to_bytes(aes_cbc_decrypt(data, key, iv))
         return zlib.decompress(decrypted_data)
 
-    def _convert_subtitles_to_srt(self, subtitles):
+    def _convert_subtitles_to_srt(self, sub_root):
         output = ''
-        for i, (start, end, text) in enumerate(re.findall(r'<event [^>]*?start="([^"]+)" [^>]*?end="([^"]+)" [^>]*?text="([^"]+)"[^>]*?>', subtitles), 1):
-            start = start.replace('.', ',')
-            end = end.replace('.', ',')
-            text = clean_html(text)
-            text = text.replace('\\N', '\n')
-            if not text:
-                continue
+
+        for i, event in enumerate(sub_root.findall('./events/event'), 1):
+            start = event.attrib['start'].replace('.', ',')
+            end = event.attrib['end'].replace('.', ',')
+            text = event.attrib['text'].replace('\\N', '\n')
             output += '%d\n%s --> %s\n%s\n\n' % (i, start, end, text)
         return output
 
-    def _convert_subtitles_to_ass(self, subtitles):
+    def _convert_subtitles_to_ass(self, sub_root):
         output = ''
 
         def ass_bool(strvalue):
@@ -130,10 +128,6 @@ class CrunchyrollIE(SubtitlesInfoExtractor):
                 assvalue = '-1'
             return assvalue
 
-        sub_root = xml.etree.ElementTree.fromstring(subtitles)
-        if not sub_root:
-            return output
-
         output = '[Script Info]\n'
         output += 'Title: %s\n' % sub_root.attrib["title"]
         output += 'ScriptType: v4.00+\n'
@@ -270,10 +264,13 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
             if not lang_code:
                 continue
+            sub_root = xml.etree.ElementTree.fromstring(subtitle)
+            if not sub_root:
+                subtitles[lang_code] = ''
             if sub_format == 'ass':
-                subtitles[lang_code] = self._convert_subtitles_to_ass(subtitle)
+                subtitles[lang_code] = self._convert_subtitles_to_ass(sub_root)
             else:
-                subtitles[lang_code] = self._convert_subtitles_to_srt(subtitle)
+                subtitles[lang_code] = self._convert_subtitles_to_srt(sub_root)
 
         if self._downloader.params.get('listsubtitles', False):
             self._list_available_subtitles(video_id, subtitles)

From feb74960eb0be0ee832ce5b07b1e1533eca518f9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 29 Oct 2014 23:29:42 +0100
Subject: [PATCH 0377/1937] release 2014.10.29

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b1f0316ee..cffa3ef82 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.10.27'
+__version__ = '2014.10.29'

From b8a618f898b42e317b2abb13e33f60641d75a762 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 30 Oct 2014 01:42:52 +0200
Subject: [PATCH 0378/1937] [ro220] Fix broken extractor and modernize (#4054)

---
 youtube_dl/extractor/ro220.py | 46 +++++++++++++++++------------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/ro220.py b/youtube_dl/extractor/ro220.py
index a6ad59465..0a3a71448 100644
--- a/youtube_dl/extractor/ro220.py
+++ b/youtube_dl/extractor/ro220.py
@@ -1,43 +1,43 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
-    clean_html,
-    compat_parse_qs,
-)
+from ..utils import compat_urllib_parse_unquote
 
 
 class Ro220IE(InfoExtractor):
     IE_NAME = '220.ro'
-    _VALID_URL = r'(?x)(?:https?://)?(?:www\.)?220\.ro/(?P<category>[^/]+)/(?P<shorttitle>[^/]+)/(?P<video_id>[^/]+)'
+    _VALID_URL = r'(?x)(?:https?://)?(?:www\.)?220\.ro/(?P<category>[^/]+)/(?P<shorttitle>[^/]+)/(?P<id>[^/]+)'
     _TEST = {
-        "url": "http://www.220.ro/sport/Luati-Le-Banii-Sez-4-Ep-1/LYV6doKo7f/",
-        'file': 'LYV6doKo7f.mp4',
+        'url': 'http://www.220.ro/sport/Luati-Le-Banii-Sez-4-Ep-1/LYV6doKo7f/',
         'md5': '03af18b73a07b4088753930db7a34add',
         'info_dict': {
-            "title": "Luati-le Banii sez 4 ep 1",
-            "description": "re:^Iata-ne reveniti dupa o binemeritata vacanta\. +Va astept si pe Facebook cu pareri si comentarii.$",
+            'id': 'LYV6doKo7f',
+            'ext': 'mp4',
+            'title': 'Luati-le Banii sez 4 ep 1',
+            'description': 're:^Iata-ne reveniti dupa o binemeritata vacanta\. +Va astept si pe Facebook cu pareri si comentarii.$',
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        flashVars_str = self._search_regex(
-            r'<param name="flashVars" value="([^"]+)"',
-            webpage, 'flashVars')
-        flashVars = compat_parse_qs(flashVars_str)
+        url = compat_urllib_parse_unquote(self._search_regex(
+            r'(?s)clip\s*:\s*{.*?url\s*:\s*\'([^\']+)\'', webpage, 'url'))
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        formats = [{
+            'format_id': 'sd',
+            'url': url,
+            'ext': 'mp4',
+        }]
 
         return {
-            '_type': 'video',
             'id': video_id,
-            'ext': 'mp4',
-            'url': flashVars['videoURL'][0],
-            'title': flashVars['title'][0],
-            'description': clean_html(flashVars['desc'][0]),
-            'thumbnail': flashVars['preview'][0],
+            'formats': formats,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
         }

From fbb21cf528fe5cc4ba72f97f691cdf7ec20ee0e6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 30 Oct 2014 09:34:13 +0100
Subject: [PATCH 0379/1937] [youtube] Add formats 298, 299 (Fixes #4056)

---
 youtube_dl/YoutubeDL.py         | 2 ++
 youtube_dl/extractor/common.py  | 1 +
 youtube_dl/extractor/youtube.py | 4 ++++
 3 files changed, 7 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index fd775b9fd..73a372df4 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1211,6 +1211,8 @@ class YoutubeDL(object):
             res += 'video@'
         if fdict.get('vbr') is not None:
             res += '%4dk' % fdict['vbr']
+        if fdict.get('fps') is not None:
+            res += ', %sfps' % fdict['fps']
         if fdict.get('acodec') is not None:
             if res:
                 res += ', '
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b77db52f7..3300734bc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -72,6 +72,7 @@ class InfoExtractor(object):
                     * acodec     Name of the audio codec in use
                     * asr        Audio sampling rate in Hertz
                     * vbr        Average video bitrate in KBit/s
+                    * fps        Frame rate
                     * vcodec     Name of the video codec in use
                     * container  Name of the container format
                     * filesize   The number of bytes, if known in advance
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 40fe4662a..fbd9311fe 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -302,6 +302,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
         '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
 
+        # Dash mov
+        '298': {'ext': 'mov', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
+        '299': {'ext': 'mov', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
+
         # RTMP (unnamed)
         '_rtmp': {'protocol': 'rtmp'},
     }

From 2c8e03d937cc08d0d924b24a95fcc2eb3076e544 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 30 Oct 2014 09:40:52 +0100
Subject: [PATCH 0380/1937] Sort formats by fps as well

---
 youtube_dl/extractor/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 3300734bc..7e4113213 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -619,6 +619,7 @@ class InfoExtractor(object):
                 f.get('vbr') if f.get('vbr') is not None else -1,
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,
+                f.get('fps') if f.get('fps') is not None else -1,
                 f.get('filesize') if f.get('filesize') is not None else -1,
                 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
                 f.get('source_preference') if f.get('source_preference') is not None else -1,

From a51d3aa001aa12e814ae31cf2abc1e3781f2b05a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 30 Oct 2014 09:43:11 +0100
Subject: [PATCH 0381/1937] [youtube] Add support for formats 302 and 303
 (Fixes #4060)

---
 youtube_dl/extractor/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index fbd9311fe..19f814125 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -297,6 +297,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
+        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
+        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
 
         # Dash webm audio
         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},

From 7b6de3728a422774c4cc3d8e22f55b0641fcd53d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 30 Oct 2014 09:53:43 +0100
Subject: [PATCH 0382/1937] [youtube] Add format 266 (Fixes #4055)

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 19f814125..6071d7245 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -307,6 +307,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         # Dash mov
         '298': {'ext': 'mov', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
         '299': {'ext': 'mov', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
+        '266': {'ext': 'mov', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'h264'},
 
         # RTMP (unnamed)
         '_rtmp': {'protocol': 'rtmp'},

From 5ec39d8b96b981dfd23044803367b2e4a05b214a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 30 Oct 2014 09:53:48 +0100
Subject: [PATCH 0383/1937] release 2014.10.30

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index cffa3ef82..87d56c164 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.10.29'
+__version__ = '2014.10.30'

From 0d2c141865389e8124353e0c88ea99da39058692 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 31 Oct 2014 11:13:02 +0100
Subject: [PATCH 0384/1937] [youtube] Detect formats 298 et al as mp4 (Fixes
 #4066)

---
 youtube_dl/extractor/youtube.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6071d7245..aad8ffbf4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -274,6 +274,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '138': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
+        '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
+        '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
+        '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'h264'},
 
         # Dash mp4 audio
         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 48, 'preference': -50},
@@ -304,11 +307,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
         '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
 
-        # Dash mov
-        '298': {'ext': 'mov', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
-        '299': {'ext': 'mov', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
-        '266': {'ext': 'mov', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'h264'},
-
         # RTMP (unnamed)
         '_rtmp': {'protocol': 'rtmp'},
     }

From 2a834bdb21b1a747ba91b27a582aa48e8f28ec0b Mon Sep 17 00:00:00 2001
From: anovicecodemonkey <anovicecodemonkey13435@mailinator.com>
Date: Fri, 31 Oct 2014 21:20:37 +1030
Subject: [PATCH 0385/1937] [FunnyOrDie] Add support for "/articles/" URLs

---
 youtube_dl/extractor/funnyordie.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index ec6d96ada..8ffcfca3b 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -8,7 +8,7 @@ from ..utils import ExtractorError
 
 
 class FunnyOrDieIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?funnyordie\.com/(?P<type>embed|videos)/(?P<id>[0-9a-f]+)(?:$|[?#/])'
+    _VALID_URL = r'https?://(?:www\.)?funnyordie\.com/(?P<type>embed|articles|videos)/(?P<id>[0-9a-f]+)(?:$|[?#/])'
     _TESTS = [{
         'url': 'http://www.funnyordie.com/videos/0732f586d7/heart-shaped-box-literal-video-version',
         'md5': 'bcd81e0c4f26189ee09be362ad6e6ba9',

From d319948b6aa52b9b7041c54ac587a4b7cdf899a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 31 Oct 2014 19:26:56 +0700
Subject: [PATCH 0386/1937] [funnyordie] Add articles URL test

---
 youtube_dl/extractor/funnyordie.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index 8ffcfca3b..487d5d1b4 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -29,6 +29,9 @@ class FunnyOrDieIE(InfoExtractor):
             'description': 'Please use this to sell something.  www.jonlajoie.com',
             'thumbnail': 're:^http:.*\.jpg$',
         },
+    }, {
+        'url': 'http://www.funnyordie.com/articles/ebf5e34fc8/10-hours-of-walking-in-nyc-as-a-man',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 72975729c89a5b0c73ecb4e5dccca38ed2dfb013 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 31 Oct 2014 20:19:30 +0700
Subject: [PATCH 0387/1937] [canalplus] Tweak extractor to support piwiplus
 (Closes #4046)

---
 youtube_dl/extractor/canalplus.py | 31 ++++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 0202078b0..4ee65c763 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -11,11 +11,15 @@ from ..utils import (
 
 
 class CanalplusIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.canalplus\.fr/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
-    _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/cplus/%s'
-    IE_NAME = 'canalplus.fr'
+    IE_DESC = 'canalplus.fr and piwiplus.fr'
+    _VALID_URL = r'https?://(?:www\.(?P<site>canal|piwi)plus\.fr/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
+    _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/%s/%s'
+    _SITE_ID_MAP = {
+        'canal': 'cplus',
+        'piwi': 'teletoon',
+    }
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.canalplus.fr/c-infos-documentaires/pid1830-c-zapping.html?vid=922470',
         'md5': '3db39fb48b9685438ecf33a1078023e4',
         'info_dict': {
@@ -25,20 +29,33 @@ class CanalplusIE(InfoExtractor):
             'description': 'Le meilleur de toutes les chaînes, tous les jours.\nEmission du 26 août 2013',
             'upload_date': '20130826',
         },
-    }
+    }, {
+        'url': 'http://www.piwiplus.fr/videos-piwi/pid1405-le-labyrinthe-boing-super-ranger.html?vid=1108190',
+        'md5': '3db39fb48b9685438ecf33a1078023e4',
+        'info_dict': {
+            'id': '1108190',
+            'ext': 'flv',
+            'title': 'Le labyrinthe - Boing super ranger',
+            'description': 'md5:',
+            'upload_date': '20140724',
+        },
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.groupdict().get('id')
 
+        site_id = self._SITE_ID_MAP[mobj.group('site') or 'canal']
+
         # Beware, some subclasses do not define an id group
         display_id = url_basename(mobj.group('path'))
 
         if video_id is None:
             webpage = self._download_webpage(url, display_id)
-            video_id = self._search_regex(r'<canal:player videoId="(\d+)"', webpage, 'video id')
+            video_id = self._search_regex(
+                r'<canal:player[^>]+?videoId="(\d+)"', webpage, 'video id')
 
-        info_url = self._VIDEO_INFO_TEMPLATE % video_id
+        info_url = self._VIDEO_INFO_TEMPLATE % (site_id, video_id)
         doc = self._download_xml(info_url, video_id, 'Downloading video XML')
 
         video_info = [video for video in doc if video.find('ID').text == video_id][0]

From 6e1cff9c33e5e35e6d01b5635e3ad08565e5941c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 31 Oct 2014 21:54:30 +0700
Subject: [PATCH 0388/1937] [canalplus] Improve and merge with d8 extractor

---
 youtube_dl/extractor/__init__.py  |  1 -
 youtube_dl/extractor/canalplus.py | 56 ++++++++++++++++++++++---------
 youtube_dl/extractor/d8.py        | 25 --------------
 3 files changed, 41 insertions(+), 41 deletions(-)
 delete mode 100644 youtube_dl/extractor/d8.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 32236f0fa..3f85c99cd 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -67,7 +67,6 @@ from .crunchyroll import (
     CrunchyrollShowPlaylistIE
 )
 from .cspan import CSpanIE
-from .d8 import D8IE
 from .dailymotion import (
     DailymotionIE,
     DailymotionPlaylistIE,
diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 4ee65c763..dddeaa59c 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -7,16 +7,18 @@ from .common import InfoExtractor
 from ..utils import (
     unified_strdate,
     url_basename,
+    qualities,
 )
 
 
 class CanalplusIE(InfoExtractor):
-    IE_DESC = 'canalplus.fr and piwiplus.fr'
-    _VALID_URL = r'https?://(?:www\.(?P<site>canal|piwi)plus\.fr/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
+    IE_DESC = 'canalplus.fr, piwiplus.fr and d8.tv'
+    _VALID_URL = r'https?://(?:www\.(?P<site>canalplus\.fr|piwiplus\.fr|d8\.tv)/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
     _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/%s/%s'
     _SITE_ID_MAP = {
-        'canal': 'cplus',
-        'piwi': 'teletoon',
+        'canalplus.fr': 'cplus',
+        'piwiplus.fr': 'teletoon',
+        'd8.tv': 'd8',
     }
 
     _TESTS = [{
@@ -31,14 +33,24 @@ class CanalplusIE(InfoExtractor):
         },
     }, {
         'url': 'http://www.piwiplus.fr/videos-piwi/pid1405-le-labyrinthe-boing-super-ranger.html?vid=1108190',
-        'md5': '3db39fb48b9685438ecf33a1078023e4',
         'info_dict': {
             'id': '1108190',
             'ext': 'flv',
             'title': 'Le labyrinthe - Boing super ranger',
-            'description': 'md5:',
+            'description': 'md5:4cea7a37153be42c1ba2c1d3064376ff',
             'upload_date': '20140724',
         },
+        'skip': 'Only works from France',
+    }, {
+        'url': 'http://www.d8.tv/d8-docs-mags/pid6589-d8-campagne-intime.html',
+        'info_dict': {
+            'id': '966289',
+            'ext': 'flv',
+            'title': 'Campagne intime - Documentaire exceptionnel',
+            'description': 'md5:d2643b799fb190846ae09c61e59a859f',
+            'upload_date': '20131108',
+        },
+        'skip': 'videos get deleted after a while',
     }]
 
     def _real_extract(self, url):
@@ -62,16 +74,30 @@ class CanalplusIE(InfoExtractor):
         media = video_info.find('MEDIA')
         infos = video_info.find('INFOS')
 
-        preferences = ['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD', 'HLS', 'HDS']
+        preference = qualities(['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD', 'HLS', 'HDS'])
 
-        formats = [
-            {
-                'url': fmt.text + '?hdcore=2.11.3' if fmt.tag == 'HDS' else fmt.text,
-                'format_id': fmt.tag,
-                'ext': 'mp4' if fmt.tag == 'HLS' else 'flv',
-                'preference': preferences.index(fmt.tag) if fmt.tag in preferences else -1,
-            } for fmt in media.find('VIDEOS') if fmt.text
-        ]
+        formats = []
+        for fmt in media.find('VIDEOS'):
+            format_url = fmt.text
+            if not format_url:
+                continue
+            format_id = fmt.tag
+            if format_id == 'HLS':
+                hls_formats = self._extract_m3u8_formats(format_url, video_id, 'flv')
+                for fmt in hls_formats:
+                    fmt['preference'] = preference(format_id)
+                formats.extend(hls_formats)
+            elif format_id == 'HDS':
+                hds_formats = self._extract_f4m_formats(format_url + '?hdcore=2.11.3', video_id)
+                for fmt in hds_formats:
+                    fmt['preference'] = preference(format_id)
+                formats.extend(hds_formats)
+            else:
+                formats.append({
+                    'url': format_url,
+                    'format_id': format_id,
+                    'preference': preference(format_id),
+                })
         self._sort_formats(formats)
 
         return {
diff --git a/youtube_dl/extractor/d8.py b/youtube_dl/extractor/d8.py
deleted file mode 100644
index 6b26ff2e3..000000000
--- a/youtube_dl/extractor/d8.py
+++ /dev/null
@@ -1,25 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-from .canalplus import CanalplusIE
-
-
-class D8IE(CanalplusIE):
-    _VALID_URL = r'https?://www\.d8\.tv/.*?/(?P<path>.*)'
-    _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/d8/%s'
-    IE_NAME = 'd8.tv'
-
-    _TEST = {
-        'url': 'http://www.d8.tv/d8-docs-mags/pid6589-d8-campagne-intime.html',
-        'file': '966289.flv',
-        'info_dict': {
-            'title': 'Campagne intime - Documentaire exceptionnel',
-            'description': 'md5:d2643b799fb190846ae09c61e59a859f',
-            'upload_date': '20131108',
-        },
-        'params': {
-            # rtmp
-            'skip_download': True,
-        },
-        'skip': 'videos get deleted after a while',
-    }

From b2a68d14cf5dc0a3732ca039b3dd67c94a628be5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Nov 2014 04:01:18 +0700
Subject: [PATCH 0389/1937] [mlb] Improve _VALID_URL (Closes #4063)

---
 youtube_dl/extractor/mlb.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index 42aa2e227..1a241aca7 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class MLBIE(InfoExtractor):
-    _VALID_URL = r'https?://m\.mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|shared/video/embed/embed\.html\?.*?\bcontent_id=)(?P<id>n?\d+)'
+    _VALID_URL = r'https?://m(?:lb)?\.mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|(?:shared/video/embed/embed\.html|[^/]+/video/play\.jsp)\?.*?\bcontent_id=)(?P<id>n?\d+)'
     _TESTS = [
         {
             'url': 'http://m.mlb.com/sea/video/topic/51231442/v34698933/nymsea-ackley-robs-a-home-run-with-an-amazing-catch/?c_id=sea',
@@ -72,6 +72,14 @@ class MLBIE(InfoExtractor):
             'url': 'http://m.mlb.com/shared/video/embed/embed.html?content_id=35692085&topic_id=6479266&width=400&height=224&property=mlb',
             'only_matching': True,
         },
+        {
+            'url': 'http://mlb.mlb.com/shared/video/embed/embed.html?content_id=36599553',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://mlb.mlb.com/es/video/play.jsp?content_id=36599553',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):

From 5263cdfcf9b9c8684959cdba307fd4cd7e1e0338 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 Nov 2014 04:01:58 +0700
Subject: [PATCH 0390/1937] [generic] Improve MLB iframe regex

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a27ec342e..3882e859c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -887,7 +887,7 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'), 'SBS')
 
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>https?://m\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
+            r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
             webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'MLB')

From ccdd0ffb805fde3bbf66d80fe765ff4e75377044 Mon Sep 17 00:00:00 2001
From: Alessandro Ghedini <alessandro@ghedini.me>
Date: Sat, 1 Nov 2014 12:04:15 +0100
Subject: [PATCH 0391/1937] [generic] indicate when a direct video has been
 extracted

Fixes #4052.
---
 youtube_dl/extractor/generic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 3882e859c..8abc340b4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -559,6 +559,7 @@ class GenericIE(InfoExtractor):
             return {
                 'id': video_id,
                 'title': os.path.splitext(url_basename(url))[0],
+                'direct': True,
                 'formats': [{
                     'format_id': m.group('format_id'),
                     'url': url,

From 08965906a8d379e2be795ae5841e8860df17cf24 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 1 Nov 2014 19:24:56 +0100
Subject: [PATCH 0392/1937] [README] Update FAQ on Ubuntu (#4078)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 230c12a88..933411fa2 100644
--- a/README.md
+++ b/README.md
@@ -381,7 +381,7 @@ Again, from then on you'll be able to update with `sudo youtube-dl -U`.
 
 YouTube changed their playlist format in March 2014 and later on, so you'll need at least youtube-dl 2014.07.25 to download all YouTube videos.
 
-If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to report bugs to the Ubuntu packaging guys - all they have to do is update the package to a somewhat recent version. See above for a way to update.
+If you have installed youtube-dl with a package manager, pip, setup.py or a tarball, please use that to update. Note that Ubuntu packages do not seem to get updated anymore. Since we are not affiliated with Ubuntu, there is little we can do. Feel free to [report bugs](https://bugs.launchpad.net/ubuntu/+source/youtube-dl/+filebug) to the [Ubuntu packaging guys](mailto:ubuntu-motu@lists.ubuntu.com?subject=outdated%20version%20of%20youtube-dl) - all they have to do is update the package to a somewhat recent version. See above for a way to update.
 
 ### Do I always have to pass in `--max-quality FORMAT`, or `-citw`?
 

From d9116714f253b2c0d75528062ee568cd844204e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 2 Nov 2014 08:01:14 +0700
Subject: [PATCH 0393/1937] [cinemassacre] Fix extraction (Closes #4083)

---
 youtube_dl/extractor/cinemassacre.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index d064a28f9..806e378b1 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -59,12 +59,9 @@ class CinemassacreIE(InfoExtractor):
 
         vidurl = self._search_regex(
             r'\'vidurl\'\s*:\s*"([^\']+)"', playerdata, 'vidurl').replace('\\/', '/')
-        vidid = self._search_regex(
-            r'\'vidid\'\s*:\s*"([^\']+)"', playerdata, 'vidid')
-        videoserver = self._html_search_regex(
-            r"'videoserver'\s*:\s*'([^']+)'", playerdata, 'videoserver')
 
-        videolist_url = 'http://%s/vod/smil:%s.smil/jwplayer.smil' % (videoserver, vidid)
+        videolist_url = self._search_regex(
+            r"file\s*:\s*'(http.+?/jwplayer\.smil)'", playerdata, 'jwplayer.smil')
         videolist = self._download_xml(videolist_url, video_id, 'Downloading videolist XML')
 
         formats = []

From ffae28ae18cfc70d2eeef93ce43e581b06c865ea Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 2 Nov 2014 09:45:51 +0100
Subject: [PATCH 0394/1937] release 2014.11.02

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 87d56c164..064113cbb 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.10.30'
+__version__ = '2014.11.02'

From c30ae9594cea215f36a736f944acd67fcec39d0d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 2 Nov 2014 10:28:21 +0100
Subject: [PATCH 0395/1937] release 2014.11.02.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 064113cbb..327560c76 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.02'
+__version__ = '2014.11.02.1'

From 9580711841d60426e2b888cde7b410ae7d512849 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 2 Nov 2014 10:50:30 +0100
Subject: [PATCH 0396/1937] [ffmpeg] Move version detection to utils

---
 youtube_dl/postprocessor/ffmpeg.py | 20 ++------------------
 youtube_dl/utils.py                | 19 +++++++++++++++++++
 2 files changed, 21 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 083c79592..338c145fa 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -11,6 +11,7 @@ from ..utils import (
     compat_subprocess_get_DEVNULL,
     encodeArgument,
     encodeFilename,
+    get_exe_version,
     is_outdated_version,
     PostProcessingError,
     prepend_extension,
@@ -19,23 +20,6 @@ from ..utils import (
 )
 
 
-def get_version(executable):
-    """ Returns the version of the specified executable,
-    or False if the executable is not present """
-    try:
-        out, err = subprocess.Popen(
-            [executable, '-version'],
-            stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
-    except OSError:
-        return False
-    firstline = out.partition(b'\n')[0].decode('ascii', 'ignore')
-    m = re.search(r'version\s+([0-9._-a-zA-Z]+)', firstline)
-    if not m:
-        return u'present'
-    else:
-        return m.group(1)
-
-
 class FFmpegPostProcessorError(PostProcessingError):
     pass
 
@@ -61,7 +45,7 @@ class FFmpegPostProcessor(PostProcessor):
     @staticmethod
     def get_versions():
         programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
-        return dict((program, get_version(program)) for program in programs)
+        return dict((p, get_exe_version(p, args=['-version'])) for p in programs)
 
     @property
     def _executable(self):
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2864e5142..fcfdadeb6 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1472,6 +1472,25 @@ def check_executable(exe, args=[]):
     return exe
 
 
+def get_exe_version(exe, args=['--version'],
+                    version_re=r'version\s+([0-9._-a-zA-Z]+)',
+                    unrecognized=u'present'):
+    """ Returns the version of the specified executable,
+    or False if the executable is not present """
+    try:
+        out, err = subprocess.Popen(
+            [exe] + args,
+            stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
+    except OSError:
+        return False
+    firstline = out.partition(b'\n')[0].decode('ascii', 'ignore')
+    m = re.search(version_re, firstline)
+    if m:
+        return m.group(1)
+    else:
+        return unrecognized
+
+
 class PagedList(object):
     def __len__(self):
         # This is only useful for tests

From 4c83c96795b1bf469b0e290fc1cbbed6f49f04bf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 2 Nov 2014 10:55:36 +0100
Subject: [PATCH 0397/1937] [YoutubeDL] Include rtmpdump in exe versions -v
 output

---
 youtube_dl/YoutubeDL.py       | 2 ++
 youtube_dl/downloader/rtmp.py | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 73a372df4..f7c996c84 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -62,6 +62,7 @@ from .utils import (
 from .cache import Cache
 from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader
+from .downloader.rtmp import rtmpdump_version
 from .postprocessor import FFmpegMergerPP, FFmpegPostProcessor
 from .version import __version__
 
@@ -1321,6 +1322,7 @@ class YoutubeDL(object):
             platform.python_version(), platform_name()))
 
         exe_versions = FFmpegPostProcessor.get_versions()
+        exe_versions['rtmpdump'] = rtmpdump_version()
         exe_str = ', '.join(
             '%s %s' % (exe, v)
             for exe, v in sorted(exe_versions.items())
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 5eb108302..17d9631fa 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -12,9 +12,15 @@ from ..utils import (
     compat_str,
     encodeFilename,
     format_bytes,
+    get_exe_version,
 )
 
 
+def rtmpdump_version():
+    return get_exe_version(
+        'rtmpdump', ['--help'], r'(?i)RTMPDump\s*v?([0-9a-zA-Z._-]+)')
+
+
 class RtmpFD(FileDownloader):
     def real_download(self, filename, info_dict):
         def run_rtmpdump(args):

From 8c25f81beea169c9d6540eea1a6f71dc045da6ed Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 2 Nov 2014 11:23:40 +0100
Subject: [PATCH 0398/1937] [util] Move compatibility functions out of util

utils is large enough without these compatibility functions.

Everything that is present in newer versions of Python (i.e. with dev Python it's just an import) goes into compat.py .
Everything else (i.e. youtube-dl-specific helpers) goes into utils.py .
---
 test/test_compat.py                           |  44 +++
 test/test_utils.py                            |  14 -
 youtube_dl/YoutubeDL.py                       |   4 +-
 youtube_dl/__init__.py                        |   4 +-
 youtube_dl/cache.py                           |   6 +-
 youtube_dl/compat.py                          | 314 ++++++++++++++++++
 youtube_dl/extractor/addanime.py              |   5 +-
 youtube_dl/extractor/cloudy.py                |  10 +-
 youtube_dl/extractor/common.py                |   5 +-
 youtube_dl/extractor/crunchyroll.py           |   1 -
 youtube_dl/extractor/dropbox.py               |   3 +-
 youtube_dl/extractor/facebook.py              |   4 +-
 youtube_dl/extractor/generic.py               |   5 +-
 youtube_dl/extractor/globo.py                 |   8 +-
 youtube_dl/extractor/laola1tv.py              |   1 +
 youtube_dl/extractor/myvideo.py               |   5 +-
 youtube_dl/extractor/ro220.py                 |   2 +-
 youtube_dl/extractor/vimeo.py                 |   4 +-
 youtube_dl/options.py                         |   4 +-
 youtube_dl/postprocessor/atomicparsley.py     |   5 +-
 youtube_dl/postprocessor/execafterdownload.py |   6 +-
 youtube_dl/postprocessor/ffmpeg.py            |   5 +-
 youtube_dl/postprocessor/xattrpp.py           |   4 +-
 youtube_dl/utils.py                           | 290 +---------------
 24 files changed, 426 insertions(+), 327 deletions(-)
 create mode 100644 test/test_compat.py
 create mode 100644 youtube_dl/compat.py

diff --git a/test/test_compat.py b/test/test_compat.py
new file mode 100644
index 000000000..d1a33ddc9
--- /dev/null
+++ b/test/test_compat.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+from youtube_dl.utils import get_filesystem_encoding
+from youtube_dl.compat import (
+    compat_getenv,
+    compat_expanduser,
+)
+
+
+class TestCompat(unittest.TestCase):
+    def test_compat_getenv(self):
+        test_str = 'тест'
+        os.environ['YOUTUBE-DL-TEST'] = (
+            test_str if sys.version_info >= (3, 0)
+            else test_str.encode(get_filesystem_encoding()))
+        self.assertEqual(compat_getenv('YOUTUBE-DL-TEST'), test_str)
+
+    def test_compat_expanduser(self):
+        test_str = 'C:\Documents and Settings\тест\Application Data'
+        os.environ['HOME'] = (
+            test_str if sys.version_info >= (3, 0)
+            else test_str.encode(get_filesystem_encoding()))
+        self.assertEqual(compat_expanduser('~'), test_str)
+
+    def test_all_present(self):
+        import youtube_dl.compat
+        all_names = youtube_dl.compat.__all__
+        present_names = set(filter(
+            lambda c: '_' in c and not c.startswith('_'),
+            dir(youtube_dl.compat)))
+        self.assertEqual(all_names, sorted(present_names))
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/test_utils.py b/test/test_utils.py
index 0b31d1a39..338701f4c 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -46,8 +46,6 @@ from youtube_dl.utils import (
     escape_url,
     js_to_json,
     get_filesystem_encoding,
-    compat_getenv,
-    compat_expanduser,
 )
 
 
@@ -359,17 +357,5 @@ class TestUtil(unittest.TestCase):
         on = js_to_json('{"abc": true}')
         self.assertEqual(json.loads(on), {'abc': True})
 
-    def test_compat_getenv(self):
-        test_str = 'тест'
-        os.environ['YOUTUBE-DL-TEST'] = (test_str if sys.version_info >= (3, 0)
-            else test_str.encode(get_filesystem_encoding()))
-        self.assertEqual(compat_getenv('YOUTUBE-DL-TEST'), test_str)
-
-    def test_compat_expanduser(self):
-        test_str = 'C:\Documents and Settings\тест\Application Data'
-        os.environ['HOME'] = (test_str if sys.version_info >= (3, 0)
-            else test_str.encode(get_filesystem_encoding()))
-        self.assertEqual(compat_expanduser('~'), test_str)
-
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index f7c996c84..8732f3db4 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -22,13 +22,15 @@ import traceback
 if os.name == 'nt':
     import ctypes
 
-from .utils import (
+from .compat import (
     compat_cookiejar,
     compat_expanduser,
     compat_http_client,
     compat_str,
     compat_urllib_error,
     compat_urllib_request,
+)
+from .utils import (
     escape_url,
     ContentTooShortError,
     date_from_str,
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 3c968082c..685dd8e5e 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -13,10 +13,12 @@ import sys
 from .options import (
     parseOpts,
 )
-from .utils import (
+from .compat import (
     compat_expanduser,
     compat_getpass,
     compat_print,
+)
+from .utils import (
     DateRange,
     DEFAULT_OUTTMPL,
     decodeOption,
diff --git a/youtube_dl/cache.py b/youtube_dl/cache.py
index ac5925d32..2d9b426cb 100644
--- a/youtube_dl/cache.py
+++ b/youtube_dl/cache.py
@@ -8,10 +8,8 @@ import re
 import shutil
 import traceback
 
-from .utils import (
-    compat_expanduser,
-    write_json_file,
-)
+from .compat import compat_expanduser
+from .utils import write_json_file
 
 
 class Cache(object):
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
new file mode 100644
index 000000000..08e8d64d7
--- /dev/null
+++ b/youtube_dl/compat.py
@@ -0,0 +1,314 @@
+import getpass
+import os
+import subprocess
+import sys
+
+
+try:
+    import urllib.request as compat_urllib_request
+except ImportError: # Python 2
+    import urllib2 as compat_urllib_request
+
+try:
+    import urllib.error as compat_urllib_error
+except ImportError: # Python 2
+    import urllib2 as compat_urllib_error
+
+try:
+    import urllib.parse as compat_urllib_parse
+except ImportError: # Python 2
+    import urllib as compat_urllib_parse
+
+try:
+    from urllib.parse import urlparse as compat_urllib_parse_urlparse
+except ImportError: # Python 2
+    from urlparse import urlparse as compat_urllib_parse_urlparse
+
+try:
+    import urllib.parse as compat_urlparse
+except ImportError: # Python 2
+    import urlparse as compat_urlparse
+
+try:
+    import http.cookiejar as compat_cookiejar
+except ImportError: # Python 2
+    import cookielib as compat_cookiejar
+
+try:
+    import html.entities as compat_html_entities
+except ImportError: # Python 2
+    import htmlentitydefs as compat_html_entities
+
+try:
+    import html.parser as compat_html_parser
+except ImportError: # Python 2
+    import HTMLParser as compat_html_parser
+
+try:
+    import http.client as compat_http_client
+except ImportError: # Python 2
+    import httplib as compat_http_client
+
+try:
+    from urllib.error import HTTPError as compat_HTTPError
+except ImportError:  # Python 2
+    from urllib2 import HTTPError as compat_HTTPError
+
+try:
+    from urllib.request import urlretrieve as compat_urlretrieve
+except ImportError:  # Python 2
+    from urllib import urlretrieve as compat_urlretrieve
+
+
+try:
+    from subprocess import DEVNULL
+    compat_subprocess_get_DEVNULL = lambda: DEVNULL
+except ImportError:
+    compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
+
+try:
+    from urllib.parse import unquote as compat_urllib_parse_unquote
+except ImportError:
+    def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
+        if string == '':
+            return string
+        res = string.split('%')
+        if len(res) == 1:
+            return string
+        if encoding is None:
+            encoding = 'utf-8'
+        if errors is None:
+            errors = 'replace'
+        # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
+        pct_sequence = b''
+        string = res[0]
+        for item in res[1:]:
+            try:
+                if not item:
+                    raise ValueError
+                pct_sequence += item[:2].decode('hex')
+                rest = item[2:]
+                if not rest:
+                    # This segment was just a single percent-encoded character.
+                    # May be part of a sequence of code units, so delay decoding.
+                    # (Stored in pct_sequence).
+                    continue
+            except ValueError:
+                rest = '%' + item
+            # Encountered non-percent-encoded characters. Flush the current
+            # pct_sequence.
+            string += pct_sequence.decode(encoding, errors) + rest
+            pct_sequence = b''
+        if pct_sequence:
+            # Flush the final pct_sequence
+            string += pct_sequence.decode(encoding, errors)
+        return string
+
+
+try:
+    from urllib.parse import parse_qs as compat_parse_qs
+except ImportError: # Python 2
+    # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
+    # Python 2's version is apparently totally broken
+
+    def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
+                encoding='utf-8', errors='replace'):
+        qs, _coerce_result = qs, unicode
+        pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
+        r = []
+        for name_value in pairs:
+            if not name_value and not strict_parsing:
+                continue
+            nv = name_value.split('=', 1)
+            if len(nv) != 2:
+                if strict_parsing:
+                    raise ValueError("bad query field: %r" % (name_value,))
+                # Handle case of a control-name with no equal sign
+                if keep_blank_values:
+                    nv.append('')
+                else:
+                    continue
+            if len(nv[1]) or keep_blank_values:
+                name = nv[0].replace('+', ' ')
+                name = compat_urllib_parse_unquote(
+                    name, encoding=encoding, errors=errors)
+                name = _coerce_result(name)
+                value = nv[1].replace('+', ' ')
+                value = compat_urllib_parse_unquote(
+                    value, encoding=encoding, errors=errors)
+                value = _coerce_result(value)
+                r.append((name, value))
+        return r
+
+    def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
+                encoding='utf-8', errors='replace'):
+        parsed_result = {}
+        pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
+                        encoding=encoding, errors=errors)
+        for name, value in pairs:
+            if name in parsed_result:
+                parsed_result[name].append(value)
+            else:
+                parsed_result[name] = [value]
+        return parsed_result
+
+try:
+    compat_str = unicode # Python 2
+except NameError:
+    compat_str = str
+
+try:
+    compat_chr = unichr # Python 2
+except NameError:
+    compat_chr = chr
+
+try:
+    from xml.etree.ElementTree import ParseError as compat_xml_parse_error
+except ImportError:  # Python 2.6
+    from xml.parsers.expat import ExpatError as compat_xml_parse_error
+
+try:
+    from shlex import quote as shlex_quote
+except ImportError:  # Python < 3.3
+    def shlex_quote(s):
+        return "'" + s.replace("'", "'\"'\"'") + "'"
+
+
+def compat_ord(c):
+    if type(c) is int: return c
+    else: return ord(c)
+
+
+if sys.version_info >= (3, 0):
+    compat_getenv = os.getenv
+    compat_expanduser = os.path.expanduser
+else:
+    # Environment variables should be decoded with filesystem encoding.
+    # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
+
+    def compat_getenv(key, default=None):
+        from .utils import get_filesystem_encoding
+        env = os.getenv(key, default)
+        if env:
+            env = env.decode(get_filesystem_encoding())
+        return env
+
+    # HACK: The default implementations of os.path.expanduser from cpython do not decode
+    # environment variables with filesystem encoding. We will work around this by
+    # providing adjusted implementations.
+    # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
+    # for different platforms with correct environment variables decoding.
+
+    if os.name == 'posix':
+        def compat_expanduser(path):
+            """Expand ~ and ~user constructions.  If user or $HOME is unknown,
+            do nothing."""
+            if not path.startswith('~'):
+                return path
+            i = path.find('/', 1)
+            if i < 0:
+                i = len(path)
+            if i == 1:
+                if 'HOME' not in os.environ:
+                    import pwd
+                    userhome = pwd.getpwuid(os.getuid()).pw_dir
+                else:
+                    userhome = compat_getenv('HOME')
+            else:
+                import pwd
+                try:
+                    pwent = pwd.getpwnam(path[1:i])
+                except KeyError:
+                    return path
+                userhome = pwent.pw_dir
+            userhome = userhome.rstrip('/')
+            return (userhome + path[i:]) or '/'
+    elif os.name == 'nt' or os.name == 'ce':
+        def compat_expanduser(path):
+            """Expand ~ and ~user constructs.
+
+            If user or $HOME is unknown, do nothing."""
+            if path[:1] != '~':
+                return path
+            i, n = 1, len(path)
+            while i < n and path[i] not in '/\\':
+                i = i + 1
+
+            if 'HOME' in os.environ:
+                userhome = compat_getenv('HOME')
+            elif 'USERPROFILE' in os.environ:
+                userhome = compat_getenv('USERPROFILE')
+            elif not 'HOMEPATH' in os.environ:
+                return path
+            else:
+                try:
+                    drive = compat_getenv('HOMEDRIVE')
+                except KeyError:
+                    drive = ''
+                userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
+
+            if i != 1: #~user
+                userhome = os.path.join(os.path.dirname(userhome), path[1:i])
+
+            return userhome + path[i:]
+    else:
+        compat_expanduser = os.path.expanduser
+
+
+if sys.version_info < (3, 0):
+    def compat_print(s):
+        from .utils import preferredencoding
+        print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
+else:
+    def compat_print(s):
+        assert type(s) == type(u'')
+        print(s)
+
+
+try:
+    subprocess_check_output = subprocess.check_output
+except AttributeError:
+    def subprocess_check_output(*args, **kwargs):
+        assert 'input' not in kwargs
+        p = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
+        output, _ = p.communicate()
+        ret = p.poll()
+        if ret:
+            raise subprocess.CalledProcessError(ret, p.args, output=output)
+        return output
+
+if sys.version_info < (3, 0) and sys.platform == 'win32':
+    def compat_getpass(prompt, *args, **kwargs):
+        if isinstance(prompt, compat_str):
+            prompt = prompt.encode(preferredencoding())
+        return getpass.getpass(prompt, *args, **kwargs)
+else:
+    compat_getpass = getpass.getpass
+
+
+__all__ = [
+    'compat_HTTPError',
+    'compat_chr',
+    'compat_cookiejar',
+    'compat_expanduser',
+    'compat_getenv',
+    'compat_getpass',
+    'compat_html_entities',
+    'compat_html_parser',
+    'compat_http_client',
+    'compat_ord',
+    'compat_parse_qs',
+    'compat_print',
+    'compat_str',
+    'compat_subprocess_get_DEVNULL',
+    'compat_urllib_error',
+    'compat_urllib_parse',
+    'compat_urllib_parse_unquote',
+    'compat_urllib_parse_urlparse',
+    'compat_urllib_request',
+    'compat_urlparse',
+    'compat_urlretrieve',
+    'compat_xml_parse_error',
+    'shlex_quote',
+    'subprocess_check_output',
+]
diff --git a/youtube_dl/extractor/addanime.py b/youtube_dl/extractor/addanime.py
index fcf296057..11f149f9e 100644
--- a/youtube_dl/extractor/addanime.py
+++ b/youtube_dl/extractor/addanime.py
@@ -3,12 +3,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_HTTPError,
     compat_str,
     compat_urllib_parse,
     compat_urllib_parse_urlparse,
-
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/cloudy.py b/youtube_dl/extractor/cloudy.py
index 386f080d2..abf8cc280 100644
--- a/youtube_dl/extractor/cloudy.py
+++ b/youtube_dl/extractor/cloudy.py
@@ -4,14 +4,16 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
+from ..compat import (
     compat_parse_qs,
     compat_urllib_parse,
-    remove_end,
-    HEADRequest,
     compat_HTTPError,
 )
+from ..utils import (
+    ExtractorError,
+    HEADRequest,
+    remove_end,
+)
 
 
 class CloudyIE(InfoExtractor):
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7e4113213..9e1d62c2b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -12,13 +12,14 @@ import sys
 import time
 import xml.etree.ElementTree
 
-from ..utils import (
+from ..compat import (
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse_urlparse,
     compat_urlparse,
     compat_str,
-
+)
+from ..utils import (
     clean_html,
     compiled_regex_type,
     ExtractorError,
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index cc612d08e..0bd0eccba 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -17,7 +17,6 @@ from ..utils import (
     bytes_to_intlist,
     intlist_to_bytes,
     unified_strdate,
-    clean_html,
     urlencode_postdata,
 )
 from ..aes import (
diff --git a/youtube_dl/extractor/dropbox.py b/youtube_dl/extractor/dropbox.py
index 5f24ac721..aefca848a 100644
--- a/youtube_dl/extractor/dropbox.py
+++ b/youtube_dl/extractor/dropbox.py
@@ -5,7 +5,8 @@ import os.path
 import re
 
 from .common import InfoExtractor
-from ..utils import compat_urllib_parse_unquote, url_basename
+from ..compat import compat_urllib_parse_unquote
+from ..utils import url_basename
 
 
 class DropboxIE(InfoExtractor):
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 3ad993751..104803563 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -5,12 +5,14 @@ import re
 import socket
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_http_client,
     compat_str,
     compat_urllib_error,
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     urlencode_postdata,
     ExtractorError,
     limit_length,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 3882e859c..babd581ea 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -7,11 +7,12 @@ import re
 
 from .common import InfoExtractor
 from .youtube import YoutubeIE
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urlparse,
     compat_xml_parse_error,
-
+)
+from ..utils import (
     determine_ext,
     ExtractorError,
     float_or_none,
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 77c3ad4fc..66ca37918 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -5,13 +5,15 @@ import random
 import math
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    float_or_none,
+from ..compat import (
     compat_str,
     compat_chr,
     compat_ord,
 )
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+)
 
 
 class GloboIE(InfoExtractor):
diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index 263f68773..102e29f7a 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -4,6 +4,7 @@ import random
 import re
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class Laola1TvIE(InfoExtractor):
diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
index ccb5959c4..a89153985 100644
--- a/youtube_dl/extractor/myvideo.py
+++ b/youtube_dl/extractor/myvideo.py
@@ -7,11 +7,12 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_ord,
     compat_urllib_parse,
     compat_urllib_request,
-
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/ro220.py b/youtube_dl/extractor/ro220.py
index 0a3a71448..962b524e9 100644
--- a/youtube_dl/extractor/ro220.py
+++ b/youtube_dl/extractor/ro220.py
@@ -1,7 +1,7 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import compat_urllib_parse_unquote
+from ..compat import compat_urllib_parse_unquote
 
 
 class Ro220IE(InfoExtractor):
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index d9cad0ea5..c744d4f04 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -7,11 +7,13 @@ import itertools
 
 from .common import InfoExtractor
 from .subtitles import SubtitlesInfoExtractor
-from ..utils import (
+from ..compat import (
     compat_HTTPError,
     compat_urllib_parse,
     compat_urllib_request,
     compat_urlparse,
+)
+from ..utils import (
     ExtractorError,
     InAdvancePagedList,
     int_or_none,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 98e20d549..997e92ad7 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -5,9 +5,11 @@ import optparse
 import shlex
 import sys
 
-from .utils import (
+from .compat import (
     compat_expanduser,
     compat_getenv,
+)
+from .utils import (
     get_term_width,
     write_string,
 )
diff --git a/youtube_dl/postprocessor/atomicparsley.py b/youtube_dl/postprocessor/atomicparsley.py
index 765b2d9ee..448ccc5f3 100644
--- a/youtube_dl/postprocessor/atomicparsley.py
+++ b/youtube_dl/postprocessor/atomicparsley.py
@@ -6,10 +6,11 @@ import os
 import subprocess
 
 from .common import PostProcessor
-
+from ..compat import (
+    compat_urlretrieve,
+)
 from ..utils import (
     check_executable,
-    compat_urlretrieve,
     encodeFilename,
     PostProcessingError,
     prepend_extension,
diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py
index 08419a3d4..baf1b1945 100644
--- a/youtube_dl/postprocessor/execafterdownload.py
+++ b/youtube_dl/postprocessor/execafterdownload.py
@@ -3,10 +3,8 @@ from __future__ import unicode_literals
 import subprocess
 
 from .common import PostProcessor
-from ..utils import (
-    shlex_quote,
-    PostProcessingError,
-)
+from ..compat import shlex_quote
+from ..utils import PostProcessingError
 
 
 class ExecAfterDownloadPP(PostProcessor):
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 338c145fa..f3f2743c0 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -1,5 +1,4 @@
 import os
-import re
 import subprocess
 import sys
 import time
@@ -7,8 +6,10 @@ import time
 
 from .common import AudioConversionError, PostProcessor
 
-from ..utils import (
+from ..compat import (
     compat_subprocess_get_DEVNULL,
+)
+from ..utils import (
     encodeArgument,
     encodeFilename,
     get_exe_version,
diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index f6940940b..b5cae41c8 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -3,10 +3,12 @@ import subprocess
 import sys
 
 from .common import PostProcessor
+from ..compat import (
+    subprocess_check_output
+)
 from ..utils import (
     check_executable,
     hyphenate_date,
-    subprocess_check_output
 )
 
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index fcfdadeb6..b0255c943 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -29,254 +29,19 @@ import traceback
 import xml.etree.ElementTree
 import zlib
 
-try:
-    import urllib.request as compat_urllib_request
-except ImportError: # Python 2
-    import urllib2 as compat_urllib_request
-
-try:
-    import urllib.error as compat_urllib_error
-except ImportError: # Python 2
-    import urllib2 as compat_urllib_error
-
-try:
-    import urllib.parse as compat_urllib_parse
-except ImportError: # Python 2
-    import urllib as compat_urllib_parse
-
-try:
-    from urllib.parse import urlparse as compat_urllib_parse_urlparse
-except ImportError: # Python 2
-    from urlparse import urlparse as compat_urllib_parse_urlparse
-
-try:
-    import urllib.parse as compat_urlparse
-except ImportError: # Python 2
-    import urlparse as compat_urlparse
-
-try:
-    import http.cookiejar as compat_cookiejar
-except ImportError: # Python 2
-    import cookielib as compat_cookiejar
-
-try:
-    import html.entities as compat_html_entities
-except ImportError: # Python 2
-    import htmlentitydefs as compat_html_entities
-
-try:
-    import html.parser as compat_html_parser
-except ImportError: # Python 2
-    import HTMLParser as compat_html_parser
-
-try:
-    import http.client as compat_http_client
-except ImportError: # Python 2
-    import httplib as compat_http_client
-
-try:
-    from urllib.error import HTTPError as compat_HTTPError
-except ImportError:  # Python 2
-    from urllib2 import HTTPError as compat_HTTPError
-
-try:
-    from urllib.request import urlretrieve as compat_urlretrieve
-except ImportError:  # Python 2
-    from urllib import urlretrieve as compat_urlretrieve
-
-
-try:
-    from subprocess import DEVNULL
-    compat_subprocess_get_DEVNULL = lambda: DEVNULL
-except ImportError:
-    compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
-
-try:
-    from urllib.parse import unquote as compat_urllib_parse_unquote
-except ImportError:
-    def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
-        if string == '':
-            return string
-        res = string.split('%')
-        if len(res) == 1:
-            return string
-        if encoding is None:
-            encoding = 'utf-8'
-        if errors is None:
-            errors = 'replace'
-        # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
-        pct_sequence = b''
-        string = res[0]
-        for item in res[1:]:
-            try:
-                if not item:
-                    raise ValueError
-                pct_sequence += item[:2].decode('hex')
-                rest = item[2:]
-                if not rest:
-                    # This segment was just a single percent-encoded character.
-                    # May be part of a sequence of code units, so delay decoding.
-                    # (Stored in pct_sequence).
-                    continue
-            except ValueError:
-                rest = '%' + item
-            # Encountered non-percent-encoded characters. Flush the current
-            # pct_sequence.
-            string += pct_sequence.decode(encoding, errors) + rest
-            pct_sequence = b''
-        if pct_sequence:
-            # Flush the final pct_sequence
-            string += pct_sequence.decode(encoding, errors)
-        return string
-
-
-try:
-    from urllib.parse import parse_qs as compat_parse_qs
-except ImportError: # Python 2
-    # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
-    # Python 2's version is apparently totally broken
-
-    def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
-                encoding='utf-8', errors='replace'):
-        qs, _coerce_result = qs, unicode
-        pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
-        r = []
-        for name_value in pairs:
-            if not name_value and not strict_parsing:
-                continue
-            nv = name_value.split('=', 1)
-            if len(nv) != 2:
-                if strict_parsing:
-                    raise ValueError("bad query field: %r" % (name_value,))
-                # Handle case of a control-name with no equal sign
-                if keep_blank_values:
-                    nv.append('')
-                else:
-                    continue
-            if len(nv[1]) or keep_blank_values:
-                name = nv[0].replace('+', ' ')
-                name = compat_urllib_parse_unquote(
-                    name, encoding=encoding, errors=errors)
-                name = _coerce_result(name)
-                value = nv[1].replace('+', ' ')
-                value = compat_urllib_parse_unquote(
-                    value, encoding=encoding, errors=errors)
-                value = _coerce_result(value)
-                r.append((name, value))
-        return r
-
-    def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
-                encoding='utf-8', errors='replace'):
-        parsed_result = {}
-        pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
-                        encoding=encoding, errors=errors)
-        for name, value in pairs:
-            if name in parsed_result:
-                parsed_result[name].append(value)
-            else:
-                parsed_result[name] = [value]
-        return parsed_result
-
-try:
-    compat_str = unicode # Python 2
-except NameError:
-    compat_str = str
-
-try:
-    compat_chr = unichr # Python 2
-except NameError:
-    compat_chr = chr
-
-try:
-    from xml.etree.ElementTree import ParseError as compat_xml_parse_error
-except ImportError:  # Python 2.6
-    from xml.parsers.expat import ExpatError as compat_xml_parse_error
-
-try:
-    from shlex import quote as shlex_quote
-except ImportError:  # Python < 3.3
-    def shlex_quote(s):
-        return "'" + s.replace("'", "'\"'\"'") + "'"
-
-
-def compat_ord(c):
-    if type(c) is int: return c
-    else: return ord(c)
-
-
-if sys.version_info >= (3, 0):
-    compat_getenv = os.getenv
-    compat_expanduser = os.path.expanduser
-else:
-    # Environment variables should be decoded with filesystem encoding.
-    # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
-
-    def compat_getenv(key, default=None):
-        env = os.getenv(key, default)
-        if env:
-            env = env.decode(get_filesystem_encoding())
-        return env
-
-    # HACK: The default implementations of os.path.expanduser from cpython do not decode
-    # environment variables with filesystem encoding. We will work around this by
-    # providing adjusted implementations.
-    # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
-    # for different platforms with correct environment variables decoding.
-
-    if os.name == 'posix':
-        def compat_expanduser(path):
-            """Expand ~ and ~user constructions.  If user or $HOME is unknown,
-            do nothing."""
-            if not path.startswith('~'):
-                return path
-            i = path.find('/', 1)
-            if i < 0:
-                i = len(path)
-            if i == 1:
-                if 'HOME' not in os.environ:
-                    import pwd
-                    userhome = pwd.getpwuid(os.getuid()).pw_dir
-                else:
-                    userhome = compat_getenv('HOME')
-            else:
-                import pwd
-                try:
-                    pwent = pwd.getpwnam(path[1:i])
-                except KeyError:
-                    return path
-                userhome = pwent.pw_dir
-            userhome = userhome.rstrip('/')
-            return (userhome + path[i:]) or '/'
-    elif os.name == 'nt' or os.name == 'ce':
-        def compat_expanduser(path):
-            """Expand ~ and ~user constructs.
-
-            If user or $HOME is unknown, do nothing."""
-            if path[:1] != '~':
-                return path
-            i, n = 1, len(path)
-            while i < n and path[i] not in '/\\':
-                i = i + 1
-
-            if 'HOME' in os.environ:
-                userhome = compat_getenv('HOME')
-            elif 'USERPROFILE' in os.environ:
-                userhome = compat_getenv('USERPROFILE')
-            elif not 'HOMEPATH' in os.environ:
-                return path
-            else:
-                try:
-                    drive = compat_getenv('HOMEDRIVE')
-                except KeyError:
-                    drive = ''
-                userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
-
-            if i != 1: #~user
-                userhome = os.path.join(os.path.dirname(userhome), path[1:i])
-
-            return userhome + path[i:]
-    else:
-        compat_expanduser = os.path.expanduser
+from .compat import (
+    compat_chr,
+    compat_getenv,
+    compat_html_entities,
+    compat_html_parser,
+    compat_parse_qs,
+    compat_str,
+    compat_urllib_error,
+    compat_urllib_parse,
+    compat_urllib_parse_urlparse,
+    compat_urllib_request,
+    compat_urlparse,
+)
 
 
 # This is not clearly defined otherwise
@@ -304,14 +69,6 @@ def preferredencoding():
 
     return pref
 
-if sys.version_info < (3,0):
-    def compat_print(s):
-        print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
-else:
-    def compat_print(s):
-        assert type(s) == type(u'')
-        print(s)
-
 
 def write_json_file(obj, fn):
     """ Encode obj as JSON and write it to fn, atomically """
@@ -1655,15 +1412,6 @@ def parse_xml(s):
     return tree
 
 
-if sys.version_info < (3, 0) and sys.platform == 'win32':
-    def compat_getpass(prompt, *args, **kwargs):
-        if isinstance(prompt, compat_str):
-            prompt = prompt.encode(preferredencoding())
-        return getpass.getpass(prompt, *args, **kwargs)
-else:
-    compat_getpass = getpass.getpass
-
-
 US_RATINGS = {
     'G': 0,
     'PG': 10,
@@ -1721,18 +1469,6 @@ def qualities(quality_ids):
 
 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
 
-try:
-    subprocess_check_output = subprocess.check_output
-except AttributeError:
-    def subprocess_check_output(*args, **kwargs):
-        assert 'input' not in kwargs
-        p = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
-        output, _ = p.communicate()
-        ret = p.poll()
-        if ret:
-            raise subprocess.CalledProcessError(ret, p.args, output=output)
-        return output
-
 
 def limit_length(s, length):
     """ Add ellipses to overly long strings """

From baa708036c64e2dce419b5f0648ee6b84ce2132e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 2 Nov 2014 11:26:40 +0100
Subject: [PATCH 0399/1937] [compat] Fix imports

---
 youtube_dl/compat.py | 1 +
 youtube_dl/utils.py  | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 08e8d64d7..3e35e2b84 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -280,6 +280,7 @@ except AttributeError:
 if sys.version_info < (3, 0) and sys.platform == 'win32':
     def compat_getpass(prompt, *args, **kwargs):
         if isinstance(prompt, compat_str):
+            from .utils import preferredencoding
             prompt = prompt.encode(preferredencoding())
         return getpass.getpass(prompt, *args, **kwargs)
 else:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b0255c943..620c4fe8a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -8,7 +8,6 @@ import ctypes
 import datetime
 import email.utils
 import errno
-import getpass
 import gzip
 import itertools
 import io

From 451948b28c4c520e3146d36b6ffb3079449f22e1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 2 Nov 2014 11:36:29 +0100
Subject: [PATCH 0400/1937] [compat] Modernize

---
 youtube_dl/compat.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 3e35e2b84..b3752634a 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import getpass
 import os
 import subprocess

From ecc0c5ee01f0e5bdd6af0c32cb5b4adcb2a2f78c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 2 Nov 2014 11:37:49 +0100
Subject: [PATCH 0401/1937] [utils] Modernize

---
 youtube_dl/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 620c4fe8a..233286de8 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from __future__ import unicode_literals
+
 import calendar
 import codecs
 import contextlib
@@ -1337,7 +1339,7 @@ def escape_rfc3986(s):
     """Escape non-ASCII characters as suggested by RFC 3986"""
     if sys.version_info < (3, 0) and isinstance(s, unicode):
         s = s.encode('utf-8')
-    return compat_urllib_parse.quote(s, "%/;:@&=+$,!~*'()?#[]")
+    return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
 
 
 def escape_url(url):

From 39f0a2a6b71c5d3cb2e2f8ef1bc3eeeaa6a96971 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 2 Nov 2014 11:41:33 +0100
Subject: [PATCH 0402/1937] [test_swfinterp] Correct compilation on modern
 mxmlc versions

---
 test/test_swfinterp.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_swfinterp.py b/test/test_swfinterp.py
index b42cd74c7..273e0f074 100644
--- a/test/test_swfinterp.py
+++ b/test/test_swfinterp.py
@@ -37,7 +37,9 @@ def _make_testfunc(testfile):
                 or os.path.getmtime(swf_file) < os.path.getmtime(as_file)):
             # Recompile
             try:
-                subprocess.check_call(['mxmlc', '-output', swf_file, as_file])
+                subprocess.check_call([
+                    'mxmlc', '-output', swf_file,
+                    '-static-link-runtime-shared-libraries', as_file])
             except OSError as ose:
                 if ose.errno == errno.ENOENT:
                     print('mxmlc not found! Skipping test.')

From 42f7d2f588f5d0276ddb89306d5371e9fdbe91a0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 2 Nov 2014 11:46:12 +0100
Subject: [PATCH 0403/1937] [test_download] Fix import

---
 test/test_download.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_download.py b/test/test_download.py
index 75e0bb289..88ce29871 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -23,10 +23,12 @@ import json
 import socket
 
 import youtube_dl.YoutubeDL
-from youtube_dl.utils import (
+from youtube_dl.compat import (
     compat_http_client,
     compat_urllib_error,
     compat_HTTPError,
+)
+from youtube_dl.utils import (
     DownloadError,
     ExtractorError,
     format_bytes,

From 982a58d049ff3b8e6127e33afbd97b55ade6ba15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Michael=20K=C3=A4ufl?= <youtube-dl@c.michael-kaeufl.de>
Date: Sun, 2 Nov 2014 16:07:40 +0100
Subject: [PATCH 0404/1937] [README] Replace links to kernel.org with links to
 git-scm.com

Unlike kernel.org, the documentation at git-scm.com is up to date and
the rest of the git documentation is easily accessible to any git
newby.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 933411fa2..9784e9c2a 100644
--- a/README.md
+++ b/README.md
@@ -500,7 +500,7 @@ If you want to add support for a new site, you can follow this quick list (assum
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want.
 8. If you can, check the code with [pyflakes](https://pypi.python.org/pypi/pyflakes) (a good idea) and [pep8](https://pypi.python.org/pypi/pep8) (optional, ignore E501).
-9. When the tests pass, [add](https://www.kernel.org/pub/software/scm/git/docs/git-add.html) the new files and [commit](https://www.kernel.org/pub/software/scm/git/docs/git-commit.html) them and [push](https://www.kernel.org/pub/software/scm/git/docs/git-push.html) the result, like this:
+9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/__init__.py
         $ git add youtube_dl/extractor/yourextractor.py

From ac35c266868ddfd39f8b708f425e1baf2555e406 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 2 Nov 2014 17:53:12 +0100
Subject: [PATCH 0405/1937] [tests] Don't auto init YoutubeDL

It would print the debug headers for each test.
And nose uses a StringIO object for stdout, which in python 2.x doesn't have the 'encoding' attribute.
---
 test/helper.py        | 2 +-
 test/test_download.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index fb8618120..325f72f0a 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -57,7 +57,7 @@ class FakeYDL(YoutubeDL):
         # Different instances of the downloader can't share the same dictionary
         # some test set the "sublang" parameter, which would break the md5 checks.
         params = get_params(override=override)
-        super(FakeYDL, self).__init__(params)
+        super(FakeYDL, self).__init__(params, auto_init=False)
         self.result = []
         
     def to_screen(self, s, skip_eol=None):
diff --git a/test/test_download.py b/test/test_download.py
index 88ce29871..87aced97c 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -96,7 +96,7 @@ def generator(test_case):
             params.setdefault('extract_flat', True)
             params.setdefault('skip_download', True)
 
-        ydl = YoutubeDL(params)
+        ydl = YoutubeDL(params, auto_init=False)
         ydl.add_default_info_extractors()
         finished_hook_called = set()
         def _hook(status):

From 4f195f55f0b734f2897319abf96c5542ce6212d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 2 Nov 2014 17:28:42 +0100
Subject: [PATCH 0406/1937] Do not override stdlib html parser
 'locatestarttagend' regex (fixes #4081)

'<a href="foo" ><img src="bar" / ></a>' wouldn't be parsed right (the problem is '/ >', '/>' worked fine).
We need to change it in python 2.6 (for example the description of youtube videos wouldn't be extracted).
---
 youtube_dl/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 233286de8..bdd637e48 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -152,7 +152,9 @@ def xpath_text(node, xpath, name=None, fatal=False):
     return n.text
 
 
-compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
+if sys.version_info < (2, 7):
+    compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
+
 class BaseHTMLParser(compat_html_parser.HTMLParser):
     def __init(self):
         compat_html_parser.HTMLParser.__init__(self)

From fdca55fe34a6d120c597641472856a296f1e80db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 3 Nov 2014 20:14:18 +0700
Subject: [PATCH 0407/1937] [trutube] Strip title

---
 youtube_dl/extractor/trutube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/trutube.py b/youtube_dl/extractor/trutube.py
index a73f3c43a..e7b79243a 100644
--- a/youtube_dl/extractor/trutube.py
+++ b/youtube_dl/extractor/trutube.py
@@ -29,7 +29,7 @@ class TruTubeIE(InfoExtractor):
 
         # filehd is always 404
         video_url = xpath_text(config, './file', 'video URL', fatal=True)
-        title = xpath_text(config, './title', 'title')
+        title = xpath_text(config, './title', 'title').strip()
         thumbnail = xpath_text(config, './image', ' thumbnail')
 
         return {

From 278143df5b026f76e52bad8500a32780b52a1e35 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 3 Nov 2014 19:12:06 +0100
Subject: [PATCH 0408/1937] [test_compat] Ignore unicode_literals

---
 test/test_compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index d1a33ddc9..4a7fc3606 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -37,7 +37,7 @@ class TestCompat(unittest.TestCase):
         all_names = youtube_dl.compat.__all__
         present_names = set(filter(
             lambda c: '_' in c and not c.startswith('_'),
-            dir(youtube_dl.compat)))
+            dir(youtube_dl.compat))) - set(['unicode_literals'])
         self.assertEqual(all_names, sorted(present_names))
 
 if __name__ == '__main__':

From b5af6fcdaddeade533b82708a137f4e5bed744df Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 3 Nov 2014 23:12:24 +0200
Subject: [PATCH 0409/1937] [brightcove] Make _VALID_URL less greedy and check
 for empty URLs (#3541)

---
 youtube_dl/extractor/brightcove.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index ad22cbafd..cc94e596c 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -23,7 +23,7 @@ from ..utils import (
 
 
 class BrightcoveIE(InfoExtractor):
-    _VALID_URL = r'https?://.*brightcove\.com/(services|viewer).*\?(?P<query>.*)'
+    _VALID_URL = r'https?://.*brightcove\.com/(services|viewer).*?\?(?P<query>.*)'
     _FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
 
     _TESTS = [
@@ -260,6 +260,8 @@ class BrightcoveIE(InfoExtractor):
             formats = []
             for rend in renditions:
                 url = rend['defaultURL']
+                if not url:
+                    continue
                 if rend['remote']:
                     # This type of renditions are served through akamaihd.net,
                     # but they don't use f4m manifests

From bdf970172905e90be7886fd5f78532bbdc924eba Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 3 Nov 2014 23:13:46 +0200
Subject: [PATCH 0410/1937] [generic/brightcove] Add a new test case for
 kijk.nl (#3541)

---
 youtube_dl/extractor/generic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index babd581ea..286f95b18 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -100,6 +100,20 @@ class GenericIE(InfoExtractor):
                 'uploader': 'Championat',
             },
         },
+        {
+            'add_ie': ['Brightcove'],
+            'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
+            'info_dict': {
+                'id': '3866516442001',
+                'ext': 'flv',
+                'title': 'Leer mij vrouwen kennen: Aflevering 1',
+                'description': 'Leer mij vrouwen kennen: Aflevering 1',
+                'uploader': 'SBS Broadcasting',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
         # Direct link to a video
         {
             'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',

From e7642ab572c1a541e7352b7407dd1c66f93ba379 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 4 Nov 2014 17:13:17 +0700
Subject: [PATCH 0411/1937] [wimp] Fix video URL regex

---
 youtube_dl/extractor/wimp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py
index 3377a543e..d6dec25ca 100644
--- a/youtube_dl/extractor/wimp.py
+++ b/youtube_dl/extractor/wimp.py
@@ -37,7 +37,7 @@ class WimpIE(InfoExtractor):
         video_id = mobj.group(1)
         webpage = self._download_webpage(url, video_id)
         video_url = self._search_regex(
-            r"'file'\s*:\s*'([^']+)'", webpage, 'video URL')
+            r"[\"']file[\"']\s*[:,]\s*[\"'](.+?)[\"']", webpage, 'video URL')
         if YoutubeIE.suitable(video_url):
             self.to_screen('Found YouTube video')
             return {

From 766306450db138873390511a856d562e0e76fe73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 4 Nov 2014 17:34:53 +0700
Subject: [PATCH 0412/1937] [played] Capture and output error message

---
 youtube_dl/extractor/played.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
index 645a1e06d..17880471d 100644
--- a/youtube_dl/extractor/played.py
+++ b/youtube_dl/extractor/played.py
@@ -6,6 +6,7 @@ import os.path
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     compat_urllib_parse,
     compat_urllib_request,
 )
@@ -29,6 +30,12 @@ class PlayedIE(InfoExtractor):
         video_id = self._match_id(url)
 
         orig_webpage = self._download_webpage(url, video_id)
+
+        m_error = re.search(
+            r'(?s)Reason for deletion:.*?<b class="err"[^>]*>(?P<msg>[^<]+)</b>', orig_webpage)
+        if m_error:
+            raise ExtractorError(m_error.group('msg'), expected=True)
+
         fields = re.findall(
             r'type="hidden" name="([^"]+)"\s+value="([^"]+)">', orig_webpage)
         data = dict(fields)

From acd40f64ed4d043ef3dbd6857fe1f82efb3a2e97 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 4 Nov 2014 22:25:08 +0100
Subject: [PATCH 0413/1937] [cnn] Modernize test definitions

---
 youtube_dl/extractor/cnn.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 78877b1cf..3826ce7e1 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -16,9 +16,10 @@ class CNNIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',
-        'file': 'sports_2013_06_09_nadal-1-on-1.cnn.mp4',
         'md5': '3e6121ea48df7e2259fe73a0628605c4',
         'info_dict': {
+            'id': 'sports_2013_06_09_nadal-1-on-1.cnn',
+            'ext': 'mp4',
             'title': 'Nadal wins 8th French Open title',
             'description': 'World Sport\'s Amanda Davies chats with 2013 French Open champion Rafael Nadal.',
             'duration': 135,
@@ -27,9 +28,10 @@ class CNNIE(InfoExtractor):
     },
     {
         "url": "http://edition.cnn.com/video/?/video/us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+rss%2Fcnn_topstories+%28RSS%3A+Top+Stories%29",
-        "file": "us_2013_08_21_sot-student-gives-epic-speech.georgia-institute-of-technology.mp4",
         "md5": "b5cc60c60a3477d185af8f19a2a26f4e",
         "info_dict": {
+            'id': 'us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology',
+            'ext': 'mp4',
             "title": "Student's epic speech stuns new freshmen",
             "description": "A Georgia Tech student welcomes the incoming freshmen with an epic speech backed by music from \"2001: A Space Odyssey.\"",
             "upload_date": "20130821",

From 94bd36131884c6b1f63052b7f435a77bae5aa83b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 4 Nov 2014 22:45:43 +0100
Subject: [PATCH 0414/1937] [youtube] Skip sts if missing (Fixes #4095, fixes
 #4103)

---
 youtube_dl/extractor/youtube.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index aad8ffbf4..b1cf15472 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -684,7 +684,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         # Get video info
         self.report_video_info_webpage_download(video_id)
         if re.search(r'player-age-gate-content">', video_webpage) is not None:
-            self.report_age_confirmation()
             age_gate = True
             # We simulate the access to the video from www.youtube.com/v/{video_id}
             # this can be viewed without login into Youtube
@@ -692,12 +691,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'video_id': video_id,
                 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
                 'sts': self._search_regex(
-                    r'"sts"\s*:\s*(\d+)', video_webpage, 'sts'),
+                    r'"sts"\s*:\s*(\d+)', video_webpage, 'sts', default=''),
             })
             video_info_url = proto + '://www.youtube.com/get_video_info?' + data
-            video_info_webpage = self._download_webpage(video_info_url, video_id,
-                                    note=False,
-                                    errnote='unable to download video info webpage')
+            video_info_webpage = self._download_webpage(
+                video_info_url, video_id,
+                note='Refetching age-gated webpage',
+                errnote='unable to download video info webpage')
             video_info = compat_parse_qs(video_info_webpage)
         else:
             age_gate = False

From a32f253112032f4b17d6d6c634358599bad4851b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 4 Nov 2014 23:04:12 +0100
Subject: [PATCH 0415/1937] [gamespot] Modernize

---
 youtube_dl/extractor/gamespot.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 3d67b9d60..908a8b2c9 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 
 class GameSpotIE(InfoExtractor):
-    _VALID_URL = r'(?:http://)?(?:www\.)?gamespot\.com/.*-(?P<page_id>\d+)/?'
+    _VALID_URL = r'(?:http://)?(?:www\.)?gamespot\.com/.*-(?P<id>\d+)/?'
     _TEST = {
         'url': 'http://www.gamespot.com/videos/arma-3-community-guide-sitrep-i/2300-6410818/',
         'md5': 'b2a30deaa8654fcccd43713a6b6a4825',
@@ -26,10 +26,10 @@ class GameSpotIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        page_id = mobj.group('page_id')
+        page_id = self._match_id(url)
         webpage = self._download_webpage(url, page_id)
-        data_video_json = self._search_regex(r'data-video=["\'](.*?)["\']', webpage, 'data video')
+        data_video_json = self._search_regex(
+            r'data-video=["\'](.*?)["\']', webpage, 'data video')
         data_video = json.loads(unescapeHTML(data_video_json))
 
         # Transform the manifest url to a link to the mp4 files
@@ -41,7 +41,8 @@ class GameSpotIE(InfoExtractor):
         http_path = f4m_path[1:].split('/', 1)[1]
         http_template = re.sub(QUALITIES_RE, r'%s', http_path)
         http_template = http_template.replace('.csmil/manifest.f4m', '')
-        http_template = compat_urlparse.urljoin('http://video.gamespotcdn.com/', http_template)
+        http_template = compat_urlparse.urljoin(
+            'http://video.gamespotcdn.com/', http_template)
         formats = []
         for q in qualities:
             formats.append({
@@ -52,8 +53,9 @@ class GameSpotIE(InfoExtractor):
 
         return {
             'id': data_video['guid'],
+            'display_id': page_id,
             'title': compat_urllib_parse.unquote(data_video['title']),
             'formats': formats,
-            'description': get_meta_content('description', webpage),
+            'description': self._html_search_meta('description', webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
         }

From 711ede6e1b8169609c4c0137af307a6da218092f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 4 Nov 2014 23:14:16 +0100
Subject: [PATCH 0416/1937] [heise] Fix description, thumbnail and format ID

---
 youtube_dl/extractor/common.py | 19 +++++++++++--------
 youtube_dl/extractor/heise.py  | 16 ++++++++++------
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9e1d62c2b..b77f0e519 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -404,7 +404,7 @@ class InfoExtractor(object):
             video_info['title'] = playlist_title
         return video_info
 
-    def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0):
+    def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
         """
         Perform a regex search on the given string, using a single or a list of
         patterns returning the first matching group.
@@ -425,8 +425,11 @@ class InfoExtractor(object):
             _name = name
 
         if mobj:
-            # return the first matching group
-            return next(g for g in mobj.groups() if g is not None)
+            if group is None:
+                # return the first matching group
+                return next(g for g in mobj.groups() if g is not None)
+            else:
+                return mobj.group(group)
         elif default is not _NO_DEFAULT:
             return default
         elif fatal:
@@ -436,11 +439,11 @@ class InfoExtractor(object):
                 'please report this issue on http://yt-dl.org/bug' % _name)
             return None
 
-    def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0):
+    def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
         """
         Like _search_regex, but strips HTML tags and unescapes entities.
         """
-        res = self._search_regex(pattern, string, name, default, fatal, flags)
+        res = self._search_regex(pattern, string, name, default, fatal, flags, group)
         if res:
             return clean_html(res).strip()
         else:
@@ -534,9 +537,9 @@ class InfoExtractor(object):
             display_name = name
         return self._html_search_regex(
             r'''(?ix)<meta
-                    (?=[^>]+(?:itemprop|name|property)=["\']?%s["\']?)
-                    [^>]+content=["\']([^"\']+)["\']''' % re.escape(name),
-            html, display_name, fatal=fatal, **kwargs)
+                    (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
+                    [^>]+content=(["\'])(?P<content>.*?)\1''' % re.escape(name),
+            html, display_name, fatal=fatal, group='content', **kwargs)
 
     def _dc_search_uploader(self, html):
         return self._html_search_meta('dc.creator', html, 'uploader')
diff --git a/youtube_dl/extractor/heise.py b/youtube_dl/extractor/heise.py
index d41c0413f..278d9f527 100644
--- a/youtube_dl/extractor/heise.py
+++ b/youtube_dl/extractor/heise.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    get_meta_content,
+    determine_ext,
     int_or_none,
     parse_iso8601,
 )
@@ -25,11 +25,11 @@ class HeiseIE(InfoExtractor):
             'title': (
                 "Podcast: c't uplink 3.3 – Owncloud / Tastaturen / Peilsender Smartphone"
             ),
-            'format_id': 'mp4_720',
+            'format_id': 'mp4_720p',
             'timestamp': 1411812600,
             'upload_date': '20140927',
             'description': 'In uplink-Episode 3.3 geht es darum, wie man sich von Cloud-Anbietern emanzipieren kann, worauf man beim Kauf einer Tastatur achten sollte und was Smartphones über uns verraten.',
-            'thumbnail': 're:https?://.*\.jpg$',
+            'thumbnail': 're:^https?://.*\.jpe?g$',
         }
     }
 
@@ -49,11 +49,12 @@ class HeiseIE(InfoExtractor):
         info = {
             'id': video_id,
             'thumbnail': self._og_search_thumbnail(webpage),
-            'timestamp': parse_iso8601(get_meta_content('date', webpage)),
+            'timestamp': parse_iso8601(
+                self._html_search_meta('date', webpage)),
             'description': self._og_search_description(webpage),
         }
 
-        title = get_meta_content('fulltitle', webpage)
+        title = self._html_search_meta('fulltitle', webpage)
         if title:
             info['title'] = title
         else:
@@ -64,9 +65,12 @@ class HeiseIE(InfoExtractor):
             label = source_node.attrib['label']
             height = int_or_none(self._search_regex(
                 r'^(.*?_)?([0-9]+)p$', label, 'height', default=None))
+            video_url = source_node.attrib['file']
+            ext = determine_ext(video_url, '')
             formats.append({
-                'url': source_node.attrib['file'],
+                'url': video_url,
                 'format_note': label,
+                'format_id': '%s_%s' % (ext, label),
                 'height': height,
             })
         self._sort_formats(formats)

From 44789f2457bab99f712e97e4ced5eb1a2fc9cbfd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 4 Nov 2014 23:15:16 +0100
Subject: [PATCH 0417/1937] [ustream] Use modern helper function instead of old
 HTML parser

---
 youtube_dl/extractor/ustream.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index cee1ea8f6..875450908 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -5,7 +5,6 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     compat_urlparse,
-    get_meta_content,
 )
 
 
@@ -79,7 +78,7 @@ class UstreamChannelIE(InfoExtractor):
         m = re.match(self._VALID_URL, url)
         display_id = m.group('slug')
         webpage = self._download_webpage(url, display_id)
-        channel_id = get_meta_content('ustream:channel_id', webpage)
+        channel_id = self._html_search_meta('ustream:channel_id', webpage)
 
         BASE = 'http://www.ustream.tv'
         next_url = '/ajax/socialstream/videos/%s/1.json' % channel_id

From 9c3e87039312e34263dbae0c9da3419e909d1001 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 4 Nov 2014 23:17:43 +0100
Subject: [PATCH 0418/1937] [gamespot] Remove unused import

---
 youtube_dl/extractor/gamespot.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 908a8b2c9..d570e3f6a 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -8,7 +8,6 @@ from ..utils import (
     compat_urllib_parse,
     compat_urlparse,
     unescapeHTML,
-    get_meta_content,
 )
 
 

From 12ea2f30cfcc381eb46cc99a0a4c3d468fe732d1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 4 Nov 2014 23:20:39 +0100
Subject: [PATCH 0419/1937] [utils] Remove unused get_meta_content function

---
 test/test_utils.py  | 12 ------------
 youtube_dl/utils.py | 12 ++----------
 2 files changed, 2 insertions(+), 22 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 338701f4c..e59547784 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -20,7 +20,6 @@ from youtube_dl.utils import (
     encodeFilename,
     find_xpath_attr,
     fix_xml_ampersands,
-    get_meta_content,
     orderedSet,
     OnDemandPagedList,
     InAdvancePagedList,
@@ -155,17 +154,6 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(find_xpath_attr(doc, './/node', 'x', 'a'), doc[1])
         self.assertEqual(find_xpath_attr(doc, './/node', 'y', 'c'), doc[2])
 
-    def test_meta_parser(self):
-        testhtml = '''
-        <head>
-            <meta name="description" content="foo &amp; bar">
-            <meta content='Plato' name='author'/>
-        </head>
-        '''
-        get_meta = lambda name: get_meta_content(name, testhtml)
-        self.assertEqual(get_meta('description'), 'foo & bar')
-        self.assertEqual(get_meta('author'), 'Plato')
-
     def test_xpath_with_ns(self):
         testxml = '''<root xmlns:media="http://example.com/">
             <media:song>
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index bdd637e48..16651bf11 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -231,10 +231,12 @@ if sys.version_info < (2, 7, 3):
         if self.rawdata[i:].startswith("</scr'+'ipt>")
         else compat_html_parser.HTMLParser.parse_endtag(self, i))
 
+
 def get_element_by_id(id, html):
     """Return the content of the tag with the specified ID in the passed HTML document"""
     return get_element_by_attribute("id", id, html)
 
+
 def get_element_by_attribute(attribute, value, html):
     """Return the content of the tag with the specified attribute in the passed HTML document"""
     parser = AttrParser(attribute, value)
@@ -265,16 +267,6 @@ class MetaParser(BaseHTMLParser):
     def get_result(self):
         return self.result
 
-def get_meta_content(name, html):
-    """
-    Return the content attribute from the meta tag with the given name attribute.
-    """
-    parser = MetaParser(name)
-    try:
-        parser.loads(html)
-    except compat_html_parser.HTMLParseError:
-        pass
-    return parser.get_result()
 
 
 def clean_html(html):

From 11fba1751d5a43491254a3f54fcd0903a40d1a35 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 4 Nov 2014 23:26:23 +0100
Subject: [PATCH 0420/1937] [imdb] Simplify

---
 youtube_dl/extractor/imdb.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index 4536db3bf..6108ed552 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -6,7 +6,6 @@ import json
 from .common import InfoExtractor
 from ..utils import (
     compat_urlparse,
-    get_element_by_attribute,
 )
 
 
@@ -27,10 +26,11 @@ class ImdbIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage('http://www.imdb.com/video/imdb/vi%s' % video_id, video_id)
-        descr = get_element_by_attribute('itemprop', 'description', webpage)
+        descr = self._html_search_regex(
+            r'(?s)<span itemprop="description">(.*?)</span>',
+            webpage, 'description', fatal=False)
         available_formats = re.findall(
             r'case \'(?P<f_id>.*?)\' :$\s+url = \'(?P<path>.*?)\'', webpage,
             flags=re.MULTILINE)
@@ -73,9 +73,7 @@ class ImdbListIE(InfoExtractor):
     }
     
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        list_id = mobj.group('id')
-
+        list_id = self._match_id(url)
         webpage = self._download_webpage(url, list_id)
         entries = [
             self.url_result('http://www.imdb.com' + m, 'Imdb')

From 3828505646a58d5e19f0ffe77821650e25d72b3a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 4 Nov 2014 23:33:43 +0100
Subject: [PATCH 0421/1937] [utils] Use a regexp instead of HTMLParser for
 get_element_by_attribute

---
 youtube_dl/utils.py | 119 ++++++--------------------------------------
 1 file changed, 15 insertions(+), 104 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 16651bf11..7c0fb1592 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -152,86 +152,6 @@ def xpath_text(node, xpath, name=None, fatal=False):
     return n.text
 
 
-if sys.version_info < (2, 7):
-    compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
-
-class BaseHTMLParser(compat_html_parser.HTMLParser):
-    def __init(self):
-        compat_html_parser.HTMLParser.__init__(self)
-        self.html = None
-
-    def loads(self, html):
-        self.html = html
-        self.feed(html)
-        self.close()
-
-class AttrParser(BaseHTMLParser):
-    """Modified HTMLParser that isolates a tag with the specified attribute"""
-    def __init__(self, attribute, value):
-        self.attribute = attribute
-        self.value = value
-        self.result = None
-        self.started = False
-        self.depth = {}
-        self.watch_startpos = False
-        self.error_count = 0
-        BaseHTMLParser.__init__(self)
-
-    def error(self, message):
-        if self.error_count > 10 or self.started:
-            raise compat_html_parser.HTMLParseError(message, self.getpos())
-        self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
-        self.error_count += 1
-        self.goahead(1)
-
-    def handle_starttag(self, tag, attrs):
-        attrs = dict(attrs)
-        if self.started:
-            self.find_startpos(None)
-        if self.attribute in attrs and attrs[self.attribute] == self.value:
-            self.result = [tag]
-            self.started = True
-            self.watch_startpos = True
-        if self.started:
-            if not tag in self.depth: self.depth[tag] = 0
-            self.depth[tag] += 1
-
-    def handle_endtag(self, tag):
-        if self.started:
-            if tag in self.depth: self.depth[tag] -= 1
-            if self.depth[self.result[0]] == 0:
-                self.started = False
-                self.result.append(self.getpos())
-
-    def find_startpos(self, x):
-        """Needed to put the start position of the result (self.result[1])
-        after the opening tag with the requested id"""
-        if self.watch_startpos:
-            self.watch_startpos = False
-            self.result.append(self.getpos())
-    handle_entityref = handle_charref = handle_data = handle_comment = \
-    handle_decl = handle_pi = unknown_decl = find_startpos
-
-    def get_result(self):
-        if self.result is None:
-            return None
-        if len(self.result) != 3:
-            return None
-        lines = self.html.split('\n')
-        lines = lines[self.result[1][0]-1:self.result[2][0]]
-        lines[0] = lines[0][self.result[1][1]:]
-        if len(lines) == 1:
-            lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
-        lines[-1] = lines[-1][:self.result[2][1]]
-        return '\n'.join(lines).strip()
-# Hack for https://github.com/rg3/youtube-dl/issues/662
-if sys.version_info < (2, 7, 3):
-    AttrParser.parse_endtag = (lambda self, i:
-        i + len("</scr'+'ipt>")
-        if self.rawdata[i:].startswith("</scr'+'ipt>")
-        else compat_html_parser.HTMLParser.parse_endtag(self, i))
-
-
 def get_element_by_id(id, html):
     """Return the content of the tag with the specified ID in the passed HTML document"""
     return get_element_by_attribute("id", id, html)
@@ -239,34 +159,25 @@ def get_element_by_id(id, html):
 
 def get_element_by_attribute(attribute, value, html):
     """Return the content of the tag with the specified attribute in the passed HTML document"""
-    parser = AttrParser(attribute, value)
-    try:
-        parser.loads(html)
-    except compat_html_parser.HTMLParseError:
-        pass
-    return parser.get_result()
 
-class MetaParser(BaseHTMLParser):
-    """
-    Modified HTMLParser that isolates a meta tag with the specified name 
-    attribute.
-    """
-    def __init__(self, name):
-        BaseHTMLParser.__init__(self)
-        self.name = name
-        self.content = None
-        self.result = None
+    m = re.search(r'''(?xs)
+        <([a-zA-Z0-9:._-]+)
+         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
+         \s+%s=['"]?%s['"]?
+         (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
+        \s*>
+        (?P<content>.*?)
+        </\1>
+    ''' % (re.escape(attribute), re.escape(value)), html)
 
-    def handle_starttag(self, tag, attrs):
-        if tag != 'meta':
-            return
-        attrs = dict(attrs)
-        if attrs.get('name') == self.name:
-            self.result = attrs.get('content')
+    if not m:
+        return None
+    res = m.group('content')
 
-    def get_result(self):
-        return self.result
+    if res.startswith('"') or res.startswith("'"):
+        res = res[1:-1]
 
+    return unescapeHTML(res)
 
 
 def clean_html(html):

From 20436c30c92796feb3a623e4ddb8d9c319971fca Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 4 Nov 2014 23:35:34 +0100
Subject: [PATCH 0422/1937] [youtube] Clarify output

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b1cf15472..c77d4056f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -696,7 +696,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             video_info_url = proto + '://www.youtube.com/get_video_info?' + data
             video_info_webpage = self._download_webpage(
                 video_info_url, video_id,
-                note='Refetching age-gated webpage',
+                note='Refetching age-gated info webpage',
                 errnote='unable to download video info webpage')
             video_info = compat_parse_qs(video_info_webpage)
         else:

From a80c96eab0b7c47ddae331ece37faa458be7bb2e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 4 Nov 2014 23:42:09 +0100
Subject: [PATCH 0423/1937] release 2014.11.04

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 327560c76..07cdb22ad 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.02.1'
+__version__ = '2014.11.04'

From 81b22aee8b874ab9069e8b9b93158ede4da85f19 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 5 Nov 2014 01:00:33 +0200
Subject: [PATCH 0424/1937] [izlesene] Update test cases and modernize

The timestamp fluctuates with DST.
---
 youtube_dl/extractor/izlesene.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/izlesene.py b/youtube_dl/extractor/izlesene.py
index 07ef682ee..d16d483ee 100644
--- a/youtube_dl/extractor/izlesene.py
+++ b/youtube_dl/extractor/izlesene.py
@@ -5,11 +5,11 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    get_element_by_id,
-    parse_iso8601,
     determine_ext,
-    int_or_none,
     float_or_none,
+    get_element_by_id,
+    int_or_none,
+    parse_iso8601,
     str_to_int,
 )
 
@@ -30,7 +30,7 @@ class IzleseneIE(InfoExtractor):
                 'description': 'md5:253753e2655dde93f59f74b572454f6d',
                 'thumbnail': 're:^http://.*\.jpg',
                 'uploader_id': 'pelikzzle',
-                'timestamp': 1404298698,
+                'timestamp': 1404302298,
                 'upload_date': '20140702',
                 'duration': 95.395,
                 'age_limit': 0,
@@ -46,7 +46,7 @@ class IzleseneIE(InfoExtractor):
                 'description': 'Tarkan Dortmund 2006 Konseri',
                 'thumbnail': 're:^http://.*\.jpg',
                 'uploader_id': 'parlayankiz',
-                'timestamp': 1163318593,
+                'timestamp': 1163322193,
                 'upload_date': '20061112',
                 'duration': 253.666,
                 'age_limit': 0,
@@ -55,10 +55,9 @@ class IzleseneIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        url = 'http://www.izlesene.com/video/%s' % video_id
+        video_id = self._match_id(url)
 
+        url = 'http://www.izlesene.com/video/%s' % video_id
         webpage = self._download_webpage(url, video_id)
 
         title = self._og_search_title(webpage)

From a316a83d2b0e0f528f28978e556b455f17324fb8 Mon Sep 17 00:00:00 2001
From: nemunaire <nemunaire@nemunai.re>
Date: Tue, 4 Nov 2014 23:20:34 +0100
Subject: [PATCH 0425/1937] [channel9] Fix extraction

---
 youtube_dl/extractor/channel9.py | 33 ++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 4f000292b..16d800512 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -27,7 +27,7 @@ class Channel9IE(InfoExtractor):
                 'title': 'Developer Kick-Off Session: Stuff We Love',
                 'description': 'md5:c08d72240b7c87fcecafe2692f80e35f',
                 'duration': 4576,
-                'thumbnail': 'http://media.ch9.ms/ch9/9d51/03902f2d-fc97-4d3c-b195-0bfe15a19d51/KOS002_220.jpg',
+                'thumbnail': 'http://video.ch9.ms/ch9/9d51/03902f2d-fc97-4d3c-b195-0bfe15a19d51/KOS002_220.jpg',
                 'session_code': 'KOS002',
                 'session_day': 'Day 1',
                 'session_room': 'Arena 1A',
@@ -43,7 +43,7 @@ class Channel9IE(InfoExtractor):
                 'title': 'Self-service BI with Power BI - nuclear testing',
                 'description': 'md5:d1e6ecaafa7fb52a2cacdf9599829f5b',
                 'duration': 1540,
-                'thumbnail': 'http://media.ch9.ms/ch9/87e1/0300391f-a455-4c72-bec3-4422f19287e1/selfservicenuk_512.jpg',
+                'thumbnail': 'http://video.ch9.ms/ch9/87e1/0300391f-a455-4c72-bec3-4422f19287e1/selfservicenuk_512.jpg',
                 'authors': [ 'Mike Wilmot' ],
             },
         }
@@ -94,7 +94,7 @@ class Channel9IE(InfoExtractor):
 
     def _extract_title(self, html):
         title = self._html_search_meta('title', html, 'title')
-        if title is None:           
+        if title is None:
             title = self._og_search_title(html)
             TITLE_SUFFIX = ' (Channel 9)'
             if title is not None and title.endswith(TITLE_SUFFIX):
@@ -115,7 +115,7 @@ class Channel9IE(InfoExtractor):
         return self._html_search_meta('description', html, 'description')
 
     def _extract_duration(self, html):
-        m = re.search(r'data-video_duration="(?P<hours>\d{2}):(?P<minutes>\d{2}):(?P<seconds>\d{2})"', html)
+        m = re.search(r'"length": *"(?P<hours>\d{2}):(?P<minutes>\d{2}):(?P<seconds>\d{2})"', html)
         return ((int(m.group('hours')) * 60 * 60) + (int(m.group('minutes')) * 60) + int(m.group('seconds'))) if m else None
 
     def _extract_slides(self, html):
@@ -167,7 +167,7 @@ class Channel9IE(InfoExtractor):
         return re.findall(r'<a href="/Events/Speakers/[^"]+">([^<]+)</a>', html)
 
     def _extract_content(self, html, content_path):
-        # Look for downloadable content        
+        # Look for downloadable content
         formats = self._formats_from_html(html)
         slides = self._extract_slides(html)
         zip_ = self._extract_zip(html)
@@ -258,16 +258,17 @@ class Channel9IE(InfoExtractor):
 
         webpage = self._download_webpage(url, content_path, 'Downloading web page')
 
-        page_type_m = re.search(r'<meta name="Search.PageType" content="(?P<pagetype>[^"]+)"/>', webpage)
-        if page_type_m is None:
-            raise ExtractorError('Search.PageType not found, don\'t know how to process this page', expected=True)
+        page_type_m = re.search(r'<meta name="WT.entryid" content="(?P<pagetype>[^:]+)[^"]+"/>', webpage)
+        if page_type_m is not None:
+            page_type = page_type_m.group('pagetype')
+            if page_type == 'Entry':      # Any 'item'-like page, may contain downloadable content
+                return self._extract_entry_item(webpage, content_path)
+            elif page_type == 'Session':  # Event session page, may contain downloadable content
+                return self._extract_session(webpage, content_path)
+            elif page_type == 'Event':
+                return self._extract_list(content_path)
+            else:
+                raise ExtractorError('Unexpected WT.entryid %s' % page_type, expected=True)
 
-        page_type = page_type_m.group('pagetype')
-        if page_type == 'List':         # List page, may contain list of 'item'-like objects
+        else: # Assuming list
             return self._extract_list(content_path)
-        elif page_type == 'Entry.Item': # Any 'item'-like page, may contain downloadable content
-            return self._extract_entry_item(webpage, content_path)
-        elif page_type == 'Session':    # Event session page, may contain downloadable content
-            return self._extract_session(webpage, content_path)
-        else:
-            raise ExtractorError('Unexpected Search.PageType %s' % page_type, expected=True)
\ No newline at end of file

From dab647a7b66646f82b12638729b36df02609f75d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 Nov 2014 21:32:46 +0700
Subject: [PATCH 0426/1937] [cinemassacre] Keep both extraction approaches and
 make more robust (Closes #4109)

---
 youtube_dl/extractor/cinemassacre.py | 78 +++++++++++++++++-----------
 1 file changed, 47 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index 806e378b1..31fe906b4 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -42,11 +42,12 @@ class CinemassacreIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
         video_date = mobj.group('date_Y') + mobj.group('date_m') + mobj.group('date_d')
-        mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?:Cinemassacre-)?(?P<video_id>.+?))"', webpage)
+        mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?P<full_video_id>(?:Cinemassacre-)?(?P<video_id>.+?)))"', webpage)
         if not mobj:
             raise ExtractorError('Can\'t extract embed url and video id')
         playerdata_url = mobj.group('embed_url')
         video_id = mobj.group('video_id')
+        full_video_id = mobj.group('full_video_id')
 
         video_title = self._html_search_regex(
             r'<title>(?P<title>.+?)\|', webpage, 'title')
@@ -60,37 +61,52 @@ class CinemassacreIE(InfoExtractor):
         vidurl = self._search_regex(
             r'\'vidurl\'\s*:\s*"([^\']+)"', playerdata, 'vidurl').replace('\\/', '/')
 
-        videolist_url = self._search_regex(
-            r"file\s*:\s*'(http.+?/jwplayer\.smil)'", playerdata, 'jwplayer.smil')
-        videolist = self._download_xml(videolist_url, video_id, 'Downloading videolist XML')
+        videolist_url = None
 
-        formats = []
-        baseurl = vidurl[:vidurl.rfind('/')+1]
-        for video in videolist.findall('.//video'):
-            src = video.get('src')
-            if not src:
-                continue
-            file_ = src.partition(':')[-1]
-            width = int_or_none(video.get('width'))
-            height = int_or_none(video.get('height'))
-            bitrate = int_or_none(video.get('system-bitrate'))
-            format = {
-                'url': baseurl + file_,
-                'format_id': src.rpartition('.')[0].rpartition('_')[-1],
-            }
-            if width or height:
-                format.update({
-                    'tbr': bitrate // 1000 if bitrate else None,
-                    'width': width,
-                    'height': height,
-                })
-            else:
-                format.update({
-                    'abr': bitrate // 1000 if bitrate else None,
-                    'vcodec': 'none',
-                })
-            formats.append(format)
-        self._sort_formats(formats)
+        mobj = re.search(r"'videoserver'\s*:\s*'(?P<videoserver>[^']+)'", playerdata)
+        if mobj:
+            videoserver = mobj.group('videoserver')
+            mobj = re.search(r'\'vidid\'\s*:\s*"(?P<vidid>[^\']+)"', playerdata)
+            vidid = mobj.group('vidid') if mobj else full_video_id
+            videolist_url = 'http://%s/vod/smil:%s.smil/jwplayer.smil' % (videoserver, vidid)
+        else:
+            mobj = re.search(r"file\s*:\s*'(?P<smil>http.+?/jwplayer\.smil)'", playerdata)
+            if mobj:
+                videolist_url = mobj.group('smil')
+
+        if videolist_url:
+            videolist = self._download_xml(videolist_url, video_id, 'Downloading videolist XML')
+            formats = []
+            baseurl = vidurl[:vidurl.rfind('/')+1]
+            for video in videolist.findall('.//video'):
+                src = video.get('src')
+                if not src:
+                    continue
+                file_ = src.partition(':')[-1]
+                width = int_or_none(video.get('width'))
+                height = int_or_none(video.get('height'))
+                bitrate = int_or_none(video.get('system-bitrate'))
+                format = {
+                    'url': baseurl + file_,
+                    'format_id': src.rpartition('.')[0].rpartition('_')[-1],
+                }
+                if width or height:
+                    format.update({
+                        'tbr': bitrate // 1000 if bitrate else None,
+                        'width': width,
+                        'height': height,
+                    })
+                else:
+                    format.update({
+                        'abr': bitrate // 1000 if bitrate else None,
+                        'vcodec': 'none',
+                    })
+                formats.append(format)
+            self._sort_formats(formats)
+        else:
+            formats = [{
+                'url': vidurl,
+            }]
 
         return {
             'id': video_id,

From 6110bbbfdd6418950beaf487e5a384cc7c483498 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 5 Nov 2014 19:52:34 +0100
Subject: [PATCH 0427/1937] [niconico] Catch deleted videos (closes #4064)

---
 youtube_dl/extractor/niconico.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 62d5707fe..45cbd4ee9 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -12,6 +12,7 @@ from ..utils import (
     unified_strdate,
     parse_duration,
     int_or_none,
+    ExtractorError,
 )
 
 
@@ -108,6 +109,9 @@ class NiconicoIE(InfoExtractor):
                 flv_info_request, video_id,
                 note='Downloading flv info', errnote='Unable to download flv info')
 
+        if 'deleted=' in flv_info_webpage:
+            raise ExtractorError('The video has been deleted.',
+                expected=True)
         video_real_url = compat_urlparse.parse_qs(flv_info_webpage)['url'][0]
 
         # Start extracting information

From 37aab27808b15bb7818d383dddbe165fe24be5d1 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 6 Nov 2014 00:14:33 +0200
Subject: [PATCH 0428/1937] [brightcove] Extract m3u8 formats (#3541)

---
 youtube_dl/extractor/brightcove.py | 15 +++++++++++----
 youtube_dl/extractor/generic.py    |  6 ++++--
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index cc94e596c..a6920685e 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -14,6 +14,7 @@ from ..utils import (
     compat_str,
     compat_urllib_request,
     compat_parse_qs,
+    compat_urllib_parse_urlparse,
 
     determine_ext,
     ExtractorError,
@@ -263,10 +264,16 @@ class BrightcoveIE(InfoExtractor):
                 if not url:
                     continue
                 if rend['remote']:
-                    # This type of renditions are served through akamaihd.net,
-                    # but they don't use f4m manifests
-                    url = url.replace('control/', '') + '?&v=3.3.0&fp=13&r=FEEFJ&g=RTSJIMBMPFPB'
-                    ext = 'flv'
+                    url_comp = compat_urllib_parse_urlparse(url)
+                    if url_comp.path.endswith('.m3u8'):
+                        formats.extend(
+                            self._extract_m3u8_formats(url, info['id'], 'mp4'))
+                        continue
+                    elif 'akamaihd.net' in url_comp.netloc:
+                        # This type of renditions are served through
+                        # akamaihd.net, but they don't use f4m manifests
+                        url = url.replace('control/', '') + '?&v=3.3.0&fp=13&r=FEEFJ&g=RTSJIMBMPFPB'
+                        ext = 'flv'
                 else:
                     ext = determine_ext(url)
                 size = rend.get('size')
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 286f95b18..8dd11e202 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -101,17 +101,19 @@ class GenericIE(InfoExtractor):
             },
         },
         {
+            # https://github.com/rg3/youtube-dl/issues/3541
             'add_ie': ['Brightcove'],
             'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
             'info_dict': {
                 'id': '3866516442001',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Leer mij vrouwen kennen: Aflevering 1',
                 'description': 'Leer mij vrouwen kennen: Aflevering 1',
                 'uploader': 'SBS Broadcasting',
             },
+            'skip': 'Restricted to Netherlands',
             'params': {
-                'skip_download': True,
+                'skip_download': True,  # m3u8 download
             },
         },
         # Direct link to a video

From 4164f0117ece057a84e67df2239b7fdaa69e27a7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 4 Nov 2014 23:51:01 +0100
Subject: [PATCH 0429/1937] [utils] Remove unused import

---
 youtube_dl/extractor/grooveshark.py | 7 ++++---
 youtube_dl/utils.py                 | 1 -
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/grooveshark.py b/youtube_dl/extractor/grooveshark.py
index 726adff77..fff74a70a 100644
--- a/youtube_dl/extractor/grooveshark.py
+++ b/youtube_dl/extractor/grooveshark.py
@@ -8,12 +8,13 @@ import re
 
 
 from .common import InfoExtractor
-from ..utils import ExtractorError, compat_urllib_request, compat_html_parser
-
-from ..utils import (
+from ..compat import (
+    compat_html_parser,
     compat_urllib_parse,
+    compat_urllib_request,
     compat_urlparse,
 )
+from ..utils import ExtractorError
 
 
 class GroovesharkHtmlParser(compat_html_parser.HTMLParser):
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7c0fb1592..d87e212ae 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -34,7 +34,6 @@ from .compat import (
     compat_chr,
     compat_getenv,
     compat_html_entities,
-    compat_html_parser,
     compat_parse_qs,
     compat_str,
     compat_urllib_error,

From 5d63b0aa9301f4e460dc5f3b08f723cbf2091e5a Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 6 Nov 2014 01:19:20 +0200
Subject: [PATCH 0430/1937] [goshgay] Fix title extraction and modernize

Also remove width and height as they are not of the actual video.
---
 youtube_dl/extractor/goshgay.py | 43 +++++++++++----------------------
 1 file changed, 14 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/goshgay.py b/youtube_dl/extractor/goshgay.py
index 7bca21ad0..18474cbb7 100644
--- a/youtube_dl/extractor/goshgay.py
+++ b/youtube_dl/extractor/goshgay.py
@@ -1,15 +1,11 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     compat_urlparse,
-    str_to_int,
     ExtractorError,
 )
-import json
 
 
 class GoshgayIE(InfoExtractor):
@@ -27,36 +23,27 @@ class GoshgayIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        title = self._search_regex(r'class="video-title"><h1>(.+?)<', webpage, 'title')
+        title = self._og_search_title(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+        family_friendly = self._html_search_meta(
+            'isFamilyFriendly', webpage, default='false')
+        config_url = self._search_regex(
+            r"'config'\s*:\s*'([^']+)'", webpage, 'config URL')
 
-        player_config = self._search_regex(
-            r'(?s)jwplayer\("player"\)\.setup\(({.+?})\)', webpage, 'config settings')
-        player_vars = json.loads(player_config.replace("'", '"'))
-        width = str_to_int(player_vars.get('width'))
-        height = str_to_int(player_vars.get('height'))
-        config_uri = player_vars.get('config')
+        config = self._download_xml(
+            config_url, video_id, 'Downloading player config XML')
 
-        if config_uri is None:
-            raise ExtractorError('Missing config URI')
-        node = self._download_xml(config_uri, video_id, 'Downloading player config XML',
-                                  errnote='Unable to download XML')
-        if node is None:
+        if config is None:
             raise ExtractorError('Missing config XML')
-        if node.tag != 'config':
+        if config.tag != 'config':
             raise ExtractorError('Missing config attribute')
-        fns = node.findall('file')
-        imgs = node.findall('image')
-        if len(fns) != 1:
+        fns = config.findall('file')
+        if len(fns) < 1:
             raise ExtractorError('Missing media URI')
         video_url = fns[0].text
-        if len(imgs) < 1:
-            thumbnail = None
-        else:
-            thumbnail = imgs[0].text
 
         url_comp = compat_urlparse.urlparse(url)
         ref = "%s://%s%s" % (url_comp[0], url_comp[1], url_comp[2])
@@ -65,9 +52,7 @@ class GoshgayIE(InfoExtractor):
             'id': video_id,
             'url': video_url,
             'title': title,
-            'width': width,
-            'height': height,
             'thumbnail': thumbnail,
             'http_referer': ref,
-            'age_limit': 18,
+            'age_limit': 0 if family_friendly == 'true' else 18,
         }

From 1fe8fb8c2022b70e6ed44d9c80570239eec74728 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 6 Nov 2014 21:44:07 +0100
Subject: [PATCH 0431/1937] [vice] Re-add extractor (fixes #4120)

The generic extraction no longer works.
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/vice.py     | 38 ++++++++++++++++++++++++++++++++
 2 files changed, 39 insertions(+)
 create mode 100644 youtube_dl/extractor/vice.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3f85c99cd..3c1807f15 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -421,6 +421,7 @@ from .vesti import VestiIE
 from .vevo import VevoIE
 from .vgtv import VGTVIE
 from .vh1 import VH1IE
+from .vice import ViceIE
 from .viddler import ViddlerIE
 from .videobam import VideoBamIE
 from .videodetective import VideoDetectiveIE
diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
new file mode 100644
index 000000000..f11ca8217
--- /dev/null
+++ b/youtube_dl/extractor/vice.py
@@ -0,0 +1,38 @@
+from __future__ import unicode_literals
+import re
+
+from .common import InfoExtractor
+from .ooyala import OoyalaIE
+from ..utils import ExtractorError
+
+
+class ViceIE(InfoExtractor):
+    _VALID_URL = r'http://www\.vice\.com/.*?/(?P<name>.+)'
+
+    _TEST = {
+        'url': 'http://www.vice.com/Fringes/cowboy-capitalists-part-1',
+        'info_dict': {
+            'id': '43cW1mYzpia9IlestBjVpd23Yu3afAfp',
+            'ext': 'mp4',
+            'title': 'VICE_COWBOYCAPITALISTS_PART01_v1_VICE_WM_1080p.mov',
+        },
+        'params': {
+            # Requires ffmpeg (m3u8 manifest)
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        name = mobj.group('name')
+        webpage = self._download_webpage(url, name)
+        try:
+            embed_code = self._search_regex(
+                r'embedCode=([^&\'"]+)', webpage,
+                'ooyala embed code')
+            ooyala_url = OoyalaIE._url_for_embed_code(embed_code)
+            print(ooyala_url)
+        except ExtractorError:
+            raise ExtractorError('The page doesn\'t contain a video', expected=True)
+        return self.url_result(ooyala_url, ie='Ooyala')
+

From b868c972d1c1bc7978e4afee51da3322abb6cc6d Mon Sep 17 00:00:00 2001
From: xantares <xantares09@hotmail.com>
Date: Fri, 7 Nov 2014 17:44:06 +0000
Subject: [PATCH 0432/1937] Add support for goldenmoustache.com

---
 youtube_dl/extractor/__init__.py        |  1 +
 youtube_dl/extractor/goldenmoustache.py | 50 +++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 youtube_dl/extractor/goldenmoustache.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3c1807f15..21853ccf4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -141,6 +141,7 @@ from .generic import GenericIE
 from .glide import GlideIE
 from .globo import GloboIE
 from .godtube import GodTubeIE
+from .goldenmoustache import GoldenMoustacheIE
 from .golem import GolemIE
 from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
diff --git a/youtube_dl/extractor/goldenmoustache.py b/youtube_dl/extractor/goldenmoustache.py
new file mode 100644
index 000000000..c25d2e779
--- /dev/null
+++ b/youtube_dl/extractor/goldenmoustache.py
@@ -0,0 +1,50 @@
+from __future__ import unicode_literals
+
+import re
+from .common import InfoExtractor
+from ..utils import (
+    parse_duration,
+    int_or_none,
+)
+
+
+class GoldenMoustacheIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?goldenmoustache\.com/(?P<display_id>[\w-]+)-(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.goldenmoustache.com/suricate-le-poker-3700/',
+        'md5': '0f904432fa07da5054d6c8beb5efb51a',
+        'info_dict': {
+            'id': '3700',
+            'ext': 'mp4',
+            'title': 'Suricate - Le Poker',
+            'description': 'md5:3d1f242f44f8c8cb0a106f1fd08e5dc9',
+            'thumbnail': 'md5:fd41386bc1f932552622da4a7e9a7242',
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_url = self._html_search_regex(r'data-src-type="mp4" data-src="([^"]+)"', webpage, 'video URL')
+
+        title = self._html_search_regex(r'<title>(.*?) - Golden Moustache</title>', webpage, 'title')
+
+        thumbnail = self._html_search_meta('og:image', webpage, 'thumbnail')
+         
+        description = self._html_search_meta('og:description', webpage, 'description')
+
+        view_count = int_or_none(self._html_search_regex(
+            r'<strong>(\d+)</strong>\s*VUES</span>', webpage, 'view count', fatal=False))
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'ext': 'mp4',
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'view_count': view_count,
+        }

From 29ed169cd63b3665a8fc4140eb698689bf656361 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 7 Nov 2014 22:53:54 +0200
Subject: [PATCH 0433/1937] [wrzuta] Add mp3 as a possible format (Closes
 #4126)

---
 youtube_dl/extractor/wrzuta.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wrzuta.py b/youtube_dl/extractor/wrzuta.py
index 34dd6d952..41756784a 100644
--- a/youtube_dl/extractor/wrzuta.py
+++ b/youtube_dl/extractor/wrzuta.py
@@ -49,7 +49,7 @@ class WrzutaIE(InfoExtractor):
 
         quality = qualities(['SD', 'MQ', 'HQ', 'HD'])
 
-        audio_table = {'flv': 'mp3', 'webm': 'ogg'}
+        audio_table = {'flv': 'mp3', 'webm': 'ogg', 'mp3': 'mp3'}
 
         embedpage = self._download_json('http://www.wrzuta.pl/npp/embed/%s/%s' % (uploader, video_id), video_id)
 

From 3898c8a7b2835b1632ef0e34481bdf5e006cee2b Mon Sep 17 00:00:00 2001
From: yac <yac@blesmrt.net>
Date: Fri, 7 Nov 2014 23:19:26 +0100
Subject: [PATCH 0434/1937] [FreeVideo] Add new extractor

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/freevideo.py | 49 +++++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+)
 create mode 100644 youtube_dl/extractor/freevideo.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3c1807f15..db76e8870 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -127,6 +127,7 @@ from .francetv import (
 )
 from .freesound import FreesoundIE
 from .freespeech import FreespeechIE
+from .freevideo import FreeVideoIE
 from .funnyordie import FunnyOrDieIE
 from .gamekings import GamekingsIE
 from .gameone import (
diff --git a/youtube_dl/extractor/freevideo.py b/youtube_dl/extractor/freevideo.py
new file mode 100644
index 000000000..f9716c4c8
--- /dev/null
+++ b/youtube_dl/extractor/freevideo.py
@@ -0,0 +1,49 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+)
+
+
+class FreeVideoIE(InfoExtractor):
+    _VALID_URL = r'^http://www.freevideo.cz/vase-videa/(?P<videoid>[^.]+)\.html$'
+
+    _TEST = {
+        'url': 'http://www.freevideo.cz/vase-videa/vysukany-zadecek-22033.html',
+        'file': 'vysukany-zadecek-22033.mp4',
+        'info_dict': {
+            "title": "vysukany-zadecek-22033",
+            "age_limit": 18,
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        if mobj is None:
+            raise ExtractorError('Invalid search query "%s"' % query)
+
+        video_id = mobj.group('videoid')
+
+        # Get webpage content
+        webpage = self._download_webpage(url, video_id)
+
+        age_limit = self._rta_search(webpage)
+        if age_limit == 0:
+            # interpret 0 as mis-detection since this site is adult-content only.
+            # However, if we get non-0, assume the rtalabel started giving proper
+            # results
+            age_limit = 18
+
+        url = re.search(r'\s+url: "(http://[a-z0-9-]+.cdn.freevideo.cz/stream/.*/video.mp4)"', webpage)
+        if url is None:
+            raise ExtractorError('ERROR: unable to extract video url')
+
+        return {
+            'id': video_id,
+            'url': url.groups()[0],
+            'title': video_id,
+            'age_limit': age_limit,
+        }

From 2fdbf27ad8f241c8f23b4a37683243a06903546d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 8 Nov 2014 14:53:23 +0100
Subject: [PATCH 0435/1937] [niconico:playlist] Use the same video url the
 webpage uses (closes #4133)

---
 youtube_dl/extractor/niconico.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 45cbd4ee9..3b5784e8f 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -175,7 +175,8 @@ class NiconicoPlaylistIE(InfoExtractor):
         entries = [{
             '_type': 'url',
             'ie_key': NiconicoIE.ie_key(),
-            'url': 'http://www.nicovideo.jp/watch/%s' % entry['item_id'],
+            'url': ('http://www.nicovideo.jp/watch/%s' %
+                entry['item_data']['video_id']),
         } for entry in entries]
 
         return {

From c2b61af54827373780415edce92b971b43ceead1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 8 Nov 2014 15:09:04 +0100
Subject: [PATCH 0436/1937] [options] Document the syntax for merging formats
 (closes #3940, closes #4132)

---
 youtube_dl/options.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 997e92ad7..cdcf2f62c 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -261,7 +261,16 @@ def parseOpts(overrideArguments=None):
     video_format.add_option(
         '-f', '--format',
         action='store', dest='format', metavar='FORMAT', default=None,
-        help='video format code, specify the order of preference using slashes: -f 22/17/18 .  -f mp4 , -f m4a and  -f flv  are also supported. You can also use the special names "best", "bestvideo", "bestaudio", "worst", "worstvideo" and "worstaudio". By default, youtube-dl will pick the best quality. Use commas to download multiple audio formats, such as  -f  136/137/mp4/bestvideo,140/m4a/bestaudio')
+        help='video format code, specify the order of preference using'
+            ' slashes: -f 22/17/18 .  -f mp4 , -f m4a and  -f flv  are also'
+            ' supported. You can also use the special names "best",'
+            ' "bestvideo", "bestaudio", "worst", "worstvideo" and'
+            ' "worstaudio". By default, youtube-dl will pick the best quality.'
+            ' Use commas to download multiple audio formats, such as'
+            ' -f  136/137/mp4/bestvideo,140/m4a/bestaudio.'
+            ' You can merge the video and audio of two formats into a single'
+            ' file using -f <video-format>+<audio-format> (requires ffmpeg or'
+            ' avconv), for example -f bestvideo+bestaudio.')
     video_format.add_option(
         '--all-formats',
         action='store_const', dest='format', const='all',

From d6fdc3868210ccb0d0d698940fdad16dbb497cb9 Mon Sep 17 00:00:00 2001
From: andi <andi@andiunx.no-ip.org>
Date: Sat, 8 Nov 2014 15:56:35 +0100
Subject: [PATCH 0437/1937] fix swrmediathek for new formats.

---
 youtube_dl/extractor/swrmediathek.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/swrmediathek.py b/youtube_dl/extractor/swrmediathek.py
index 13c6ea677..bf430d870 100644
--- a/youtube_dl/extractor/swrmediathek.py
+++ b/youtube_dl/extractor/swrmediathek.py
@@ -80,7 +80,7 @@ class SWRMediathekIE(InfoExtractor):
 
             if media_type == 'Video':
                 fmt.update({
-                    'format_note': ['144p', '288p', '544p'][quality-1],
+                    'format_note': ['144p', '288p', '544p', '720p'][quality-1],
                     'vcodec': codec,
                 })
             elif media_type == 'Audio':
@@ -101,4 +101,4 @@ class SWRMediathekIE(InfoExtractor):
             'uploader': attr['channel_title'],
             'uploader_id': attr['channel_idkey'],
             'formats': formats,
-        }
\ No newline at end of file
+        }

From f866e474f313d262b2c9696f84aee4432d75f7a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 9 Nov 2014 10:59:56 +0100
Subject: [PATCH 0438/1937] [YoutubeDL] Don't dowload formats for merging if
 the first doesn't contain the video (#4132)

---
 youtube_dl/YoutubeDL.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8732f3db4..172180d14 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -835,6 +835,13 @@ class YoutubeDL(object):
                         format_1, format_2 = rf.split('+')
                         formats_info = (self.select_format(format_1, formats),
                             self.select_format(format_2, formats))
+                        # The first format must contain the video and the
+                        # second the audio
+                        if formats_info[0].get('vcodec') == 'none':
+                            self.report_error('The first format must contain '
+                                'the video, try using "-f %s+%s"' %
+                                (format_2, format_1))
+                            return
                         if all(formats_info):
                             selected_format = {
                                 'requested_formats': formats_info,

From a6dae6c09c2c6be7a6d090ca0a7741674bf1fd98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 9 Nov 2014 11:15:50 +0100
Subject: [PATCH 0439/1937] [ndr] Improve video url regex (fixes #4140)

---
 youtube_dl/extractor/ndr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index add4b3e5d..c31ff8160 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -67,7 +67,7 @@ class NDRIE(InfoExtractor):
 
         thumbnail = None
 
-        video_url = re.search(r'''3: \{src:'(?P<video>.+?)\.hi\.mp4', type:"video/mp4"},''', page)
+        video_url = re.search(r'''3: \{src:'(?P<video>.+?)\.(lo|hi|hq)\.mp4', type:"video/mp4"},''', page)
         if video_url:
             thumbnails = re.findall(r'''\d+: \{src: "([^"]+)"(?: \|\| '[^']+')?, quality: '([^']+)'}''', page)
             if thumbnails:

From 1a92e086a7cee10c4461cbb92b13fa0df195ada5 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 9 Nov 2014 15:01:12 +0200
Subject: [PATCH 0440/1937] [tapely] Add Referer header (Closes #4138)

---
 youtube_dl/extractor/tapely.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/tapely.py b/youtube_dl/extractor/tapely.py
index 77e056242..283e11350 100644
--- a/youtube_dl/extractor/tapely.py
+++ b/youtube_dl/extractor/tapely.py
@@ -50,6 +50,7 @@ class TapelyIE(InfoExtractor):
         request = compat_urllib_request.Request(playlist_url)
         request.add_header('X-Requested-With', 'XMLHttpRequest')
         request.add_header('Accept', 'application/json')
+        request.add_header('Referer', url)
 
         playlist = self._download_json(request, display_id)
 

From 3c6af203cc615f835e8ff95508e04d908950249c Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 9 Nov 2014 22:00:51 +0200
Subject: [PATCH 0441/1937] [streamcloud] Match URLs without fname (Closes
 #4144)

Also, modernize a bit.
---
 youtube_dl/extractor/streamcloud.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dl/extractor/streamcloud.py
index 172def221..c1178f26d 100644
--- a/youtube_dl/extractor/streamcloud.py
+++ b/youtube_dl/extractor/streamcloud.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 class StreamcloudIE(InfoExtractor):
     IE_NAME = 'streamcloud.eu'
-    _VALID_URL = r'https?://streamcloud\.eu/(?P<id>[a-zA-Z0-9_-]+)/(?P<fname>[^#?]*)\.html'
+    _VALID_URL = r'https?://streamcloud\.eu/(?P<id>[a-zA-Z0-9_-]+)(?:/(?P<fname>[^#?]*)\.html)?'
 
     _TEST = {
         'url': 'http://streamcloud.eu/skp9j99s4bpz/youtube-dl_test_video_____________-BaW_jenozKc.mp4.html',
@@ -27,8 +27,8 @@ class StreamcloudIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
+        url = 'http://streamcloud.eu/%s' % video_id
 
         orig_webpage = self._download_webpage(url, video_id)
 

From a1cf99d03a6a842aa4e53c211c9f38dd27db5971 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 9 Nov 2014 22:32:26 +0100
Subject: [PATCH 0442/1937] [YoutubeDL] Add playlist_id and playlist_title
 fields (Fixes #4139)

---
 youtube_dl/YoutubeDL.py         | 2 ++
 youtube_dl/extractor/youtube.py | 1 +
 youtube_dl/options.py           | 6 ++++--
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 172180d14..9fdeb4cfb 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -658,6 +658,8 @@ class YoutubeDL(object):
                 extra = {
                     'n_entries': n_entries,
                     'playlist': playlist,
+                    'playlist_id': ie_result.get('id'),
+                    'playlist_title': ie_result.get('title'),
                     'playlist_index': i + playliststart,
                     'extractor': ie_result['extractor'],
                     'webpage_url': ie_result['webpage_url'],
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c77d4056f..5406de50b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1043,6 +1043,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'url': 'https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
         'info_dict': {
             'title': 'ytdl test PL',
+            'id': 'PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
         },
         'playlist_count': 3,
     }, {
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index cdcf2f62c..c182abfdc 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -490,10 +490,12 @@ def parseOpts(overrideArguments=None):
               '%(format_id)s for the unique id of the format (like Youtube\'s itags: "137"), '
               '%(upload_date)s for the upload date (YYYYMMDD), '
               '%(extractor)s for the provider (youtube, metacafe, etc), '
-              '%(id)s for the video id, %(playlist)s for the playlist the video is in, '
-              '%(playlist_index)s for the position in the playlist and %% for a literal percent. '
+              '%(id)s for the video id, '
+              '%(playlist_title)s, %(playlist_id)s, or %(playlist)s (=title if present, ID otherwise) for the playlist the video is in, '
+              '%(playlist_index)s for the position in the playlist. '
               '%(height)s and %(width)s for the width and height of the video format. '
               '%(resolution)s for a textual description of the resolution of the video format. '
+              '%% for a literal percent. '
               'Use - to output to stdout. Can also be used to download to a different directory, '
               'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
     filesystem.add_option(

From 1d4df56d096a387e924a344d02f8ddb576e5ba88 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 9 Nov 2014 22:32:41 +0100
Subject: [PATCH 0443/1937] release 2014.11.09

---
 README.md             | 29 ++++++++++++++++++-----------
 youtube_dl/version.py |  2 +-
 2 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index 933411fa2..cae61a855 100644
--- a/README.md
+++ b/README.md
@@ -131,17 +131,19 @@ which means you can modify it, redistribute it or use it however you like.
                                      %(upload_date)s for the upload date
                                      (YYYYMMDD), %(extractor)s for the provider
                                      (youtube, metacafe, etc), %(id)s for the
-                                     video id, %(playlist)s for the playlist the
+                                     video id, %(playlist_title)s,
+                                     %(playlist_id)s, or %(playlist)s (=title if
+                                     present, ID otherwise) for the playlist the
                                      video is in, %(playlist_index)s for the
-                                     position in the playlist and %% for a
-                                     literal percent. %(height)s and %(width)s
-                                     for the width and height of the video
-                                     format. %(resolution)s for a textual
+                                     position in the playlist. %(height)s and
+                                     %(width)s for the width and height of the
+                                     video format. %(resolution)s for a textual
                                      description of the resolution of the video
-                                     format. Use - to output to stdout. Can also
-                                     be used to download to a different
-                                     directory, for example with -o '/my/downloa
-                                     ds/%(uploader)s/%(title)s-%(id)s.%(ext)s' .
+                                     format. %% for a literal percent. Use - to
+                                     output to stdout. Can also be used to
+                                     download to a different directory, for
+                                     example with -o '/my/downloads/%(uploader)s
+                                     /%(title)s-%(id)s.%(ext)s' .
     --autonumber-size NUMBER         Specifies the number of digits in
                                      %(autonumber)s when it is present in output
                                      filename template or --auto-number option
@@ -239,8 +241,13 @@ which means you can modify it, redistribute it or use it however you like.
                                      "worst", "worstvideo" and "worstaudio". By
                                      default, youtube-dl will pick the best
                                      quality. Use commas to download multiple
-                                     audio formats, such as  -f
-                                     136/137/mp4/bestvideo,140/m4a/bestaudio
+                                     audio formats, such as -f
+                                     136/137/mp4/bestvideo,140/m4a/bestaudio.
+                                     You can merge the video and audio of two
+                                     formats into a single file using -f <video-
+                                     format>+<audio-format> (requires ffmpeg or
+                                     avconv), for example -f
+                                     bestvideo+bestaudio.
     --all-formats                    download all available video formats
     --prefer-free-formats            prefer free video formats unless a specific
                                      one is requested
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 07cdb22ad..9b2888b1b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.04'
+__version__ = '2014.11.09'

From 60526d6bcb33d9db76a7df85f89b08ea4ee9a038 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 11 Nov 2014 16:23:06 +0200
Subject: [PATCH 0444/1937] [wrzuta] Fix audio extension lookup (Closes #4156)

Also, replace deleted test case
---
 youtube_dl/extractor/wrzuta.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/wrzuta.py b/youtube_dl/extractor/wrzuta.py
index 41756784a..1735ea20c 100644
--- a/youtube_dl/extractor/wrzuta.py
+++ b/youtube_dl/extractor/wrzuta.py
@@ -27,15 +27,15 @@ class WrzutaIE(InfoExtractor):
             'description': 'md5:7fb5ef3c21c5893375fda51d9b15d9cd',
         },
     }, {
-        'url': 'http://w729.wrzuta.pl/audio/9oXJqdcndqv/david_guetta_amp_showtek_ft._vassy_-_bad',
-        'md5': '1e546a18e1c22ac6e9adce17b8961ff5',
+        'url': 'http://jolka85.wrzuta.pl/audio/063jOPX5ue2/liber_natalia_szroeder_-_teraz_ty',
+        'md5': 'bc78077859bea7bcfe4295d7d7fc9025',
         'info_dict': {
-            'id': '9oXJqdcndqv',
+            'id': '063jOPX5ue2',
             'ext': 'ogg',
-            'title': 'David Guetta & Showtek ft. Vassy - Bad',
-            'duration': 270,
-            'uploader_id': 'w729',
-            'description': 'md5:4628f01c666bbaaecefa83476cfa794a',
+            'title': 'Liber & Natalia Szroeder - Teraz Ty',
+            'duration': 203,
+            'uploader_id': 'jolka85',
+            'description': 'md5:2d2b6340f9188c8c4cd891580e481096',
         },
     }]
 
@@ -49,16 +49,17 @@ class WrzutaIE(InfoExtractor):
 
         quality = qualities(['SD', 'MQ', 'HQ', 'HD'])
 
-        audio_table = {'flv': 'mp3', 'webm': 'ogg', 'mp3': 'mp3'}
+        audio_table = {'flv': 'mp3', 'webm': 'ogg'}
 
         embedpage = self._download_json('http://www.wrzuta.pl/npp/embed/%s/%s' % (uploader, video_id), video_id)
 
         formats = []
         for media in embedpage['url']:
+            fmt = media['type'].split('@')[0]
             if typ == 'audio':
-                ext = audio_table[media['type'].split('@')[0]]
+                ext = audio_table.get(fmt, fmt)
             else:
-                ext = media['type'].split('@')[0]
+                ext = fmt
 
             formats.append({
                 'format_id': '%s_%s' % (ext, media['quality'].lower()),

From bbcc21efd1e5efd81dbd1fdfa16661cbb609c152 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 11 Nov 2014 16:47:54 +0200
Subject: [PATCH 0445/1937] [wrzuta] Fallback to mp3 on unknown media type
 (#4156)

---
 youtube_dl/extractor/wrzuta.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wrzuta.py b/youtube_dl/extractor/wrzuta.py
index 1735ea20c..c42764921 100644
--- a/youtube_dl/extractor/wrzuta.py
+++ b/youtube_dl/extractor/wrzuta.py
@@ -49,7 +49,7 @@ class WrzutaIE(InfoExtractor):
 
         quality = qualities(['SD', 'MQ', 'HQ', 'HD'])
 
-        audio_table = {'flv': 'mp3', 'webm': 'ogg'}
+        audio_table = {'flv': 'mp3', 'webm': 'ogg', '???': 'mp3'}
 
         embedpage = self._download_json('http://www.wrzuta.pl/npp/embed/%s/%s' % (uploader, video_id), video_id)
 

From 2774852c2f69284966b84b00b221852c3041db62 Mon Sep 17 00:00:00 2001
From: Pascal Brax <pascalbrax+github@gmail.com>
Date: Wed, 12 Nov 2014 01:10:08 +0100
Subject: [PATCH 0446/1937] Fix MTV/GameTrailers "Bad Request" error

Bugfix for bug #4123 & #4153
---
 youtube_dl/extractor/mtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 228b42d2b..f0e28e66a 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -33,7 +33,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp\..+?/.*)$', rtmp_video_url)
         if not m:
             return rtmp_video_url
-        base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/'
+        base = 'http://viacommtvstrmfs.fplive.net/'
         return base + m.group('finalid')
 
     def _get_feed_url(self, uri):

From 7a8cbc72b23e312e1d9b00709c7cc68d5222dd2c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 12 Nov 2014 08:46:34 +0100
Subject: [PATCH 0447/1937] release 2014.11.12

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9b2888b1b..2f13dde4d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.09'
+__version__ = '2014.11.12'

From eb4cb42a028f1050f8c64d9efdd865b2b782e929 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 12 Nov 2014 09:30:57 +0100
Subject: [PATCH 0448/1937] [ted] Extract duration (closes #4155)

---
 youtube_dl/extractor/ted.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index cd4af96fd..855038077 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -38,6 +38,7 @@ class TEDIE(SubtitlesInfoExtractor):
                 'actively fooling us.'),
             'uploader': 'Dan Dennett',
             'width': 854,
+            'duration': 1308,
         }
     }, {
         'url': 'http://www.ted.com/watch/ted-institute/ted-bcg/vishal-sikka-the-beauty-and-power-of-algorithms',
@@ -57,6 +58,7 @@ class TEDIE(SubtitlesInfoExtractor):
             'title': 'Be passionate. Be courageous. Be your best.',
             'uploader': 'Gabby Giffords and Mark Kelly',
             'description': 'md5:5174aed4d0f16021b704120360f72b92',
+            'duration': 1128,
         },
     }, {
         'url': 'http://www.ted.com/playlists/who_are_the_hackers',
@@ -178,6 +180,7 @@ class TEDIE(SubtitlesInfoExtractor):
             'description': self._og_search_description(webpage),
             'subtitles': video_subtitles,
             'formats': formats,
+            'duration': talk_info.get('duration'),
         }
 
     def _get_available_subtitles(self, video_id, talk_info):

From c295490830d46379e89ce2128dc18a6cb2db3b72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 12 Nov 2014 09:42:35 +0100
Subject: [PATCH 0449/1937] [YoutubeDL] Fix bug in the detection of formats
 that don't contain video (fixes #4150)

If the format requested was not available, we called the method '.get' in None.
---
 youtube_dl/YoutubeDL.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 9fdeb4cfb..7951001fa 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -837,14 +837,14 @@ class YoutubeDL(object):
                         format_1, format_2 = rf.split('+')
                         formats_info = (self.select_format(format_1, formats),
                             self.select_format(format_2, formats))
-                        # The first format must contain the video and the
-                        # second the audio
-                        if formats_info[0].get('vcodec') == 'none':
-                            self.report_error('The first format must contain '
-                                'the video, try using "-f %s+%s"' %
-                                (format_2, format_1))
-                            return
                         if all(formats_info):
+                            # The first format must contain the video and the
+                            # second the audio
+                            if formats_info[0].get('vcodec') == 'none':
+                                self.report_error('The first format must '
+                                    'contain the video, try using '
+                                    '"-f %s+%s"' % (format_2, format_1))
+                                return
                             selected_format = {
                                 'requested_formats': formats_info,
                                 'format': rf,

From 8f6ec4bbe6e867f3ec6adcdcc8d48ebbe89eec7b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 12 Nov 2014 11:44:26 +0100
Subject: [PATCH 0450/1937] release 2014.11.12.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2f13dde4d..473916f7a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.12'
+__version__ = '2014.11.12.1'

From 13ba3a6461a2f02ae918f6aa7df0a37854dddef6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 12 Nov 2014 15:00:54 +0100
Subject: [PATCH 0451/1937] [bandcamp:album] Fix test case

---
 youtube_dl/extractor/bandcamp.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index c13446665..1b8da43ca 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -110,20 +110,25 @@ class BandcampAlbumIE(InfoExtractor):
         'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
         'playlist': [
             {
-                'file': '1353101989.mp3',
                 'md5': '39bc1eded3476e927c724321ddf116cf',
                 'info_dict': {
+                    'id': '1353101989',
+                    'ext': 'mp3',
                     'title': 'Intro',
                 }
             },
             {
-                'file': '38097443.mp3',
                 'md5': '1a2c32e2691474643e912cc6cd4bffaa',
                 'info_dict': {
+                    'id': '38097443',
+                    'ext': 'mp3',
                     'title': 'Kero One - Keep It Alive (Blazo remix)',
                 }
             },
         ],
+        'info_dict': {
+            'title': 'Jazz Format Mixtape vol.1',
+        },
         'params': {
             'playlistend': 2
         },

From ef89dba58f2ea60cce8d952941355182449bc301 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 12 Nov 2014 15:01:48 +0100
Subject: [PATCH 0452/1937] [myspass] Modernize test case

---
 youtube_dl/extractor/myspass.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/myspass.py b/youtube_dl/extractor/myspass.py
index 4fa0575f8..51e540814 100644
--- a/youtube_dl/extractor/myspass.py
+++ b/youtube_dl/extractor/myspass.py
@@ -13,9 +13,10 @@ class MySpassIE(InfoExtractor):
     _VALID_URL = r'http://www\.myspass\.de/.*'
     _TEST = {
         'url': 'http://www.myspass.de/myspass/shows/tvshows/absolute-mehrheit/Absolute-Mehrheit-vom-17022013-Die-Highlights-Teil-2--/11741/',
-        'file': '11741.mp4',
         'md5': '0b49f4844a068f8b33f4b7c88405862b',
         'info_dict': {
+            'id': '11741',
+            'ext': 'mp4',
             "description": "Wer kann in die Fu\u00dfstapfen von Wolfgang Kubicki treten und die Mehrheit der Zuschauer hinter sich versammeln? Wird vielleicht sogar die Absolute Mehrheit geknackt und der Jackpot von 200.000 Euro mit nach Hause genommen?",
             "title": "Absolute Mehrheit vom 17.02.2013 - Die Highlights, Teil 2",
         },

From 28465df1ff4ad643bc201ed9a0b330327e79469b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 12 Nov 2014 15:19:23 +0100
Subject: [PATCH 0453/1937] [youjizz] Modernize (#4131)

---
 youtube_dl/extractor/youjizz.py | 32 +++++++++++---------------------
 1 file changed, 11 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/youjizz.py b/youtube_dl/extractor/youjizz.py
index b86331e3c..c642075dc 100644
--- a/youtube_dl/extractor/youjizz.py
+++ b/youtube_dl/extractor/youjizz.py
@@ -9,40 +9,30 @@ from ..utils import (
 
 
 class YouJizzIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:\w+\.)?youjizz\.com/videos/(?P<videoid>[^.]+)\.html$'
+    _VALID_URL = r'https?://(?:\w+\.)?youjizz\.com/videos/[^/#?]+-(?P<id>[0-9]+)\.html(?:$|[?#])'
     _TEST = {
         'url': 'http://www.youjizz.com/videos/zeichentrick-1-2189178.html',
-        'file': '2189178.flv',
         'md5': '07e15fa469ba384c7693fd246905547c',
         'info_dict': {
+            'id': '2189178',
+            'ext': 'flv',
             "title": "Zeichentrick 1",
             "age_limit": 18,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-
-        video_id = mobj.group('videoid')
-
-        # Get webpage content
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-
         age_limit = self._rta_search(webpage)
+        video_title = self._html_search_regex(
+            r'<title>\s*(.*)\s*</title>', webpage, 'title')
 
-        # Get the video title
-        video_title = self._html_search_regex(r'<title>(?P<title>.*)</title>',
-            webpage, 'title').strip()
-
-        # Get the embed page
-        result = re.search(r'https?://www.youjizz.com/videos/embed/(?P<videoid>[0-9]+)', webpage)
-        if result is None:
-            raise ExtractorError('ERROR: unable to extract embed page')
-
-        embed_page_url = result.group(0).strip()
-        video_id = result.group('videoid')
-
-        webpage = self._download_webpage(embed_page_url, video_id)
+        embed_page_url = self._search_regex(
+            r'(https?://www.youjizz.com/videos/embed/[0-9]+)',
+            webpage, 'embed page')
+        webpage = self._download_webpage(
+            embed_page_url, video_id, note='downloading embed page')
 
         # Get the video URL
         m_playlist = re.search(r'so.addVariable\("playlist", ?"(?P<playlist>.+?)"\);', webpage)

From 498942f187d248271366068f684755a64cc62481 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Michael=20K=C3=A4ufl?= <youtube-dl@c.michael-kaeufl.de>
Date: Wed, 5 Nov 2014 01:47:49 +0100
Subject: [PATCH 0454/1937] [test_youtube_signature] Fix import

Broken in commit 8c25f81beea169c9d6540eea1a6f71dc045da6ed
---
 test/test_youtube_signature.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index df2cb09f2..13d228cd8 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -14,7 +14,7 @@ import re
 import string
 
 from youtube_dl.extractor import YoutubeIE
-from youtube_dl.utils import compat_str, compat_urlretrieve
+from youtube_dl.compat import compat_str, compat_urlretrieve
 
 _TESTS = [
     (

From 7009a9047ac57b4826623b0c22c6956ebf9ac902 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Michael=20K=C3=A4ufl?= <youtube-dl@c.michael-kaeufl.de>
Date: Wed, 5 Nov 2014 15:43:53 +0100
Subject: [PATCH 0455/1937] [byutv] Update test

---
 youtube_dl/extractor/byutv.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/byutv.py b/youtube_dl/extractor/byutv.py
index cf19b7b0c..6252be05b 100644
--- a/youtube_dl/extractor/byutv.py
+++ b/youtube_dl/extractor/byutv.py
@@ -10,12 +10,12 @@ from ..utils import ExtractorError
 class BYUtvIE(InfoExtractor):
     _VALID_URL = r'^https?://(?:www\.)?byutv.org/watch/[0-9a-f-]+/(?P<video_id>[^/?#]+)'
     _TEST = {
-        'url': 'http://www.byutv.org/watch/44e80f7b-e3ba-43ba-8c51-b1fd96c94a79/granite-flats-talking',
+        'url': 'http://www.byutv.org/watch/6587b9a3-89d2-42a6-a7f7-fd2f81840a7d/studio-c-season-5-episode-5',
         'info_dict': {
-            'id': 'granite-flats-talking',
+            'id': 'studio-c-season-5-episode-5',
             'ext': 'mp4',
-            'description': 'md5:4e9a7ce60f209a33eca0ac65b4918e1c',
-            'title': 'Talking',
+            'description': 'md5:5438d33774b6bdc662f9485a340401cc',
+            'title': 'Season 5 Episode 5',
             'thumbnail': 're:^https?://.*promo.*'
         },
         'params': {

From 9d2a4dae90383e5da73f7e30ef83479b82c446b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Michael=20K=C3=A4ufl?= <youtube-dl@c.michael-kaeufl.de>
Date: Wed, 5 Nov 2014 14:39:18 +0100
Subject: [PATCH 0456/1937] [allocine] Update test

---
 youtube_dl/extractor/allocine.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/allocine.py b/youtube_dl/extractor/allocine.py
index 7bd797884..398e93bfb 100644
--- a/youtube_dl/extractor/allocine.py
+++ b/youtube_dl/extractor/allocine.py
@@ -22,7 +22,7 @@ class AllocineIE(InfoExtractor):
             'id': '19546517',
             'ext': 'mp4',
             'title': 'Astérix - Le Domaine des Dieux Teaser VF',
-            'description': 'md5:4a754271d9c6f16c72629a8a993ee884',
+            'description': 'md5:abcd09ce503c6560512c14ebfdb720d2',
             'thumbnail': 're:http://.*\.jpg',
         },
     }, {

From 732c848c14af4b0c1c80176cc99588049dc837b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Michael=20K=C3=A4ufl?= <youtube-dl@c.michael-kaeufl.de>
Date: Wed, 5 Nov 2014 14:30:39 +0100
Subject: [PATCH 0457/1937] [abc] Update test case

Old video has expired.
---
 youtube_dl/extractor/abc.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py
index 69f89320c..dc0fb85d6 100644
--- a/youtube_dl/extractor/abc.py
+++ b/youtube_dl/extractor/abc.py
@@ -11,13 +11,13 @@ class ABCIE(InfoExtractor):
     _VALID_URL = r'http://www\.abc\.net\.au/news/[^/]+/[^/]+/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://www.abc.net.au/news/2014-07-25/bringing-asylum-seekers-to-australia-would-give/5624716',
-        'md5': 'dad6f8ad011a70d9ddf887ce6d5d0742',
+        'url': 'http://www.abc.net.au/news/2014-11-05/australia-to-staff-ebola-treatment-centre-in-sierra-leone/5868334',
+        'md5': 'cb3dd03b18455a661071ee1e28344d9f',
         'info_dict': {
-            'id': '5624716',
+            'id': '5868334',
             'ext': 'mp4',
-            'title': 'Bringing asylum seekers to Australia would give them right to asylum claims: professor',
-            'description': 'md5:ba36fa5e27e5c9251fd929d339aea4af',
+            'title': 'Australia to help staff Ebola treatment centre in Sierra Leone',
+            'description': 'md5:809ad29c67a05f54eb41f2a105693a67',
         },
     }
 

From c6afed48ff0a5b42d14227bd7e52e009345776e6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 12 Nov 2014 15:30:26 +0100
Subject: [PATCH 0458/1937] [YoutubeDL] guard against strange sys.stdouts

---
 youtube_dl/YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 7951001fa..5c875b497 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1306,11 +1306,13 @@ class YoutubeDL(object):
             self.report_warning(
                 'Your Python is broken! Update to a newer and supported version')
 
+        stdout_encoding = getattr(
+            sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
         encoding_str = (
             '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
                 locale.getpreferredencoding(),
                 sys.getfilesystemencoding(),
-                sys.stdout.encoding,
+                stdout_encoding,
                 self.get_encoding()))
         write_string(encoding_str, encoding=None)
 

From ffe38646cafe890aafa1f6627d14945adc8cf0b3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 12 Nov 2014 15:33:15 +0100
Subject: [PATCH 0459/1937] [funnyordie] Remove test md5sum (Fixes #4113)

---
 youtube_dl/extractor/funnyordie.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index 487d5d1b4..a49fc1151 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -21,7 +21,6 @@ class FunnyOrDieIE(InfoExtractor):
         },
     }, {
         'url': 'http://www.funnyordie.com/embed/e402820827',
-        'md5': '29f4c5e5a61ca39dfd7e8348a75d0aad',
         'info_dict': {
             'id': 'e402820827',
             'ext': 'mp4',

From 5ccaddf5b150fc113f5b423f6d2f8b7e9a30cc83 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 12 Nov 2014 15:36:59 +0100
Subject: [PATCH 0460/1937] [goldenmoustache] Simplify (#4128)

---
 youtube_dl/extractor/goldenmoustache.py | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/goldenmoustache.py b/youtube_dl/extractor/goldenmoustache.py
index c25d2e779..f2e231bf4 100644
--- a/youtube_dl/extractor/goldenmoustache.py
+++ b/youtube_dl/extractor/goldenmoustache.py
@@ -18,26 +18,24 @@ class GoldenMoustacheIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Suricate - Le Poker',
             'description': 'md5:3d1f242f44f8c8cb0a106f1fd08e5dc9',
-            'thumbnail': 'md5:fd41386bc1f932552622da4a7e9a7242',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'view_count': int,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        video_url = self._html_search_regex(r'data-src-type="mp4" data-src="([^"]+)"', webpage, 'video URL')
-
-        title = self._html_search_regex(r'<title>(.*?) - Golden Moustache</title>', webpage, 'title')
-
-        thumbnail = self._html_search_meta('og:image', webpage, 'thumbnail')
-         
-        description = self._html_search_meta('og:description', webpage, 'description')
-
+        video_url = self._html_search_regex(
+            r'data-src-type="mp4" data-src="([^"]+)"', webpage, 'video URL')
+        title = self._html_search_regex(
+            r'<title>(.*?) - Golden Moustache</title>', webpage, 'title')
+        thumbnail = self._og_search_thumbnail(webpage)
+        description = self._og_search_description(webpage)
         view_count = int_or_none(self._html_search_regex(
-            r'<strong>(\d+)</strong>\s*VUES</span>', webpage, 'view count', fatal=False))
+            r'<strong>([0-9]+)</strong>\s*VUES</span>',
+            webpage, 'view count', fatal=False))
 
         return {
             'id': video_id,

From 4b50ba0989f4c3ab82321ede0eb08391efd0e39f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 12 Nov 2014 15:53:00 +0100
Subject: [PATCH 0461/1937] Credit @xantares for goldenmoustache (#4128)

---
 AUTHORS | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/AUTHORS b/AUTHORS
index a574949b3..48189e26c 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -79,4 +79,5 @@ Dennis Scheiba
 Damon Timm
 winwon
 Xavier Beynon
-Gabriel Schubiner
\ No newline at end of file
+Gabriel Schubiner
+xantares

From 73689dafbfb394a7820a9917b044d37f6392ce01 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 12 Nov 2014 19:26:00 +0200
Subject: [PATCH 0462/1937] [tvplay] Fix f4m URL extraction (Closes #4119)

Add query parameters which are needed by AkamaiHD F4M player.
Also, modernize a bit.
---
 youtube_dl/extractor/tvplay.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 445e0ec41..eb9473754 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -4,9 +4,9 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
-    compat_str,
     parse_iso8601,
     qualities,
 )
@@ -176,8 +176,7 @@ class TVPlayIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         video = self._download_json(
             'http://playapi.mtgx.tv/v1/videos/%s' % video_id, video_id, 'Downloading video JSON')
@@ -208,6 +207,10 @@ class TVPlayIE(InfoExtractor):
                     'app': m.group('app'),
                     'play_path': m.group('playpath'),
                 })
+            elif video_url.endswith('.f4m'):
+                formats.extend(self._extract_f4m_formats(
+                    video_url + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81', video_id))
+                continue
             else:
                 fmt.update({
                     'url': video_url,

From bbd5f2de5ee970599b03796e029676ec03f510e4 Mon Sep 17 00:00:00 2001
From: peugeot <lucien.judert@gmail.com>
Date: Wed, 12 Nov 2014 20:41:13 +0100
Subject: [PATCH 0463/1937] [sexu] initial support

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/sexu.py     | 61 ++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)
 create mode 100644 youtube_dl/extractor/sexu.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 21853ccf4..c15b1add8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -324,6 +324,7 @@ from .sbs import SBSIE
 from .scivee import SciVeeIE
 from .screencast import ScreencastIE
 from .servingsys import ServingSysIE
+from .sexu import SexuIE
 from .sexykarma import SexyKarmaIE
 from .shared import SharedIE
 from .sharesix import ShareSixIE
diff --git a/youtube_dl/extractor/sexu.py b/youtube_dl/extractor/sexu.py
new file mode 100644
index 000000000..7002643bc
--- /dev/null
+++ b/youtube_dl/extractor/sexu.py
@@ -0,0 +1,61 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class SexuIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?sexu\.com/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://sexu.com/961791/',
+        'md5': 'ff615aca9691053c94f8f10d96cd7884',
+        'info_dict': {
+            'id': '961791',
+            'ext': 'mp4',
+            'title': str, # NSFW
+            'description': 'md5:c5ed8625eb386855d5a7967bd7b77a54',
+            'categories': list,  # NSFW
+            'thumbnail': 're:https?://.*\.jpg$',
+            'age_limit': 18,
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        webpage = self._download_webpage(url, video_id)
+
+        quality_arr = self._search_regex(r'sources:\s*\[([^\]]+)\]', webpage, 'quality formats')
+
+        formats = [{
+            'url': fmt[0].replace('\\', ''),
+            'format_id': fmt[1],
+            'height': int(fmt[1][:3]),
+        } for fmt in re.findall(r'"file":"([^"]+)","label":"([^"]+)"', quality_arr)]
+        self._sort_formats(formats)
+
+        title = self._html_search_regex(
+            r'<title>([^<]+)\s*-\s*Sexu.Com</title>', webpage, 'title')
+
+        description = self._html_search_meta('description', webpage, 'description')
+
+        thumbnail = self._html_search_regex(
+            r'image:\s*"([^"]+)"',
+            webpage, 'thumbnail', fatal=False)
+
+        categories_str = self._html_search_meta('keywords', webpage, 'categories', fatal=False)
+        categories = (
+            None if categories_str is None
+            else categories_str.split(','))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'categories': categories,
+            'formats': formats,
+            'age_limit': 18,
+        }

From 9e05d039e0e0eff90622ead186299192f3f87b18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 12 Nov 2014 23:32:27 +0100
Subject: [PATCH 0464/1937] [dailymotion] Fix extraction of vevo videos (fixes
 #4168)

---
 youtube_dl/extractor/dailymotion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index dbcf5d6a7..fd4bc75b2 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -94,7 +94,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
 
         # It may just embed a vevo video:
         m_vevo = re.search(
-            r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?videoId=(?P<id>[\w]*)',
+            r'<link rel="video_src" href="[^"]*?vevo.com[^"]*?video=(?P<id>[\w]*)',
             webpage)
         if m_vevo is not None:
             vevo_id = m_vevo.group('id')

From 894dd8682e07be0d85d1eb2401b9f99ce056e809 Mon Sep 17 00:00:00 2001
From: Aaron Wojnowski <aaronwojnowski@gmail.com>
Date: Thu, 13 Nov 2014 00:33:27 -0600
Subject: [PATCH 0465/1937] Fix YouTube signature extraction.

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5406de50b..cea3d5c95 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -510,7 +510,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
     def _parse_sig_js(self, jscode):
         funcname = self._search_regex(
-            r'signature=([$a-zA-Z]+)', jscode,
+            r'\.sig\|\|([a-zA-Z0-9]+)\(', jscode,
              'Initial JS player signature function name')
 
         jsi = JSInterpreter(jscode)

From 2d42905b6863cab63d6ce456cb2e0de865bb6950 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 13 Nov 2014 09:57:58 +0100
Subject: [PATCH 0466/1937] release 2014.11.13

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 473916f7a..d0e215393 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.12.1'
+__version__ = '2014.11.13'

From d91d12408165a045909c8c63a19dd9dc43d72d7f Mon Sep 17 00:00:00 2001
From: peugeot <lucien.judert@gmail.com>
Date: Thu, 13 Nov 2014 13:57:10 +0100
Subject: [PATCH 0467/1937] fix python 2 test

---
 youtube_dl/extractor/sexu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sexu.py b/youtube_dl/extractor/sexu.py
index 7002643bc..f45d17089 100644
--- a/youtube_dl/extractor/sexu.py
+++ b/youtube_dl/extractor/sexu.py
@@ -13,7 +13,7 @@ class SexuIE(InfoExtractor):
         'info_dict': {
             'id': '961791',
             'ext': 'mp4',
-            'title': str, # NSFW
+            'title': 'md5:033cf4a71907aac2758e82f55894a129',
             'description': 'md5:c5ed8625eb386855d5a7967bd7b77a54',
             'categories': list,  # NSFW
             'thumbnail': 're:https?://.*\.jpg$',

From 437f68d868176e7e3d56a8e2c258384b984baeec Mon Sep 17 00:00:00 2001
From: peugeot <lucien.judert@gmail.com>
Date: Thu, 13 Nov 2014 14:02:53 +0100
Subject: [PATCH 0468/1937] Update sexu.py

---
 youtube_dl/extractor/sexu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sexu.py b/youtube_dl/extractor/sexu.py
index f45d17089..45b33b3ea 100644
--- a/youtube_dl/extractor/sexu.py
+++ b/youtube_dl/extractor/sexu.py
@@ -13,7 +13,7 @@ class SexuIE(InfoExtractor):
         'info_dict': {
             'id': '961791',
             'ext': 'mp4',
-            'title': 'md5:033cf4a71907aac2758e82f55894a129',
+            'title': 'md5:4d05a19a5fc049a63dbbaf05fb71d91b',
             'description': 'md5:c5ed8625eb386855d5a7967bd7b77a54',
             'categories': list,  # NSFW
             'thumbnail': 're:https?://.*\.jpg$',

From 0e15e725a09e04b8ba47ac7ddfe7d26683db584c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 13 Nov 2014 14:45:17 +0100
Subject: [PATCH 0469/1937] [spiegel] Modernize

---
 youtube_dl/extractor/spiegel.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index 9ed7d3b39..4b4da43d1 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -8,7 +8,7 @@ from ..utils import compat_urlparse
 
 
 class SpiegelIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<videoID>[0-9]+)(?:\.html)?(?:#.*)?$'
+    _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<id>[0-9]+)(?:\.html)?(?:#.*)?$'
     _TESTS = [{
         'url': 'http://www.spiegel.de/video/vulkan-tungurahua-in-ecuador-ist-wieder-aktiv-video-1259285.html',
         'md5': '2c2754212136f35fb4b19767d242f66e',
@@ -32,9 +32,7 @@ class SpiegelIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('videoID')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(
@@ -90,8 +88,7 @@ class SpiegelArticleIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
         video_link = self._search_regex(

From 3e6e4999ca09352fa513c20bfb5ba7b37811819b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 13 Nov 2014 14:55:45 +0100
Subject: [PATCH 0470/1937] [test/helper] Improve output

---
 test/helper.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index 325f72f0a..8be37a183 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -145,7 +145,8 @@ def expect_info_dict(self, expected_dict, got_dict):
         info_dict_str = ''.join(
             '    %s: %s,\n' % (_repr(k), _repr(v))
             for k, v in test_info_dict.items())
-        write_string('\n"info_dict": {\n' + info_dict_str + '}\n', out=sys.stderr)
+        write_string(
+            '\n\'info_dict\': {\n' + info_dict_str + '}\n', out=sys.stderr)
         self.assertFalse(
             missing_keys,
             'Missing keys in test definition: %s' % (

From e4bdb37ec6c463df236cf7178046b8653c70b78e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 13 Nov 2014 15:02:31 +0100
Subject: [PATCH 0471/1937] [spiegel] Add support for embeds

---
 test/test_utils.py              |  5 ++++
 youtube_dl/extractor/spiegel.py | 53 ++++++++++++++++++++++++---------
 2 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index e59547784..0c11d0438 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -16,6 +16,7 @@ import json
 import xml.etree.ElementTree
 
 from youtube_dl.utils import (
+    clean_html,
     DateRange,
     encodeFilename,
     find_xpath_attr,
@@ -345,5 +346,9 @@ class TestUtil(unittest.TestCase):
         on = js_to_json('{"abc": true}')
         self.assertEqual(json.loads(on), {'abc': True})
 
+    def test_clean_html(self):
+        self.assertEqual(clean_html('a:\nb'), 'a: b')
+        self.assertEqual(clean_html('a:\n   "b"'), 'a:    "b"')
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index 4b4da43d1..9586a7da2 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -4,11 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import compat_urlparse
+from ..compat import compat_urlparse
 
 
 class SpiegelIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<id>[0-9]+)(?:\.html)?(?:#.*)?$'
+    _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<id>[0-9]+)(?:-embed)?(?:\.html)?(?:#.*)?$'
     _TESTS = [{
         'url': 'http://www.spiegel.de/video/vulkan-tungurahua-in-ecuador-ist-wieder-aktiv-video-1259285.html',
         'md5': '2c2754212136f35fb4b19767d242f66e',
@@ -29,14 +29,24 @@ class SpiegelIE(InfoExtractor):
             'description': 'md5:c2322b65e58f385a820c10fa03b2d088',
             'duration': 983,
         },
+    }, {
+        'url': 'http://www.spiegel.de/video/astronaut-alexander-gerst-von-der-iss-station-beantwortet-fragen-video-1519126-embed.html',
+        'md5': 'd8eeca6bfc8f1cd6f490eb1f44695d51',
+        'info_dict': {
+            'id': '1519126',
+            'ext': 'mp4',
+            'description': 'SPIEGEL ONLINE-Nutzer durften den deutschen Astronauten Alexander Gerst über sein Leben auf der ISS-Station befragen. Hier kommen seine Antworten auf die besten sechs Fragen.',
+            'title': 'Fragen an Astronaut Alexander Gerst: "Bekommen Sie die Tageszeiten mit?"',
+        }
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(
-            r'<div class="module-title">(.*?)</div>', webpage, 'title')
+        title = re.sub(r'\s+', ' ', self._html_search_regex(
+            r'(?s)<(?:h1|div) class="module-title"[^>]*>(.*?)</(?:h1|div)>',
+            webpage, 'title'))
         description = self._html_search_meta('description', webpage, 'description')
 
         base_url = self._search_regex(
@@ -77,7 +87,7 @@ class SpiegelArticleIE(InfoExtractor):
     _VALID_URL = 'https?://www\.spiegel\.de/(?!video/)[^?#]*?-(?P<id>[0-9]+)\.html'
     IE_NAME = 'Spiegel:Article'
     IE_DESC = 'Articles on spiegel.de'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.spiegel.de/sport/sonst/badminton-wm-die-randsportart-soll-populaerer-werden-a-987092.html',
         'info_dict': {
             'id': '1516455',
@@ -85,19 +95,34 @@ class SpiegelArticleIE(InfoExtractor):
             'title': 'Faszination Badminton: Nennt es bloß nicht Federball',
             'description': 're:^Patrick Kämnitz gehört.{100,}',
         },
-    }
+    }, {
+        'url': 'http://www.spiegel.de/wissenschaft/weltall/astronaut-alexander-gerst-antwortet-spiegel-online-lesern-a-989876.html',
+        'info_dict': {
+
+        },
+        'playlist_count': 6,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         webpage = self._download_webpage(url, video_id)
+
+        # Single video on top of the page
         video_link = self._search_regex(
             r'<a href="([^"]+)" onclick="return spOpenVideo\(this,', webpage,
-            'video page URL')
-        video_url = compat_urlparse.urljoin(
-            self.http_scheme() + '//spiegel.de/', video_link)
+            'video page URL', default=None)
+        if video_link:
+            video_url = compat_urlparse.urljoin(
+                self.http_scheme() + '//spiegel.de/', video_link)
+            return self.url_result(video_url)
 
-        return {
-            '_type': 'url',
-            'url': video_url,
-        }
+        # Multiple embedded videos
+        embeds = re.findall(
+            r'<div class="vid_holder[0-9]+.*?</div>\s*.*?url\s*=\s*"([^"]+)"',
+            webpage)
+        entries = [
+            self.url_result(compat_urlparse.urljoin(
+                self.http_scheme() + '//spiegel.de/', embed_path))
+            for embed_path in embeds
+        ]
+        return self.playlist_result(entries)

From 3fa5bb38028b38a6e65779f0b38407e6a46d15ea Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 13 Nov 2014 15:20:49 +0100
Subject: [PATCH 0472/1937] [sexu] Modernize (#4171)

---
 youtube_dl/extractor/sexu.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/sexu.py b/youtube_dl/extractor/sexu.py
index 45b33b3ea..6365a8779 100644
--- a/youtube_dl/extractor/sexu.py
+++ b/youtube_dl/extractor/sexu.py
@@ -22,13 +22,11 @@ class SexuIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        quality_arr = self._search_regex(r'sources:\s*\[([^\]]+)\]', webpage, 'quality formats')
-
+        quality_arr = self._search_regex(
+            r'sources:\s*\[([^\]]+)\]', webpage, 'forrmat string')
         formats = [{
             'url': fmt[0].replace('\\', ''),
             'format_id': fmt[1],
@@ -37,15 +35,17 @@ class SexuIE(InfoExtractor):
         self._sort_formats(formats)
 
         title = self._html_search_regex(
-            r'<title>([^<]+)\s*-\s*Sexu.Com</title>', webpage, 'title')
+            r'<title>([^<]+)\s*-\s*Sexu\.Com</title>', webpage, 'title')
 
-        description = self._html_search_meta('description', webpage, 'description')
+        description = self._html_search_meta(
+            'description', webpage, 'description')
 
         thumbnail = self._html_search_regex(
             r'image:\s*"([^"]+)"',
             webpage, 'thumbnail', fatal=False)
 
-        categories_str = self._html_search_meta('keywords', webpage, 'categories', fatal=False)
+        categories_str = self._html_search_meta(
+            'keywords', webpage, 'categories')
         categories = (
             None if categories_str is None
             else categories_str.split(','))

From 4c0924bb24c32e18d1e3c030f8d1faef3c3c56c4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 13 Nov 2014 15:28:42 +0100
Subject: [PATCH 0473/1937] [utils] Fix intlist_to_bytes in Python 2 (#4181)

---
 test/test_utils.py  | 6 ++++++
 youtube_dl/utils.py | 5 +----
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 0c11d0438..a9649397f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -46,6 +46,7 @@ from youtube_dl.utils import (
     escape_url,
     js_to_json,
     get_filesystem_encoding,
+    intlist_to_bytes,
 )
 
 
@@ -350,5 +351,10 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(clean_html('a:\nb'), 'a: b')
         self.assertEqual(clean_html('a:\n   "b"'), 'a:    "b"')
 
+    def test_intlist_to_bytes(self):
+        self.assertEqual(
+            intlist_to_bytes([0, 1, 127, 128, 255]),
+            b'\x00\x01\x7f\x80\xff')
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d87e212ae..0b2ba39e2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -843,10 +843,7 @@ def bytes_to_intlist(bs):
 def intlist_to_bytes(xs):
     if not xs:
         return b''
-    if isinstance(chr(0), bytes):  # Python 2
-        return ''.join([chr(x) for x in xs])
-    else:
-        return bytes(xs)
+    return struct.pack('%dB' % len(xs), *xs)
 
 
 # Cross-platform file locking

From c3d582985f049f611ddd35e6bfde4026cc98da57 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 13 Nov 2014 15:42:48 +0100
Subject: [PATCH 0474/1937] release 2014.11.13.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d0e215393..ed0c828ad 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.13'
+__version__ = '2014.11.13.1'

From 3deed1e91ad4ede72ab0eb222dfd6c54fe963413 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 13 Nov 2014 15:59:22 +0100
Subject: [PATCH 0475/1937] [freevideo] Simplify and raise error for foreigners
 (Fixes #4131)

---
 youtube_dl/extractor/freevideo.py | 45 ++++++++++++-------------------
 1 file changed, 17 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/freevideo.py b/youtube_dl/extractor/freevideo.py
index f9716c4c8..f755e3c4a 100644
--- a/youtube_dl/extractor/freevideo.py
+++ b/youtube_dl/extractor/freevideo.py
@@ -1,49 +1,38 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-)
+from ..utils import ExtractorError
 
 
 class FreeVideoIE(InfoExtractor):
-    _VALID_URL = r'^http://www.freevideo.cz/vase-videa/(?P<videoid>[^.]+)\.html$'
+    _VALID_URL = r'^http://www.freevideo.cz/vase-videa/(?P<id>[^.]+)\.html(?:$|[?#])'
 
     _TEST = {
         'url': 'http://www.freevideo.cz/vase-videa/vysukany-zadecek-22033.html',
-        'file': 'vysukany-zadecek-22033.mp4',
         'info_dict': {
+            'id': 'vysukany-zadecek-22033',
+            'ext': 'mp4',
             "title": "vysukany-zadecek-22033",
             "age_limit": 18,
-        }
+        },
+        'skip': 'Blocked outside .cz',
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError('Invalid search query "%s"' % query)
+        video_id = self._match_id(url)
+        webpage, handle = self._download_webpage_handle(url, video_id)
+        if '//www.czechav.com/' in handle.geturl():
+            raise ExtractorError(
+                'Access to freevideo is blocked from your location',
+                expected=True)
 
-        video_id = mobj.group('videoid')
-
-        # Get webpage content
-        webpage = self._download_webpage(url, video_id)
-
-        age_limit = self._rta_search(webpage)
-        if age_limit == 0:
-            # interpret 0 as mis-detection since this site is adult-content only.
-            # However, if we get non-0, assume the rtalabel started giving proper
-            # results
-            age_limit = 18
-
-        url = re.search(r'\s+url: "(http://[a-z0-9-]+.cdn.freevideo.cz/stream/.*/video.mp4)"', webpage)
-        if url is None:
-            raise ExtractorError('ERROR: unable to extract video url')
+        video_url = self._search_regex(
+            r'\s+url: "(http://[a-z0-9-]+.cdn.freevideo.cz/stream/.*?/video.mp4)"',
+            webpage, 'video URL')
 
         return {
             'id': video_id,
-            'url': url.groups()[0],
+            'url': video_url,
             'title': video_id,
-            'age_limit': age_limit,
+            'age_limit': 18,
         }

From 68d1d41c036b60a34f980a07e0f0adef66dc7864 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 13 Nov 2014 15:59:48 +0100
Subject: [PATCH 0476/1937] Credit @yaccz for freevideo (#4131)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 48189e26c..9f4a4fd03 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -81,3 +81,4 @@ winwon
 Xavier Beynon
 Gabriel Schubiner
 xantares
+Jan Matějka

From af63fed7d87c136a90e10d5632ce5f2e3f2ce8d8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 13 Nov 2014 16:12:51 +0100
Subject: [PATCH 0477/1937] [generic] Add support for livestream embeds (Fixes
 #4185)

---
 youtube_dl/extractor/generic.py    | 18 +++++++++++++++++-
 youtube_dl/extractor/livestream.py |  5 ++++-
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 01d6a57f8..af769ab61 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -434,7 +434,17 @@ class GenericIE(InfoExtractor):
                 'title': 'Chet Chat 171 - Oct 29, 2014',
                 'upload_date': '20141029',
             }
-        }
+        },
+        # Livestream embed
+        {
+            'url': 'http://www.esa.int/Our_Activities/Space_Science/Rosetta/Philae_comet_touch-down_webcast',
+            'info_dict': {
+                'id': '67864563',
+                'ext': 'flv',
+                'upload_date': '20141112',
+                'title': 'Rosetta #CometLanding webcast HL 10',
+            }
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -916,6 +926,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')
 
+        mobj = re.search(
+            r'<iframe[^>]+src="(?P<url>https?://new\.livestream\.com/[^"]+/player[^"]+)"',
+            webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'), 'Livestream')
+
         def check_video(vurl):
             vpath = compat_urlparse.urlparse(vurl).path
             vext = determine_ext(vpath)
diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 363a12ad0..03c4691c6 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -18,7 +18,7 @@ from ..utils import (
 
 class LivestreamIE(InfoExtractor):
     IE_NAME = 'livestream'
-    _VALID_URL = r'http://new\.livestream\.com/.*?/(?P<event_name>.*?)(/videos/(?P<id>\d+))?/?$'
+    _VALID_URL = r'https?://new\.livestream\.com/.*?/(?P<event_name>.*?)(/videos/(?P<id>[0-9]+)(?:/player)?)?/?(?:$|[?#])'
     _TESTS = [{
         'url': 'http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370',
         'md5': '53274c76ba7754fb0e8d072716f2292b',
@@ -37,6 +37,9 @@ class LivestreamIE(InfoExtractor):
             'title': 'TEDCity2.0 (English)',
         },
         'playlist_mincount': 4,
+    }, {
+        'url': 'https://new.livestream.com/accounts/362/events/3557232/videos/67864563/player?autoPlay=false&height=360&mute=false&width=640',
+        'only_matching': True,
     }]
 
     def _parse_smil(self, video_id, smil_url):

From bf951c5e29548cfed80480389762edd29fcc8825 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 13 Nov 2014 16:12:54 +0100
Subject: [PATCH 0478/1937] release 2014.11.13.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index ed0c828ad..3e0849a8b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.13.1'
+__version__ = '2014.11.13.2'

From 609a61e3e6fffce3d45e845f33ae2c5fa2d432ac Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 13 Nov 2014 16:28:05 +0100
Subject: [PATCH 0479/1937] [npo] Improve npo.nl (Fixes #4173)

---
 test/test_utils.py          | 4 ++++
 youtube_dl/extractor/npo.py | 3 ++-
 youtube_dl/utils.py         | 3 ++-
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index a9649397f..edd95c3e3 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -284,6 +284,10 @@ class TestUtil(unittest.TestCase):
         d = json.loads(stripped)
         self.assertEqual(d, [{"id": "532cb", "x": 3}])
 
+        stripped = strip_jsonp('parseMetadata({"STATUS":"OK"})\n\n\n//epc')
+        d = json.loads(stripped)
+        self.assertEqual(d, {'STATUS': 'OK'})
+
     def test_uppercase_escape(self):
         self.assertEqual(uppercase_escape('aä'), 'aä')
         self.assertEqual(uppercase_escape('\\U0001d550'), '𝕐')
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index f36d446d2..ce31694a5 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -7,6 +7,7 @@ from ..utils import (
     unified_strdate,
     parse_duration,
     qualities,
+    strip_jsonp,
     url_basename,
 )
 
@@ -63,7 +64,7 @@ class NPOIE(InfoExtractor):
             'http://e.omroep.nl/metadata/aflevering/%s' % video_id,
             video_id,
             # We have to remove the javascript callback
-            transform_source=lambda j: re.sub(r'parseMetadata\((.*?)\);\n//.*$', r'\1', j)
+            transform_source=strip_jsonp,
         )
         token_page = self._download_webpage(
             'http://ida.omroep.nl/npoplayer/i.js',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 0b2ba39e2..5786fa699 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1331,7 +1331,8 @@ def parse_age_limit(s):
 
 
 def strip_jsonp(code):
-    return re.sub(r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?\s*$', r'\1', code)
+    return re.sub(
+        r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
 
 
 def js_to_json(code):

From 69ede8ef813bc686384a230bf461adfdd7f218c0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 13 Nov 2014 16:28:24 +0100
Subject: [PATCH 0480/1937] release 2014.11.13.3

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3e0849a8b..6af578493 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.13.2'
+__version__ = '2014.11.13.3'

From eb4157fd173ced14753b92c943dc3c3ab0b5b67f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 14 Nov 2014 00:39:32 +0100
Subject: [PATCH 0481/1937] [utils] Fix struct.pack call on very old Python
 versions (#4181)

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5786fa699..7460d8701 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -843,7 +843,7 @@ def bytes_to_intlist(bs):
 def intlist_to_bytes(xs):
     if not xs:
         return b''
-    return struct.pack('%dB' % len(xs), *xs)
+    return struct_pack('%dB' % len(xs), *xs)
 
 
 # Cross-platform file locking

From 5fee0eeac04d0d63c1c1b78a35c5ed8b086f7d0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 14 Nov 2014 18:34:44 +0100
Subject: [PATCH 0482/1937] [ComedyCentralShows] Use the rtmp urls transform
 function from the MTV IE (fixes #3364)

It produces the right mp4 urls, so we stop prefering the rtmp urls.
---
 youtube_dl/extractor/comedycentral.py | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 035046120..49b978b4e 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -31,7 +31,7 @@ class ComedyCentralIE(MTVServicesInfoExtractor):
     }
 
 
-class ComedyCentralShowsIE(InfoExtractor):
+class ComedyCentralShowsIE(MTVServicesInfoExtractor):
     IE_DESC = 'The Daily Show / The Colbert Report'
     # urls can be abbreviations like :thedailyshow or :colbert
     # urls for episodes like:
@@ -109,14 +109,6 @@ class ComedyCentralShowsIE(InfoExtractor):
         '400': (384, 216),
     }
 
-    @staticmethod
-    def _transform_rtmp_url(rtmp_video_url):
-        m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp\.comedystor/.*)$', rtmp_video_url)
-        if not m:
-            raise ExtractorError('Cannot transform RTMP url')
-        base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/'
-        return base + m.group('finalid')
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url, re.VERBOSE)
         if mobj is None:
@@ -212,9 +204,6 @@ class ComedyCentralShowsIE(InfoExtractor):
                     'ext': self._video_extensions.get(format, 'mp4'),
                     'height': h,
                     'width': w,
-
-                    'format_note': 'HTTP 400 at the moment (patches welcome!)',
-                    'preference': -100,
                 })
                 formats.append({
                     'format_id': 'rtmp-%s' % format,

From 71f8c7ce7a03ec3d7a94f6733c5bde5536ce2bd8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 14 Nov 2014 19:02:18 +0100
Subject: [PATCH 0483/1937] [mtvservices:embedded] Improve config url (fixes
 #4092)

---
 youtube_dl/extractor/mtv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index f0e28e66a..474bdff7d 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -186,7 +186,8 @@ class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):
     def _get_feed_url(self, uri):
         video_id = self._id_from_uri(uri)
         site_id = uri.replace(video_id, '')
-        config_url = 'http://media.mtvnservices.com/pmt/e1/players/{0}/config.xml'.format(site_id)
+        config_url = ('http://media.mtvnservices.com/pmt/e1/players/{0}/'
+            'context4/context5/config.xml'.format(site_id))
         config_doc = self._download_xml(config_url, video_id)
         feed_node = config_doc.find('.//feed')
         feed_url = feed_node.text.strip().split('?')[0]

From c735450e079d7b53ffe39ba76de472b797b1c4d8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 14 Nov 2014 22:27:56 +0100
Subject: [PATCH 0484/1937] release 2014.11.14

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6af578493..023c7e691 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.13.3'
+__version__ = '2014.11.14'

From 0cc8888038b31c74c4b3dfa14f01cefc52c21335 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 15 Nov 2014 00:34:03 +0100
Subject: [PATCH 0485/1937] [crunchyroll] Remove NOP code (#2782)

---
 youtube_dl/extractor/crunchyroll.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 0bd0eccba..fe1324fe3 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -264,8 +264,6 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             if not lang_code:
                 continue
             sub_root = xml.etree.ElementTree.fromstring(subtitle)
-            if not sub_root:
-                subtitles[lang_code] = ''
             if sub_format == 'ass':
                 subtitles[lang_code] = self._convert_subtitles_to_ass(sub_root)
             else:

From 00d9ef0b708de7723c7119eada93038ad15155bb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 15 Nov 2014 11:01:10 +0100
Subject: [PATCH 0486/1937] [mailru] Adapt to new data format (Fixes #4201)

---
 youtube_dl/extractor/mailru.py | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/mailru.py b/youtube_dl/extractor/mailru.py
index 7460d81cd..54a14cb94 100644
--- a/youtube_dl/extractor/mailru.py
+++ b/youtube_dl/extractor/mailru.py
@@ -16,7 +16,7 @@ class MailRuIE(InfoExtractor):
             'url': 'http://my.mail.ru/video/top#video=/mail/sonypicturesrus/75/76',
             'md5': 'dea205f03120046894db4ebb6159879a',
             'info_dict': {
-                'id': '46301138',
+                'id': '46301138_76',
                 'ext': 'mp4',
                 'title': 'Новый Человек-Паук. Высокое напряжение. Восстание Электро',
                 'timestamp': 1393232740,
@@ -30,7 +30,7 @@ class MailRuIE(InfoExtractor):
             'url': 'http://my.mail.ru/corp/hitech/video/news_hi-tech_mail_ru/1263.html',
             'md5': '00a91a58c3402204dcced523777b475f',
             'info_dict': {
-                'id': '46843144',
+                'id': '46843144_1263',
                 'ext': 'mp4',
                 'title': 'Samsung Galaxy S5 Hammer Smash Fail Battery Explosion',
                 'timestamp': 1397217632,
@@ -54,33 +54,36 @@ class MailRuIE(InfoExtractor):
 
         author = video_data['author']
         uploader = author['name']
-        uploader_id = author['id']
+        uploader_id = author.get('id') or author.get('email')
+        view_count = video_data.get('views_count')
 
-        movie = video_data['movie']
-        content_id = str(movie['contentId'])
-        title = movie['title']
+        meta_data = video_data['meta']
+        content_id = '%s_%s' % (
+            meta_data.get('accId', ''), meta_data['itemId'])
+        title = meta_data['title']
         if title.endswith('.mp4'):
             title = title[:-4]
-        thumbnail = movie['poster']
-        duration = movie['duration']
-
-        view_count = video_data['views_count']
+        thumbnail = meta_data['poster']
+        duration = meta_data['duration']
+        timestamp = meta_data['timestamp']
 
         formats = [
             {
                 'url': video['url'],
-                'format_id': video['name'],
+                'format_id': video['key'],
+                'height': int(video['key'].rstrip('p'))
             } for video in video_data['videos']
         ]
+        self._sort_formats(formats)
 
         return {
             'id': content_id,
             'title': title,
             'thumbnail': thumbnail,
-            'timestamp': video_data['timestamp'],
+            'timestamp': timestamp,
             'uploader': uploader,
             'uploader_id': uploader_id,
             'duration': duration,
             'view_count': view_count,
             'formats': formats,
-        }
\ No newline at end of file
+        }

From a0155d93d9659cfc4f5f723b3e2cd7a97c0a4dc2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 15 Nov 2014 11:01:54 +0100
Subject: [PATCH 0487/1937] release 2014.11.15

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 023c7e691..68b90d7b0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.14'
+__version__ = '2014.11.15'

From c7b0add86f67942b8e68c1b9dd537132e9383d2f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 15 Nov 2014 15:17:19 +0100
Subject: [PATCH 0488/1937] [compat] Work around kwargs bugs in old 2.6 Python
 releases (Fixes #3813)

---
 youtube_dl/compat.py  | 9 +++++++++
 youtube_dl/options.py | 3 ++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index b3752634a..385924803 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -288,6 +288,14 @@ if sys.version_info < (3, 0) and sys.platform == 'win32':
 else:
     compat_getpass = getpass.getpass
 
+# Old 2.6 and 2.7 releases require kwargs to be bytes
+try:
+    (lambda x: x)(**{'x': 0})
+except TypeError:
+    def compat_kwargs(kwargs):
+        return dict((bytes(k), v) for k, v in kwargs.items())
+else:
+    compat_kwargs = lambda kwargs: kwargs
 
 __all__ = [
     'compat_HTTPError',
@@ -299,6 +307,7 @@ __all__ = [
     'compat_html_entities',
     'compat_html_parser',
     'compat_http_client',
+    'compat_kwargs',
     'compat_ord',
     'compat_parse_qs',
     'compat_print',
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index c182abfdc..16aa6ffca 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -8,6 +8,7 @@ import sys
 from .compat import (
     compat_expanduser,
     compat_getenv,
+    compat_kwargs,
 )
 from .utils import (
     get_term_width,
@@ -112,7 +113,7 @@ def parseOpts(overrideArguments=None):
         'conflict_handler': 'resolve',
     }
 
-    parser = optparse.OptionParser(**kw)
+    parser = optparse.OptionParser(**compat_kwargs(kw))
 
     general = optparse.OptionGroup(parser, 'General Options')
     general.add_option(

From bbefcf04bf953a24d3cfaa3ac0eeab04589aed73 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 15 Nov 2014 15:21:34 +0100
Subject: [PATCH 0489/1937] [goldenmoustache] Fix title (Closes #4203)

---
 youtube_dl/extractor/goldenmoustache.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/goldenmoustache.py b/youtube_dl/extractor/goldenmoustache.py
index f2e231bf4..7e13b131b 100644
--- a/youtube_dl/extractor/goldenmoustache.py
+++ b/youtube_dl/extractor/goldenmoustache.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 class GoldenMoustacheIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?goldenmoustache\.com/(?P<display_id>[\w-]+)-(?P<id>\d+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.goldenmoustache.com/suricate-le-poker-3700/',
         'md5': '0f904432fa07da5054d6c8beb5efb51a',
         'info_dict': {
@@ -21,7 +21,18 @@ class GoldenMoustacheIE(InfoExtractor):
             'thumbnail': 're:^https?://.*\.jpg$',
             'view_count': int,
         }
-    }
+    }, {
+        'url': 'http://www.goldenmoustache.com/le-lab-tout-effacer-mc-fly-et-carlito-55249/',
+        'md5': '27f0c50fb4dd5f01dc9082fc67cd5700',
+        'info_dict': {
+            'id': '55249',
+            'ext': 'mp4',
+            'title': 'Le LAB - Tout Effacer (Mc Fly et Carlito)',
+            'description': 'md5:9b7fbf11023fb2250bd4b185e3de3b2a',
+            'thumbnail': 're:^https?://.*\.(?:png|jpg)$',
+            'view_count': int,
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -30,7 +41,7 @@ class GoldenMoustacheIE(InfoExtractor):
         video_url = self._html_search_regex(
             r'data-src-type="mp4" data-src="([^"]+)"', webpage, 'video URL')
         title = self._html_search_regex(
-            r'<title>(.*?) - Golden Moustache</title>', webpage, 'title')
+            r'<title>(.*?)(?: - Golden Moustache)?</title>', webpage, 'title')
         thumbnail = self._og_search_thumbnail(webpage)
         description = self._og_search_description(webpage)
         view_count = int_or_none(self._html_search_regex(

From e2548b5b25da17f5931db1be9a7bddfcb4bab0b6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 15 Nov 2014 15:21:50 +0100
Subject: [PATCH 0490/1937] release 2014.11.15.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 68b90d7b0..6a011846c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.15'
+__version__ = '2014.11.15.1'

From 8caa0c97799dda29fff7d0b0e717e09beee851d3 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 15 Nov 2014 16:56:04 +0200
Subject: [PATCH 0491/1937] [bliptv] Fix the resolve of lookup ID (Closes
 #4197)

---
 youtube_dl/extractor/bliptv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index 57d17bea3..2370c24b0 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -71,11 +71,12 @@ class BlipTVIE(SubtitlesInfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         lookup_id = mobj.group('lookup_id')
 
-        # See https://github.com/rg3/youtube-dl/issues/857
+        # See https://github.com/rg3/youtube-dl/issues/857 and
+        # https://github.com/rg3/youtube-dl/issues/4197
         if lookup_id:
             info_page = self._download_webpage(
                 'http://blip.tv/play/%s.x?p=1' % lookup_id, lookup_id, 'Resolving lookup id')
-            video_id = self._search_regex(r'data-episode-id="([0-9]+)', info_page, 'video_id')
+            video_id = self._search_regex(r'config\.id\s*=\s*"([0-9]+)', info_page, 'video_id')
         else:
             video_id = mobj.group('id')
 

From ec5f601670dfb6c39d3a4669898284bb2782dd0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 15 Nov 2014 22:00:32 +0100
Subject: [PATCH 0492/1937] [utils] Fix "write_json_file" for unicode names in
 python 2.x (fixes #4125)

---
 youtube_dl/utils.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7460d8701..67be4a9ae 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -73,10 +73,22 @@ def preferredencoding():
 def write_json_file(obj, fn):
     """ Encode obj as JSON and write it to fn, atomically """
 
+    if sys.version_info < (3, 0):
+        encoding = get_filesystem_encoding()
+        # os.path.basename returns a bytes object, but NamedTemporaryFile
+        # will fail if the filename contains non ascii characters unless we
+        # use a unicode object
+        path_basename = lambda f: os.path.basename(fn).decode(encoding)
+        # the same for os.path.dirname
+        path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
+    else:
+        path_basename = os.path.basename
+        path_dirname = os.path.dirname
+
     args = {
         'suffix': '.tmp',
-        'prefix': os.path.basename(fn) + '.',
-        'dir': os.path.dirname(fn),
+        'prefix': path_basename(fn) + '.',
+        'dir': path_dirname(fn),
         'delete': False,
     }
 

From 89c15fe0b3044d39caf1c97df523acc33107f399 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 16 Nov 2014 00:33:51 +0100
Subject: [PATCH 0493/1937] [spiegeltv] Modernize

---
 youtube_dl/extractor/spiegeltv.py | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index 7f388aced..06838f78d 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -1,13 +1,13 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
 from .common import InfoExtractor
+from ..utils import float_or_none
 
 
 class SpiegeltvIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?spiegel\.tv/filme/(?P<id>[\-a-z0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.spiegel.tv/filme/flug-mh370/',
         'info_dict': {
             'id': 'flug-mh370',
@@ -20,12 +20,10 @@ class SpiegeltvIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         }
-    }
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         title = self._html_search_regex(r'<h1.*?>(.*?)</h1>', webpage, 'title')
 
@@ -61,12 +59,8 @@ class SpiegeltvIE(InfoExtractor):
             })
 
         description = media_json['subtitle']
-        duration = media_json['duration_in_ms'] / 1000.
-
-        if is_wide:
-            format = '16x9'
-        else:
-            format = '4x3'
+        duration = float_or_none(media_json.get('duration_in_ms'), scale=1000)
+        format = '16x9' if is_wide else '4x3'
 
         url = server + 'mp4:' + uuid + '_spiegeltv_0500_' + format + '.m4v'
 

From 939fe70de064130330b137da86a11c1f7a78af24 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 16 Nov 2014 00:40:09 +0100
Subject: [PATCH 0494/1937] [spiegeltv] Match hash-style URLs (Closes #4210)

---
 youtube_dl/extractor/spiegeltv.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index 06838f78d..ef5e7c08b 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -6,7 +6,7 @@ from ..utils import float_or_none
 
 
 class SpiegeltvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?spiegel\.tv/filme/(?P<id>[\-a-z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?spiegel\.tv/(?:#/)?filme/(?P<id>[\-a-z0-9]+)'
     _TESTS = [{
         'url': 'http://www.spiegel.tv/filme/flug-mh370/',
         'info_dict': {
@@ -20,9 +20,14 @@ class SpiegeltvIE(InfoExtractor):
             # rtmp download
             'skip_download': True,
         }
+    }, {
+        'url': 'http://www.spiegel.tv/#/filme/alleskino-die-wahrheit-ueber-maenner/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
+        if '/#/' in url:
+            url = url.replace('/#/', '/')
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         title = self._html_search_regex(r'<h1.*?>(.*?)</h1>', webpage, 'title')

From 2707b50ffe540c81629374ea673eea7fb1d1c1a0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 16 Nov 2014 00:51:31 +0100
Subject: [PATCH 0495/1937] [spiegel] Correct handling of redirects to
 spiegel.tv (Closes #4211)

---
 youtube_dl/extractor/spiegel.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index 9586a7da2..1e55a9ffb 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..compat import compat_urlparse
+from .spiegeltv import SpiegeltvIE
 
 
 class SpiegelIE(InfoExtractor):
@@ -42,7 +43,11 @@ class SpiegelIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        webpage, handle = self._download_webpage_handle(url, video_id)
+
+        # 302 to spiegel.tv, like http://www.spiegel.de/video/der-film-zum-wochenende-die-wahrheit-ueber-maenner-video-99003272.html
+        if SpiegeltvIE.suitable(handle.geturl()):
+            return self.url_result(handle.geturl(), 'Spiegeltv')
 
         title = re.sub(r'\s+', ' ', self._html_search_regex(
             r'(?s)<(?:h1|div) class="module-title"[^>]*>(.*?)</(?:h1|div)>',

From 0cf166ad4fe7e26dcb2d07feb2339da3276fa81f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 16 Nov 2014 00:51:46 +0100
Subject: [PATCH 0496/1937] release 2014.11.16

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6a011846c..34bf665ad 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.15.1'
+__version__ = '2014.11.16'

From 6a68bb574acaf4df3f3f01d3330498cad5101773 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 16 Nov 2014 14:55:22 +0100
Subject: [PATCH 0497/1937] [eporner] Fix duration (Closes #4188)

---
 test/test_utils.py              | 1 +
 youtube_dl/extractor/eporner.py | 6 ++----
 youtube_dl/utils.py             | 7 ++++++-
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index edd95c3e3..0fa873147 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -218,6 +218,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_duration('0m0s'), 0)
         self.assertEqual(parse_duration('0s'), 0)
         self.assertEqual(parse_duration('01:02:03.05'), 3723.05)
+        self.assertEqual(parse_duration('T30M38S'), 1838)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
diff --git a/youtube_dl/extractor/eporner.py b/youtube_dl/extractor/eporner.py
index bb231ecb1..4de8d4bc5 100644
--- a/youtube_dl/extractor/eporner.py
+++ b/youtube_dl/extractor/eporner.py
@@ -20,7 +20,7 @@ class EpornerIE(InfoExtractor):
             'display_id': 'Infamous-Tiffany-Teen-Strip-Tease-Video',
             'ext': 'mp4',
             'title': 'Infamous Tiffany Teen Strip Tease Video',
-            'duration': 194,
+            'duration': 1838,
             'view_count': int,
             'age_limit': 18,
         }
@@ -57,9 +57,7 @@ class EpornerIE(InfoExtractor):
             formats.append(fmt)
         self._sort_formats(formats)
 
-        duration = parse_duration(self._search_regex(
-            r'class="mbtim">([0-9:]+)</div>', webpage, 'duration',
-            fatal=False))
+        duration = parse_duration(self._html_search_meta('duration', webpage))
         view_count = str_to_int(self._search_regex(
             r'id="cinemaviews">\s*([0-9,]+)\s*<small>views',
             webpage, 'view count', fatal=False))
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 67be4a9ae..58897231d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1115,7 +1115,12 @@ def parse_duration(s):
     s = s.strip()
 
     m = re.match(
-        r'(?i)(?:(?:(?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*)?(?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?$', s)
+        r'''(?ix)T?
+            (?:
+                (?:(?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*)?
+                (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
+            )?
+            (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?$''', s)
     if not m:
         return None
     res = int(m.group('secs'))

From b6b70730bf810ca5a7f8f68ba0e43f0d3760014d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 16 Nov 2014 15:06:59 +0100
Subject: [PATCH 0498/1937] [downloader/common] Modernize

---
 youtube_dl/downloader/common.py | 44 +++++++++++++++++----------------
 1 file changed, 23 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index f85f0c94e..7c33004b1 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import os
 import re
 import sys
@@ -159,14 +161,14 @@ class FileDownloader(object):
 
     def temp_name(self, filename):
         """Returns a temporary filename for the given filename."""
-        if self.params.get('nopart', False) or filename == u'-' or \
+        if self.params.get('nopart', False) or filename == '-' or \
                 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
             return filename
-        return filename + u'.part'
+        return filename + '.part'
 
     def undo_temp_name(self, filename):
-        if filename.endswith(u'.part'):
-            return filename[:-len(u'.part')]
+        if filename.endswith('.part'):
+            return filename[:-len('.part')]
         return filename
 
     def try_rename(self, old_filename, new_filename):
@@ -175,7 +177,7 @@ class FileDownloader(object):
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
         except (IOError, OSError) as err:
-            self.report_error(u'unable to rename file: %s' % compat_str(err))
+            self.report_error('unable to rename file: %s' % compat_str(err))
 
     def try_utime(self, filename, last_modified_hdr):
         """Try to set the last-modified time of the given file."""
@@ -200,10 +202,10 @@ class FileDownloader(object):
 
     def report_destination(self, filename):
         """Report destination filename."""
-        self.to_screen(u'[download] Destination: ' + filename)
+        self.to_screen('[download] Destination: ' + filename)
 
     def _report_progress_status(self, msg, is_last_line=False):
-        fullmsg = u'[download] ' + msg
+        fullmsg = '[download] ' + msg
         if self.params.get('progress_with_newline', False):
             self.to_screen(fullmsg)
         else:
@@ -211,13 +213,13 @@ class FileDownloader(object):
                 prev_len = getattr(self, '_report_progress_prev_line_length',
                                    0)
                 if prev_len > len(fullmsg):
-                    fullmsg += u' ' * (prev_len - len(fullmsg))
+                    fullmsg += ' ' * (prev_len - len(fullmsg))
                 self._report_progress_prev_line_length = len(fullmsg)
-                clear_line = u'\r'
+                clear_line = '\r'
             else:
-                clear_line = (u'\r\x1b[K' if sys.stderr.isatty() else u'\r')
+                clear_line = ('\r\x1b[K' if sys.stderr.isatty() else '\r')
             self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
-        self.to_console_title(u'youtube-dl ' + msg)
+        self.to_console_title('youtube-dl ' + msg)
 
     def report_progress(self, percent, data_len_str, speed, eta):
         """Report download progress."""
@@ -233,7 +235,7 @@ class FileDownloader(object):
             percent_str = 'Unknown %'
         speed_str = self.format_speed(speed)
 
-        msg = (u'%s of %s at %s ETA %s' %
+        msg = ('%s of %s at %s ETA %s' %
                (percent_str, data_len_str, speed_str, eta_str))
         self._report_progress_status(msg)
 
@@ -243,37 +245,37 @@ class FileDownloader(object):
         downloaded_str = format_bytes(downloaded_data_len)
         speed_str = self.format_speed(speed)
         elapsed_str = FileDownloader.format_seconds(elapsed)
-        msg = u'%s at %s (%s)' % (downloaded_str, speed_str, elapsed_str)
+        msg = '%s at %s (%s)' % (downloaded_str, speed_str, elapsed_str)
         self._report_progress_status(msg)
 
     def report_finish(self, data_len_str, tot_time):
         """Report download finished."""
         if self.params.get('noprogress', False):
-            self.to_screen(u'[download] Download completed')
+            self.to_screen('[download] Download completed')
         else:
             self._report_progress_status(
-                (u'100%% of %s in %s' %
+                ('100%% of %s in %s' %
                  (data_len_str, self.format_seconds(tot_time))),
                 is_last_line=True)
 
     def report_resuming_byte(self, resume_len):
         """Report attempt to resume at given byte."""
-        self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
+        self.to_screen('[download] Resuming download at byte %s' % resume_len)
 
     def report_retry(self, count, retries):
         """Report retry in case of HTTP error 5xx"""
-        self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
+        self.to_screen('[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
 
     def report_file_already_downloaded(self, file_name):
         """Report file has already been fully downloaded."""
         try:
-            self.to_screen(u'[download] %s has already been downloaded' % file_name)
+            self.to_screen('[download] %s has already been downloaded' % file_name)
         except UnicodeEncodeError:
-            self.to_screen(u'[download] The file has already been downloaded')
+            self.to_screen('[download] The file has already been downloaded')
 
     def report_unable_to_resume(self):
         """Report it was impossible to resume download."""
-        self.to_screen(u'[download] Unable to resume')
+        self.to_screen('[download] Unable to resume')
 
     def download(self, filename, info_dict):
         """Download to a filename using the info from info_dict
@@ -293,7 +295,7 @@ class FileDownloader(object):
 
     def real_download(self, filename, info_dict):
         """Real download process. Redefine in subclasses."""
-        raise NotImplementedError(u'This method must be implemented by subclasses')
+        raise NotImplementedError('This method must be implemented by subclasses')
 
     def _hook_progress(self, status):
         for ph in self._progress_hooks:

From a4bc4336196c62bfc7e34046172c442108b19d55 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 16 Nov 2014 15:08:34 +0100
Subject: [PATCH 0499/1937] [__init__] Modernize

---
 youtube_dl/__init__.py | 76 ++++++++++++++++++++++--------------------
 1 file changed, 39 insertions(+), 37 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 685dd8e5e..8e186a0db 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from __future__ import unicode_literals
+
 __license__ = 'Public Domain'
 
 import codecs
@@ -55,7 +57,7 @@ def _real_main(argv=None):
         # https://github.com/rg3/youtube-dl/issues/820
         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 
-    setproctitle(u'youtube-dl')
+    setproctitle('youtube-dl')
 
     parser, opts, args = parseOpts(argv)
 
@@ -71,10 +73,10 @@ def _real_main(argv=None):
     if opts.headers is not None:
         for h in opts.headers:
             if h.find(':', 1) < 0:
-                parser.error(u'wrong header formatting, it should be key:value, not "%s"'%h)
+                parser.error('wrong header formatting, it should be key:value, not "%s"'%h)
             key, value = h.split(':', 2)
             if opts.verbose:
-                write_string(u'[debug] Adding header from command line option %s:%s\n'%(key, value))
+                write_string('[debug] Adding header from command line option %s:%s\n'%(key, value))
             std_headers[key] = value
 
     # Dump user agent
@@ -92,9 +94,9 @@ def _real_main(argv=None):
                 batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
             batch_urls = read_batch_urls(batchfd)
             if opts.verbose:
-                write_string(u'[debug] Batch file urls: ' + repr(batch_urls) + u'\n')
+                write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n')
         except IOError:
-            sys.exit(u'ERROR: batch file could not be read')
+            sys.exit('ERROR: batch file could not be read')
     all_urls = batch_urls + args
     all_urls = [url.strip() for url in all_urls]
     _enc = preferredencoding()
@@ -107,7 +109,7 @@ def _real_main(argv=None):
             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
             matchedUrls = [url for url in all_urls if ie.suitable(url)]
             for mu in matchedUrls:
-                compat_print(u'  ' + mu)
+                compat_print('  ' + mu)
         sys.exit(0)
     if opts.list_extractor_descriptions:
         for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
@@ -117,63 +119,63 @@ def _real_main(argv=None):
             if desc is False:
                 continue
             if hasattr(ie, 'SEARCH_KEY'):
-                _SEARCHES = (u'cute kittens', u'slithering pythons', u'falling cat', u'angry poodle', u'purple fish', u'running tortoise', u'sleeping bunny')
-                _COUNTS = (u'', u'5', u'10', u'all')
-                desc += u' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
+                _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny')
+                _COUNTS = ('', '5', '10', 'all')
+                desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
             compat_print(desc)
         sys.exit(0)
 
 
     # Conflicting, missing and erroneous options
     if opts.usenetrc and (opts.username is not None or opts.password is not None):
-        parser.error(u'using .netrc conflicts with giving username/password')
+        parser.error('using .netrc conflicts with giving username/password')
     if opts.password is not None and opts.username is None:
-        parser.error(u'account username missing\n')
+        parser.error('account username missing\n')
     if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid):
-        parser.error(u'using output template conflicts with using title, video ID or auto number')
+        parser.error('using output template conflicts with using title, video ID or auto number')
     if opts.usetitle and opts.useid:
-        parser.error(u'using title conflicts with using video ID')
+        parser.error('using title conflicts with using video ID')
     if opts.username is not None and opts.password is None:
-        opts.password = compat_getpass(u'Type account password and press [Return]: ')
+        opts.password = compat_getpass('Type account password and press [Return]: ')
     if opts.ratelimit is not None:
         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
         if numeric_limit is None:
-            parser.error(u'invalid rate limit specified')
+            parser.error('invalid rate limit specified')
         opts.ratelimit = numeric_limit
     if opts.min_filesize is not None:
         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
         if numeric_limit is None:
-            parser.error(u'invalid min_filesize specified')
+            parser.error('invalid min_filesize specified')
         opts.min_filesize = numeric_limit
     if opts.max_filesize is not None:
         numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
         if numeric_limit is None:
-            parser.error(u'invalid max_filesize specified')
+            parser.error('invalid max_filesize specified')
         opts.max_filesize = numeric_limit
     if opts.retries is not None:
         try:
             opts.retries = int(opts.retries)
         except (TypeError, ValueError):
-            parser.error(u'invalid retry count specified')
+            parser.error('invalid retry count specified')
     if opts.buffersize is not None:
         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
         if numeric_buffersize is None:
-            parser.error(u'invalid buffer size specified')
+            parser.error('invalid buffer size specified')
         opts.buffersize = numeric_buffersize
     if opts.playliststart <= 0:
-        raise ValueError(u'Playlist start must be positive')
+        raise ValueError('Playlist start must be positive')
     if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart:
-        raise ValueError(u'Playlist end must be greater than playlist start')
+        raise ValueError('Playlist end must be greater than playlist start')
     if opts.extractaudio:
         if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']:
-            parser.error(u'invalid audio format specified')
+            parser.error('invalid audio format specified')
     if opts.audioquality:
         opts.audioquality = opts.audioquality.strip('k').strip('K')
         if not opts.audioquality.isdigit():
-            parser.error(u'invalid audio quality specified')
+            parser.error('invalid audio quality specified')
     if opts.recodevideo is not None:
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
-            parser.error(u'invalid video recode format specified')
+            parser.error('invalid video recode format specified')
     if opts.date is not None:
         date = DateRange.day(opts.date)
     else:
@@ -193,17 +195,17 @@ def _real_main(argv=None):
         if opts.outtmpl is not None:
             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
     outtmpl =((opts.outtmpl is not None and opts.outtmpl)
-            or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s')
-            or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
-            or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
-            or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s')
-            or (opts.useid and u'%(id)s.%(ext)s')
-            or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
+            or (opts.format == '-1' and opts.usetitle and '%(title)s-%(id)s-%(format)s.%(ext)s')
+            or (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s')
+            or (opts.usetitle and opts.autonumber and '%(autonumber)s-%(title)s-%(id)s.%(ext)s')
+            or (opts.usetitle and '%(title)s-%(id)s.%(ext)s')
+            or (opts.useid and '%(id)s.%(ext)s')
+            or (opts.autonumber and '%(autonumber)s-%(id)s.%(ext)s')
             or DEFAULT_OUTTMPL)
     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
-        parser.error(u'Cannot download a video and extract audio into the same'
-                     u' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
-                     u' template'.format(outtmpl))
+        parser.error('Cannot download a video and extract audio into the same'
+                     ' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
+                     ' template'.format(outtmpl))
 
     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
     download_archive_fn = compat_expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
@@ -330,7 +332,7 @@ def _real_main(argv=None):
         # Maybe do nothing
         if (len(all_urls) < 1) and (opts.load_info_filename is None):
             if not (opts.update_self or opts.rm_cachedir):
-                parser.error(u'you must provide at least one URL')
+                parser.error('you must provide at least one URL')
             else:
                 sys.exit()
 
@@ -340,7 +342,7 @@ def _real_main(argv=None):
             else:
                 retcode = ydl.download(all_urls)
         except MaxDownloadsReached:
-            ydl.to_screen(u'--max-download limit reached, aborting.')
+            ydl.to_screen('--max-download limit reached, aborting.')
             retcode = 101
 
     sys.exit(retcode)
@@ -352,6 +354,6 @@ def main(argv=None):
     except DownloadError:
         sys.exit(1)
     except SameFileError:
-        sys.exit(u'ERROR: fixed output name but more than one file to download')
+        sys.exit('ERROR: fixed output name but more than one file to download')
     except KeyboardInterrupt:
-        sys.exit(u'\nERROR: Interrupted by user')
+        sys.exit('\nERROR: Interrupted by user')

From a0f59cdcb405b0556bff9884a7a82f3b808263dd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 16 Nov 2014 15:17:48 +0100
Subject: [PATCH 0500/1937] [tests] Modernize

---
 test/test_age_restriction.py |  3 ++-
 test/test_download.py        |  6 ++++--
 test/test_execution.py       |  7 ++++++-
 test/test_subtitles.py       | 39 ++++++++++++++++++------------------
 test/test_swfinterp.py       |  1 +
 5 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/test/test_age_restriction.py b/test/test_age_restriction.py
index 71e80b037..5be065c43 100644
--- a/test/test_age_restriction.py
+++ b/test/test_age_restriction.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import unicode_literals
 
 # Allow direct execution
 import os
@@ -19,7 +20,7 @@ def _download_restricted(url, filename, age):
         'age_limit': age,
         'skip_download': True,
         'writeinfojson': True,
-        "outtmpl": "%(id)s.%(ext)s",
+        'outtmpl': '%(id)s.%(ext)s',
     }
     ydl = YoutubeDL(params)
     ydl.add_default_info_extractors()
diff --git a/test/test_download.py b/test/test_download.py
index 87aced97c..12cfb5cbe 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+from __future__ import unicode_literals
+
 # Allow direct execution
 import os
 import sys
@@ -210,9 +212,9 @@ for n, test_case in enumerate(defs):
     tname = 'test_' + str(test_case['name'])
     i = 1
     while hasattr(TestDownload, tname):
-        tname = 'test_'  + str(test_case['name']) + '_' + str(i)
+        tname = 'test_%s_%d' % (test_case['name'], i)
         i += 1
-    test_method.__name__ = tname
+    test_method.__name__ = str(tname)
     setattr(TestDownload, test_method.__name__, test_method)
     del test_method
 
diff --git a/test/test_execution.py b/test/test_execution.py
index 2b115fb31..60df187de 100644
--- a/test/test_execution.py
+++ b/test/test_execution.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
 import unittest
 
 import sys
@@ -6,17 +9,19 @@ import subprocess
 
 rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
+
 try:
     _DEV_NULL = subprocess.DEVNULL
 except AttributeError:
     _DEV_NULL = open(os.devnull, 'wb')
 
+
 class TestExecution(unittest.TestCase):
     def test_import(self):
         subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
 
     def test_module_exec(self):
-        if sys.version_info >= (2,7): # Python 2.6 doesn't support package execution
+        if sys.version_info >= (2, 7):  # Python 2.6 doesn't support package execution
             subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
 
     def test_main_exec(self):
diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 8f4602e5f..94e3290db 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import unicode_literals
 
 # Allow direct execution
 import os
@@ -74,7 +75,7 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['en']), '3cb210999d3e021bd6c7f0ea751eab06')
 
     def test_youtube_list_subtitles(self):
-        self.DL.expect_warning(u'Video doesn\'t have automatic captions')
+        self.DL.expect_warning('Video doesn\'t have automatic captions')
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
@@ -87,7 +88,7 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
         self.assertTrue(subtitles['it'] is not None)
 
     def test_youtube_nosubtitles(self):
-        self.DL.expect_warning(u'video doesn\'t have subtitles')
+        self.DL.expect_warning('video doesn\'t have subtitles')
         self.url = 'n5BB19UTcdA'
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
@@ -101,7 +102,7 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
         self.DL.params['subtitleslangs'] = langs
         subtitles = self.getSubtitles()
         for lang in langs:
-            self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
+            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
 
 class TestDailymotionSubtitles(BaseTestSubtitles):
@@ -130,20 +131,20 @@ class TestDailymotionSubtitles(BaseTestSubtitles):
         self.assertEqual(len(subtitles.keys()), 5)
 
     def test_list_subtitles(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
     def test_automatic_captions(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['writeautomaticsub'] = True
         self.DL.params['subtitleslang'] = ['en']
         subtitles = self.getSubtitles()
         self.assertTrue(len(subtitles.keys()) == 0)
 
     def test_nosubtitles(self):
-        self.DL.expect_warning(u'video doesn\'t have subtitles')
+        self.DL.expect_warning('video doesn\'t have subtitles')
         self.url = 'http://www.dailymotion.com/video/x12u166_le-zapping-tele-star-du-08-aout-2013_tv'
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
@@ -156,7 +157,7 @@ class TestDailymotionSubtitles(BaseTestSubtitles):
         self.DL.params['subtitleslangs'] = langs
         subtitles = self.getSubtitles()
         for lang in langs:
-            self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
+            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
 
 class TestTedSubtitles(BaseTestSubtitles):
@@ -185,13 +186,13 @@ class TestTedSubtitles(BaseTestSubtitles):
         self.assertTrue(len(subtitles.keys()) >= 28)
 
     def test_list_subtitles(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
     def test_automatic_captions(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['writeautomaticsub'] = True
         self.DL.params['subtitleslang'] = ['en']
         subtitles = self.getSubtitles()
@@ -203,7 +204,7 @@ class TestTedSubtitles(BaseTestSubtitles):
         self.DL.params['subtitleslangs'] = langs
         subtitles = self.getSubtitles()
         for lang in langs:
-            self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
+            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
 
 class TestBlipTVSubtitles(BaseTestSubtitles):
@@ -211,13 +212,13 @@ class TestBlipTVSubtitles(BaseTestSubtitles):
     IE = BlipTVIE
 
     def test_list_subtitles(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
     def test_allsubtitles(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
@@ -251,20 +252,20 @@ class TestVimeoSubtitles(BaseTestSubtitles):
         self.assertEqual(set(subtitles.keys()), set(['de', 'en', 'es', 'fr']))
 
     def test_list_subtitles(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
     def test_automatic_captions(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['writeautomaticsub'] = True
         self.DL.params['subtitleslang'] = ['en']
         subtitles = self.getSubtitles()
         self.assertTrue(len(subtitles.keys()) == 0)
 
     def test_nosubtitles(self):
-        self.DL.expect_warning(u'video doesn\'t have subtitles')
+        self.DL.expect_warning('video doesn\'t have subtitles')
         self.url = 'http://vimeo.com/56015672'
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
@@ -277,7 +278,7 @@ class TestVimeoSubtitles(BaseTestSubtitles):
         self.DL.params['subtitleslangs'] = langs
         subtitles = self.getSubtitles()
         for lang in langs:
-            self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
+            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
 
 class TestWallaSubtitles(BaseTestSubtitles):
@@ -285,13 +286,13 @@ class TestWallaSubtitles(BaseTestSubtitles):
     IE = WallaIE
 
     def test_list_subtitles(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
     def test_allsubtitles(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
@@ -299,7 +300,7 @@ class TestWallaSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['heb']), 'e758c5d7cb982f6bef14f377ec7a3920')
 
     def test_nosubtitles(self):
-        self.DL.expect_warning(u'video doesn\'t have subtitles')
+        self.DL.expect_warning('video doesn\'t have subtitles')
         self.url = 'http://vod.walla.co.il/movie/2642630/one-direction-all-for-one'
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
diff --git a/test/test_swfinterp.py b/test/test_swfinterp.py
index 273e0f074..9f18055e6 100644
--- a/test/test_swfinterp.py
+++ b/test/test_swfinterp.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import unicode_literals
 
 # Allow direct execution
 import os

From b7558d9881ae0dede7c13e1c082eaf89aefed27e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 01:27:15 +0100
Subject: [PATCH 0501/1937] [swfinterp] Allow function patching

---
 youtube_dl/swfinterp.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index b63c65b20..e5deb2c14 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -150,6 +150,7 @@ def _read_byte(reader):
 
 class SWFInterpreter(object):
     def __init__(self, file_contents):
+        self._patched_functions = {}
         code_tag = next(tag
                         for tag_code, tag in _extract_tags(file_contents)
                         if tag_code == 82)
@@ -354,6 +355,9 @@ class SWFInterpreter(object):
 
         assert p + code_reader.tell() == len(code_tag)
 
+    def patch_function(self, avm_class, func_name, f):
+        self._patched_functions[(avm_class, func_name)] = f
+
     def extract_class(self, class_name):
         try:
             return self._classes_by_name[class_name]
@@ -361,6 +365,9 @@ class SWFInterpreter(object):
             raise ExtractorError('Class %r not found' % class_name)
 
     def extract_function(self, avm_class, func_name):
+        p = self._patched_functions.get((avm_class, func_name))
+        if p:
+            return p
         if func_name in avm_class.method_pyfunctions:
             return avm_class.method_pyfunctions[func_name]
         if func_name in self._classes_by_name:

From 7f3e33a1475cd4ac11c73108e03d3405b86262b8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 01:27:34 +0100
Subject: [PATCH 0502/1937] [swfinterp] Implement member assignment

---
 test/swftests/MemberAssignment.as | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 test/swftests/MemberAssignment.as

diff --git a/test/swftests/MemberAssignment.as b/test/swftests/MemberAssignment.as
new file mode 100644
index 000000000..dcba5e3ff
--- /dev/null
+++ b/test/swftests/MemberAssignment.as
@@ -0,0 +1,22 @@
+// input: [1]
+// output: 2
+
+package {
+public class MemberAssignment {
+    public var v:int;
+
+    public function g():int {
+        return this.v;
+    }
+
+    public function f(a:int):int{
+        this.v = a;
+        return this.v + this.g();
+    }
+
+    public static function main(a:int): int {
+        var v:MemberAssignment = new MemberAssignment();
+        return v.f(a);
+    }
+}
+}

From 4baafa229d5e013a62d636fa60920cdc1a876a5a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 03:46:23 +0100
Subject: [PATCH 0503/1937] [swfinterp] Intepret more multinames

---
 test/swftests/DictCall.as | 10 ++++++++++
 youtube_dl/swfinterp.py   |  9 +++++++++
 2 files changed, 19 insertions(+)
 create mode 100644 test/swftests/DictCall.as

diff --git a/test/swftests/DictCall.as b/test/swftests/DictCall.as
new file mode 100644
index 000000000..236efd848
--- /dev/null
+++ b/test/swftests/DictCall.as
@@ -0,0 +1,10 @@
+// input: [{"x": 1, "y": 2}]
+// output: 3
+
+package {
+public class DictCall {
+    public static function main(d:Object):int{
+    	return d.x + d.y;
+    }
+}
+}
diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index e5deb2c14..7c0ee1e61 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -213,6 +213,10 @@ class SWFInterpreter(object):
                 u30()  # namespace_idx
                 name_idx = u30()
                 self.multinames.append(self.constant_strings[name_idx])
+            elif kind == 0x09:
+                name_idx = u30()
+                u30()
+                self.multinames.append(self.constant_strings[name_idx])
             else:
                 self.multinames.append(_Multiname(kind))
                 for _c2 in range(MULTINAME_SIZES[kind]):
@@ -557,6 +561,11 @@ class SWFInterpreter(object):
                         obj = stack.pop()
                         assert isinstance(obj, list)
                         stack.append(len(obj))
+                    elif isinstance(pname, compat_str):  # Member access
+                        obj = stack.pop()
+                        assert isinstance(obj, (dict, _ScopeDict)), \
+                            'Accessing member on %r' % obj
+                        stack.append(obj[pname])
                     else:  # Assume attribute access
                         idx = stack.pop()
                         assert isinstance(idx, int)

From 0ab1ca5501108b8038090750e8455bfb5d72c443 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 03:53:32 +0100
Subject: [PATCH 0504/1937] [swfinterp] Better error message

---
 test/swftests/DictCall.as | 2 +-
 youtube_dl/swfinterp.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/swftests/DictCall.as b/test/swftests/DictCall.as
index 236efd848..c2d174cc2 100644
--- a/test/swftests/DictCall.as
+++ b/test/swftests/DictCall.as
@@ -4,7 +4,7 @@
 package {
 public class DictCall {
     public static function main(d:Object):int{
-    	return d.x + d.y;
+        return d.x + d.y;
     }
 }
 }
diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index 7c0ee1e61..58da6c586 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -564,7 +564,7 @@ class SWFInterpreter(object):
                     elif isinstance(pname, compat_str):  # Member access
                         obj = stack.pop()
                         assert isinstance(obj, (dict, _ScopeDict)), \
-                            'Accessing member on %r' % obj
+                            'Accessing member %r on %r' % (pname, obj)
                         stack.append(obj[pname])
                     else:  # Assume attribute access
                         idx = stack.pop()

From e983cf52775e493b7deedbe48d2f50f598c9da4e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 04:00:31 +0100
Subject: [PATCH 0505/1937] [swfinterp] Interpret yet more opcodes

---
 test/swftests/NeOperator.as | 24 ++++++++++++++++++++++++
 youtube_dl/swfinterp.py     | 19 ++++++++++++++++++-
 2 files changed, 42 insertions(+), 1 deletion(-)
 create mode 100644 test/swftests/NeOperator.as

diff --git a/test/swftests/NeOperator.as b/test/swftests/NeOperator.as
new file mode 100644
index 000000000..61dcbc4e9
--- /dev/null
+++ b/test/swftests/NeOperator.as
@@ -0,0 +1,24 @@
+// input: []
+// output: 123
+
+package {
+public class NeOperator {
+    public static function main(): int {
+        var res:int = 0;
+        if (1 != 2) {
+            res += 3;
+        } else {
+            res += 4;
+        }
+        if (2 != 2) {
+            res += 10;
+        } else {
+            res += 20;
+        }
+        if (9 == 9) {
+            res += 100;
+        }
+        return res;
+    }
+}
+}
diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index 58da6c586..85efde592 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -393,7 +393,10 @@ class SWFInterpreter(object):
                 self._classes_by_name, avm_class.variables])
             while True:
                 opcode = _read_byte(coder)
-                if opcode == 17:  # iftrue
+                if opcode == 16:  # jump
+                    offset = s24()
+                    coder.seek(coder.tell() + offset)
+                elif opcode == 17:  # iftrue
                     offset = s24()
                     value = stack.pop()
                     if value:
@@ -403,6 +406,20 @@ class SWFInterpreter(object):
                     value = stack.pop()
                     if not value:
                         coder.seek(coder.tell() + offset)
+                elif opcode == 19:  # ifeq
+                    offset = s24()
+                    value2 = stack.pop()
+                    value1 = stack.pop()
+                    if value2 == value1:
+                        coder.seek(coder.tell() + offset)
+                elif opcode == 20:  # ifne
+                    offset = s24()
+                    value2 = stack.pop()
+                    value1 = stack.pop()
+                    if value2 != value1:
+                        coder.seek(coder.tell() + offset)
+                elif opcode == 32:  # pushnull
+                    stack.append(None)
                 elif opcode == 36:  # pushbyte
                     v = _read_byte(coder)
                     stack.append(v)

From 3cbcff8a2dacf6d4f10f00af36b9761ef833e6ea Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 04:25:04 +0100
Subject: [PATCH 0506/1937] [swfinterp] Implement String basics

---
 test/swftests/StringBasics.as     | 11 +++++++++++
 test/swftests/StringConversion.as | 11 +++++++++++
 youtube_dl/swfinterp.py           | 21 +++++++++++++++++++--
 3 files changed, 41 insertions(+), 2 deletions(-)
 create mode 100644 test/swftests/StringBasics.as
 create mode 100644 test/swftests/StringConversion.as

diff --git a/test/swftests/StringBasics.as b/test/swftests/StringBasics.as
new file mode 100644
index 000000000..d27430b13
--- /dev/null
+++ b/test/swftests/StringBasics.as
@@ -0,0 +1,11 @@
+// input: []
+// output: 3
+
+package {
+public class StringBasics {
+    public static function main():int{
+        var s:String = "abc";
+        return s.length;
+    }
+}
+}
diff --git a/test/swftests/StringConversion.as b/test/swftests/StringConversion.as
new file mode 100644
index 000000000..c976f5042
--- /dev/null
+++ b/test/swftests/StringConversion.as
@@ -0,0 +1,11 @@
+// input: []
+// output: 2
+
+package {
+public class StringConversion {
+    public static function main():int{
+        var s:String = String(99);
+        return s.length;
+    }
+}
+}
diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index 85efde592..0ce8474ca 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -148,6 +148,9 @@ def _read_byte(reader):
     return res
 
 
+StringClass = _AVMClass('(no name idx)', 'String')
+
+
 class SWFInterpreter(object):
     def __init__(self, file_contents):
         self._patched_functions = {}
@@ -483,6 +486,17 @@ class SWFInterpreter(object):
                             res = args[0].join(obj)
                             stack.append(res)
                             continue
+                    elif obj == StringClass:
+                        if mname == 'String':
+                            assert len(args) == 1
+                            assert isinstance(args[0], (int, compat_str))
+                            res = compat_str(args[0])
+                            stack.append(res)
+                            continue
+                        else:
+                            raise NotImplementedError(
+                                'Function String.%s is not yet implemented'
+                                % mname)
                     raise NotImplementedError(
                         'Unsupported property %r on %r'
                         % (mname, obj))
@@ -532,7 +546,10 @@ class SWFInterpreter(object):
                             break
                     else:
                         res = scopes[0]
-                    stack.append(res[mname])
+                    if mname not in res and mname == 'String':
+                        stack.append(StringClass)
+                    else:
+                        stack.append(res[mname])
                 elif opcode == 94:  # findproperty
                     index = u30()
                     mname = self.multinames[index]
@@ -576,7 +593,7 @@ class SWFInterpreter(object):
                     pname = self.multinames[index]
                     if pname == 'length':
                         obj = stack.pop()
-                        assert isinstance(obj, list)
+                        assert isinstance(obj, (compat_str, list))
                         stack.append(len(obj))
                     elif isinstance(pname, compat_str):  # Member access
                         obj = stack.pop()

From eb5376044ca2bb86a61c70ce1905992a0e5d25bd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 04:27:51 +0100
Subject: [PATCH 0507/1937] [swfinterp] Implement equals opcode

---
 test/swftests/EqualsOperator.as | 10 ++++++++++
 youtube_dl/swfinterp.py         |  5 +++++
 2 files changed, 15 insertions(+)
 create mode 100644 test/swftests/EqualsOperator.as

diff --git a/test/swftests/EqualsOperator.as b/test/swftests/EqualsOperator.as
new file mode 100644
index 000000000..837a69a46
--- /dev/null
+++ b/test/swftests/EqualsOperator.as
@@ -0,0 +1,10 @@
+// input: []
+// output: false
+
+package {
+public class EqualsOperator {
+    public static function main():Boolean{
+        return 1 == 2;
+    }
+}
+}
diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index 0ce8474ca..1e8b26ecc 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -629,6 +629,11 @@ class SWFInterpreter(object):
                     value1 = stack.pop()
                     res = value1 % value2
                     stack.append(res)
+                elif opcode == 171:  # equals
+                    value2 = stack.pop()
+                    value1 = stack.pop()
+                    result = value1 == value2
+                    stack.append(result)
                 elif opcode == 175:  # greaterequals
                     value2 = stack.pop()
                     value1 = stack.pop()

From a4bb83956cf3aa4f569c30b4d144b4fb327c7b56 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 04:29:34 +0100
Subject: [PATCH 0508/1937] [swfinterp] Implement pushtrue and pushfalse
 opcodes

---
 youtube_dl/swfinterp.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index 1e8b26ecc..dd4dd060a 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -426,6 +426,10 @@ class SWFInterpreter(object):
                 elif opcode == 36:  # pushbyte
                     v = _read_byte(coder)
                     stack.append(v)
+                elif opcode == 38:  # pushtrue
+                    stack.append(True)
+                elif opcode == 39:  # pushfalse
+                    stack.append(False)
                 elif opcode == 42:  # dup
                     value = stack[-1]
                     stack.append(value)

From 8d05f2c16a79e4127963d28bb41587105680e72c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 04:36:23 +0100
Subject: [PATCH 0509/1937] [swfinterp] Add support for void methods

---
 test/swftests/PrivateVoidCall.as | 22 ++++++++++++++++++++++
 youtube_dl/swfinterp.py          | 17 ++++++++++++++++-
 2 files changed, 38 insertions(+), 1 deletion(-)
 create mode 100644 test/swftests/PrivateVoidCall.as

diff --git a/test/swftests/PrivateVoidCall.as b/test/swftests/PrivateVoidCall.as
new file mode 100644
index 000000000..2cc016797
--- /dev/null
+++ b/test/swftests/PrivateVoidCall.as
@@ -0,0 +1,22 @@
+// input: []
+// output: 9
+
+package {
+public class PrivateVoidCall {
+    public static function main():int{
+        var f:OtherClass = new OtherClass();
+        f.func();
+        return 9;
+    }
+}
+}
+
+class OtherClass {
+    private function pf():void {
+        ;
+    }
+
+    public function func():void {
+        this.pf();
+    }
+}
diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index dd4dd060a..f4ee022f4 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -504,6 +504,9 @@ class SWFInterpreter(object):
                     raise NotImplementedError(
                         'Unsupported property %r on %r'
                         % (mname, obj))
+                elif opcode == 71:  # returnvoid
+                    res = None
+                    return res
                 elif opcode == 72:  # returnvalue
                     res = stack.pop()
                     return res
@@ -527,6 +530,17 @@ class SWFInterpreter(object):
                     args = list(reversed(
                         [stack.pop() for _ in range(arg_count)]))
                     obj = stack.pop()
+                    if isinstance(obj, _AVMClass_Object):
+                        func = self.extract_function(obj.avm_class, mname)
+                        res = func(args)
+                        assert res is None
+                        continue
+                    if isinstance(obj, _ScopeDict):
+                        assert mname in obj.avm_class.method_names
+                        func = self.extract_function(obj.avm_class, mname)
+                        res = func(args)
+                        assert res is None
+                        continue
                     if mname == 'reverse':
                         assert isinstance(obj, list)
                         obj.reverse()
@@ -603,7 +617,8 @@ class SWFInterpreter(object):
                         obj = stack.pop()
                         assert isinstance(obj, (dict, _ScopeDict)), \
                             'Accessing member %r on %r' % (pname, obj)
-                        stack.append(obj[pname])
+                        res = obj.get(pname, None)
+                        stack.append(res)
                     else:  # Assume attribute access
                         idx = stack.pop()
                         assert isinstance(idx, int)

From 4686ae4b64c3c0a88b9a8a3fb7a7657df3753c5e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 04:45:12 +0100
Subject: [PATCH 0510/1937] [swfinterp] Implement various opcodes

---
 youtube_dl/swfinterp.py | 46 +++++++++++++++++++++++++++++++++++------
 1 file changed, 40 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index f4ee022f4..008cd76c7 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -151,6 +151,16 @@ def _read_byte(reader):
 StringClass = _AVMClass('(no name idx)', 'String')
 
 
+class _Undefined(object):
+    def __boolean__(self):
+        return False
+
+    def __hash__(self):
+        return 0
+
+undefined = _Undefined()
+
+
 class SWFInterpreter(object):
     def __init__(self, file_contents):
         self._patched_functions = {}
@@ -423,6 +433,8 @@ class SWFInterpreter(object):
                         coder.seek(coder.tell() + offset)
                 elif opcode == 32:  # pushnull
                     stack.append(None)
+                elif opcode == 33:  # pushundefined
+                    stack.append(undefined)
                 elif opcode == 36:  # pushbyte
                     v = _read_byte(coder)
                     stack.append(v)
@@ -430,6 +442,8 @@ class SWFInterpreter(object):
                     stack.append(True)
                 elif opcode == 39:  # pushfalse
                     stack.append(False)
+                elif opcode == 40:  # pushnan
+                    stack.append(float('NaN'))
                 elif opcode == 42:  # dup
                     value = stack[-1]
                     stack.append(value)
@@ -493,8 +507,12 @@ class SWFInterpreter(object):
                     elif obj == StringClass:
                         if mname == 'String':
                             assert len(args) == 1
-                            assert isinstance(args[0], (int, compat_str))
-                            res = compat_str(args[0])
+                            assert isinstance(args[0], (
+                                int, compat_str, _Undefined))
+                            if args[0] == undefined:
+                                res = 'undefined'
+                            else:
+                                res = compat_str(args[0])
                             stack.append(res)
                             continue
                         else:
@@ -505,7 +523,7 @@ class SWFInterpreter(object):
                         'Unsupported property %r on %r'
                         % (mname, obj))
                 elif opcode == 71:  # returnvoid
-                    res = None
+                    res = undefined
                     return res
                 elif opcode == 72:  # returnvalue
                     res = stack.pop()
@@ -533,13 +551,13 @@ class SWFInterpreter(object):
                     if isinstance(obj, _AVMClass_Object):
                         func = self.extract_function(obj.avm_class, mname)
                         res = func(args)
-                        assert res is None
+                        assert res is undefined
                         continue
                     if isinstance(obj, _ScopeDict):
                         assert mname in obj.avm_class.method_names
                         func = self.extract_function(obj.avm_class, mname)
                         res = func(args)
-                        assert res is None
+                        assert res is undefined
                         continue
                     if mname == 'reverse':
                         assert isinstance(obj, list)
@@ -617,7 +635,7 @@ class SWFInterpreter(object):
                         obj = stack.pop()
                         assert isinstance(obj, (dict, _ScopeDict)), \
                             'Accessing member %r on %r' % (pname, obj)
-                        res = obj.get(pname, None)
+                        res = obj.get(pname, undefined)
                         stack.append(res)
                     else:  # Assume attribute access
                         idx = stack.pop()
@@ -631,8 +649,24 @@ class SWFInterpreter(object):
                     stack.append(intvalue)
                 elif opcode == 128:  # coerce
                     u30()
+                elif opcode == 130:  # coerce_a
+                    value = stack.pop()
+                    # um, yes, it's any value
+                    stack.append(value)
                 elif opcode == 133:  # coerce_s
                     assert isinstance(stack[-1], (type(None), compat_str))
+                elif opcode == 147:  # decrement
+                    value = stack.pop()
+                    assert isinstance(value, int)
+                    stack.append(value - 1)
+                elif opcode == 149:  # typeof
+                    value = stack.pop()
+                    return {
+                        _Undefined: 'undefined',
+                        compat_str: 'String',
+                        int: 'Number',
+                        float: 'Number',
+                    }[type(value)]
                 elif opcode == 160:  # add
                     value2 = stack.pop()
                     value1 = stack.pop()

From 6b592d93a29163664d3125761fbf6d7c9fe5f56c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 04:54:54 +0100
Subject: [PATCH 0511/1937] [swfinterp] Formalize built-in classes

---
 youtube_dl/swfinterp.py | 46 +++++++++++++++++++++++++----------------
 1 file changed, 28 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index 008cd76c7..4b47df29d 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -149,6 +149,11 @@ def _read_byte(reader):
 
 
 StringClass = _AVMClass('(no name idx)', 'String')
+ByteArrayClass = _AVMClass('(no name idx)', 'ByteArray')
+_builtin_classes = {
+    StringClass.name: StringClass,
+    ByteArrayClass.name: ByteArrayClass,
+}
 
 
 class _Undefined(object):
@@ -468,11 +473,31 @@ class SWFInterpreter(object):
                         [stack.pop() for _ in range(arg_count)]))
                     obj = stack.pop()
 
-                    if isinstance(obj, _AVMClass_Object):
+                    if obj == StringClass:
+                        if mname == 'String':
+                            assert len(args) == 1
+                            assert isinstance(args[0], (
+                                int, compat_str, _Undefined))
+                            if args[0] == undefined:
+                                res = 'undefined'
+                            else:
+                                res = compat_str(args[0])
+                            stack.append(res)
+                            continue
+                        else:
+                            raise NotImplementedError(
+                                'Function String.%s is not yet implemented'
+                                % mname)
+                    elif isinstance(obj, _AVMClass_Object):
                         func = self.extract_function(obj.avm_class, mname)
                         res = func(args)
                         stack.append(res)
                         continue
+                    elif isinstance(obj, _AVMClass):
+                        func = self.extract_function(obj, mname)
+                        res = func(args)
+                        stack.append(res)
+                        continue
                     elif isinstance(obj, _ScopeDict):
                         if mname in obj.avm_class.method_names:
                             func = self.extract_function(obj.avm_class, mname)
@@ -504,21 +529,6 @@ class SWFInterpreter(object):
                             res = args[0].join(obj)
                             stack.append(res)
                             continue
-                    elif obj == StringClass:
-                        if mname == 'String':
-                            assert len(args) == 1
-                            assert isinstance(args[0], (
-                                int, compat_str, _Undefined))
-                            if args[0] == undefined:
-                                res = 'undefined'
-                            else:
-                                res = compat_str(args[0])
-                            stack.append(res)
-                            continue
-                        else:
-                            raise NotImplementedError(
-                                'Function String.%s is not yet implemented'
-                                % mname)
                     raise NotImplementedError(
                         'Unsupported property %r on %r'
                         % (mname, obj))
@@ -582,8 +592,8 @@ class SWFInterpreter(object):
                             break
                     else:
                         res = scopes[0]
-                    if mname not in res and mname == 'String':
-                        stack.append(StringClass)
+                    if mname not in res and mname in _builtin_classes:
+                        stack.append(_builtin_classes[mname])
                     else:
                         stack.append(res[mname])
                 elif opcode == 94:  # findproperty

From 33a266f4bae7b53ad12bd79293bcc1f11ae6d407 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 05:03:46 +0100
Subject: [PATCH 0512/1937] [swfinterp] Implement charCodeAt

---
 test/swftests/StringCharCodeAt.as | 11 +++++++++++
 youtube_dl/swfinterp.py           | 26 +++++++++++++++++++++++++-
 2 files changed, 36 insertions(+), 1 deletion(-)
 create mode 100644 test/swftests/StringCharCodeAt.as

diff --git a/test/swftests/StringCharCodeAt.as b/test/swftests/StringCharCodeAt.as
new file mode 100644
index 000000000..c20d74d65
--- /dev/null
+++ b/test/swftests/StringCharCodeAt.as
@@ -0,0 +1,11 @@
+// input: []
+// output: 9897
+
+package {
+public class StringCharCodeAt {
+    public static function main():int{
+        var s:String = "abc";
+        return s.charCodeAt(1) * 100 + s.charCodeAt();
+    }
+}
+}
diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index 4b47df29d..7369c94fc 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -411,7 +411,9 @@ class SWFInterpreter(object):
                 self._classes_by_name, avm_class.variables])
             while True:
                 opcode = _read_byte(coder)
-                if opcode == 16:  # jump
+                if opcode == 9:  # label
+                    pass  # Spec says: "Do nothing."
+                elif opcode == 16:  # jump
                     offset = s24()
                     coder.seek(coder.tell() + offset)
                 elif opcode == 17:  # iftrue
@@ -436,6 +438,12 @@ class SWFInterpreter(object):
                     value1 = stack.pop()
                     if value2 != value1:
                         coder.seek(coder.tell() + offset)
+                elif opcode == 21:  # iflt
+                    offset = s24()
+                    value2 = stack.pop()
+                    value1 = stack.pop()
+                    if value1 < value2:
+                        coder.seek(coder.tell() + offset)
                 elif opcode == 32:  # pushnull
                     stack.append(None)
                 elif opcode == 33:  # pushundefined
@@ -516,6 +524,13 @@ class SWFInterpreter(object):
                                 res = obj.split(args[0])
                             stack.append(res)
                             continue
+                        elif mname == 'charCodeAt':
+                            assert len(args) <= 1
+                            idx = 0 if len(args) == 0 else args[0]
+                            assert isinstance(idx, int)
+                            res = ord(obj[idx])
+                            stack.append(res)
+                            continue
                     elif isinstance(obj, list):
                         if mname == 'slice':
                             assert len(args) == 1
@@ -687,6 +702,11 @@ class SWFInterpreter(object):
                     value1 = stack.pop()
                     res = value1 - value2
                     stack.append(res)
+                elif opcode == 162:  # multiply
+                    value2 = stack.pop()
+                    value1 = stack.pop()
+                    res = value1 * value2
+                    stack.append(res)
                 elif opcode == 164:  # modulo
                     value2 = stack.pop()
                     value1 = stack.pop()
@@ -702,6 +722,10 @@ class SWFInterpreter(object):
                     value1 = stack.pop()
                     result = value1 >= value2
                     stack.append(result)
+                elif opcode == 192:  # increment_i
+                    value = stack.pop()
+                    assert isinstance(value, int)
+                    stack.append(value + 1)
                 elif opcode == 208:  # getlocal_0
                     stack.append(registers[0])
                 elif opcode == 209:  # getlocal_1

From 162f54eca6b9c0dc2e4fe36bd2905190ba0185fa Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 05:08:39 +0100
Subject: [PATCH 0513/1937] [swfinterp] Implement bitand and pushshort
 operators

---
 youtube_dl/swfinterp.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index 7369c94fc..fc704603a 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -451,6 +451,9 @@ class SWFInterpreter(object):
                 elif opcode == 36:  # pushbyte
                     v = _read_byte(coder)
                     stack.append(v)
+                elif opcode == 37:  # pushshort
+                    v = u30()
+                    stack.append(v)
                 elif opcode == 38:  # pushtrue
                     stack.append(True)
                 elif opcode == 39:  # pushfalse
@@ -712,6 +715,13 @@ class SWFInterpreter(object):
                     value1 = stack.pop()
                     res = value1 % value2
                     stack.append(res)
+                elif opcode == 168:  # bitand
+                    value2 = stack.pop()
+                    value1 = stack.pop()
+                    assert isinstance(value1, int)
+                    assert isinstance(value2, int)
+                    res = value1 & value2
+                    stack.append(res)
                 elif opcode == 171:  # equals
                     value2 = stack.pop()
                     value1 = stack.pop()

From cd9ad1d7e8970c4e5d468d873ede88d26fd99bd2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 07:14:02 +0100
Subject: [PATCH 0514/1937] [swfinterp] Basic support for constants (only ints
 for now)

---
 test/swftests/ConstantInt.as | 12 ++++++++++
 youtube_dl/swfinterp.py      | 43 +++++++++++++++++++++++++++---------
 2 files changed, 45 insertions(+), 10 deletions(-)
 create mode 100644 test/swftests/ConstantInt.as

diff --git a/test/swftests/ConstantInt.as b/test/swftests/ConstantInt.as
new file mode 100644
index 000000000..e0bbb6166
--- /dev/null
+++ b/test/swftests/ConstantInt.as
@@ -0,0 +1,12 @@
+// input: []
+// output: 2
+
+package {
+public class ConstantInt {
+	private static const x:int = 2;
+
+    public static function main():int{
+        return x;
+    }
+}
+}
diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index fc704603a..5d17fb1ac 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -71,6 +71,7 @@ class _AVMClass(object):
         self.method_pyfunctions = {}
 
         self.variables = _ScopeDict(self)
+        self.constants = {}
 
     def make_object(self):
         return _AVMClass_Object(self)
@@ -189,11 +190,13 @@ class SWFInterpreter(object):
 
         # Constant pool
         int_count = u30()
+        self.constant_ints = [0]
         for _c in range(1, int_count):
-            s32()
+            self.constant_ints.append(s32())
+        self.constant_uints = [0]
         uint_count = u30()
         for _c in range(1, uint_count):
-            u32()
+            self.constant_uints.append(u32())
         double_count = u30()
         read_bytes(max(0, (double_count - 1)) * 8)
         string_count = u30()
@@ -281,13 +284,28 @@ class SWFInterpreter(object):
             kind = kind_full & 0x0f
             attrs = kind_full >> 4
             methods = {}
-            if kind in [0x00, 0x06]:  # Slot or Const
+            constants = None
+            if kind == 0x00:  # Slot
                 u30()  # Slot id
                 u30()  # type_name_idx
                 vindex = u30()
                 if vindex != 0:
                     read_byte()  # vkind
-            elif kind in [0x01, 0x02, 0x03]:  # Method / Getter / Setter
+            elif kind == 0x06:  # Const
+                u30()  # Slot id
+                u30()  # type_name_idx
+                vindex = u30()
+                vkind = 'any'
+                if vindex != 0:
+                    vkind = read_byte()
+                if vkind == 0x03:  # Constant_Int
+                    value = self.constant_ints[vindex]
+                elif vkind == 0x04:  # Constant_UInt
+                    value = self.constant_uints[vindex]
+                else:
+                    return {}, None  # Ignore silently for now
+                constants = {self.multinames[trait_name_idx]: value}
+            elif kind in (0x01, 0x02, 0x03):  # Method / Getter / Setter
                 u30()  # disp_id
                 method_idx = u30()
                 methods[self.multinames[trait_name_idx]] = method_idx
@@ -306,7 +324,7 @@ class SWFInterpreter(object):
                 for _c3 in range(metadata_count):
                     u30()  # metadata index
 
-            return methods
+            return methods, constants
 
         # Classes
         class_count = u30()
@@ -328,8 +346,9 @@ class SWFInterpreter(object):
             u30()  # iinit
             trait_count = u30()
             for _c2 in range(trait_count):
-                trait_methods = parse_traits_info()
+                trait_methods, constants = parse_traits_info()
                 avm_class.register_methods(trait_methods)
+                assert constants is None
 
         assert len(classes) == class_count
         self._classes_by_name = dict((c.name, c) for c in classes)
@@ -338,8 +357,10 @@ class SWFInterpreter(object):
             u30()  # cinit
             trait_count = u30()
             for _c2 in range(trait_count):
-                trait_methods = parse_traits_info()
+                trait_methods, trait_constants = parse_traits_info()
                 avm_class.register_methods(trait_methods)
+                if trait_constants:
+                    avm_class.constants.update(trait_constants)
 
         # Scripts
         script_count = u30()
@@ -633,9 +654,11 @@ class SWFInterpreter(object):
                             break
                     else:
                         scope = avm_class.variables
-                    # I cannot find where static variables are initialized
-                    # so let's just return None
-                    res = scope.get(mname)
+
+                    if mname in scope:
+                        res = scope[mname]
+                    else:
+                        res = avm_class.constants[mname]
                     stack.append(res)
                 elif opcode == 97:  # setproperty
                     index = u30()

From 28e614de5ccfd846fe4dafe37d4834ef3488b403 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 07:16:12 +0100
Subject: [PATCH 0515/1937] [utils] Remove stray u'

---
 youtube_dl/utils.py | 62 ++++++++++++++++++++++-----------------------
 1 file changed, 31 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 58897231d..50e515a04 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -63,7 +63,7 @@ def preferredencoding():
     """
     try:
         pref = locale.getpreferredencoding()
-        u'TEST'.encode(pref)
+        'TEST'.encode(pref)
     except:
         pref = 'UTF-8'
 
@@ -215,7 +215,7 @@ def sanitize_open(filename, open_mode):
     It returns the tuple (stream, definitive_file_name).
     """
     try:
-        if filename == u'-':
+        if filename == '-':
             if sys.platform == 'win32':
                 import msvcrt
                 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
@@ -228,7 +228,7 @@ def sanitize_open(filename, open_mode):
 
         # In case of error, try to remove win32 forbidden chars
         alt_filename = os.path.join(
-                        re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', path_part)
+                        re.sub('[/<>:"\\|\\\\?\\*]', '#', path_part)
                         for path_part in os.path.split(filename)
                        )
         if alt_filename == filename:
@@ -267,7 +267,7 @@ def sanitize_filename(s, restricted=False, is_id=False):
             return '_'
         return char
 
-    result = u''.join(map(replace_insane, s))
+    result = ''.join(map(replace_insane, s))
     if not is_id:
         while '__' in result:
             result = result.replace('__', '_')
@@ -297,15 +297,15 @@ def _htmlentity_transform(entity):
     mobj = re.match(r'#(x?[0-9]+)', entity)
     if mobj is not None:
         numstr = mobj.group(1)
-        if numstr.startswith(u'x'):
+        if numstr.startswith('x'):
             base = 16
-            numstr = u'0%s' % numstr
+            numstr = '0%s' % numstr
         else:
             base = 10
         return compat_chr(int(numstr, base))
 
     # Unknown entity in name, return its literal representation
-    return (u'&%s;' % entity)
+    return ('&%s;' % entity)
 
 
 def unescapeHTML(s):
@@ -329,7 +329,7 @@ def encodeFilename(s, for_subprocess=False):
         return s
 
     if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
-        # Pass u'' directly to use Unicode APIs on Windows 2000 and up
+        # Pass '' directly to use Unicode APIs on Windows 2000 and up
         # (Detecting Windows NT 4 is tricky because 'major >= 4' would
         # match Windows 9x series as well. Besides, NT 4 is obsolete.)
         if not for_subprocess:
@@ -424,9 +424,9 @@ class ExtractorError(Exception):
         if video_id is not None:
             msg = video_id + ': ' + msg
         if cause:
-            msg += u' (caused by %r)' % cause
+            msg += ' (caused by %r)' % cause
         if not expected:
-            msg = msg + u'; please report this issue on https://yt-dl.org/bug . Be sure to call youtube-dl with the --verbose flag and include its complete output. Make sure you are using the latest version; type  youtube-dl -U  to update.'
+            msg = msg + '; please report this issue on https://yt-dl.org/bug . Be sure to call youtube-dl with the --verbose flag and include its complete output. Make sure you are using the latest version; type  youtube-dl -U  to update.'
         super(ExtractorError, self).__init__(msg)
 
         self.traceback = tb
@@ -437,7 +437,7 @@ class ExtractorError(Exception):
     def format_traceback(self):
         if self.traceback is None:
             return None
-        return u''.join(traceback.format_tb(self.traceback))
+        return ''.join(traceback.format_tb(self.traceback))
 
 
 class RegexNotFoundError(ExtractorError):
@@ -665,17 +665,17 @@ def unified_strdate(date_str):
             upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
     return upload_date
 
-def determine_ext(url, default_ext=u'unknown_video'):
+def determine_ext(url, default_ext='unknown_video'):
     if url is None:
         return default_ext
-    guess = url.partition(u'?')[0].rpartition(u'.')[2]
+    guess = url.partition('?')[0].rpartition('.')[2]
     if re.match(r'^[A-Za-z0-9]+$', guess):
         return guess
     else:
         return default_ext
 
 def subtitles_filename(filename, sub_lang, sub_format):
-    return filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
+    return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
 
 def date_from_str(date_str):
     """
@@ -967,7 +967,7 @@ def shell_quote(args):
             # We may get a filename encoded with 'encodeFilename'
             a = a.decode(encoding)
         quoted_args.append(pipes.quote(a))
-    return u' '.join(quoted_args)
+    return ' '.join(quoted_args)
 
 
 def takewhile_inclusive(pred, seq):
@@ -983,31 +983,31 @@ def smuggle_url(url, data):
     """ Pass additional data in a URL for internal use. """
 
     sdata = compat_urllib_parse.urlencode(
-        {u'__youtubedl_smuggle': json.dumps(data)})
-    return url + u'#' + sdata
+        {'__youtubedl_smuggle': json.dumps(data)})
+    return url + '#' + sdata
 
 
 def unsmuggle_url(smug_url, default=None):
     if not '#__youtubedl_smuggle' in smug_url:
         return smug_url, default
-    url, _, sdata = smug_url.rpartition(u'#')
-    jsond = compat_parse_qs(sdata)[u'__youtubedl_smuggle'][0]
+    url, _, sdata = smug_url.rpartition('#')
+    jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
     data = json.loads(jsond)
     return url, data
 
 
 def format_bytes(bytes):
     if bytes is None:
-        return u'N/A'
+        return 'N/A'
     if type(bytes) is str:
         bytes = float(bytes)
     if bytes == 0.0:
         exponent = 0
     else:
         exponent = int(math.log(bytes, 1024.0))
-    suffix = [u'B', u'KiB', u'MiB', u'GiB', u'TiB', u'PiB', u'EiB', u'ZiB', u'YiB'][exponent]
+    suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
     converted = float(bytes) / float(1024 ** exponent)
-    return u'%.2f%s' % (converted, suffix)
+    return '%.2f%s' % (converted, suffix)
 
 
 def get_term_width():
@@ -1030,8 +1030,8 @@ def month_by_name(name):
     """ Return the number of a month by (locale-independently) English name """
 
     ENGLISH_NAMES = [
-        u'January', u'February', u'March', u'April', u'May', u'June',
-        u'July', u'August', u'September', u'October', u'November', u'December']
+        'January', 'February', 'March', 'April', 'May', 'June',
+        'July', 'August', 'September', 'October', 'November', 'December']
     try:
         return ENGLISH_NAMES.index(name) + 1
     except ValueError:
@@ -1042,7 +1042,7 @@ def fix_xml_ampersands(xml_str):
     """Replace all the '&' by '&amp;' in XML"""
     return re.sub(
         r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
-        u'&amp;',
+        '&amp;',
         xml_str)
 
 
@@ -1075,7 +1075,7 @@ def remove_end(s, end):
 
 def url_basename(url):
     path = compat_urlparse.urlparse(url).path
-    return path.strip(u'/').split(u'/')[-1]
+    return path.strip('/').split('/')[-1]
 
 
 class HEADRequest(compat_urllib_request.Request):
@@ -1100,7 +1100,7 @@ def str_to_int(int_str):
     """ A more relaxed version of int_or_none """
     if int_str is None:
         return None
-    int_str = re.sub(r'[,\.\+]', u'', int_str)
+    int_str = re.sub(r'[,\.\+]', '', int_str)
     return int(int_str)
 
 
@@ -1135,7 +1135,7 @@ def parse_duration(s):
 
 def prepend_extension(filename, ext):
     name, real_ext = os.path.splitext(filename) 
-    return u'{0}.{1}{2}'.format(name, ext, real_ext)
+    return '{0}.{1}{2}'.format(name, ext, real_ext)
 
 
 def check_executable(exe, args=[]):
@@ -1150,7 +1150,7 @@ def check_executable(exe, args=[]):
 
 def get_exe_version(exe, args=['--version'],
                     version_re=r'version\s+([0-9._-a-zA-Z]+)',
-                    unrecognized=u'present'):
+                    unrecognized='present'):
     """ Returns the version of the specified executable,
     or False if the executable is not present """
     try:
@@ -1271,7 +1271,7 @@ def escape_url(url):
     ).geturl()
 
 try:
-    struct.pack(u'!I', 0)
+    struct.pack('!I', 0)
 except TypeError:
     # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
     def struct_pack(spec, *args):
@@ -1292,7 +1292,7 @@ def read_batch_urls(batch_fd):
     def fixup(url):
         if not isinstance(url, compat_str):
             url = url.decode('utf-8', 'replace')
-        BOM_UTF8 = u'\xef\xbb\xbf'
+        BOM_UTF8 = '\xef\xbb\xbf'
         if url.startswith(BOM_UTF8):
             url = url[len(BOM_UTF8):]
         url = url.strip()

From 1921b24551424d4bb17356c4c7c1ad374d4f899b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 07:31:22 +0100
Subject: [PATCH 0516/1937] [swfinterp] Add support for more complicated
 constants

---
 youtube_dl/swfinterp.py | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index 5d17fb1ac..e9bf178e4 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -354,7 +354,7 @@ class SWFInterpreter(object):
         self._classes_by_name = dict((c.name, c) for c in classes)
 
         for avm_class in classes:
-            u30()  # cinit
+            avm_class.cinit_idx = u30()
             trait_count = u30()
             for _c2 in range(trait_count):
                 trait_methods, trait_constants = parse_traits_info()
@@ -373,6 +373,7 @@ class SWFInterpreter(object):
         # Method bodies
         method_body_count = u30()
         Method = collections.namedtuple('Method', ['code', 'local_count'])
+        self._all_methods = []
         for _c in range(method_body_count):
             method_idx = u30()
             u30()  # max_stack
@@ -381,9 +382,10 @@ class SWFInterpreter(object):
             u30()  # max_scope_depth
             code_length = u30()
             code = read_bytes(code_length)
+            m = Method(code, local_count)
+            self._all_methods.append(m)
             for avm_class in classes:
                 if method_idx in avm_class.method_idxs:
-                    m = Method(code, local_count)
                     avm_class.methods[avm_class.method_idxs[method_idx]] = m
             exception_count = u30()
             for _c2 in range(exception_count):
@@ -401,12 +403,20 @@ class SWFInterpreter(object):
     def patch_function(self, avm_class, func_name, f):
         self._patched_functions[(avm_class, func_name)] = f
 
-    def extract_class(self, class_name):
+    def extract_class(self, class_name, call_cinit=True):
         try:
-            return self._classes_by_name[class_name]
+            res = self._classes_by_name[class_name]
         except KeyError:
             raise ExtractorError('Class %r not found' % class_name)
 
+        if call_cinit and hasattr(res, 'cinit_idx'):
+            res.register_methods({'$cinit': res.cinit_idx})
+            res.methods['$cinit'] = self._all_methods[res.cinit_idx]
+            cinit = self.extract_function(res, '$cinit')
+            cinit([])
+
+        return res
+
     def extract_function(self, avm_class, func_name):
         p = self._patched_functions.get((avm_class, func_name))
         if p:
@@ -694,6 +704,14 @@ class SWFInterpreter(object):
                         obj = stack.pop()
                         assert isinstance(obj, list)
                         stack.append(obj[idx])
+                elif opcode == 104:  # initproperty
+                    index = u30()
+                    value = stack.pop()
+                    idx = self.multinames[index]
+                    if isinstance(idx, _Multiname):
+                        idx = stack.pop()
+                    obj = stack.pop()
+                    obj[idx] = value
                 elif opcode == 115:  # convert_
                     value = stack.pop()
                     intvalue = int(value)

From fbf94a78158ea2ea70dcbc1599036a64bb3321c3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 07:54:06 +0100
Subject: [PATCH 0517/1937] [swfinterp] Add more builtins and improve static
 variables

---
 test/swftests/ConstArrayAccess.as | 18 ++++++++++++++
 youtube_dl/swfinterp.py           | 39 ++++++++++++++++++++++++-------
 2 files changed, 49 insertions(+), 8 deletions(-)
 create mode 100644 test/swftests/ConstArrayAccess.as

diff --git a/test/swftests/ConstArrayAccess.as b/test/swftests/ConstArrayAccess.as
new file mode 100644
index 000000000..07dc3f460
--- /dev/null
+++ b/test/swftests/ConstArrayAccess.as
@@ -0,0 +1,18 @@
+// input: []
+// output: 4
+
+package {
+public class ConstArrayAccess {
+	private static const x:int = 2;
+	private static const ar:Array = ["42", "3411"];
+
+    public static function main():int{
+        var c:ConstArrayAccess = new ConstArrayAccess();
+        return c.f();
+    }
+
+    public function f(): int {
+    	return ar[1].length;
+    }
+}
+}
diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index e9bf178e4..6f3c3bf95 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -62,13 +62,14 @@ class _ScopeDict(dict):
 
 
 class _AVMClass(object):
-    def __init__(self, name_idx, name):
+    def __init__(self, name_idx, name, static_properties=None):
         self.name_idx = name_idx
         self.name = name
         self.method_names = {}
         self.method_idxs = {}
         self.methods = {}
         self.method_pyfunctions = {}
+        self.static_properties = static_properties if static_properties else {}
 
         self.variables = _ScopeDict(self)
         self.constants = {}
@@ -151,15 +152,20 @@ def _read_byte(reader):
 
 StringClass = _AVMClass('(no name idx)', 'String')
 ByteArrayClass = _AVMClass('(no name idx)', 'ByteArray')
+TimerClass = _AVMClass('(no name idx)', 'Timer')
+TimerEventClass = _AVMClass('(no name idx)', 'TimerEvent', {'TIMER': 'timer'})
 _builtin_classes = {
     StringClass.name: StringClass,
     ByteArrayClass.name: ByteArrayClass,
+    TimerClass.name: TimerClass,
+    TimerEventClass.name: TimerEventClass,
 }
 
 
 class _Undefined(object):
-    def __boolean__(self):
+    def __bool__(self):
         return False
+    __nonzero__ = __bool__
 
     def __hash__(self):
         return 0
@@ -169,7 +175,9 @@ undefined = _Undefined()
 
 class SWFInterpreter(object):
     def __init__(self, file_contents):
-        self._patched_functions = {}
+        self._patched_functions = {
+            (TimerClass, 'addEventListener'): lambda params: undefined,
+        }
         code_tag = next(tag
                         for tag_code, tag in _extract_tags(file_contents)
                         if tag_code == 82)
@@ -346,9 +354,10 @@ class SWFInterpreter(object):
             u30()  # iinit
             trait_count = u30()
             for _c2 in range(trait_count):
-                trait_methods, constants = parse_traits_info()
+                trait_methods, trait_constants = parse_traits_info()
                 avm_class.register_methods(trait_methods)
-                assert constants is None
+                if trait_constants:
+                    avm_class.constants.update(trait_constants)
 
         assert len(classes) == class_count
         self._classes_by_name = dict((c.name, c) for c in classes)
@@ -439,7 +448,7 @@ class SWFInterpreter(object):
             registers = [avm_class.variables] + list(args) + [None] * m.local_count
             stack = []
             scopes = collections.deque([
-                self._classes_by_name, avm_class.variables])
+                self._classes_by_name, avm_class.constants, avm_class.variables])
             while True:
                 opcode = _read_byte(coder)
                 if opcode == 9:  # label
@@ -587,6 +596,12 @@ class SWFInterpreter(object):
                 elif opcode == 72:  # returnvalue
                     res = stack.pop()
                     return res
+                elif opcode == 73:  # constructsuper
+                    # Not yet implemented, just hope it works without it
+                    arg_count = u30()
+                    args = list(reversed(
+                        [stack.pop() for _ in range(arg_count)]))
+                    obj = stack.pop()
                 elif opcode == 74:  # constructproperty
                     index = u30()
                     arg_count = u30()
@@ -667,8 +682,11 @@ class SWFInterpreter(object):
 
                     if mname in scope:
                         res = scope[mname]
+                    elif mname in _builtin_classes:
+                        res = _builtin_classes[mname]
                     else:
-                        res = avm_class.constants[mname]
+                        # Assume unitialized
+                        res = undefined
                     stack.append(res)
                 elif opcode == 97:  # setproperty
                     index = u30()
@@ -694,7 +712,12 @@ class SWFInterpreter(object):
                         stack.append(len(obj))
                     elif isinstance(pname, compat_str):  # Member access
                         obj = stack.pop()
-                        assert isinstance(obj, (dict, _ScopeDict)), \
+                        if isinstance(obj, _AVMClass):
+                            res = obj.static_properties[pname]
+                            stack.append(res)
+                            continue
+
+                        assert isinstance(obj, (dict, _ScopeDict)),\
                             'Accessing member %r on %r' % (pname, obj)
                         res = obj.get(pname, undefined)
                         stack.append(res)

From f48d3e9bbcd00b436f9d81233b1e89ad53f7d6a7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 08:02:48 +0100
Subject: [PATCH 0518/1937] [swfinterp] Improve undefined representation

---
 youtube_dl/swfinterp.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index 6f3c3bf95..99220940f 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -170,6 +170,10 @@ class _Undefined(object):
     def __hash__(self):
         return 0
 
+    def __str__(self):
+        return 'undefined'
+    __repr__ = __str__
+
 undefined = _Undefined()
 
 
@@ -686,6 +690,7 @@ class SWFInterpreter(object):
                         res = _builtin_classes[mname]
                     else:
                         # Assume unitialized
+                        # TODO warn here
                         res = undefined
                     stack.append(res)
                 elif opcode == 97:  # setproperty

From 23ad44b57bb62a76414daf630d85c7544e0b2728 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 17 Nov 2014 20:12:52 +0100
Subject: [PATCH 0519/1937] [youtube] Better error message for DASH manifest

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index cea3d5c95..08f63be96 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -991,7 +991,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                         existing_format.update(f)
 
             except (ExtractorError, KeyError) as e:
-                self.report_warning('Skipping DASH manifest: %s' % e, video_id)
+                self.report_warning('Skipping DASH manifest: %r' % e, video_id)
 
         self._sort_formats(formats)
 

From 469d4c89686afca46333d85442bb770e6010518c Mon Sep 17 00:00:00 2001
From: Will Sewell <me@willsewell.name>
Date: Mon, 17 Nov 2014 17:52:00 -0500
Subject: [PATCH 0520/1937] [vk] Added a new information extractor for pages
 that are a list of a user\'s videos on vk.com. It works in a same way to
 playlist style pages for the YT information extractors.

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/vk.py       | 37 +++++++++++++++++++++++++++++++-
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f45ce05ab..b687a56b4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -452,7 +452,10 @@ from .vine import (
     VineUserIE,
 )
 from .viki import VikiIE
-from .vk import VKIE
+from .vk import (
+    VKIE,
+    VKUserVideosIE,
+)
 from .vodlocker import VodlockerIE
 from .vporn import VpornIE
 from .vrt import VRTIE
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 36cd7e52e..5223e5e2c 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 class VKIE(InfoExtractor):
     IE_NAME = 'vk.com'
-    _VALID_URL = r'https?://(?:m\.)?vk\.com/(?:video_ext\.php\?.*?\boid=(?P<oid>-?\d+).*?\bid=(?P<id>\d+)|(?:.+?\?.*?z=)?video(?P<videoid>.*?)(?:\?|%2F|$))'
+    _VALID_URL = r'https?://(?:m\.)?vk\.com/(?:video_ext\.php\?.*?\boid=(?P<oid>-?\d+).*?\bid=(?P<id>\d+)|(?:.+?\?.*?z=)?video(?P<videoid>[^s].*?)(?:\?|%2F|$))'
     _NETRC_MACHINE = 'vk'
 
     _TESTS = [
@@ -185,3 +185,38 @@ class VKIE(InfoExtractor):
             'uploader': data.get('md_author'),
             'duration': data.get('duration')
         }
+
+
+class VKUserVideosIE(InfoExtractor):
+    IE_NAME = 'vk.com:user-videos'
+    IE_DESC = 'All of a user\'s videos'
+    _VALID_URL = r'https?://(?:m\.)?vk\.com/videos([0-9]+)'
+    _TEMPLATE_URL = 'https://vk.com/videos'
+    _TEST = {
+        'url': 'http://vk.com/videos205387401',
+        'playlist_mincount': 4,
+    }
+
+    def extract_videos_from_page(self, page):
+        ids_in_page = []
+        for mobj in re.finditer(r'href="/video([0-9_]+)"', page):
+            if mobj.group(1) not in ids_in_page:
+                ids_in_page.append(mobj.group(1))
+        return ids_in_page
+
+    def _real_extract(self, url):
+        # Extract page id
+        mobj = re.match(self._VALID_URL, url)
+        if mobj is None:
+            raise ExtractorError('Invalid URL: %s' % url)
+
+        # Download page and get video ids
+        page_id = mobj.group(1)
+        page = self._download_webpage(url, page_id)
+        video_ids = self.extract_videos_from_page(page)
+
+        self._downloader.to_screen('[vk] User videos %s: Found %i videos' % (page_id, len(video_ids)))
+
+        url_entries = [self.url_result('http://vk.com/video' + video_id, 'VK', video_id=video_id)
+                       for video_id in video_ids]
+        return self.playlist_result(url_entries, page_id)
\ No newline at end of file

From 0857baade31dd0ce9ce220148ef914cdc63030cb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 18 Nov 2014 11:06:09 +0100
Subject: [PATCH 0521/1937] [youtube] Add webm audio formats (Fixes #4229)

---
 youtube_dl/extractor/youtube.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 08f63be96..461271d3b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -307,6 +307,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
         '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
 
+        # Dash webm audio with opus inside
+        '249': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50, 'preference': -50},
+        '250': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70, 'preference': -50},
+        '251': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160, 'preference': -50},
+
         # RTMP (unnamed)
         '_rtmp': {'protocol': 'rtmp'},
     }

From 6fcd6e0e21839ae4b1753995dc44d2a93f72ac1f Mon Sep 17 00:00:00 2001
From: Will Sewell <me@willsewell.name>
Date: Tue, 18 Nov 2014 19:34:12 +0000
Subject: [PATCH 0522/1937] [vk] Updated the regex for matching user video
 pages. It now matches optional URL parameters too.

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 5223e5e2c..3bcf50e28 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -190,7 +190,7 @@ class VKIE(InfoExtractor):
 class VKUserVideosIE(InfoExtractor):
     IE_NAME = 'vk.com:user-videos'
     IE_DESC = 'All of a user\'s videos'
-    _VALID_URL = r'https?://(?:m\.)?vk\.com/videos([0-9]+)'
+    _VALID_URL = r'https?://(?:m\.)?vk\.com/videos([0-9]+)(?:m\?.*)?'
     _TEMPLATE_URL = 'https://vk.com/videos'
     _TEST = {
         'url': 'http://vk.com/videos205387401',

From 02a12f9fe69508525c9cad06782151f5cf950671 Mon Sep 17 00:00:00 2001
From: Will Sewell <me@willsewell.name>
Date: Tue, 18 Nov 2014 20:19:56 +0000
Subject: [PATCH 0523/1937] [vk] date_added is now extracted from the video
 page.

---
 youtube_dl/extractor/vk.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 3bcf50e28..deaad6c3e 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -11,7 +11,7 @@ from ..utils import (
     compat_urllib_parse,
     compat_str,
     unescapeHTML,
-)
+    unified_strdate)
 
 
 class VKIE(InfoExtractor):
@@ -169,6 +169,12 @@ class VKIE(InfoExtractor):
         data_json = self._search_regex(r'var vars = ({.*?});', info_page, 'vars')
         data = json.loads(data_json)
 
+        # Extract upload date
+        upload_date = None
+        mobj = re.search(r'id="mv_date_wrap".*?Added ([a-zA-Z]+ [0-9]+), ([0-9]+) at', info_page)
+        if mobj is not None:
+            upload_date = unified_strdate(mobj.group(1) + ' ' + mobj.group(2))
+
         formats = [{
             'format_id': k,
             'url': v,
@@ -183,7 +189,8 @@ class VKIE(InfoExtractor):
             'title': unescapeHTML(data['md_title']),
             'thumbnail': data.get('jpg'),
             'uploader': data.get('md_author'),
-            'duration': data.get('duration')
+            'duration': data.get('duration'),
+            'upload_date': upload_date,
         }
 
 

From 37eddd314319d42f509307698196e4e9a8e84c7e Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 19 Nov 2014 00:26:44 +0200
Subject: [PATCH 0524/1937] [rtlxl] Use m3u8 streams instead of f4m (#4115,
 #4118)

---
 youtube_dl/extractor/rtlnl.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 0ab1eb69c..599952512 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -28,8 +28,9 @@ class RtlXlIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         uuid = mobj.group('uuid')
 
+        # Use m3u8 streams (see https://github.com/rg3/youtube-dl/issues/4118)
         info = self._download_json(
-            'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=flash/' % uuid,
+            'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/d=pc/fmt=adaptive/' % uuid,
             uuid)
 
         material = info['material'][0]
@@ -39,11 +40,11 @@ class RtlXlIE(InfoExtractor):
         subtitle = material['title'] or info['episodes'][0]['name']
 
         videopath = material['videopath']
-        f4m_url = 'http://manifest.us.rtl.nl' + videopath
+        m3u8_url = 'http://manifest.us.rtl.nl' + videopath
 
-        formats = self._extract_f4m_formats(f4m_url, uuid)
+        formats = self._extract_m3u8_formats(m3u8_url, uuid, ext='mp4')
 
-        video_urlpart = videopath.split('/flash/')[1][:-4]
+        video_urlpart = videopath.split('/adaptive/')[1][:-4]
         PG_URL_TEMPLATE = 'http://pg.us.rtl.nl/rtlxl/network/%s/progressive/%s.mp4'
 
         formats.extend([
@@ -57,6 +58,8 @@ class RtlXlIE(InfoExtractor):
             }
         ])
 
+        self._sort_formats(formats)
+
         return {
             'id': uuid,
             'title': '%s - %s' % (progname, subtitle),

From 92120217eb27d820bfba0f6a50837f111cbf80ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 18 Nov 2014 23:28:42 +0100
Subject: [PATCH 0525/1937] [cache] Fix writing to paths with unicode
 characters

* Use "compat_getenv"
* "write_json_file" now expects the filename to be a string
---
 youtube_dl/YoutubeDL.py | 2 +-
 youtube_dl/cache.py     | 4 ++--
 youtube_dl/utils.py     | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5c875b497..94c50903c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1001,7 +1001,7 @@ class YoutubeDL(object):
             else:
                 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
                 try:
-                    write_json_file(info_dict, encodeFilename(infofn))
+                    write_json_file(info_dict, infofn)
                 except (OSError, IOError):
                     self.report_error('Cannot write metadata to JSON file ' + infofn)
                     return
diff --git a/youtube_dl/cache.py b/youtube_dl/cache.py
index 2d9b426cb..5fe839eb1 100644
--- a/youtube_dl/cache.py
+++ b/youtube_dl/cache.py
@@ -8,7 +8,7 @@ import re
 import shutil
 import traceback
 
-from .compat import compat_expanduser
+from .compat import compat_expanduser, compat_getenv
 from .utils import write_json_file
 
 
@@ -19,7 +19,7 @@ class Cache(object):
     def _get_root_dir(self):
         res = self._ydl.params.get('cachedir')
         if res is None:
-            cache_root = os.environ.get('XDG_CACHE_HOME', '~/.cache')
+            cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
             res = os.path.join(cache_root, 'youtube-dl')
         return compat_expanduser(res)
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 50e515a04..94b496dd0 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -73,6 +73,7 @@ def preferredencoding():
 def write_json_file(obj, fn):
     """ Encode obj as JSON and write it to fn, atomically """
 
+    fn = encodeFilename(fn)
     if sys.version_info < (3, 0):
         encoding = get_filesystem_encoding()
         # os.path.basename returns a bytes object, but NamedTemporaryFile

From f56875f2719868d191a51a1444720717c2e7e5c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 18 Nov 2014 23:34:46 +0100
Subject: [PATCH 0526/1937] [test/test_compat] Restore the old value of the
 HOME environment variable
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

If the test was run before the YoutubeIE tests (for example by running
"nosetests -v test/test_compat.py test/test_download.py -m 'Youtube_1|compat_expand'"),
it wrote the signatures cache to the 'C:\Documents and Settings\тест\Application Data' folder.
It failed due to a problem in the cache code and the write_json_file function (fixed in f03e33b89a622af13fa5275c46b63aaa4814c499)
---
 test/test_compat.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/test_compat.py b/test/test_compat.py
index 4a7fc3606..1eb454e06 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -26,11 +26,13 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(compat_getenv('YOUTUBE-DL-TEST'), test_str)
 
     def test_compat_expanduser(self):
+        old_home = os.environ.get('HOME')
         test_str = 'C:\Documents and Settings\тест\Application Data'
         os.environ['HOME'] = (
             test_str if sys.version_info >= (3, 0)
             else test_str.encode(get_filesystem_encoding()))
         self.assertEqual(compat_expanduser('~'), test_str)
+        os.environ['HOME'] = old_home
 
     def test_all_present(self):
         import youtube_dl.compat

From 480b7c32a94db050bb216519d3b7fd4589d1e8fb Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 19 Nov 2014 01:20:42 +0200
Subject: [PATCH 0527/1937] [rtlxl] Fix format order

---
 youtube_dl/extractor/rtlnl.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 599952512..5daef2fc5 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -55,6 +55,7 @@ class RtlXlIE(InfoExtractor):
             {
                 'url': PG_URL_TEMPLATE % ('a3m', video_urlpart),
                 'format_id': 'pg-hd',
+                'quality': 0,
             }
         ])
 

From e07e931375df3984e4f92610413a2f52764bf432 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 19 Nov 2014 18:21:58 +0100
Subject: [PATCH 0528/1937] Work around 2.7.0 deficencies (Fixes #4223)

---
 youtube_dl/__init__.py |  3 +++
 youtube_dl/compat.py   | 22 ++++++++++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 8e186a0db..c1323b4f3 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -19,6 +19,7 @@ from .compat import (
     compat_expanduser,
     compat_getpass,
     compat_print,
+    workaround_optparse_bug9161,
 )
 from .utils import (
     DateRange,
@@ -57,6 +58,8 @@ def _real_main(argv=None):
         # https://github.com/rg3/youtube-dl/issues/820
         codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
 
+    workaround_optparse_bug9161()
+
     setproctitle('youtube-dl')
 
     parser, opts, args = parseOpts(argv)
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 385924803..64a975489 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import getpass
+import optparse
 import os
 import subprocess
 import sys
@@ -297,6 +298,26 @@ except TypeError:
 else:
     compat_kwargs = lambda kwargs: kwargs
 
+
+# Fix https://github.com/rg3/youtube-dl/issues/4223
+# See http://bugs.python.org/issue9161 for what is broken
+def workaround_optparse_bug9161():
+    try:
+        optparse.OptionGroup('foo').add_option('-t')
+    except TypeError:
+        real_add_option = optparse.OptionGroup.add_option
+
+        def _compat_add_option(self, *args, **kwargs):
+            enc = lambda v: (
+                v.encode('ascii', 'replace') if isinstance(v, compat_str)
+                else v)
+            bargs = [enc(a) for a in args]
+            bkwargs = dict(
+                (k, enc(v)) for k, v in kwargs.items())
+            return real_add_option(self, *bargs, **bkwargs)
+        optparse.OptionGroup.add_option = _compat_add_option
+
+
 __all__ = [
     'compat_HTTPError',
     'compat_chr',
@@ -323,4 +344,5 @@ __all__ = [
     'compat_xml_parse_error',
     'shlex_quote',
     'subprocess_check_output',
+    'workaround_optparse_bug9161',
 ]

From 07e378fa18b1c6b8becd46f6cf2d476e29a70a59 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 20 Nov 2014 07:21:12 +0100
Subject: [PATCH 0529/1937] [compat] correct OptionGroup invocation for Python
 3 (fixes #4243)

---
 youtube_dl/compat.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 64a975489..adf81e7ab 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -302,9 +302,11 @@ else:
 # Fix https://github.com/rg3/youtube-dl/issues/4223
 # See http://bugs.python.org/issue9161 for what is broken
 def workaround_optparse_bug9161():
+    op = optparse.OptionParser()
+    og = optparse.OptionGroup(op, 'foo')
     try:
-        optparse.OptionGroup('foo').add_option('-t')
-    except TypeError:
+        og.add_option('-t')
+    except TypeError as te:
         real_add_option = optparse.OptionGroup.add_option
 
         def _compat_add_option(self, *args, **kwargs):

From 61ee5aeb7363d16c16cdec0914ade78af1afd8fd Mon Sep 17 00:00:00 2001
From: Ivan Kozik <ivan@ludios.org>
Date: Thu, 20 Nov 2014 06:04:04 +0000
Subject: [PATCH 0530/1937] Fix UnicodeEncodeError with --write-info-json on
 Python 2.7 + Windows

Fixes #4244
---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 94b496dd0..f339fcb31 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -74,7 +74,7 @@ def write_json_file(obj, fn):
     """ Encode obj as JSON and write it to fn, atomically """
 
     fn = encodeFilename(fn)
-    if sys.version_info < (3, 0):
+    if sys.version_info < (3, 0) and sys.platform != 'win32':
         encoding = get_filesystem_encoding()
         # os.path.basename returns a bytes object, but NamedTemporaryFile
         # will fail if the filename contains non ascii characters unless we

From 1394646a0a531f6b13a4af7b1e3eec9951a6f9fb Mon Sep 17 00:00:00 2001
From: Ivan Kozik <ivan@ludios.org>
Date: Thu, 20 Nov 2014 06:05:39 +0000
Subject: [PATCH 0531/1937] Fix "ERROR: Cannot write metadata to JSON file" on
 Windows

Fixes #4246
---
 youtube_dl/utils.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f339fcb31..bfe88b40b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -71,7 +71,7 @@ def preferredencoding():
 
 
 def write_json_file(obj, fn):
-    """ Encode obj as JSON and write it to fn, atomically """
+    """ Encode obj as JSON and write it to fn, atomically if possible """
 
     fn = encodeFilename(fn)
     if sys.version_info < (3, 0) and sys.platform != 'win32':
@@ -108,6 +108,13 @@ def write_json_file(obj, fn):
     try:
         with tf:
             json.dump(obj, tf)
+        if sys.platform == 'win32':
+            # Need to remove existing file on Windows, else os.rename raises
+            # WindowsError or FileExistsError.
+            try:
+                os.unlink(fn)
+            except OSError:
+                pass
         os.rename(tf.name, fn)
     except:
         try:

From 0cd64bd077fb378ae3afc5b2a8e091e4f50df9c2 Mon Sep 17 00:00:00 2001
From: Ivan Kozik <ivan@ludios.org>
Date: Thu, 20 Nov 2014 06:25:12 +0000
Subject: [PATCH 0532/1937] Copy the mtime from the oldest source file to the
 file created by ffmpeg

Fixes #4245
---
 youtube_dl/postprocessor/ffmpeg.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index f3f2743c0..d44de84cf 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -79,6 +79,7 @@ class FFmpegPostProcessor(PostProcessor):
         files_cmd = []
         for path in input_paths:
             files_cmd.extend(['-i', encodeFilename(path, True)])
+        oldest_mtime = min(os.stat(path).st_mtime for path in input_paths)
         cmd = ([self._executable, '-y'] + files_cmd
                + [encodeArgument(o) for o in opts] +
                [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])
@@ -91,6 +92,7 @@ class FFmpegPostProcessor(PostProcessor):
             stderr = stderr.decode('utf-8', 'replace')
             msg = stderr.strip().split('\n')[-1]
             raise FFmpegPostProcessorError(msg)
+        os.utime(out_path, (oldest_mtime, oldest_mtime))
         if self._deletetempfiles:
             for ipath in input_paths:
                 os.remove(ipath)

From 3b9f631c416fb31db1b4489b82e0f97927ae2971 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 20 Nov 2014 08:55:56 +0100
Subject: [PATCH 0533/1937] release 2014.11.20

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 34bf665ad..f7965d907 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.16'
+__version__ = '2014.11.20'

From aff2f4f4f56e15976c539211def26236a4cd55ef Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 20 Nov 2014 12:06:33 +0100
Subject: [PATCH 0534/1937] [arte] Clean up format sorting mess

We now use our standard sorting facilities. As a side effect, it's finally possible to download German videos from French URLs and vice versa.
---
 youtube_dl/extractor/arte.py   | 113 +++++++++++++--------------------
 youtube_dl/extractor/common.py |   6 ++
 2 files changed, 50 insertions(+), 69 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index b9a9440c0..aafc3a24d 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -8,10 +8,10 @@ from ..utils import (
     ExtractorError,
     find_xpath_attr,
     unified_strdate,
-    determine_ext,
     get_element_by_id,
     get_element_by_attribute,
     int_or_none,
+    qualities,
 )
 
 # There are different sources of video in arte.tv, the extraction process 
@@ -102,79 +102,54 @@ class ArteTVPlus7IE(InfoExtractor):
             'upload_date': unified_strdate(upload_date_str),
             'thumbnail': player_info.get('programImage') or player_info.get('VTU', {}).get('IUR'),
         }
+        qfunc = qualities(['HQ', 'MQ', 'EQ', 'SQ'])
 
-        all_formats = []
+        formats = []
         for format_id, format_dict in player_info['VSR'].items():
-            fmt = dict(format_dict)
-            fmt['format_id'] = format_id
-            all_formats.append(fmt)
-        # Some formats use the m3u8 protocol
-        all_formats = list(filter(lambda f: f.get('videoFormat') != 'M3U8', all_formats))
-        def _match_lang(f):
-            if f.get('versionCode') is None:
-                return True
-            # Return true if that format is in the language of the url
-            if lang == 'fr':
-                l = 'F'
-            elif lang == 'de':
-                l = 'A'
-            else:
-                l = lang
-            regexes = [r'VO?%s' % l, r'VO?.-ST%s' % l]
-            return any(re.match(r, f['versionCode']) for r in regexes)
-        # Some formats may not be in the same language as the url
-        # TODO: Might want not to drop videos that does not match requested language
-        # but to process those formats with lower precedence
-        formats = filter(_match_lang, all_formats)
-        formats = list(formats)  # in python3 filter returns an iterator
-        if not formats:
-            # Some videos are only available in the 'Originalversion'
-            # they aren't tagged as being in French or German
-            # Sometimes there are neither videos of requested lang code
-            # nor original version videos available
-            # For such cases we just take all_formats as is
-            formats = all_formats
-            if not formats:
-                raise ExtractorError('The formats list is empty')
+            f = dict(format_dict)
+            versionCode = f.get('versionCode')
 
-        if re.match(r'[A-Z]Q', formats[0]['quality']) is not None:
-            def sort_key(f):
-                return ['HQ', 'MQ', 'EQ', 'SQ'].index(f['quality'])
-        else:
-            def sort_key(f):
-                versionCode = f.get('versionCode')
-                if versionCode is None:
-                    versionCode = ''
-                return (
-                    # Sort first by quality
-                    int(f.get('height', -1)),
-                    int(f.get('bitrate', -1)),
-                    # The original version with subtitles has lower relevance
-                    re.match(r'VO-ST(F|A)', versionCode) is None,
-                    # The version with sourds/mal subtitles has also lower relevance
-                    re.match(r'VO?(F|A)-STM\1', versionCode) is None,
-                    # Prefer http downloads over m3u8
-                    0 if f['url'].endswith('m3u8') else 1,
-                )
-        formats = sorted(formats, key=sort_key)
-        def _format(format_info):
-            info = {
-                'format_id': format_info['format_id'],
-                'format_note': '%s, %s' % (format_info.get('versionCode'), format_info.get('versionLibelle')),
-                'width': int_or_none(format_info.get('width')),
-                'height': int_or_none(format_info.get('height')),
-                'tbr': int_or_none(format_info.get('bitrate')),
+            langcode = {
+                'fr': 'F',
+                'de': 'A',
+            }.get(lang, lang)
+            lang_rexs = [r'VO?%s' % langcode, r'VO?.-ST%s' % langcode]
+            lang_pref = (
+                None if versionCode is None else (
+                    10 if any(re.match(r, versionCode) for r in lang_rexs)
+                    else -10))
+            source_pref = 0
+            if versionCode is not None:
+                # The original version with subtitles has lower relevance
+                if re.match(r'VO-ST(F|A)', versionCode):
+                    source_pref -= 10
+                # The version with sourds/mal subtitles has also lower relevance
+                elif re.match(r'VO?(F|A)-STM\1', versionCode):
+                    source_pref -= 9
+            format = {
+                'format_id': format_id,
+                'preference': -10 if f.get('videoFormat') == 'M3U8' else None,
+                'language_preference': lang_pref,
+                'format_note': '%s, %s' % (f.get('versionCode'), f.get('versionLibelle')),
+                'width': int_or_none(f.get('width')),
+                'height': int_or_none(f.get('height')),
+                'tbr': int_or_none(f.get('bitrate')),
+                'quality': qfunc(f['quality']),
+                'source_preference': source_pref,
             }
-            if format_info['mediaType'] == 'rtmp':
-                info['url'] = format_info['streamer']
-                info['play_path'] = 'mp4:' + format_info['url']
-                info['ext'] = 'flv'
-            else:
-                info['url'] = format_info['url']
-                info['ext'] = determine_ext(info['url'])
-            return info
-        info_dict['formats'] = [_format(f) for f in formats]
 
+            if f.get('mediaType') == 'rtmp':
+                format['url'] = f['streamer']
+                format['play_path'] = 'mp4:' + f['url']
+                format['ext'] = 'flv'
+            else:
+                format['url'] = f['url']
+
+            formats.append(format)
+
+        self._sort_formats(formats)
+
+        info_dict['formats'] = formats
         return info_dict
 
 
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b77f0e519..7f627c44c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -87,6 +87,11 @@ class InfoExtractor(object):
                                  by this field, regardless of all other values.
                                  -1 for default (order by other properties),
                                  -2 or smaller for less than default.
+                    * language_preference  Is this in the correct requested
+                                 language?
+                                 10 if it's what the URL is about,
+                                 -1 for default (don't know),
+                                 -10 otherwise, other values reserved for now.
                     * quality    Order number of the video quality of this
                                  format, irrespective of the file format.
                                  -1 for default (order by other properties),
@@ -615,6 +620,7 @@ class InfoExtractor(object):
 
             return (
                 preference,
+                f.get('language_preference') if f.get('language_preference') is not None else -1,
                 f.get('quality') if f.get('quality') is not None else -1,
                 f.get('height') if f.get('height') is not None else -1,
                 f.get('width') if f.get('width') is not None else -1,

From 732ea2f09bbcf250e376d6a16dfa0ae8c406ff34 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 20 Nov 2014 12:14:28 +0100
Subject: [PATCH 0535/1937] [utils] Improve update on error message somewhat

We still may want to implement a bulletproof check for the current version, and a better place to add this message so that it works for all kind of other errors too.
---
 youtube_dl/utils.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index bfe88b40b..5be7cf992 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -420,6 +420,7 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
             pass  # Python < 3.4
         return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
 
+
 class ExtractorError(Exception):
     """Error during info extraction."""
     def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
@@ -434,7 +435,13 @@ class ExtractorError(Exception):
         if cause:
             msg += ' (caused by %r)' % cause
         if not expected:
-            msg = msg + '; please report this issue on https://yt-dl.org/bug . Be sure to call youtube-dl with the --verbose flag and include its complete output. Make sure you are using the latest version; type  youtube-dl -U  to update.'
+            if ytdl_is_updateable():
+                update_cmd = 'type  youtube-dl -U  to update'
+            else:
+                update_cmd = 'see  https://yt-dl.org/update  on how to update'
+            msg += '; please report this issue on https://yt-dl.org/bug .'
+            msg += ' Make sure you are using the latest version; %s.' % update_cmd
+            msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
         super(ExtractorError, self).__init__(msg)
 
         self.traceback = tb
@@ -1419,3 +1426,10 @@ def is_outdated_version(version, limit, assume_new=True):
         return version_tuple(version) < version_tuple(limit)
     except ValueError:
         return not assume_new
+
+
+def ytdl_is_updateable():
+    """ Returns if youtube-dl can be updated with -U """
+    from zipimport import zipimporter
+
+    return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')

From 84437adfa334a33daa147ad4d8a7f78a8ea9a7f5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 20 Nov 2014 12:20:57 +0100
Subject: [PATCH 0536/1937] release 2014.11.20.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f7965d907..e7505aa7e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.20'
+__version__ = '2014.11.20.1'

From 5239075bb684bc2c19978dde70e3954bbe7a62de Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 20 Nov 2014 16:25:19 +0100
Subject: [PATCH 0537/1937] [mtv] Return a proper playlist result (#4254)

---
 youtube_dl/extractor/mtv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 474bdff7d..5f0f476b6 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -145,7 +145,8 @@ class MTVServicesInfoExtractor(InfoExtractor):
         idoc = self._download_xml(
             feed_url + '?' + data, video_id,
             'Downloading info', transform_source=fix_xml_ampersands)
-        return [self._get_video_info(item) for item in idoc.findall('.//item')]
+        return self.playlist_result(
+            [self._get_video_info(item) for item in idoc.findall('.//item')])
 
     def _real_extract(self, url):
         title = url_basename(url)

From c9bf41145fc7c722eb9b5ae528642976177c1b14 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 20 Nov 2014 16:29:31 +0100
Subject: [PATCH 0538/1937] [YoutubeDL] Warn if an extractor returns
 compat_list

---
 youtube_dl/YoutubeDL.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 94c50903c..79f5e995e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -679,6 +679,9 @@ class YoutubeDL(object):
             ie_result['entries'] = playlist_results
             return ie_result
         elif result_type == 'compat_list':
+            self.report_warning(
+                'Extractor %s returned a compat_list result. '
+                'It needs to be updated.' % ie_result.get('extractor'))
             def _fixup(r):
                 self.add_extra_info(r,
                     {

From f42c190769869278cdd5f7ca62d2bd29e2a9d709 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 20 Nov 2014 16:34:54 +0100
Subject: [PATCH 0539/1937] [stanfordoc] Modernize

---
 youtube_dl/extractor/stanfordoc.py | 88 +++++++++++++-----------------
 1 file changed, 37 insertions(+), 51 deletions(-)

diff --git a/youtube_dl/extractor/stanfordoc.py b/youtube_dl/extractor/stanfordoc.py
index 44c52c718..5feb4ff83 100644
--- a/youtube_dl/extractor/stanfordoc.py
+++ b/youtube_dl/extractor/stanfordoc.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import re
 
 from .common import InfoExtractor
@@ -9,24 +11,23 @@ from ..utils import (
 
 
 class StanfordOpenClassroomIE(InfoExtractor):
-    IE_NAME = u'stanfordoc'
-    IE_DESC = u'Stanford Open ClassRoom'
-    _VALID_URL = r'^(?:https?://)?openclassroom\.stanford\.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
+    IE_NAME = 'stanfordoc'
+    IE_DESC = 'Stanford Open ClassRoom'
+    _VALID_URL = r'https?://openclassroom\.stanford\.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
     _TEST = {
-        u'url': u'http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100',
-        u'file': u'PracticalUnix_intro-environment.mp4',
-        u'md5': u'544a9468546059d4e80d76265b0443b8',
-        u'info_dict': {
-            u"title": u"Intro Environment"
+        'url': 'http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100',
+        'md5': '544a9468546059d4e80d76265b0443b8',
+        'info_dict': {
+            'id': 'PracticalUnix_intro-environment',
+            'ext': 'mp4',
+            'title': 'Intro Environment',
         }
     }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
 
-        if mobj.group('course') and mobj.group('video'): # A specific video
+        if mobj.group('course') and mobj.group('video'):  # A specific video
             course = mobj.group('course')
             video = mobj.group('video')
             info = {
@@ -35,7 +36,6 @@ class StanfordOpenClassroomIE(InfoExtractor):
                 'upload_date': None,
             }
 
-            self.report_extraction(info['id'])
             baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
             xmlUrl = baseUrl + video + '.xml'
             mdoc = self._download_xml(xmlUrl, info['id'])
@@ -43,63 +43,49 @@ class StanfordOpenClassroomIE(InfoExtractor):
                 info['title'] = mdoc.findall('./title')[0].text
                 info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text
             except IndexError:
-                raise ExtractorError(u'Invalid metadata XML file')
-            info['ext'] = info['url'].rpartition('.')[2]
-            return [info]
-        elif mobj.group('course'): # A course page
+                raise ExtractorError('Invalid metadata XML file')
+            return info
+        elif mobj.group('course'):  # A course page
             course = mobj.group('course')
             info = {
                 'id': course,
-                'type': 'playlist',
+                '_type': 'playlist',
                 'uploader': None,
                 'upload_date': None,
             }
 
-            coursepage = self._download_webpage(url, info['id'],
-                                        note='Downloading course info page',
-                                        errnote='Unable to download course info page')
+            coursepage = self._download_webpage(
+                url, info['id'],
+                note='Downloading course info page',
+                errnote='Unable to download course info page')
 
-            info['title'] = self._html_search_regex('<h1>([^<]+)</h1>', coursepage, 'title', default=info['id'])
+            info['title'] = self._html_search_regex(
+                r'<h1>([^<]+)</h1>', coursepage, 'title', default=info['id'])
 
-            info['description'] = self._html_search_regex('<description>([^<]+)</description>',
-                coursepage, u'description', fatal=False)
+            info['description'] = self._html_search_regex(
+                r'(?s)<description>([^<]+)</description>',
+                coursepage, 'description', fatal=False)
 
             links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage))
-            info['list'] = [
-                {
-                    'type': 'reference',
-                    'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage),
-                }
-                    for vpage in links]
-            results = []
-            for entry in info['list']:
-                assert entry['type'] == 'reference'
-                results += self.extract(entry['url'])
-            return results
-        else: # Root page
+            info['entries'] = [self.url_result(
+                'http://openclassroom.stanford.edu/MainFolder/%s' % unescapeHTML(l)
+            ) for l in links]
+            return info
+        else:  # Root page
             info = {
                 'id': 'Stanford OpenClassroom',
-                'type': 'playlist',
+                '_type': 'playlist',
                 'uploader': None,
                 'upload_date': None,
             }
+            info['title'] = info['id']
 
             rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
             rootpage = self._download_webpage(rootURL, info['id'],
-                errnote=u'Unable to download course info page')
-
-            info['title'] = info['id']
+                errnote='Unable to download course info page')
 
             links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
-            info['list'] = [
-                {
-                    'type': 'reference',
-                    'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage),
-                }
-                    for cpage in links]
-
-            results = []
-            for entry in info['list']:
-                assert entry['type'] == 'reference'
-                results += self.extract(entry['url'])
-            return results
+            info['entries'] = [self.url_result(
+                'http://openclassroom.stanford.edu/MainFolder/%s' % unescapeHTML(l)
+            ) for l in links]
+            return info

From b244b5c3f9082a307f0e1c4ee561594bd389aeb2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 20 Nov 2014 16:35:55 +0100
Subject: [PATCH 0540/1937] remove unused imports

---
 youtube_dl/compat.py                    | 2 +-
 youtube_dl/extractor/arte.py            | 1 -
 youtube_dl/extractor/comedycentral.py   | 1 -
 youtube_dl/extractor/goldenmoustache.py | 2 --
 4 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index adf81e7ab..9d33a8ec5 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -306,7 +306,7 @@ def workaround_optparse_bug9161():
     og = optparse.OptionGroup(op, 'foo')
     try:
         og.add_option('-t')
-    except TypeError as te:
+    except TypeError:
         real_add_option = optparse.OptionGroup.add_option
 
         def _compat_add_option(self, *args, **kwargs):
diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index aafc3a24d..3a57ce527 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -5,7 +5,6 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
     find_xpath_attr,
     unified_strdate,
     get_element_by_id,
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 49b978b4e..f0da1844f 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -2,7 +2,6 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
 from .mtv import MTVServicesInfoExtractor
 from ..utils import (
     compat_str,
diff --git a/youtube_dl/extractor/goldenmoustache.py b/youtube_dl/extractor/goldenmoustache.py
index 7e13b131b..10001d4d9 100644
--- a/youtube_dl/extractor/goldenmoustache.py
+++ b/youtube_dl/extractor/goldenmoustache.py
@@ -1,9 +1,7 @@
 from __future__ import unicode_literals
 
-import re
 from .common import InfoExtractor
 from ..utils import (
-    parse_duration,
     int_or_none,
 )
 

From 6adeffa7c686de97c8e259a8ef474bc807d611d7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 20 Nov 2014 16:36:53 +0100
Subject: [PATCH 0541/1937] [comedycentral] Modernize

---
 youtube_dl/extractor/comedycentral.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index f0da1844f..2e3ef3fda 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -109,9 +109,7 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url, re.VERBOSE)
-        if mobj is None:
-            raise ExtractorError('Invalid URL: %s' % url)
+        mobj = re.match(self._VALID_URL, url)
 
         if mobj.group('shortname'):
             if mobj.group('shortname') in ('tds', 'thedailyshow'):

From fed5d0326042900449443c2519327f8a6334d872 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 20 Nov 2014 16:47:59 +0100
Subject: [PATCH 0542/1937] [extractor/common] Document _type values (Motivated
 by #4254)

---
 youtube_dl/extractor/common.py | 38 +++++++++++++++++++++++++++++++++-
 1 file changed, 37 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7f627c44c..93a5a3d57 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -43,7 +43,11 @@ class InfoExtractor(object):
     information possibly downloading the video to the file system, among
     other possible outcomes.
 
-    The dictionaries must include the following fields:
+    The type field determines the the type of the result.
+    By far the most common value (and the default if _type is missing) is
+    "video", which indicates a single video.
+
+    For a video, the dictionaries must include the following fields:
 
     id:             Video identifier.
     title:          Video title, unescaped.
@@ -151,6 +155,38 @@ class InfoExtractor(object):
 
     Unless mentioned otherwise, None is equivalent to absence of information.
 
+
+    _type "playlist" indicates multiple videos.
+    There must be a key "entries", which is a list or a PagedList object, each
+    element of which is a valid dictionary under this specfication.
+
+    Additionally, playlists can have "title" and "id" attributes with the same
+    semantics as videos (see above).
+
+
+    _type "multi_video" indicates that there are multiple videos that
+    form a single show, for examples multiple acts of an opera or TV episode.
+    It must have an entries key like a playlist and contain all the keys
+    required for a video at the same time.
+
+
+    _type "url" indicates that the video must be extracted from another
+    location, possibly by a different extractor. Its only required key is:
+    "url" - the next URL to extract.
+
+    Additionally, it may have properties believed to be identical to the
+    resolved entity, for example "title" if the title of the referred video is
+    known ahead of time.
+
+
+    _type "url_transparent" entities have the same specification as "url", but
+    indicate that the given additional information is more precise than the one
+    associated with the resolved URL.
+    This is useful when a site employs a video service that hosts the video and
+    its technical metadata, but that video service does not embed a useful
+    title, description etc.
+
+
     Subclasses of this one should re-define the _real_initialize() and
     _real_extract() methods and define a _VALID_URL regexp.
     Probably, they should also be added to the list of extractors.

From 8569f3d629c1ec2e4b1d8472238aedb1d877d5f0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 20 Nov 2014 16:51:33 +0100
Subject: [PATCH 0543/1937] [vh1] Modernize

---
 youtube_dl/extractor/vh1.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vh1.py b/youtube_dl/extractor/vh1.py
index 2f77e3898..6be3774b7 100644
--- a/youtube_dl/extractor/vh1.py
+++ b/youtube_dl/extractor/vh1.py
@@ -121,4 +121,7 @@ class VH1IE(MTVIE):
         idoc = self._download_xml(
             doc_url, video_id,
             'Downloading info', transform_source=fix_xml_ampersands)
-        return [self._get_video_info(item) for item in idoc.findall('.//item')]
+        return self.playlist_result(
+            [self._get_video_info(item) for item in idoc.findall('.//item')],
+            playlist_id=video_id,
+        )

From d6f714f321d41169d26ccd23bcc8f3987fa6baae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 20 Nov 2014 18:37:08 +0100
Subject: [PATCH 0544/1937] [brightcove] Remove the namespace from the
 BrightcoveExperience html object

---
 youtube_dl/extractor/brightcove.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index a6920685e..8d5acef14 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -111,6 +111,8 @@ class BrightcoveIE(InfoExtractor):
                             lambda m: m.group(1) + '/>', object_str)
         # Fix up some stupid XML, see https://github.com/rg3/youtube-dl/issues/1608
         object_str = object_str.replace('<--', '<!--')
+        # remove namespace to simplify extraction
+        object_str = re.sub(r'(<object[^>]*)(xmlns=".*?")', r'\1', object_str)
         object_str = fix_xml_ampersands(object_str)
 
         object_doc = xml.etree.ElementTree.fromstring(object_str.encode('utf-8'))

From 4d46c1c68ceae04de6195f6c7bf7b91bed06c162 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 20 Nov 2014 18:44:54 +0100
Subject: [PATCH 0545/1937] [brightcove] Improve error message detection
 (#4256)

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 8d5acef14..2db7f9fef 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -221,7 +221,7 @@ class BrightcoveIE(InfoExtractor):
         webpage = self._download_webpage(req, video_id)
 
         error_msg = self._html_search_regex(
-            r"<h1>We're sorry.</h1>\s*<p>(.*?)</p>", webpage,
+            r"<h1>We're sorry.</h1>([\s\n]*<p>.*?</p>)+", webpage,
             'error message', default=None)
         if error_msg is not None:
             raise ExtractorError(

From be53e2a73734bfa3f0f5c0d0146b08e577a0b998 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 21 Nov 2014 00:25:13 +0100
Subject: [PATCH 0546/1937] [blip.tv:user] Modernize and add a test

---
 youtube_dl/extractor/bliptv.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index 2370c24b0..f2b02643d 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -166,9 +166,17 @@ class BlipTVIE(SubtitlesInfoExtractor):
 
 
 class BlipTVUserIE(InfoExtractor):
-    _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)(?!api\.swf)([^/]+)/*$'
+    _VALID_URL = r'(?:(?:https?://(?:\w+\.)?blip\.tv/)|bliptvuser:)(?!api\.swf)([^/]+)/*$'
     _PAGE_SIZE = 12
     IE_NAME = 'blip.tv:user'
+    _TEST = {
+        'url': 'http://blip.tv/actone',
+        'info_dict': {
+            'id': 'actone',
+            'title': 'Act One: The Series',
+        },
+        'playlist_count': 5,
+    }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -179,6 +187,7 @@ class BlipTVUserIE(InfoExtractor):
         page = self._download_webpage(url, username, 'Downloading user page')
         mobj = re.search(r'data-users-id="([^"]+)"', page)
         page_base = page_base % mobj.group(1)
+        title = self._og_search_title(page)
 
         # Download video ids using BlipTV Ajax calls. Result size per
         # query is limited (currently to 12 videos) so we need to query
@@ -215,4 +224,5 @@ class BlipTVUserIE(InfoExtractor):
 
         urls = ['http://blip.tv/%s' % video_id for video_id in video_ids]
         url_entries = [self.url_result(vurl, 'BlipTV') for vurl in urls]
-        return [self.playlist_result(url_entries, playlist_title=username)]
+        return self.playlist_result(
+            url_entries, playlist_title=title, playlist_id=username)

From dfd5313afdf00250d9b8a4e103bd7647713f1af3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 21 Nov 2014 00:25:46 +0100
Subject: [PATCH 0547/1937] [YoutubeDL] Support new _type multi_video

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 79f5e995e..0a3569304 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -624,7 +624,7 @@ class YoutubeDL(object):
 
             return self.process_ie_result(
                 new_result, download=download, extra_info=extra_info)
-        elif result_type == 'playlist':
+        elif result_type == 'playlist' or playlist == 'multi_video':
             # We process each entry in the playlist
             playlist = ie_result.get('title', None) or ie_result.get('id', None)
             self.to_screen('[download] Downloading playlist: %s' % playlist)

From 85b9275517869bb7fb150ff65b9021ca49009b8b Mon Sep 17 00:00:00 2001
From: tinybug <tinybug1991@gmail.com>
Date: Fri, 21 Nov 2014 17:09:22 +0800
Subject: [PATCH 0548/1937] Update jsinterp.py

http://s.ytimg.com/yts/jsbin/html5player-zh_HK-vfl1NK6PR/html5player.js
fix raise ExtractorError
---
 youtube_dl/jsinterp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index c40cd376d..b4617fbad 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -61,7 +61,7 @@ class JSInterpreter(object):
             pass
 
         m = re.match(
-            r'^(?P<var>[a-zA-Z0-9_]+)\.(?P<member>[^(]+)(?:\(+(?P<args>[^()]*)\))?$',
+            r'^(?P<var>[$a-zA-Z0-9_]+)\.(?P<member>[^(]+)(?:\(+(?P<args>[^()]*)\))?$',
             expr)
         if m:
             variable = m.group('var')

From ec06f0f610f85aa8b14a7ed30e85b81314af4cda Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 21 Nov 2014 10:41:18 +0100
Subject: [PATCH 0549/1937] release 2014.11.21

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e7505aa7e..95b05fa95 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.20.1'
+__version__ = '2014.11.21'

From 9b32eca3ce6cda72449592c9ab26f73311a613f5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 21 Nov 2014 22:01:25 +0100
Subject: [PATCH 0550/1937] [generic] Add support for single quotes in HTML5
 videos (Fixes #4265)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index af769ab61..c7a824c29 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -979,7 +979,7 @@ class GenericIE(InfoExtractor):
                 found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
         if not found:
             # HTML5 video
-            found = re.findall(r'(?s)<video[^<]*(?:>.*?<source[^>]*)?\s+src="([^"]+)"', webpage)
+            found = re.findall(r'(?s)<video[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']', webpage)
         if not found:
             found = re.search(
                 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'

From f3391db88925782e4814e8eaa3eb9808c30d6f01 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 21 Nov 2014 22:02:16 +0100
Subject: [PATCH 0551/1937] [sztvhu] Modernize

---
 youtube_dl/extractor/sztvhu.py | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/sztvhu.py b/youtube_dl/extractor/sztvhu.py
index c9359fafb..804a25e22 100644
--- a/youtube_dl/extractor/sztvhu.py
+++ b/youtube_dl/extractor/sztvhu.py
@@ -1,27 +1,25 @@
 # -*- coding: utf-8 -*-
-
-import re
+from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import determine_ext
 
 
 class SztvHuIE(InfoExtractor):
-    _VALID_URL = r'(?:http://)?(?:(?:www\.)?sztv\.hu|www\.tvszombathely\.hu)/(?:[^/]+)/.+-(?P<id>[0-9]+)'
+    _VALID_URL = r'http://(?:(?:www\.)?sztv\.hu|www\.tvszombathely\.hu)/(?:[^/]+)/.+-(?P<id>[0-9]+)'
     _TEST = {
-        u'url': u'http://sztv.hu/hirek/cserkeszek-nepszerusitettek-a-kornyezettudatos-eletmodot-a-savaria-teren-20130909',
-        u'file': u'20130909.mp4',
-        u'md5': u'a6df607b11fb07d0e9f2ad94613375cb',
-        u'info_dict': {
-            u"title": u"Cserkészek népszerűsítették a környezettudatos életmódot a Savaria téren",
-            u"description": u'A zöld nap játékos ismeretterjesztő programjait a Magyar Cserkész Szövetség szervezte, akik az ország nyolc városában adják át tudásukat az érdeklődőknek. A PET...',
+        'url': 'http://sztv.hu/hirek/cserkeszek-nepszerusitettek-a-kornyezettudatos-eletmodot-a-savaria-teren-20130909',
+        'md5': 'a6df607b11fb07d0e9f2ad94613375cb',
+        'info_dict': {
+            'id': '20130909',
+            'ext': 'mp4',
+            'title': 'Cserkészek népszerűsítették a környezettudatos életmódot a Savaria téren',
+            'description': 'A zöld nap játékos ismeretterjesztő programjait a Magyar Cserkész Szövetség szervezte, akik az ország nyolc városában adják át tudásukat az érdeklődőknek. A PET...',
         },
-        u'skip': u'Service temporarily disabled as of 2013-11-20'
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         video_file = self._search_regex(
             r'file: "...:(.*?)",', webpage, 'video file')

From 71069d215791a85dc627ec3dcfa0a87f3d5643eb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 21 Nov 2014 22:03:29 +0100
Subject: [PATCH 0552/1937] [sztv] Remove useless determine_ext call

---
 youtube_dl/extractor/sztvhu.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/sztvhu.py b/youtube_dl/extractor/sztvhu.py
index 804a25e22..aa5964acb 100644
--- a/youtube_dl/extractor/sztvhu.py
+++ b/youtube_dl/extractor/sztvhu.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import determine_ext
 
 
 class SztvHuIE(InfoExtractor):
@@ -37,7 +36,6 @@ class SztvHuIE(InfoExtractor):
             'id': video_id,
             'url': video_url,
             'title': title,
-            'ext': determine_ext(video_url),
             'description': description,
             'thumbnail': thumbnail,
         }

From 6127693ed9b7157bff55dd495a1da66e0f61c4d1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 21 Nov 2014 22:36:24 +0100
Subject: [PATCH 0553/1937] [folketinget] Add extractor (Fixes #4262)

---
 youtube_dl/extractor/__init__.py    |  1 +
 youtube_dl/extractor/folketinget.py | 75 +++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)
 create mode 100644 youtube_dl/extractor/folketinget.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f45ce05ab..70708e41b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -115,6 +115,7 @@ from .fktv import (
     FKTVPosteckeIE,
 )
 from .flickr import FlickrIE
+from .folketinget import FolketingetIE
 from .fourtube import FourTubeIE
 from .franceculture import FranceCultureIE
 from .franceinter import FranceInterIE
diff --git a/youtube_dl/extractor/folketinget.py b/youtube_dl/extractor/folketinget.py
new file mode 100644
index 000000000..68e2db943
--- /dev/null
+++ b/youtube_dl/extractor/folketinget.py
@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_parse_qs
+from ..utils import (
+    int_or_none,
+    parse_duration,
+    parse_iso8601,
+    xpath_text,
+)
+
+
+class FolketingetIE(InfoExtractor):
+    IE_DESC = 'Folketinget (ft.dk; Danish parliament)'
+    _VALID_URL = r'https?://(?:www\.)?ft\.dk/webtv/video/[^?#]*?\.(?P<id>[0-9]+)\.aspx'
+    _TEST = {
+        'url': 'http://www.ft.dk/webtv/video/20141/eru/td.1165642.aspx?as=1#player',
+        'info_dict': {
+            'id': '1165642',
+            'ext': 'mp4',
+            'title': 'Åbent samråd i Erhvervsudvalget',
+            'description': 'Åbent samråd med erhvervs- og vækstministeren om regeringens politik på teleområdet',
+            'view_count': int,
+            'width': 768,
+            'height': 432,
+            'tbr': 928000,
+            'timestamp': 1416493800,
+            'upload_date': '20141120',
+            'duration': 3960,
+        },
+        'params': {
+            'skip_download': 'rtmpdump required',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(webpage)
+        description = self._html_search_regex(
+            r'(?s)<div class="video-item-agenda"[^>]*>(.*?)<',
+            webpage, 'description', fatal=False)
+
+        player_params = compat_parse_qs(self._search_regex(
+            r'<embed src="http://ft\.arkena\.tv/flash/ftplayer\.swf\?([^"]+)"',
+            webpage, 'player params'))
+        xml_url = player_params['xml'][0]
+        doc = self._download_xml(xml_url, video_id)
+
+        timestamp = parse_iso8601(xpath_text(doc, './/date'))
+        duration = parse_duration(xpath_text(doc, './/duration'))
+        width = int_or_none(xpath_text(doc, './/width'))
+        height = int_or_none(xpath_text(doc, './/height'))
+        view_count = int_or_none(xpath_text(doc, './/views'))
+
+        formats = [{
+            'format_id': n.attrib['bitrate'],
+            'url': xpath_text(n, './url', fatal=True),
+            'tbr': int_or_none(n.attrib['bitrate']),
+        } for n in doc.findall('.//streams/stream')]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'description': description,
+            'timestamp': timestamp,
+            'width': width,
+            'height': height,
+            'duration': duration,
+            'view_count': view_count,
+        }

From 42e12102a979a03f156e71fc86006f61c905f7dd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 21 Nov 2014 22:38:16 +0100
Subject: [PATCH 0554/1937] [YoutubeDL] Fix multi_video check

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0a3569304..fde026fbf 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -624,7 +624,7 @@ class YoutubeDL(object):
 
             return self.process_ie_result(
                 new_result, download=download, extra_info=extra_info)
-        elif result_type == 'playlist' or playlist == 'multi_video':
+        elif result_type == 'playlist' or result_type == 'multi_video':
             # We process each entry in the playlist
             playlist = ie_result.get('title', None) or ie_result.get('id', None)
             self.to_screen('[download] Downloading playlist: %s' % playlist)

From 2c64b8ba635ce30593aeb7b1890c1201442006af Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 21 Nov 2014 22:47:23 +0100
Subject: [PATCH 0555/1937] release 2014.11.21.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 95b05fa95..a283afbe3 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.21'
+__version__ = '2014.11.21.1'

From 42e1ff8665ceb6eddb6b2067f57239be3a8ab209 Mon Sep 17 00:00:00 2001
From: Will Sewell <me@willsewell.name>
Date: Fri, 21 Nov 2014 22:23:39 +0000
Subject: [PATCH 0556/1937] [vk.com] Added upload_date variable to the test
 cases that still work.

---
 youtube_dl/extractor/vk.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index deaad6c3e..de87bee1f 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -29,6 +29,7 @@ class VKIE(InfoExtractor):
                 'title': 'ProtivoGunz - Хуёвая песня',
                 'uploader': 're:Noize MC.*',
                 'duration': 195,
+                'upload_date': '20120212',
             },
         },
         {
@@ -52,6 +53,7 @@ class VKIE(InfoExtractor):
                 'uploader': 'Vladimir Gavrin',
                 'title': 'Lin Dan',
                 'duration': 101,
+                'upload_date': '20120730',
             }
         },
         {
@@ -88,6 +90,7 @@ class VKIE(InfoExtractor):
                 'uploader': 'Киномания - лучшее из мира кино',
                 'title': ' ',
                 'duration': 7291,
+                'upload_date': '20140328',
             },
             'skip': 'Requires vk account credentials',
         },
@@ -100,6 +103,7 @@ class VKIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Книга Илая',
                 'duration': 6771,
+                'upload_date': '20140626',
             },
             'skip': 'Only works from Russia',
         },

From c52331f30c15ff715431cf1ca5fceec505efe599 Mon Sep 17 00:00:00 2001
From: Will Sewell <me@willsewell.name>
Date: Fri, 21 Nov 2014 22:52:01 +0000
Subject: [PATCH 0557/1937] [vk.com] Updated a test video that has been
 removed, and added a comment for others to update two other test videos that
 are also now removed.

---
 youtube_dl/extractor/vk.py | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index de87bee1f..d6632cbb7 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -33,14 +33,15 @@ class VKIE(InfoExtractor):
             },
         },
         {
-            'url': 'http://vk.com/video4643923_163339118',
-            'md5': 'f79bccb5cd182b1f43502ca5685b2b36',
+            'url': 'http://vk.com/video205387401_165548505',
+            'md5': '6c0aeb2e90396ba97035b9cbde548700',
             'info_dict': {
-                'id': '163339118',
+                'id': '165548505',
                 'ext': 'mp4',
-                'uploader': 'Elya Iskhakova',
-                'title': 'Dream Theater - Hollow Years Live at Budokan 720*',
-                'duration': 558,
+                'uploader': 'Tom Cruise',
+                'title': 'No name',
+                'duration': 9,
+                'upload_date': '20130721'
             }
         },
         {
@@ -57,6 +58,8 @@ class VKIE(InfoExtractor):
             }
         },
         {
+            # VIDEO NOW REMOVED
+            # please update if you find a video whose URL follows the same pattern
             'url': 'http://vk.com/video-8871596_164049491',
             'md5': 'a590bcaf3d543576c9bd162812387666',
             'note': 'Only available for registered users',
@@ -66,10 +69,13 @@ class VKIE(InfoExtractor):
                 'uploader': 'Триллеры',
                 'title': '► Бойцовский клуб / Fight Club 1999 [HD 720]',
                 'duration': 8352,
+                'upload_date': '20121218'
             },
             'skip': 'Requires vk account credentials',
         },
         {
+            # VIDEO NOW REMOVED
+            # please update if you find a video whose URL follows the same pattern
             'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a',
             'md5': 'd82c22e449f036282d1d3f7f4d276869',
             'info_dict': {
@@ -78,6 +84,7 @@ class VKIE(InfoExtractor):
                 'uploader': 'Киномания - лучшее из мира кино',
                 'title': 'Запах женщины (1992)',
                 'duration': 9392,
+                'upload_date': '20130914'
             },
             'skip': 'Requires vk account credentials',
         },
@@ -177,6 +184,7 @@ class VKIE(InfoExtractor):
         upload_date = None
         mobj = re.search(r'id="mv_date_wrap".*?Added ([a-zA-Z]+ [0-9]+), ([0-9]+) at', info_page)
         if mobj is not None:
+            x = mobj.group(1) + ' ' + mobj.group(2)
             upload_date = unified_strdate(mobj.group(1) + ' ' + mobj.group(2))
 
         formats = [{

From cad985ab4d78ef00a77932e87cee5aba8a1e3320 Mon Sep 17 00:00:00 2001
From: Will Sewell <me@willsewell.name>
Date: Fri, 21 Nov 2014 22:54:11 +0000
Subject: [PATCH 0558/1937] [vk.com] Updated the description to include vk.com.

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index d6632cbb7..d96abd16e 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -208,7 +208,7 @@ class VKIE(InfoExtractor):
 
 class VKUserVideosIE(InfoExtractor):
     IE_NAME = 'vk.com:user-videos'
-    IE_DESC = 'All of a user\'s videos'
+    IE_DESC = 'vk.com:All of a user\'s videos'
     _VALID_URL = r'https?://(?:m\.)?vk\.com/videos([0-9]+)(?:m\?.*)?'
     _TEMPLATE_URL = 'https://vk.com/videos'
     _TEST = {

From 53d1cd1f779201af426548fc77e1724b6c70abd9 Mon Sep 17 00:00:00 2001
From: Will Sewell <me@willsewell.name>
Date: Fri, 21 Nov 2014 23:03:31 +0000
Subject: [PATCH 0559/1937] [vk.com] Updated the _VALID_URL regex for the
 playlist IE. Removed optional m, and named the id group.

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index d96abd16e..7136a0cb2 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -209,7 +209,7 @@ class VKIE(InfoExtractor):
 class VKUserVideosIE(InfoExtractor):
     IE_NAME = 'vk.com:user-videos'
     IE_DESC = 'vk.com:All of a user\'s videos'
-    _VALID_URL = r'https?://(?:m\.)?vk\.com/videos([0-9]+)(?:m\?.*)?'
+    _VALID_URL = r'https?://vk\.com/videos(?P<id>[0-9]+)(?:m\?.*)?'
     _TEMPLATE_URL = 'https://vk.com/videos'
     _TEST = {
         'url': 'http://vk.com/videos205387401',

From e1e8b6897b2e2610c45eb53fe44e1e07c3c39e82 Mon Sep 17 00:00:00 2001
From: Will Sewell <me@willsewell.name>
Date: Fri, 21 Nov 2014 23:16:12 +0000
Subject: [PATCH 0560/1937] [vk.com] Updated the extract_videos_from_page
 function with a much simpler 1-liner.

---
 youtube_dl/extractor/vk.py | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 7136a0cb2..e8d7cdeae 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -11,7 +11,8 @@ from ..utils import (
     compat_urllib_parse,
     compat_str,
     unescapeHTML,
-    unified_strdate)
+    unified_strdate,
+    orderedSet)
 
 
 class VKIE(InfoExtractor):
@@ -216,13 +217,6 @@ class VKUserVideosIE(InfoExtractor):
         'playlist_mincount': 4,
     }
 
-    def extract_videos_from_page(self, page):
-        ids_in_page = []
-        for mobj in re.finditer(r'href="/video([0-9_]+)"', page):
-            if mobj.group(1) not in ids_in_page:
-                ids_in_page.append(mobj.group(1))
-        return ids_in_page
-
     def _real_extract(self, url):
         # Extract page id
         mobj = re.match(self._VALID_URL, url)
@@ -232,7 +226,7 @@ class VKUserVideosIE(InfoExtractor):
         # Download page and get video ids
         page_id = mobj.group(1)
         page = self._download_webpage(url, page_id)
-        video_ids = self.extract_videos_from_page(page)
+        video_ids = orderedSet(m.group(1) for m in re.finditer(r'href="/video([0-9_]+)"', page))
 
         self._downloader.to_screen('[vk] User videos %s: Found %i videos' % (page_id, len(video_ids)))
 

From 021a0db8f787521757919fde5a2303205312abff Mon Sep 17 00:00:00 2001
From: Will Sewell <me@willsewell.name>
Date: Fri, 21 Nov 2014 23:21:03 +0000
Subject: [PATCH 0561/1937] [vk.com] Simplified the page_id acquisition by
 using the id matched in the URL earlier on.

---
 youtube_dl/extractor/vk.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index e8d7cdeae..b4e261fad 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -218,13 +218,7 @@ class VKUserVideosIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        # Extract page id
-        mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError('Invalid URL: %s' % url)
-
-        # Download page and get video ids
-        page_id = mobj.group(1)
+        page_id = self._match_id(url)
         page = self._download_webpage(url, page_id)
         video_ids = orderedSet(m.group(1) for m in re.finditer(r'href="/video([0-9_]+)"', page))
 

From b9272e8f8f785fee4d77258bdb1cf1d85f947ecf Mon Sep 17 00:00:00 2001
From: Will Sewell <me@willsewell.name>
Date: Fri, 21 Nov 2014 23:22:52 +0000
Subject: [PATCH 0562/1937] [vk.com] Removed redundant log message -- this
 information is already being logged.

---
 youtube_dl/extractor/vk.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index b4e261fad..248b7d54d 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -221,9 +221,6 @@ class VKUserVideosIE(InfoExtractor):
         page_id = self._match_id(url)
         page = self._download_webpage(url, page_id)
         video_ids = orderedSet(m.group(1) for m in re.finditer(r'href="/video([0-9_]+)"', page))
-
-        self._downloader.to_screen('[vk] User videos %s: Found %i videos' % (page_id, len(video_ids)))
-
         url_entries = [self.url_result('http://vk.com/video' + video_id, 'VK', video_id=video_id)
                        for video_id in video_ids]
         return self.playlist_result(url_entries, page_id)
\ No newline at end of file

From 9262867e86cb415ed1994f9f956536eba39fc457 Mon Sep 17 00:00:00 2001
From: Will Sewell <me@willsewell.name>
Date: Fri, 21 Nov 2014 23:25:05 +0000
Subject: [PATCH 0563/1937] [vk.com] Added newline at the end of the file.

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 248b7d54d..daf615af9 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -223,4 +223,4 @@ class VKUserVideosIE(InfoExtractor):
         video_ids = orderedSet(m.group(1) for m in re.finditer(r'href="/video([0-9_]+)"', page))
         url_entries = [self.url_result('http://vk.com/video' + video_id, 'VK', video_id=video_id)
                        for video_id in video_ids]
-        return self.playlist_result(url_entries, page_id)
\ No newline at end of file
+        return self.playlist_result(url_entries, page_id)

From 2cead7e7bca9643a2f7c92cf5654ac5da512d6bc Mon Sep 17 00:00:00 2001
From: nulloz <sebastien@mauroy.net>
Date: Sat, 22 Nov 2014 13:34:29 +0100
Subject: [PATCH 0564/1937] telebruxelles Add new extractor

---
 youtube_dl/extractor/__init__.py      |  1 +
 youtube_dl/extractor/telebruxelles.py | 52 +++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 youtube_dl/extractor/telebruxelles.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f45ce05ab..7275d247a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -379,6 +379,7 @@ from .teachingchannel import TeachingChannelIE
 from .teamcoco import TeamcocoIE
 from .techtalks import TechTalksIE
 from .ted import TEDIE
+from .telebruxelles import TeleBruxellesIE
 from .telecinco import TelecincoIE
 from .telemb import TeleMBIE
 from .tenplay import TenPlayIE
diff --git a/youtube_dl/extractor/telebruxelles.py b/youtube_dl/extractor/telebruxelles.py
new file mode 100644
index 000000000..39852dd0e
--- /dev/null
+++ b/youtube_dl/extractor/telebruxelles.py
@@ -0,0 +1,52 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+
+
+class TeleBruxellesIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?telebruxelles\.be/(news|sport|dernier-jt)/?(?P<title>[^\?]+)'
+    _TESTS = [{
+        'url': r'http://www.telebruxelles.be/news/auditions-devant-parlement-francken-galant-tres-attendus/',
+        'md5': '59439e568c9ee42fb77588b2096b214f', 
+        'info_dict': {
+            'id': '11942',
+            'ext': 'flv',
+            'title': 're:Parlement : Francken et Galant répondent aux interpellations*',
+			'description': 're:Les auditions des ministres se poursuivent*'
+        }
+    }, {
+        'url': r'http://www.telebruxelles.be/sport/basket-brussels-bat-mons-80-74/',
+        'md5': '181d3fbdcf20b909309e5aef5c6c6047', 
+        'info_dict': {
+            'id': '10091',
+            'ext': 'flv',
+            'title': 'Basket : le Brussels bat Mons 80-74',
+			'description': 're:Ils l\u2019on fait ! En basket, le B*'
+        }
+	}]
+
+    def _real_extract(self, url):
+		mobj = re.match(self._VALID_URL, url)
+		title = mobj.group('title')
+
+		webpage = self._download_webpage(url, title)
+		
+		article_id = self._html_search_regex(r"<article id=\"post-(\d+)\"", webpage, '0')
+		title = self._html_search_regex(r'<h1 class=\"entry-title\">(.*?)</h1>', webpage, 'title')
+		description = self._html_search_regex(r"property=\"og:description\" content=\"(.*?)\"", webpage, 'description', fatal=False)
+		
+		rtmp_url = self._html_search_regex(r"file: \"(rtmp://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/vod/mp4:\" \+ \"\w+\" \+ \".mp4)\"", webpage, 'url')
+		rtmp_url = rtmp_url.replace("\" + \"", "")
+
+		return {
+			'id': article_id,
+			'title': title,
+			'description': description,
+			'url': rtmp_url,
+			'ext': 'flv',
+			'rtmp_live': True              # if rtmpdump is not called with "--live" argument, the download is blocked and can be completed
+		}
\ No newline at end of file

From 49f0da7ae1ec292132477730f0eba242ad52b6e2 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 22 Nov 2014 21:06:45 +0200
Subject: [PATCH 0565/1937] [rtlxl] Use unencrypted m3u8 streams (#4115)

---
 youtube_dl/extractor/rtlnl.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 5daef2fc5..4a188e5d4 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -28,9 +28,8 @@ class RtlXlIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         uuid = mobj.group('uuid')
 
-        # Use m3u8 streams (see https://github.com/rg3/youtube-dl/issues/4118)
         info = self._download_json(
-            'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/d=pc/fmt=adaptive/' % uuid,
+            'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=flash/' % uuid,
             uuid)
 
         material = info['material'][0]
@@ -39,12 +38,13 @@ class RtlXlIE(InfoExtractor):
         progname = info['abstracts'][0]['name']
         subtitle = material['title'] or info['episodes'][0]['name']
 
-        videopath = material['videopath']
+        # Use unencrypted m3u8 streams (See https://github.com/rg3/youtube-dl/issues/4118)
+        videopath = material['videopath'].replace('.f4m', '.m3u8')
         m3u8_url = 'http://manifest.us.rtl.nl' + videopath
 
         formats = self._extract_m3u8_formats(m3u8_url, uuid, ext='mp4')
 
-        video_urlpart = videopath.split('/adaptive/')[1][:-4]
+        video_urlpart = videopath.split('/flash/')[1][:-4]
         PG_URL_TEMPLATE = 'http://pg.us.rtl.nl/rtlxl/network/%s/progressive/%s.mp4'
 
         formats.extend([

From dd60be2bf9755f65d0a78535d020c8b8a9b76fcb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Nov 2014 09:44:42 +0100
Subject: [PATCH 0566/1937] [telebruxelles] Simplify (#4270)

---
 youtube_dl/extractor/telebruxelles.py | 72 +++++++++++++++------------
 1 file changed, 40 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/telebruxelles.py b/youtube_dl/extractor/telebruxelles.py
index 39852dd0e..a3d05f97d 100644
--- a/youtube_dl/extractor/telebruxelles.py
+++ b/youtube_dl/extractor/telebruxelles.py
@@ -1,52 +1,60 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-import json
-
 from .common import InfoExtractor
 
 
 class TeleBruxellesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?telebruxelles\.be/(news|sport|dernier-jt)/?(?P<title>[^\?]+)'
+    _VALID_URL = r'https?://(?:www\.)?telebruxelles\.be/(news|sport|dernier-jt)/?(?P<id>[^/#?]+)'
     _TESTS = [{
-        'url': r'http://www.telebruxelles.be/news/auditions-devant-parlement-francken-galant-tres-attendus/',
-        'md5': '59439e568c9ee42fb77588b2096b214f', 
+        'url': 'http://www.telebruxelles.be/news/auditions-devant-parlement-francken-galant-tres-attendus/',
+        'md5': '59439e568c9ee42fb77588b2096b214f',
         'info_dict': {
             'id': '11942',
+            'display_id': 'auditions-devant-parlement-francken-galant-tres-attendus',
             'ext': 'flv',
-            'title': 're:Parlement : Francken et Galant répondent aux interpellations*',
-			'description': 're:Les auditions des ministres se poursuivent*'
-        }
+            'title': 'Parlement : Francken et Galant répondent aux interpellations de l’opposition',
+            'description': 're:Les auditions des ministres se poursuivent*'
+        },
+        'params': {
+            'skip_download': 'requires rtmpdump'
+        },
     }, {
-        'url': r'http://www.telebruxelles.be/sport/basket-brussels-bat-mons-80-74/',
-        'md5': '181d3fbdcf20b909309e5aef5c6c6047', 
+        'url': 'http://www.telebruxelles.be/sport/basket-brussels-bat-mons-80-74/',
+        'md5': '181d3fbdcf20b909309e5aef5c6c6047',
         'info_dict': {
             'id': '10091',
+            'display_id': 'basket-brussels-bat-mons-80-74',
             'ext': 'flv',
             'title': 'Basket : le Brussels bat Mons 80-74',
-			'description': 're:Ils l\u2019on fait ! En basket, le B*'
-        }
-	}]
+            'description': 're:^Ils l\u2019on fait ! En basket, le B*',
+        },
+        'params': {
+            'skip_download': 'requires rtmpdump'
+        },
+    }]
 
     def _real_extract(self, url):
-		mobj = re.match(self._VALID_URL, url)
-		title = mobj.group('title')
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
 
-		webpage = self._download_webpage(url, title)
-		
-		article_id = self._html_search_regex(r"<article id=\"post-(\d+)\"", webpage, '0')
-		title = self._html_search_regex(r'<h1 class=\"entry-title\">(.*?)</h1>', webpage, 'title')
-		description = self._html_search_regex(r"property=\"og:description\" content=\"(.*?)\"", webpage, 'description', fatal=False)
-		
-		rtmp_url = self._html_search_regex(r"file: \"(rtmp://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/vod/mp4:\" \+ \"\w+\" \+ \".mp4)\"", webpage, 'url')
-		rtmp_url = rtmp_url.replace("\" + \"", "")
+        article_id = self._html_search_regex(
+            r"<article id=\"post-(\d+)\"", webpage, 'article ID')
+        title = self._html_search_regex(
+            r'<h1 class=\"entry-title\">(.*?)</h1>', webpage, 'title')
+        description = self._og_search_description(webpage)
 
-		return {
-			'id': article_id,
-			'title': title,
-			'description': description,
-			'url': rtmp_url,
-			'ext': 'flv',
-			'rtmp_live': True              # if rtmpdump is not called with "--live" argument, the download is blocked and can be completed
-		}
\ No newline at end of file
+        rtmp_url = self._html_search_regex(
+            r"file: \"(rtmp://\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/vod/mp4:\" \+ \"\w+\" \+ \".mp4)\"",
+            webpage, 'RTMP url')
+        rtmp_url = rtmp_url.replace("\" + \"", "")
+
+        return {
+            'id': article_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'url': rtmp_url,
+            'ext': 'flv',
+            'rtmp_live': True  # if rtmpdump is not called with "--live" argument, the download is blocked and can be completed
+        }

From 88125905cfab51bb97cb626c64dc4af95373d239 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Nov 2014 09:49:15 +0100
Subject: [PATCH 0567/1937] Credit @nulloz for telebruxelles (#4270)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 9f4a4fd03..7aa5a7a1b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -82,3 +82,4 @@ Xavier Beynon
 Gabriel Schubiner
 xantares
 Jan Matějka
+Mauroy Sébastien

From aa79ac0c82a80c5f2e8dcfbea0ceeda0d8463a81 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Nov 2014 09:59:02 +0100
Subject: [PATCH 0568/1937] [youtube] Support controversy videos (Fixes #4275)

---
 youtube_dl/extractor/youtube.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 461271d3b..0cb837afc 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -406,6 +406,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'format': '141',
             },
         },
+        # Controversy video
+        {
+            'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
+            'info_dict': {
+                'id': 'T4XJQO3qol8',
+                'ext': 'mp4',
+                'upload_date': '20100909',
+                'uploader': 'The Amazing Atheist',
+                'uploader_id': 'TheAmazingAtheist',
+                'title': 'Burning Everyone\'s Koran',
+                'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
+            }
+        }
     ]
 
     def __init__(self, *args, **kwargs):
@@ -666,7 +679,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         video_id = self.extract_id(url)
 
         # Get video webpage
-        url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id
+        url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
         pref_cookies = [
             c for c in self._downloader.cookiejar
             if c.domain == '.youtube.com' and c.name == 'PREF']

From b9042def9d2878df037c8c11eeb8ea90c8c52dd0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Nov 2014 09:59:42 +0100
Subject: [PATCH 0569/1937] release 2014.11.23

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a283afbe3..6be5d07c4 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.21.1'
+__version__ = '2014.11.23'

From a8363f3ab7eb56d20738bcc9d99cc78a33874016 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Nov 2014 10:11:04 +0100
Subject: [PATCH 0570/1937] [vk] Clarify test

---
 youtube_dl/extractor/vk.py | 20 +++++---------------
 1 file changed, 5 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index daf615af9..0e7d4dff2 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -74,21 +74,6 @@ class VKIE(InfoExtractor):
             },
             'skip': 'Requires vk account credentials',
         },
-        {
-            # VIDEO NOW REMOVED
-            # please update if you find a video whose URL follows the same pattern
-            'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a',
-            'md5': 'd82c22e449f036282d1d3f7f4d276869',
-            'info_dict': {
-                'id': '166094326',
-                'ext': 'mp4',
-                'uploader': 'Киномания - лучшее из мира кино',
-                'title': 'Запах женщины (1992)',
-                'duration': 9392,
-                'upload_date': '20130914'
-            },
-            'skip': 'Requires vk account credentials',
-        },
         {
             'url': 'http://vk.com/hd_kino_mania?z=video-43215063_168067957%2F15c66b9b533119788d',
             'md5': '4d7a5ef8cf114dfa09577e57b2993202',
@@ -115,6 +100,11 @@ class VKIE(InfoExtractor):
             },
             'skip': 'Only works from Russia',
         },
+        {
+            # removed video, just testing that we match the pattern
+            'url': 'http://vk.com/feed?z=video-43215063_166094326%2Fbb50cacd3177146d7a',
+            'only_matching': True,
+        },
     ]
 
     def _login(self):

From d16abf434aa22ee20aefd26491894d4600c48af0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Nov 2014 10:11:52 +0100
Subject: [PATCH 0571/1937] [vk] Some PEP8 love

---
 youtube_dl/extractor/vk.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 0e7d4dff2..9d6ec4382 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -210,7 +210,10 @@ class VKUserVideosIE(InfoExtractor):
     def _real_extract(self, url):
         page_id = self._match_id(url)
         page = self._download_webpage(url, page_id)
-        video_ids = orderedSet(m.group(1) for m in re.finditer(r'href="/video([0-9_]+)"', page))
-        url_entries = [self.url_result('http://vk.com/video' + video_id, 'VK', video_id=video_id)
-                       for video_id in video_ids]
+        video_ids = orderedSet(
+            m.group(1) for m in re.finditer(r'href="/video([0-9_]+)"', page))
+        url_entries = [
+            self.url_result(
+                'http://vk.com/video' + video_id, 'VK', video_id=video_id)
+            for video_id in video_ids]
         return self.playlist_result(url_entries, page_id)

From d37cab2a9d1da8056460613007f3099e7a9d8f7e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Nov 2014 10:12:35 +0100
Subject: [PATCH 0572/1937] Credit @WillSewell for vk:user (#4233)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 7aa5a7a1b..5ec2f1dc7 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -83,3 +83,4 @@ Gabriel Schubiner
 xantares
 Jan Matějka
 Mauroy Sébastien
+William Sewell

From 7d4111ed14848c3e72d55d47f11cd7e9fadea403 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Nov 2014 10:49:19 +0100
Subject: [PATCH 0573/1937] Provide guidance when called with a YouTube ID
 starting with a dash.

Reported at https://news.ycombinator.com/item?id=8648121
---
 test/test_utils.py      |  7 +++++++
 youtube_dl/YoutubeDL.py | 19 +++++++++++++++++++
 youtube_dl/__init__.py  |  7 ++++---
 youtube_dl/compat.py    |  6 +++++-
 youtube_dl/utils.py     |  6 ++++++
 5 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 0fa873147..9a62322f0 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -47,6 +47,7 @@ from youtube_dl.utils import (
     js_to_json,
     get_filesystem_encoding,
     intlist_to_bytes,
+    args_to_str,
 )
 
 
@@ -361,5 +362,11 @@ class TestUtil(unittest.TestCase):
             intlist_to_bytes([0, 1, 127, 128, 255]),
             b'\x00\x01\x7f\x80\xff')
 
+    def test_args_to_str(self):
+        self.assertEqual(
+            args_to_str(['foo', 'ba/r', '-baz', '2 be', '']),
+            'foo ba/r -baz \'2 be\' \'\''
+        )
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index fde026fbf..bfa0c6d43 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -29,6 +29,7 @@ from .compat import (
     compat_str,
     compat_urllib_error,
     compat_urllib_request,
+    shlex_quote,
 )
 from .utils import (
     escape_url,
@@ -60,6 +61,7 @@ from .utils import (
     write_string,
     YoutubeDLHandler,
     prepend_extension,
+    args_to_str,
 )
 from .cache import Cache
 from .extractor import get_info_extractor, gen_extractors
@@ -253,6 +255,22 @@ class YoutubeDL(object):
             self.print_debug_header()
             self.add_default_info_extractors()
 
+    def warn_if_short_id(self, argv):
+        # short YouTube ID starting with dash?
+        idxs = [
+            i for i, a in enumerate(argv)
+            if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
+        if idxs:
+            correct_argv = (
+                ['youtube-dl'] +
+                [a for i, a in enumerate(argv) if i not in idxs] +
+                ['--'] + [argv[i] for i in idxs]
+            )
+            self.report_warning(
+                'Long argument string detected. '
+                'Use -- to separate parameters and URLs, like this:\n%s\n' %
+                args_to_str(correct_argv))
+
     def add_info_extractor(self, ie):
         """Add an InfoExtractor object to the end of the list."""
         self._ies.append(ie)
@@ -1410,3 +1428,4 @@ class YoutubeDL(object):
         if encoding is None:
             encoding = preferredencoding()
         return encoding
+
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index c1323b4f3..f519fae3e 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -334,11 +334,12 @@ def _real_main(argv=None):
 
         # Maybe do nothing
         if (len(all_urls) < 1) and (opts.load_info_filename is None):
-            if not (opts.update_self or opts.rm_cachedir):
-                parser.error('you must provide at least one URL')
-            else:
+            if opts.update_self or opts.rm_cachedir:
                 sys.exit()
 
+            ydl.warn_if_short_id(sys.argv[1:] if argv is None else argv)
+            parser.error('you must provide at least one URL')
+
         try:
             if opts.load_info_filename is not None:
                 retcode = ydl.download_with_info_file(opts.load_info_filename)
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9d33a8ec5..549206534 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 import getpass
 import optparse
 import os
+import re
 import subprocess
 import sys
 
@@ -174,7 +175,10 @@ try:
     from shlex import quote as shlex_quote
 except ImportError:  # Python < 3.3
     def shlex_quote(s):
-        return "'" + s.replace("'", "'\"'\"'") + "'"
+        if re.match(r'^[-_\w./]+$', s):
+            return s
+        else:
+            return "'" + s.replace("'", "'\"'\"'") + "'"
 
 
 def compat_ord(c):
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5be7cf992..c3d8bf8e9 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -41,6 +41,7 @@ from .compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urlparse,
+    shlex_quote,
 )
 
 
@@ -1433,3 +1434,8 @@ def ytdl_is_updateable():
     from zipimport import zipimporter
 
     return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
+
+
+def args_to_str(args):
+    # Get a short string representation for a subprocess command
+    return ' '.join(shlex_quote(a) for a in args)

From 835a22ef3fc4f8f5b6a414a870561f6492a51de4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Nov 2014 10:51:16 +0100
Subject: [PATCH 0574/1937] release 2014.11.23.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6be5d07c4..6fff4fcc5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.23'
+__version__ = '2014.11.23.1'

From 4698b14b768e099955c58e2dc2ee458e390dcbd7 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 23 Nov 2014 13:28:09 +0200
Subject: [PATCH 0575/1937] [rtlxl] Strip additional dot from video URL (#4115)

---
 youtube_dl/extractor/rtlnl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index 4a188e5d4..d029b0ec5 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -44,7 +44,7 @@ class RtlXlIE(InfoExtractor):
 
         formats = self._extract_m3u8_formats(m3u8_url, uuid, ext='mp4')
 
-        video_urlpart = videopath.split('/flash/')[1][:-4]
+        video_urlpart = videopath.split('/flash/')[1][:-5]
         PG_URL_TEMPLATE = 'http://pg.us.rtl.nl/rtlxl/network/%s/progressive/%s.mp4'
 
         formats.extend([

From 598c218f7b5c3e78f98dad40f45646c0c9ec773e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 23 Nov 2014 23:53:41 +0600
Subject: [PATCH 0576/1937] [smotri] Adapt to new API and modernize

---
 youtube_dl/extractor/smotri.py | 149 ++++++++++-----------------------
 1 file changed, 43 insertions(+), 106 deletions(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 9bd5defa7..6c1d82dd7 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -1,7 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import os.path
 import re
 import json
 import hashlib
@@ -12,15 +11,15 @@ from ..utils import (
     compat_urllib_parse,
     compat_urllib_request,
     ExtractorError,
-    url_basename,
     int_or_none,
+    unified_strdate,
 )
 
 
 class SmotriIE(InfoExtractor):
     IE_DESC = 'Smotri.com'
     IE_NAME = 'smotri'
-    _VALID_URL = r'^https?://(?:www\.)?(?:smotri\.com/video/view/\?id=|pics\.smotri\.com/(?:player|scrubber_custom8)\.swf\?file=)(?P<videoid>v(?P<realvideoid>[0-9]+)[a-z0-9]{4})'
+    _VALID_URL = r'^https?://(?:www\.)?(?:smotri\.com/video/view/\?id=|pics\.smotri\.com/(?:player|scrubber_custom8)\.swf\?file=)(?P<id>v(?P<realvideoid>[0-9]+)[a-z0-9]{4})'
     _NETRC_MACHINE = 'smotri'
 
     _TESTS = [
@@ -35,7 +34,6 @@ class SmotriIE(InfoExtractor):
                 'uploader': 'rbc2008',
                 'uploader_id': 'rbc08',
                 'upload_date': '20131118',
-                'description': 'катастрофа с камер видеонаблюдения, видео катастрофа с камер видеонаблюдения',
                 'thumbnail': 'http://frame6.loadup.ru/8b/a9/2610366.3.3.jpg',
             },
         },
@@ -50,7 +48,6 @@ class SmotriIE(InfoExtractor):
                 'uploader': 'Support Photofile@photofile',
                 'uploader_id': 'support-photofile',
                 'upload_date': '20070704',
-                'description': 'test, видео test',
                 'thumbnail': 'http://frame4.loadup.ru/03/ed/57591.2.3.jpg',
             },
         },
@@ -66,7 +63,6 @@ class SmotriIE(InfoExtractor):
                 'uploader_id': 'timoxa40',
                 'upload_date': '20100404',
                 'thumbnail': 'http://frame7.loadup.ru/af/3f/1390466.3.3.jpg',
-                'description': 'TOCCA_A_NOI_-_LE_COSE_NON_VANNO_CAMBIAMOLE_ORA-1, видео TOCCA_A_NOI_-_LE_COSE_NON_VANNO_CAMBIAMOLE_ORA-1',
             },
             'params': {
                 'videopassword': 'qwerty',
@@ -85,7 +81,6 @@ class SmotriIE(InfoExtractor):
                 'upload_date': '20101001',
                 'thumbnail': 'http://frame3.loadup.ru/75/75/1540889.1.3.jpg',
                 'age_limit': 18,
-                'description': 'этот ролик не покажут по ТВ, видео этот ролик не покажут по ТВ',
             },
             'params': {
                 'videopassword': '333'
@@ -102,17 +97,11 @@ class SmotriIE(InfoExtractor):
                 'uploader': 'HannahL',
                 'uploader_id': 'lisaha95',
                 'upload_date': '20090331',
-                'description': 'Shakira - Don\'t Bother, видео Shakira - Don\'t Bother',
                 'thumbnail': 'http://frame8.loadup.ru/44/0b/918809.7.3.jpg',
             },
         },
     ]
 
-    _SUCCESS = 0
-    _PASSWORD_NOT_VERIFIED = 1
-    _PASSWORD_DETECTED = 2
-    _VIDEO_NOT_FOUND = 3
-
     @classmethod
     def _extract_url(cls, webpage):
         mobj = re.search(
@@ -137,44 +126,44 @@ class SmotriIE(InfoExtractor):
         return self._html_search_meta(name, html, display_name)
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
-        real_video_id = mobj.group('realvideoid')
+        video_id = self._match_id(url)
 
-        # Download video JSON data
-        video_json_url = 'http://smotri.com/vt.php?id=%s' % real_video_id
-        video_json_page = self._download_webpage(video_json_url, video_id, 'Downloading video JSON')
-        video_json = json.loads(video_json_page)
+        video_form = {
+            'ticket': video_id,
+            'video_url': '1',
+            'frame_url': '1',
+            'devid': 'LoadupFlashPlayer',
+            'getvideoinfo': '1',
+        }
 
-        status = video_json['status']
-        if status == self._VIDEO_NOT_FOUND:
+        request = compat_urllib_request.Request(
+            'http://smotri.com/video/view/url/bot/', compat_urllib_parse.urlencode(video_form))
+        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+
+        video = self._download_json(request, video_id, 'Downloading video JSON')
+
+        if video.get('_moderate_no') or not video.get('moderated'):
+            raise ExtractorError('Video %s has not been approved by moderator' % video_id, expected=True)
+
+        if video.get('error'):
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
-        elif status == self._PASSWORD_DETECTED: # The video is protected by a password, retry with
-                                                # video-password set
-            video_password = self._downloader.params.get('videopassword', None)
-            if not video_password:
-                raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
-            video_json_url += '&md5pass=%s' % hashlib.md5(video_password.encode('utf-8')).hexdigest()
-            video_json_page = self._download_webpage(video_json_url, video_id, 'Downloading video JSON (video-password set)')
-            video_json = json.loads(video_json_page)
-            status = video_json['status']
-            if status == self._PASSWORD_NOT_VERIFIED:
-                raise ExtractorError('Video password is invalid', expected=True)
 
-        if status != self._SUCCESS:
-            raise ExtractorError('Unexpected status value %s' % status)
-
-        # Extract the URL of the video
-        video_url = video_json['file_data']
+        video_url = video.get('_vidURL') or video.get('_vidURL_mp4')
+        title = video['title']
+        thumbnail = video['_imgURL']
+        upload_date = unified_strdate(video['added'])
+        uploader = video['userNick']
+        uploader_id = video['userLogin']
+        duration = int_or_none(video['duration'])
 
         # Video JSON does not provide enough meta data
         # We will extract some from the video web page instead
-        video_page_url = 'http://smotri.com/video/view/?id=%s' % video_id
-        video_page = self._download_webpage(video_page_url, video_id, 'Downloading video page')
+        webpage_url = 'http://smotri.com/video/view/?id=%s' % video_id
+        webpage = self._download_webpage(webpage_url, video_id, 'Downloading video page')
 
         # Warning if video is unavailable
         warning = self._html_search_regex(
-            r'<div class="videoUnModer">(.*?)</div>', video_page,
+            r'<div class="videoUnModer">(.*?)</div>', webpage,
             'warning message', default=None)
         if warning is not None:
             self._downloader.report_warning(
@@ -182,84 +171,32 @@ class SmotriIE(InfoExtractor):
                 (video_id, warning))
 
         # Adult content
-        if re.search('EroConfirmText">', video_page) is not None:
+        if re.search('EroConfirmText">', webpage) is not None:
             self.report_age_confirmation()
             confirm_string = self._html_search_regex(
                 r'<a href="/video/view/\?id=%s&confirm=([^"]+)" title="[^"]+">' % video_id,
-                video_page, 'confirm string')
-            confirm_url = video_page_url + '&confirm=%s' % confirm_string
-            video_page = self._download_webpage(confirm_url, video_id, 'Downloading video page (age confirmed)')
+                webpage, 'confirm string')
+            confirm_url = webpage_url + '&confirm=%s' % confirm_string
+            webpage = self._download_webpage(confirm_url, video_id, 'Downloading video page (age confirmed)')
             adult_content = True
         else:
             adult_content = False
 
-        # Extract the rest of meta data
-        video_title = self._search_meta('name', video_page, 'title')
-        if not video_title:
-            video_title = os.path.splitext(url_basename(video_url))[0]
-
-        video_description = self._search_meta('description', video_page)
-        END_TEXT = ' на сайте Smotri.com'
-        if video_description and video_description.endswith(END_TEXT):
-            video_description = video_description[:-len(END_TEXT)]
-        START_TEXT = 'Смотреть онлайн ролик '
-        if video_description and video_description.startswith(START_TEXT):
-            video_description = video_description[len(START_TEXT):]
-        video_thumbnail = self._search_meta('thumbnail', video_page)
-
-        upload_date_str = self._search_meta('uploadDate', video_page, 'upload date')
-        if upload_date_str:
-            upload_date_m = re.search(r'(?P<year>\d{4})\.(?P<month>\d{2})\.(?P<day>\d{2})T', upload_date_str)
-            video_upload_date = (
-                (
-                    upload_date_m.group('year') +
-                    upload_date_m.group('month') +
-                    upload_date_m.group('day')
-                )
-                if upload_date_m else None
-            )
-        else:
-            video_upload_date = None
-
-        duration_str = self._search_meta('duration', video_page)
-        if duration_str:
-            duration_m = re.search(r'T(?P<hours>[0-9]{2})H(?P<minutes>[0-9]{2})M(?P<seconds>[0-9]{2})S', duration_str)
-            video_duration = (
-                (
-                    (int(duration_m.group('hours')) * 60 * 60) +
-                    (int(duration_m.group('minutes')) * 60) +
-                    int(duration_m.group('seconds'))
-                )
-                if duration_m else None
-            )
-        else:
-            video_duration = None
-
-        video_uploader = self._html_search_regex(
-            '<div class="DescrUser"><div>Автор.*?onmouseover="popup_user_info[^"]+">(.*?)</a>',
-            video_page, 'uploader', fatal=False, flags=re.MULTILINE|re.DOTALL)
-
-        video_uploader_id = self._html_search_regex(
-            '<div class="DescrUser"><div>Автор.*?onmouseover="popup_user_info\\(.*?\'([^\']+)\'\\);">',
-            video_page, 'uploader id', fatal=False, flags=re.MULTILINE|re.DOTALL)
-
-        video_view_count = self._html_search_regex(
+        view_count = self._html_search_regex(
             'Общее количество просмотров.*?<span class="Number">(\\d+)</span>',
-            video_page, 'view count', fatal=False, flags=re.MULTILINE|re.DOTALL)
+            webpage, 'view count', fatal=False, flags=re.MULTILINE|re.DOTALL)
 
         return {
             'id': video_id,
             'url': video_url,
-            'title': video_title,
-            'thumbnail': video_thumbnail,
-            'description': video_description,
-            'uploader': video_uploader,
-            'upload_date': video_upload_date,
-            'uploader_id': video_uploader_id,
-            'duration': video_duration,
-            'view_count': int_or_none(video_view_count),
+            'title': title,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'upload_date': upload_date,
+            'uploader_id': uploader_id,
+            'duration': duration,
+            'view_count': int_or_none(view_count),
             'age_limit': 18 if adult_content else 0,
-            'video_page_url': video_page_url
         }
 
 

From f3a34072266c0a7595f73fa032685cf8a50d2ab4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Nov 2014 20:09:10 +0100
Subject: [PATCH 0577/1937] [youtube] Clarify keywords

---
 youtube_dl/extractor/youtube.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0cb837afc..b4bbaf902 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1558,20 +1558,20 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
         return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE)
 
 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
-    IE_DESC = 'YouTube.com recommended videos, "ytrec" keyword (requires authentication)'
+    IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
     _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
     _FEED_NAME = 'recommended'
     _PLAYLIST_TITLE = 'Youtube Recommended videos'
 
 class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor):
-    IE_DESC = 'Youtube watch later list, "ytwatchlater" keyword (requires authentication)'
+    IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
     _VALID_URL = r'https?://www\.youtube\.com/feed/watch_later|:ytwatchlater'
     _FEED_NAME = 'watch_later'
     _PLAYLIST_TITLE = 'Youtube Watch Later'
     _PERSONAL_FEED = True
 
 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
-    IE_DESC = 'Youtube watch history, "ythistory" keyword (requires authentication)'
+    IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
     _VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
     _FEED_NAME = 'history'
     _PERSONAL_FEED = True
@@ -1579,7 +1579,7 @@ class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
 
 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
     IE_NAME = 'youtube:favorites'
-    IE_DESC = 'YouTube.com favourite videos, "ytfav" keyword (requires authentication)'
+    IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
     _VALID_URL = r'https?://www\.youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
     _LOGIN_REQUIRED = True
 

From 5f6a1245ffa9276c1af59b0835afeef67e2fb5b1 Mon Sep 17 00:00:00 2001
From: Jouke Waleson <jouke.waleson@mendix.com>
Date: Sun, 23 Nov 2014 20:41:03 +0100
Subject: [PATCH 0578/1937] PEP8 applied

---
 devscripts/bash-completion.py                 |   5 +-
 devscripts/buildserver.py                     |   3 +-
 devscripts/fish-completion.py                 |   1 +
 devscripts/gh-pages/sign-versions.py          |  18 +--
 devscripts/gh-pages/update-copyright.py       |   2 +-
 devscripts/gh-pages/update-feed.py            |   1 -
 devscripts/gh-pages/update-sites.py           |   1 +
 devscripts/transition_helper.py               |   8 +-
 devscripts/transition_helper_exe/setup.py     |   2 +-
 .../transition_helper_exe/youtube-dl.py       |  22 ++-
 test/helper.py                                |   6 +-
 test/test_YoutubeDL.py                        |   1 +
 test/test_all_urls.py                         |   6 +-
 test/test_download.py                         |  13 +-
 test/test_subtitles.py                        |   1 +
 test/test_utils.py                            |   6 +-
 test/test_write_annotations.py                |  26 ++--
 test/test_youtube_lists.py                    |   2 +-
 youtube_dl/YoutubeDL.py                       |  10 +-
 youtube_dl/__init__.py                        |   4 +-
 youtube_dl/aes.py                             | 125 ++++++++++--------
 youtube_dl/compat.py                          |  32 ++---
 youtube_dl/downloader/common.py               |   6 +-
 youtube_dl/downloader/hls.py                  |   1 -
 youtube_dl/downloader/rtmp.py                 |   2 +-
 youtube_dl/extractor/adultswim.py             |   1 +
 youtube_dl/extractor/aparat.py                |   2 +-
 youtube_dl/extractor/appletrailers.py         |   2 +
 youtube_dl/extractor/ard.py                   |   1 -
 youtube_dl/extractor/arte.py                  |   2 +-
 youtube_dl/extractor/audiomack.py             |   8 +-
 youtube_dl/extractor/bambuser.py              |   2 +-
 youtube_dl/extractor/bandcamp.py              |   4 +-
 youtube_dl/extractor/bbccouk.py               |   2 +-
 youtube_dl/extractor/beeg.py                  |   2 +-
 youtube_dl/extractor/bild.py                  |   2 +-
 youtube_dl/extractor/canalplus.py             |   2 +-
 youtube_dl/extractor/cbsnews.py               |   2 +-
 youtube_dl/extractor/ceskatelevize.py         |   2 +-
 youtube_dl/extractor/channel9.py              |  15 ++-
 youtube_dl/extractor/clipsyndicate.py         |   1 +
 youtube_dl/extractor/common.py                |   7 +-
 youtube_dl/extractor/cracked.py               |   4 +-
 youtube_dl/extractor/crunchyroll.py           |   6 +-
 youtube_dl/extractor/dailymotion.py           |   4 +-
 youtube_dl/extractor/defense.py               |   4 +-
 youtube_dl/extractor/dotsub.py                |   2 +-
 youtube_dl/extractor/fc2.py                   |   2 +-
 youtube_dl/extractor/firsttv.py               |   2 +-
 youtube_dl/extractor/flickr.py                |   4 +-
 youtube_dl/extractor/fourtube.py              |   2 +-
 youtube_dl/extractor/generic.py               |   3 +-
 youtube_dl/extractor/globo.py                 |   2 +-
 youtube_dl/extractor/gorillavid.py            |   2 +-
 youtube_dl/extractor/hornbunny.py             |   2 +-
 youtube_dl/extractor/howcast.py               |   2 +-
 youtube_dl/extractor/imdb.py                  |   2 +-
 youtube_dl/extractor/internetvideoarchive.py  |   2 +-
 youtube_dl/extractor/ivi.py                   |  10 +-
 youtube_dl/extractor/jadorecettepub.py        |   1 -
 youtube_dl/extractor/jeuxvideo.py             |   4 +-
 youtube_dl/extractor/kankan.py                |   2 +-
 youtube_dl/extractor/kontrtube.py             |   2 +-
 youtube_dl/extractor/ku6.py                   |   1 -
 youtube_dl/extractor/laola1tv.py              |   1 -
 youtube_dl/extractor/lifenews.py              |   4 +-
 youtube_dl/extractor/lynda.py                 |   8 +-
 youtube_dl/extractor/m6.py                    |   2 +-
 youtube_dl/extractor/malemotion.py            |   1 +
 youtube_dl/extractor/mdr.py                   |   2 +-
 youtube_dl/extractor/mojvideo.py              |   2 +-
 youtube_dl/extractor/moniker.py               |   2 +-
 youtube_dl/extractor/mooshare.py              |   2 +-
 youtube_dl/extractor/motherless.py            |   2 +-
 youtube_dl/extractor/moviezine.py             |   2 +-
 youtube_dl/extractor/movshare.py              |   2 +-
 youtube_dl/extractor/mpora.py                 |   2 +-
 youtube_dl/extractor/mtv.py                   |   6 +-
 youtube_dl/extractor/muenchentv.py            |   1 -
 youtube_dl/extractor/musicplayon.py           |   2 +-
 youtube_dl/extractor/muzu.py                  |   2 +-
 youtube_dl/extractor/myvideo.py               |   7 +-
 youtube_dl/extractor/naver.py                 |   4 +-
 youtube_dl/extractor/nba.py                   |   1 -
 youtube_dl/extractor/ndr.py                   |   2 +-
 youtube_dl/extractor/newgrounds.py            |   4 +-
 youtube_dl/extractor/newstube.py              |   2 +-
 youtube_dl/extractor/nfb.py                   |   2 +-
 youtube_dl/extractor/nhl.py                   |   2 +-
 youtube_dl/extractor/noco.py                  |   2 +-
 youtube_dl/extractor/novamov.py               |   2 +-
 youtube_dl/extractor/nowvideo.py              |   2 +-
 youtube_dl/extractor/ntv.py                   |   2 +-
 youtube_dl/extractor/nuvid.py                 |   2 +-
 youtube_dl/extractor/nytimes.py               |   2 +-
 youtube_dl/extractor/ooyala.py                |   1 -
 youtube_dl/extractor/orf.py                   |   2 +-
 youtube_dl/extractor/podomatic.py             |   1 +
 youtube_dl/extractor/pornhub.py               |   2 +-
 youtube_dl/extractor/pornotube.py             |   2 +-
 youtube_dl/extractor/prosiebensat1.py         |   2 +-
 youtube_dl/extractor/rai.py                   |   2 +-
 youtube_dl/extractor/ringtv.py                |   1 -
 youtube_dl/extractor/rtlnow.py                |   4 +-
 youtube_dl/extractor/rtve.py                  |   1 -
 youtube_dl/extractor/rutv.py                  |   2 +-
 youtube_dl/extractor/scivee.py                |   2 +-
 youtube_dl/extractor/servingsys.py            |   2 -
 youtube_dl/extractor/shared.py                |   2 +-
 youtube_dl/extractor/smotri.py                |   2 +-
 youtube_dl/extractor/soundcloud.py            |   4 +-
 youtube_dl/extractor/spiegeltv.py             |   2 +-
 youtube_dl/extractor/sport5.py                |   2 +-
 youtube_dl/extractor/sportdeutschland.py      |   1 -
 youtube_dl/extractor/subtitles.py             |   2 +-
 youtube_dl/extractor/teachertube.py           |   2 +-
 youtube_dl/extractor/teamcoco.py              |   2 +-
 youtube_dl/extractor/telecinco.py             |   2 +-
 youtube_dl/extractor/theplatform.py           |   4 +-
 youtube_dl/extractor/thisav.py                |   4 +-
 youtube_dl/extractor/tinypic.py               |   4 +-
 youtube_dl/extractor/tnaflix.py               |   2 +-
 youtube_dl/extractor/traileraddict.py         |   6 +-
 youtube_dl/extractor/trilulilu.py             |   3 +-
 youtube_dl/extractor/tudou.py                 |   2 +-
 youtube_dl/extractor/tvigle.py                |   2 +-
 youtube_dl/extractor/udemy.py                 |   2 +-
 youtube_dl/extractor/vesti.py                 |   2 +-
 youtube_dl/extractor/vgtv.py                  |   2 +-
 youtube_dl/extractor/vice.py                  |   1 -
 youtube_dl/extractor/videobam.py              |   2 +-
 youtube_dl/extractor/videofyme.py             |   5 +-
 youtube_dl/extractor/videott.py               |   2 +-
 youtube_dl/extractor/videoweed.py             |   2 +-
 youtube_dl/extractor/vidzi.py                 |   7 +-
 youtube_dl/extractor/vrt.py                   |   4 +-
 youtube_dl/extractor/wdr.py                   |   2 +-
 youtube_dl/extractor/worldstarhiphop.py       |   1 -
 youtube_dl/extractor/xbef.py                  |   1 -
 youtube_dl/extractor/xhamster.py              |   2 +-
 youtube_dl/extractor/ynet.py                  |   2 +-
 youtube_dl/extractor/youku.py                 |  14 +-
 youtube_dl/extractor/youporn.py               |   4 +-
 youtube_dl/extractor/youtube.py               |  30 +++--
 youtube_dl/options.py                         |   2 +-
 youtube_dl/postprocessor/execafterdownload.py |   1 -
 youtube_dl/postprocessor/ffmpeg.py            |  18 +--
 youtube_dl/postprocessor/xattrpp.py           |   1 -
 youtube_dl/swfinterp.py                       |   1 -
 youtube_dl/update.py                          |  45 +++++--
 youtube_dl/utils.py                           |  32 ++++-
 151 files changed, 419 insertions(+), 343 deletions(-)

diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py
index 49287724d..70a3f552c 100755
--- a/devscripts/bash-completion.py
+++ b/devscripts/bash-completion.py
@@ -9,16 +9,17 @@ import youtube_dl
 BASH_COMPLETION_FILE = "youtube-dl.bash-completion"
 BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in"
 
+
 def build_completion(opt_parser):
     opts_flag = []
     for group in opt_parser.option_groups:
         for option in group.option_list:
-            #for every long flag
+            # for every long flag
             opts_flag.append(option.get_opt_string())
     with open(BASH_COMPLETION_TEMPLATE) as f:
         template = f.read()
     with open(BASH_COMPLETION_FILE, "w") as f:
-        #just using the special char
+        # just using the special char
         filled_template = template.replace("{{flags}}", " ".join(opts_flag))
         f.write(filled_template)
 
diff --git a/devscripts/buildserver.py b/devscripts/buildserver.py
index e0c3cc83e..42ee2b5cb 100644
--- a/devscripts/buildserver.py
+++ b/devscripts/buildserver.py
@@ -233,6 +233,7 @@ def rmtree(path):
 
 #==============================================================================
 
+
 class BuildError(Exception):
     def __init__(self, output, code=500):
         self.output = output
@@ -369,7 +370,7 @@ class Builder(PythonBuilder, GITBuilder, YoutubeDLBuilder, DownloadBuilder, Clea
 
 
 class BuildHTTPRequestHandler(BaseHTTPRequestHandler):
-    actionDict = { 'build': Builder, 'download': Builder } # They're the same, no more caching.
+    actionDict = {'build': Builder, 'download': Builder}  # They're the same, no more caching.
 
     def do_GET(self):
         path = urlparse.urlparse(self.path)
diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py
index f4aaf0201..2185d5522 100755
--- a/devscripts/fish-completion.py
+++ b/devscripts/fish-completion.py
@@ -23,6 +23,7 @@ EXTRA_ARGS = {
     'batch-file': ['--require-parameter'],
 }
 
+
 def build_completion(opt_parser):
     commands = []
 
diff --git a/devscripts/gh-pages/sign-versions.py b/devscripts/gh-pages/sign-versions.py
index 8a824df56..953a5162e 100755
--- a/devscripts/gh-pages/sign-versions.py
+++ b/devscripts/gh-pages/sign-versions.py
@@ -11,22 +11,22 @@ except NameError:
 
 versions_info = json.load(open('update/versions.json'))
 if 'signature' in versions_info:
-	del versions_info['signature']
+    del versions_info['signature']
 
 print('Enter the PKCS1 private key, followed by a blank line:')
 privkey = b''
 while True:
-	try:
-		line = input()
-	except EOFError:
-		break
-	if line == '':
-		break
-	privkey += line.encode('ascii') + b'\n'
+    try:
+        line = input()
+    except EOFError:
+        break
+    if line == '':
+        break
+    privkey += line.encode('ascii') + b'\n'
 privkey = rsa.PrivateKey.load_pkcs1(privkey)
 
 signature = hexlify(rsa.pkcs1.sign(json.dumps(versions_info, sort_keys=True).encode('utf-8'), privkey, 'SHA-256')).decode()
 print('signature: ' + signature)
 
 versions_info['signature'] = signature
-json.dump(versions_info, open('update/versions.json', 'w'), indent=4, sort_keys=True)
\ No newline at end of file
+json.dump(versions_info, open('update/versions.json', 'w'), indent=4, sort_keys=True)
diff --git a/devscripts/gh-pages/update-copyright.py b/devscripts/gh-pages/update-copyright.py
index 12c2a9194..5bc5c6514 100755
--- a/devscripts/gh-pages/update-copyright.py
+++ b/devscripts/gh-pages/update-copyright.py
@@ -5,7 +5,7 @@ from __future__ import with_statement
 
 import datetime
 import glob
-import io # For Python 2 compatibilty
+import io  # For Python 2 compatibilty
 import os
 import re
 
diff --git a/devscripts/gh-pages/update-feed.py b/devscripts/gh-pages/update-feed.py
index 0ba15ae0f..46373a8db 100755
--- a/devscripts/gh-pages/update-feed.py
+++ b/devscripts/gh-pages/update-feed.py
@@ -73,4 +73,3 @@ atom_template = atom_template.replace('@ENTRIES@', entries_str)
 
 with io.open('update/releases.atom', 'w', encoding='utf-8') as atom_file:
     atom_file.write(atom_template)
-
diff --git a/devscripts/gh-pages/update-sites.py b/devscripts/gh-pages/update-sites.py
index 153e15c8a..0d526784d 100755
--- a/devscripts/gh-pages/update-sites.py
+++ b/devscripts/gh-pages/update-sites.py
@@ -9,6 +9,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(
 
 import youtube_dl
 
+
 def main():
     with open('supportedsites.html.in', 'r', encoding='utf-8') as tmplf:
         template = tmplf.read()
diff --git a/devscripts/transition_helper.py b/devscripts/transition_helper.py
index d5ca2d4ba..cfd21919b 100644
--- a/devscripts/transition_helper.py
+++ b/devscripts/transition_helper.py
@@ -4,7 +4,7 @@ import sys, os
 
 try:
     import urllib.request as compat_urllib_request
-except ImportError: # Python 2
+except ImportError:  # Python 2
     import urllib2 as compat_urllib_request
 
 sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')
@@ -12,9 +12,9 @@ sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorr
 sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n')
 
 try:
-	raw_input()
-except NameError: # Python 3
-	input()
+    raw_input()
+except NameError:  # Python 3
+    input()
 
 filename = sys.argv[0]
 
diff --git a/devscripts/transition_helper_exe/setup.py b/devscripts/transition_helper_exe/setup.py
index aaf5c2983..bb3264af0 100644
--- a/devscripts/transition_helper_exe/setup.py
+++ b/devscripts/transition_helper_exe/setup.py
@@ -9,4 +9,4 @@ py2exe_options = {
     "dll_excludes": ['w9xpopen.exe']
 }
 
-setup(console=['youtube-dl.py'], options={ "py2exe": py2exe_options }, zipfile=None)
\ No newline at end of file
+setup(console=['youtube-dl.py'], options={"py2exe": py2exe_options}, zipfile=None)
diff --git a/devscripts/transition_helper_exe/youtube-dl.py b/devscripts/transition_helper_exe/youtube-dl.py
index 6297dfd40..edb449fb3 100644
--- a/devscripts/transition_helper_exe/youtube-dl.py
+++ b/devscripts/transition_helper_exe/youtube-dl.py
@@ -4,13 +4,17 @@ import sys, os
 import urllib2
 import json, hashlib
 
+
 def rsa_verify(message, signature, key):
     from struct import pack
     from hashlib import sha256
     from sys import version_info
+
     def b(x):
-        if version_info[0] == 2: return x
-        else: return x.encode('latin1')
+        if version_info[0] == 2:
+            return x
+        else:
+            return x.encode('latin1')
     assert(type(message) == type(b('')))
     block_size = 0
     n = key[0]
@@ -23,13 +27,17 @@ def rsa_verify(message, signature, key):
         raw_bytes.insert(0, pack("B", signature & 0xFF))
         signature >>= 8
     signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
-    if signature[0:2] != b('\x00\x01'): return False
+    if signature[0:2] != b('\x00\x01'):
+        return False
     signature = signature[2:]
-    if not b('\x00') in signature: return False
+    if not b('\x00') in signature:
+        return False
     signature = signature[signature.index(b('\x00'))+1:]
-    if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False
+    if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')):
+        return False
     signature = signature[19:]
-    if signature != sha256(message).digest(): return False
+    if signature != sha256(message).digest():
+        return False
     return True
 
 sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')
@@ -92,7 +100,7 @@ echo Updating youtube-dl...
 ping 127.0.0.1 -n 5 -w 1000 > NUL
 move /Y "%s.new" "%s"
 del "%s"
-    \n""" %(exe, exe, bat))
+    \n""" % (exe, exe, bat))
     b.close()
 
     os.startfile(bat)
diff --git a/test/helper.py b/test/helper.py
index 8be37a183..91822935f 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -59,7 +59,7 @@ class FakeYDL(YoutubeDL):
         params = get_params(override=override)
         super(FakeYDL, self).__init__(params, auto_init=False)
         self.result = []
-        
+
     def to_screen(self, s, skip_eol=None):
         print(s)
 
@@ -72,8 +72,10 @@ class FakeYDL(YoutubeDL):
     def expect_warning(self, regex):
         # Silence an expected warning matching a regex
         old_report_warning = self.report_warning
+
         def report_warning(self, message):
-            if re.match(regex, message): return
+            if re.match(regex, message):
+                return
             old_report_warning(message)
         self.report_warning = types.MethodType(report_warning, self)
 
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index ab61e1976..f8e4f930e 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -266,6 +266,7 @@ class TestFormatSelection(unittest.TestCase):
             'ext': 'mp4',
             'width': None,
         }
+
         def fname(templ):
             ydl = YoutubeDL({'outtmpl': templ})
             return ydl.prepare_filename(info)
diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index 965e5d8a5..bd4fe17bf 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -32,19 +32,19 @@ class TestAllURLsMatching(unittest.TestCase):
     def test_youtube_playlist_matching(self):
         assertPlaylist = lambda url: self.assertMatch(url, ['youtube:playlist'])
         assertPlaylist('ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
-        assertPlaylist('UUBABnxM4Ar9ten8Mdjj1j0Q') #585
+        assertPlaylist('UUBABnxM4Ar9ten8Mdjj1j0Q')  # 585
         assertPlaylist('PL63F0C78739B09958')
         assertPlaylist('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q')
         assertPlaylist('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
         assertPlaylist('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC')
-        assertPlaylist('https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012') #668
+        assertPlaylist('https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012')  # 668
         self.assertFalse('youtube:playlist' in self.matching_ies('PLtS2H6bU1M'))
         # Top tracks
         assertPlaylist('https://www.youtube.com/playlist?list=MCUS.20142101')
 
     def test_youtube_matching(self):
         self.assertTrue(YoutubeIE.suitable('PLtS2H6bU1M'))
-        self.assertFalse(YoutubeIE.suitable('https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012')) #668
+        self.assertFalse(YoutubeIE.suitable('https://www.youtube.com/watch?v=AV6J6_AeFEQ&playnext=1&list=PL4023E734DA416012'))  # 668
         self.assertMatch('http://youtu.be/BaW_jenozKc', ['youtube'])
         self.assertMatch('http://www.youtube.com/v/BaW_jenozKc', ['youtube'])
         self.assertMatch('https://youtube.googleapis.com/v/BaW_jenozKc', ['youtube'])
diff --git a/test/test_download.py b/test/test_download.py
index 12cfb5cbe..b2615f338 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -40,18 +40,22 @@ from youtube_dl.extractor import get_info_extractor
 
 RETRIES = 3
 
+
 class YoutubeDL(youtube_dl.YoutubeDL):
     def __init__(self, *args, **kwargs):
         self.to_stderr = self.to_screen
         self.processed_info_dicts = []
         super(YoutubeDL, self).__init__(*args, **kwargs)
+
     def report_warning(self, message):
         # Don't accept warnings during tests
         raise ExtractorError(message)
+
     def process_info(self, info_dict):
         self.processed_info_dicts.append(info_dict)
         return super(YoutubeDL, self).process_info(info_dict)
 
+
 def _file_md5(fn):
     with open(fn, 'rb') as f:
         return hashlib.md5(f.read()).hexdigest()
@@ -61,10 +65,13 @@ defs = gettestcases()
 
 class TestDownload(unittest.TestCase):
     maxDiff = None
+
     def setUp(self):
         self.defs = defs
 
-### Dynamically generate tests
+# Dynamically generate tests
+
+
 def generator(test_case):
 
     def test_template(self):
@@ -101,6 +108,7 @@ def generator(test_case):
         ydl = YoutubeDL(params, auto_init=False)
         ydl.add_default_info_extractors()
         finished_hook_called = set()
+
         def _hook(status):
             if status['status'] == 'finished':
                 finished_hook_called.add(status['filename'])
@@ -111,6 +119,7 @@ def generator(test_case):
             return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
 
         res_dict = None
+
         def try_rm_tcs_files(tcs=None):
             if tcs is None:
                 tcs = test_cases
@@ -206,7 +215,7 @@ def generator(test_case):
 
     return test_template
 
-### And add them to TestDownload
+# And add them to TestDownload
 for n, test_case in enumerate(defs):
     test_method = generator(test_case)
     tname = 'test_' + str(test_case['name'])
diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 94e3290db..2eeb31bc6 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -23,6 +23,7 @@ from youtube_dl.extractor import (
 class BaseTestSubtitles(unittest.TestCase):
     url = None
     IE = None
+
     def setUp(self):
         self.DL = FakeYDL()
         self.ie = self.IE(self.DL)
diff --git a/test/test_utils.py b/test/test_utils.py
index 9a62322f0..380d1059d 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -120,16 +120,16 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
         self.assertEqual(orderedSet([]), [])
         self.assertEqual(orderedSet([1]), [1])
-        #keep the list ordered
+        # keep the list ordered
         self.assertEqual(orderedSet([135, 1, 1, 1]), [135, 1])
 
     def test_unescape_html(self):
         self.assertEqual(unescapeHTML('%20;'), '%20;')
         self.assertEqual(
             unescapeHTML('&eacute;'), 'é')
-        
+
     def test_daterange(self):
-        _20century = DateRange("19000101","20000101")
+        _20century = DateRange("19000101", "20000101")
         self.assertFalse("17890714" in _20century)
         _ac = DateRange("00010101")
         self.assertTrue("19690721" in _ac)
diff --git a/test/test_write_annotations.py b/test/test_write_annotations.py
index eac53b285..852553ada 100644
--- a/test/test_write_annotations.py
+++ b/test/test_write_annotations.py
@@ -31,19 +31,18 @@ params = get_params({
 })
 
 
-
 TEST_ID = 'gr51aVj-mLg'
 ANNOTATIONS_FILE = TEST_ID + '.flv.annotations.xml'
 EXPECTED_ANNOTATIONS = ['Speech bubble', 'Note', 'Title', 'Spotlight', 'Label']
 
+
 class TestAnnotations(unittest.TestCase):
     def setUp(self):
         # Clear old files
         self.tearDown()
 
-
     def test_info_json(self):
-        expected = list(EXPECTED_ANNOTATIONS) #Two annotations could have the same text.
+        expected = list(EXPECTED_ANNOTATIONS)  # Two annotations could have the same text.
         ie = youtube_dl.extractor.YoutubeIE()
         ydl = YoutubeDL(params)
         ydl.add_info_extractor(ie)
@@ -51,7 +50,7 @@ class TestAnnotations(unittest.TestCase):
         self.assertTrue(os.path.exists(ANNOTATIONS_FILE))
         annoxml = None
         with io.open(ANNOTATIONS_FILE, 'r', encoding='utf-8') as annof:
-                annoxml = xml.etree.ElementTree.parse(annof)
+            annoxml = xml.etree.ElementTree.parse(annof)
         self.assertTrue(annoxml is not None, 'Failed to parse annotations XML')
         root = annoxml.getroot()
         self.assertEqual(root.tag, 'document')
@@ -59,18 +58,17 @@ class TestAnnotations(unittest.TestCase):
         self.assertEqual(annotationsTag.tag, 'annotations')
         annotations = annotationsTag.findall('annotation')
 
-        #Not all the annotations have TEXT children and the annotations are returned unsorted.
+        # Not all the annotations have TEXT children and the annotations are returned unsorted.
         for a in annotations:
-                self.assertEqual(a.tag, 'annotation')
-                if a.get('type') == 'text':
-                        textTag = a.find('TEXT')
-                        text = textTag.text
-                        self.assertTrue(text in expected) #assertIn only added in python 2.7
-                        #remove the first occurance, there could be more than one annotation with the same text
-                        expected.remove(text)
-        #We should have seen (and removed) all the expected annotation texts.
+            self.assertEqual(a.tag, 'annotation')
+            if a.get('type') == 'text':
+                textTag = a.find('TEXT')
+                text = textTag.text
+                self.assertTrue(text in expected)  # assertIn only added in python 2.7
+                # remove the first occurance, there could be more than one annotation with the same text
+                expected.remove(text)
+        # We should have seen (and removed) all the expected annotation texts.
         self.assertEqual(len(expected), 0, 'Not all expected annotations were found.')
-        
 
     def tearDown(self):
         try_rm(ANNOTATIONS_FILE)
diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index 410f9edc2..452d342ae 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -31,7 +31,7 @@ class TestYoutubeLists(unittest.TestCase):
         result = ie.extract('https://www.youtube.com/watch?v=FXxLjLQi3Fg&list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
         self.assertEqual(result['_type'], 'url')
         self.assertEqual(YoutubeIE().extract_id(result['url']), 'FXxLjLQi3Fg')
-    
+
     def test_youtube_course(self):
         dl = FakeYDL()
         ie = YoutubePlaylistIE(dl)
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index bfa0c6d43..c1a529f13 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -552,7 +552,7 @@ class YoutubeDL(object):
 
             try:
                 ie_result = ie.extract(url)
-                if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
+                if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
                     break
                 if isinstance(ie_result, list):
                     # Backwards compatibility: old IE result format
@@ -565,7 +565,7 @@ class YoutubeDL(object):
                     return self.process_ie_result(ie_result, download, extra_info)
                 else:
                     return ie_result
-            except ExtractorError as de: # An error we somewhat expected
+            except ExtractorError as de:  # An error we somewhat expected
                 self.report_error(compat_str(de), de.format_traceback())
                 break
             except MaxDownloadsReached:
@@ -700,6 +700,7 @@ class YoutubeDL(object):
             self.report_warning(
                 'Extractor %s returned a compat_list result. '
                 'It needs to be updated.' % ie_result.get('extractor'))
+
             def _fixup(r):
                 self.add_extra_info(r,
                     {
@@ -1010,7 +1011,7 @@ class YoutubeDL(object):
                     else:
                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
-                                subfile.write(sub)
+                            subfile.write(sub)
                 except (OSError, IOError):
                     self.report_error('Cannot write subtitles file ' + sub_filename)
                     return
@@ -1111,7 +1112,7 @@ class YoutubeDL(object):
 
         for url in url_list:
             try:
-                #It also downloads the videos
+                # It also downloads the videos
                 res = self.extract_info(url)
             except UnavailableVideoError:
                 self.report_error('unable to download video')
@@ -1428,4 +1429,3 @@ class YoutubeDL(object):
         if encoding is None:
             encoding = preferredencoding()
         return encoding
-
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index f519fae3e..ee3067134 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -128,7 +128,6 @@ def _real_main(argv=None):
             compat_print(desc)
         sys.exit(0)
 
-
     # Conflicting, missing and erroneous options
     if opts.usenetrc and (opts.username is not None or opts.password is not None):
         parser.error('using .netrc conflicts with giving username/password')
@@ -197,7 +196,7 @@ def _real_main(argv=None):
         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
         if opts.outtmpl is not None:
             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
-    outtmpl =((opts.outtmpl is not None and opts.outtmpl)
+    outtmpl = ((opts.outtmpl is not None and opts.outtmpl)
             or (opts.format == '-1' and opts.usetitle and '%(title)s-%(id)s-%(format)s.%(ext)s')
             or (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s')
             or (opts.usetitle and opts.autonumber and '%(autonumber)s-%(title)s-%(id)s.%(ext)s')
@@ -317,7 +316,6 @@ def _real_main(argv=None):
                 ydl.add_post_processor(FFmpegAudioFixPP())
             ydl.add_post_processor(AtomicParsleyPP())
 
-
         # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
         # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
         if opts.exec_cmd:
diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py
index e9c5e2152..ccfd73770 100644
--- a/youtube_dl/aes.py
+++ b/youtube_dl/aes.py
@@ -7,10 +7,11 @@ from .utils import bytes_to_intlist, intlist_to_bytes
 
 BLOCK_SIZE_BYTES = 16
 
+
 def aes_ctr_decrypt(data, key, counter):
     """
     Decrypt with aes in counter mode
-    
+
     @param {int[]} data        cipher
     @param {int[]} key         16/24/32-Byte cipher key
     @param {instance} counter  Instance whose next_value function (@returns {int[]}  16-Byte block)
@@ -19,23 +20,24 @@ def aes_ctr_decrypt(data, key, counter):
     """
     expanded_key = key_expansion(key)
     block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
-    
-    decrypted_data=[]
+
+    decrypted_data = []
     for i in range(block_count):
         counter_block = counter.next_value()
-        block = data[i*BLOCK_SIZE_BYTES : (i+1)*BLOCK_SIZE_BYTES]
+        block = data[i*BLOCK_SIZE_BYTES: (i+1)*BLOCK_SIZE_BYTES]
         block += [0]*(BLOCK_SIZE_BYTES - len(block))
-        
+
         cipher_counter_block = aes_encrypt(counter_block, expanded_key)
         decrypted_data += xor(block, cipher_counter_block)
     decrypted_data = decrypted_data[:len(data)]
-    
+
     return decrypted_data
 
+
 def aes_cbc_decrypt(data, key, iv):
     """
     Decrypt with aes in CBC mode
-    
+
     @param {int[]} data        cipher
     @param {int[]} key         16/24/32-Byte cipher key
     @param {int[]} iv          16-Byte IV
@@ -43,60 +45,62 @@ def aes_cbc_decrypt(data, key, iv):
     """
     expanded_key = key_expansion(key)
     block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
-    
-    decrypted_data=[]
+
+    decrypted_data = []
     previous_cipher_block = iv
     for i in range(block_count):
-        block = data[i*BLOCK_SIZE_BYTES : (i+1)*BLOCK_SIZE_BYTES]
+        block = data[i*BLOCK_SIZE_BYTES: (i+1)*BLOCK_SIZE_BYTES]
         block += [0]*(BLOCK_SIZE_BYTES - len(block))
-        
+
         decrypted_block = aes_decrypt(block, expanded_key)
         decrypted_data += xor(decrypted_block, previous_cipher_block)
         previous_cipher_block = block
     decrypted_data = decrypted_data[:len(data)]
-    
+
     return decrypted_data
 
+
 def key_expansion(data):
     """
     Generate key schedule
-    
+
     @param {int[]} data  16/24/32-Byte cipher key
-    @returns {int[]}     176/208/240-Byte expanded key 
+    @returns {int[]}     176/208/240-Byte expanded key
     """
-    data = data[:] # copy
+    data = data[:]  # copy
     rcon_iteration = 1
     key_size_bytes = len(data)
     expanded_key_size_bytes = (key_size_bytes // 4 + 7) * BLOCK_SIZE_BYTES
-    
+
     while len(data) < expanded_key_size_bytes:
         temp = data[-4:]
         temp = key_schedule_core(temp, rcon_iteration)
         rcon_iteration += 1
-        data += xor(temp, data[-key_size_bytes : 4-key_size_bytes])
-        
+        data += xor(temp, data[-key_size_bytes: 4-key_size_bytes])
+
         for _ in range(3):
             temp = data[-4:]
-            data += xor(temp, data[-key_size_bytes : 4-key_size_bytes])
-        
+            data += xor(temp, data[-key_size_bytes: 4-key_size_bytes])
+
         if key_size_bytes == 32:
             temp = data[-4:]
             temp = sub_bytes(temp)
-            data += xor(temp, data[-key_size_bytes : 4-key_size_bytes])
-        
+            data += xor(temp, data[-key_size_bytes: 4-key_size_bytes])
+
         for _ in range(3 if key_size_bytes == 32  else 2 if key_size_bytes == 24 else 0):
             temp = data[-4:]
-            data += xor(temp, data[-key_size_bytes : 4-key_size_bytes])
+            data += xor(temp, data[-key_size_bytes: 4-key_size_bytes])
     data = data[:expanded_key_size_bytes]
-    
+
     return data
 
+
 def aes_encrypt(data, expanded_key):
     """
     Encrypt one block with aes
-    
+
     @param {int[]} data          16-Byte state
-    @param {int[]} expanded_key  176/208/240-Byte expanded key 
+    @param {int[]} expanded_key  176/208/240-Byte expanded key
     @returns {int[]}             16-Byte cipher
     """
     rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1
@@ -107,30 +111,32 @@ def aes_encrypt(data, expanded_key):
         data = shift_rows(data)
         if i != rounds:
             data = mix_columns(data)
-        data = xor(data, expanded_key[i*BLOCK_SIZE_BYTES : (i+1)*BLOCK_SIZE_BYTES])
+        data = xor(data, expanded_key[i*BLOCK_SIZE_BYTES: (i+1)*BLOCK_SIZE_BYTES])
 
     return data
 
+
 def aes_decrypt(data, expanded_key):
     """
     Decrypt one block with aes
-    
+
     @param {int[]} data          16-Byte cipher
     @param {int[]} expanded_key  176/208/240-Byte expanded key
     @returns {int[]}             16-Byte state
     """
     rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1
-    
+
     for i in range(rounds, 0, -1):
-        data = xor(data, expanded_key[i*BLOCK_SIZE_BYTES : (i+1)*BLOCK_SIZE_BYTES])
+        data = xor(data, expanded_key[i*BLOCK_SIZE_BYTES: (i+1)*BLOCK_SIZE_BYTES])
         if i != rounds:
             data = mix_columns_inv(data)
         data = shift_rows_inv(data)
         data = sub_bytes_inv(data)
     data = xor(data, expanded_key[:BLOCK_SIZE_BYTES])
-    
+
     return data
 
+
 def aes_decrypt_text(data, password, key_size_bytes):
     """
     Decrypt text
@@ -138,33 +144,34 @@ def aes_decrypt_text(data, password, key_size_bytes):
     - The cipher key is retrieved by encrypting the first 16 Byte of 'password'
       with the first 'key_size_bytes' Bytes from 'password' (if necessary filled with 0's)
     - Mode of operation is 'counter'
-    
+
     @param {str} data                    Base64 encoded string
     @param {str,unicode} password        Password (will be encoded with utf-8)
     @param {int} key_size_bytes          Possible values: 16 for 128-Bit, 24 for 192-Bit or 32 for 256-Bit
     @returns {str}                       Decrypted data
     """
     NONCE_LENGTH_BYTES = 8
-    
+
     data = bytes_to_intlist(base64.b64decode(data))
     password = bytes_to_intlist(password.encode('utf-8'))
-    
+
     key = password[:key_size_bytes] + [0]*(key_size_bytes - len(password))
     key = aes_encrypt(key[:BLOCK_SIZE_BYTES], key_expansion(key)) * (key_size_bytes // BLOCK_SIZE_BYTES)
-    
+
     nonce = data[:NONCE_LENGTH_BYTES]
     cipher = data[NONCE_LENGTH_BYTES:]
-    
+
     class Counter:
         __value = nonce + [0]*(BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES)
+
         def next_value(self):
             temp = self.__value
             self.__value = inc(self.__value)
             return temp
-    
+
     decrypted_data = aes_ctr_decrypt(cipher, key, Counter())
     plaintext = intlist_to_bytes(decrypted_data)
-    
+
     return plaintext
 
 RCON = (0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36)
@@ -200,14 +207,14 @@ SBOX_INV = (0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x
             0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef,
             0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61,
             0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d)
-MIX_COLUMN_MATRIX = ((0x2,0x3,0x1,0x1),
-                     (0x1,0x2,0x3,0x1),
-                     (0x1,0x1,0x2,0x3),
-                     (0x3,0x1,0x1,0x2))
-MIX_COLUMN_MATRIX_INV = ((0xE,0xB,0xD,0x9),
-                         (0x9,0xE,0xB,0xD),
-                         (0xD,0x9,0xE,0xB),
-                         (0xB,0xD,0x9,0xE))
+MIX_COLUMN_MATRIX = ((0x2, 0x3, 0x1, 0x1),
+                     (0x1, 0x2, 0x3, 0x1),
+                     (0x1, 0x1, 0x2, 0x3),
+                     (0x3, 0x1, 0x1, 0x2))
+MIX_COLUMN_MATRIX_INV = ((0xE, 0xB, 0xD, 0x9),
+                         (0x9, 0xE, 0xB, 0xD),
+                         (0xD, 0x9, 0xE, 0xB),
+                         (0xB, 0xD, 0x9, 0xE))
 RIJNDAEL_EXP_TABLE = (0x01, 0x03, 0x05, 0x0F, 0x11, 0x33, 0x55, 0xFF, 0x1A, 0x2E, 0x72, 0x96, 0xA1, 0xF8, 0x13, 0x35,
                       0x5F, 0xE1, 0x38, 0x48, 0xD8, 0x73, 0x95, 0xA4, 0xF7, 0x02, 0x06, 0x0A, 0x1E, 0x22, 0x66, 0xAA,
                       0xE5, 0x34, 0x5C, 0xE4, 0x37, 0x59, 0xEB, 0x26, 0x6A, 0xBE, 0xD9, 0x70, 0x90, 0xAB, 0xE6, 0x31,
@@ -241,30 +248,37 @@ RIJNDAEL_LOG_TABLE = (0x00, 0x00, 0x19, 0x01, 0x32, 0x02, 0x1a, 0xc6, 0x4b, 0xc7
                       0x44, 0x11, 0x92, 0xd9, 0x23, 0x20, 0x2e, 0x89, 0xb4, 0x7c, 0xb8, 0x26, 0x77, 0x99, 0xe3, 0xa5,
                       0x67, 0x4a, 0xed, 0xde, 0xc5, 0x31, 0xfe, 0x18, 0x0d, 0x63, 0x8c, 0x80, 0xc0, 0xf7, 0x70, 0x07)
 
+
 def sub_bytes(data):
     return [SBOX[x] for x in data]
 
+
 def sub_bytes_inv(data):
     return [SBOX_INV[x] for x in data]
 
+
 def rotate(data):
     return data[1:] + [data[0]]
 
+
 def key_schedule_core(data, rcon_iteration):
     data = rotate(data)
     data = sub_bytes(data)
     data[0] = data[0] ^ RCON[rcon_iteration]
-    
+
     return data
 
+
 def xor(data1, data2):
     return [x^y for x, y in zip(data1, data2)]
 
+
 def rijndael_mul(a, b):
-    if(a==0 or b==0):
+    if(a == 0 or b == 0):
         return 0
     return RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[a] + RIJNDAEL_LOG_TABLE[b]) % 0xFF]
 
+
 def mix_column(data, matrix):
     data_mixed = []
     for row in range(4):
@@ -275,33 +289,38 @@ def mix_column(data, matrix):
         data_mixed.append(mixed)
     return data_mixed
 
+
 def mix_columns(data, matrix=MIX_COLUMN_MATRIX):
     data_mixed = []
     for i in range(4):
-        column = data[i*4 : (i+1)*4]
+        column = data[i*4: (i+1)*4]
         data_mixed += mix_column(column, matrix)
     return data_mixed
 
+
 def mix_columns_inv(data):
     return mix_columns(data, MIX_COLUMN_MATRIX_INV)
 
+
 def shift_rows(data):
     data_shifted = []
     for column in range(4):
         for row in range(4):
-            data_shifted.append( data[((column + row) & 0b11) * 4 + row] )
+            data_shifted.append(data[((column + row) & 0b11) * 4 + row])
     return data_shifted
 
+
 def shift_rows_inv(data):
     data_shifted = []
     for column in range(4):
         for row in range(4):
-            data_shifted.append( data[((column - row) & 0b11) * 4 + row] )
+            data_shifted.append(data[((column - row) & 0b11) * 4 + row])
     return data_shifted
 
+
 def inc(data):
-    data = data[:] # copy
-    for i in range(len(data)-1,-1,-1):
+    data = data[:]  # copy
+    for i in range(len(data)-1, -1, -1):
         if data[i] == 255:
             data[i] = 0
         else:
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 549206534..9087b4f85 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -10,47 +10,47 @@ import sys
 
 try:
     import urllib.request as compat_urllib_request
-except ImportError: # Python 2
+except ImportError:  # Python 2
     import urllib2 as compat_urllib_request
 
 try:
     import urllib.error as compat_urllib_error
-except ImportError: # Python 2
+except ImportError:  # Python 2
     import urllib2 as compat_urllib_error
 
 try:
     import urllib.parse as compat_urllib_parse
-except ImportError: # Python 2
+except ImportError:  # Python 2
     import urllib as compat_urllib_parse
 
 try:
     from urllib.parse import urlparse as compat_urllib_parse_urlparse
-except ImportError: # Python 2
+except ImportError:  # Python 2
     from urlparse import urlparse as compat_urllib_parse_urlparse
 
 try:
     import urllib.parse as compat_urlparse
-except ImportError: # Python 2
+except ImportError:  # Python 2
     import urlparse as compat_urlparse
 
 try:
     import http.cookiejar as compat_cookiejar
-except ImportError: # Python 2
+except ImportError:  # Python 2
     import cookielib as compat_cookiejar
 
 try:
     import html.entities as compat_html_entities
-except ImportError: # Python 2
+except ImportError:  # Python 2
     import htmlentitydefs as compat_html_entities
 
 try:
     import html.parser as compat_html_parser
-except ImportError: # Python 2
+except ImportError:  # Python 2
     import HTMLParser as compat_html_parser
 
 try:
     import http.client as compat_http_client
-except ImportError: # Python 2
+except ImportError:  # Python 2
     import httplib as compat_http_client
 
 try:
@@ -111,7 +111,7 @@ except ImportError:
 
 try:
     from urllib.parse import parse_qs as compat_parse_qs
-except ImportError: # Python 2
+except ImportError:  # Python 2
     # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
     # Python 2's version is apparently totally broken
 
@@ -157,12 +157,12 @@ except ImportError: # Python 2
         return parsed_result
 
 try:
-    compat_str = unicode # Python 2
+    compat_str = unicode  # Python 2
 except NameError:
     compat_str = str
 
 try:
-    compat_chr = unichr # Python 2
+    compat_chr = unichr  # Python 2
 except NameError:
     compat_chr = chr
 
@@ -182,8 +182,10 @@ except ImportError:  # Python < 3.3
 
 
 def compat_ord(c):
-    if type(c) is int: return c
-    else: return ord(c)
+    if type(c) is int:
+        return c
+    else:
+        return ord(c)
 
 
 if sys.version_info >= (3, 0):
@@ -254,7 +256,7 @@ else:
                     drive = ''
                 userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
 
-            if i != 1: #~user
+            if i != 1:  # ~user
                 userhome = os.path.join(os.path.dirname(userhome), path[1:i])
 
             return userhome + path[i:]
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 7c33004b1..c0af50c59 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -81,7 +81,7 @@ class FileDownloader(object):
         if total is None:
             return None
         dif = now - start
-        if current == 0 or dif < 0.001: # One millisecond
+        if current == 0 or dif < 0.001:  # One millisecond
             return None
         rate = float(current) / dif
         return int((float(total) - float(current)) / rate)
@@ -95,7 +95,7 @@ class FileDownloader(object):
     @staticmethod
     def calc_speed(start, now, bytes):
         dif = now - start
-        if bytes == 0 or dif < 0.001: # One millisecond
+        if bytes == 0 or dif < 0.001:  # One millisecond
             return None
         return float(bytes) / dif
 
@@ -108,7 +108,7 @@ class FileDownloader(object):
     @staticmethod
     def best_block_size(elapsed_time, bytes):
         new_min = max(bytes / 2.0, 1.0)
-        new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
+        new_max = min(max(bytes * 2.0, 1.0), 4194304)  # Do not surpass 4 MB
         if elapsed_time < 0.001:
             return int(new_max)
         rate = bytes / elapsed_time
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 68eafa403..0c33d39ee 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -101,4 +101,3 @@ class NativeHlsFD(FileDownloader):
         })
         self.try_rename(tmpfilename, filename)
         return True
-
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 17d9631fa..b8e104c59 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -180,7 +180,7 @@ class RtmpFD(FileDownloader):
         while (retval == RD_INCOMPLETE or retval == RD_FAILED) and not test and not live:
             prevsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen('[rtmpdump] %s bytes' % prevsize)
-            time.sleep(5.0) # This seems to be needed
+            time.sleep(5.0)  # This seems to be needed
             retval = run_rtmpdump(basic_args + ['-e'] + [[], ['-k', '1']][retval == RD_FAILED])
             cursize = os.path.getsize(encodeFilename(tmpfilename))
             if prevsize == cursize and retval == RD_FAILED:
diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index b4b40f2d4..0d05cbb4b 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -5,6 +5,7 @@ import re
 
 from .common import InfoExtractor
 
+
 class AdultSwimIE(InfoExtractor):
     _VALID_URL = r'https?://video\.adultswim\.com/(?P<path>.+?)(?:\.html)?(?:\?.*)?(?:#.*)?$'
     _TEST = {
diff --git a/youtube_dl/extractor/aparat.py b/youtube_dl/extractor/aparat.py
index 748608826..56775289b 100644
--- a/youtube_dl/extractor/aparat.py
+++ b/youtube_dl/extractor/aparat.py
@@ -1,4 +1,4 @@
-#coding: utf-8
+# coding: utf-8
 
 from __future__ import unicode_literals
 
diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 4359b88d1..567a76cf0 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -70,11 +70,13 @@ class AppleTrailersIE(InfoExtractor):
         uploader_id = mobj.group('company')
 
         playlist_url = compat_urlparse.urljoin(url, 'includes/playlists/itunes.inc')
+
         def fix_html(s):
             s = re.sub(r'(?s)<script[^<]*?>.*?</script>', '', s)
             s = re.sub(r'<img ([^<]*?)>', r'<img \1/>', s)
             # The ' in the onClick attributes are not escaped, it couldn't be parsed
             # like: http://trailers.apple.com/trailers/wb/gravity/
+
             def _clean_json(m):
                 return 'iTunes.playURL(%s);' % m.group(1).replace('\'', '&#39;')
             s = re.sub(self._JSON_RE, _clean_json, s)
diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 630b1faa9..967bd865c 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -192,4 +192,3 @@ class ARDIE(InfoExtractor):
             'upload_date': upload_date,
             'thumbnail': thumbnail,
         }
-
diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 3a57ce527..219631b9b 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -13,7 +13,7 @@ from ..utils import (
     qualities,
 )
 
-# There are different sources of video in arte.tv, the extraction process 
+# There are different sources of video in arte.tv, the extraction process
 # is different for each one. The videos usually expire in 7 days, so we can't
 # add tests.
 
diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 6232d2cd0..eeeec768f 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -12,17 +12,17 @@ class AudiomackIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?audiomack\.com/song/(?P<id>[\w/-]+)'
     IE_NAME = 'audiomack'
     _TESTS = [
-        #hosted on audiomack
+        # hosted on audiomack
         {
             'url': 'http://www.audiomack.com/song/roosh-williams/extraordinary',
             'info_dict':
             {
-                'id' : 'roosh-williams/extraordinary',
+                'id': 'roosh-williams/extraordinary',
                 'ext': 'mp3',
                 'title': 'Roosh Williams - Extraordinary'
             }
         },
-        #hosted on soundcloud via audiomack
+        # hosted on soundcloud via audiomack
         {
             'url': 'http://www.audiomack.com/song/xclusiveszone/take-kare',
             'file': '172419696.mp3',
@@ -49,7 +49,7 @@ class AudiomackIE(InfoExtractor):
             raise ExtractorError("Unable to deduce api url of song")
         realurl = api_response["url"]
 
-        #Audiomack wraps a lot of soundcloud tracks in their branded wrapper
+        # Audiomack wraps a lot of soundcloud tracks in their branded wrapper
         # - if so, pass the work off to the soundcloud extractor
         if SoundcloudIE.suitable(realurl):
             return {'_type': 'url', 'url': realurl, 'ie_key': 'Soundcloud'}
diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index de5d4faf3..eab99faaa 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -18,7 +18,7 @@ class BambuserIE(InfoExtractor):
     _TEST = {
         'url': 'http://bambuser.com/v/4050584',
         # MD5 seems to be flaky, see https://travis-ci.org/rg3/youtube-dl/jobs/14051016#L388
-        #u'md5': 'fba8f7693e48fd4e8641b3fd5539a641',
+        # u'md5': 'fba8f7693e48fd4e8641b3fd5539a641',
         'info_dict': {
             'id': '4050584',
             'ext': 'flv',
diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 1b8da43ca..acddbc8f1 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -83,12 +83,12 @@ class BandcampIE(InfoExtractor):
         initial_url = mp3_info['url']
         re_url = r'(?P<server>http://(.*?)\.bandcamp\.com)/download/track\?enc=mp3-320&fsig=(?P<fsig>.*?)&id=(?P<id>.*?)&ts=(?P<ts>.*)$'
         m_url = re.match(re_url, initial_url)
-        #We build the url we will use to get the final track url
+        # We build the url we will use to get the final track url
         # This url is build in Bandcamp in the script download_bunde_*.js
         request_url = '%s/statdownload/track?enc=mp3-320&fsig=%s&id=%s&ts=%s&.rand=665028774616&.vrs=1' % (m_url.group('server'), m_url.group('fsig'), video_id, m_url.group('ts'))
         final_url_webpage = self._download_webpage(request_url, video_id, 'Requesting download url')
         # If we could correctly generate the .rand field the url would be
-        #in the "download_url" key
+        # in the "download_url" key
         final_url = re.search(r'"retry_url":"(.*?)"', final_url_webpage).group(1)
 
         return {
diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 75e608f99..89476f032 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -220,4 +220,4 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
             'duration': duration,
             'formats': formats,
             'subtitles': subtitles,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index 314e37f8b..4e79fea8f 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -40,7 +40,7 @@ class BeegIE(InfoExtractor):
 
         title = self._html_search_regex(
             r'<title>([^<]+)\s*-\s*beeg\.?</title>', webpage, 'title')
-        
+
         description = self._html_search_regex(
             r'<meta name="description" content="([^"]*)"',
             webpage, 'description', fatal=False)
diff --git a/youtube_dl/extractor/bild.py b/youtube_dl/extractor/bild.py
index 0269d1174..77b562d99 100644
--- a/youtube_dl/extractor/bild.py
+++ b/youtube_dl/extractor/bild.py
@@ -1,4 +1,4 @@
-#coding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index dddeaa59c..9873728df 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -112,4 +112,4 @@ class CanalplusIE(InfoExtractor):
             'like_count': int(infos.find('NB_LIKES').text),
             'comment_count': int(infos.find('NB_COMMENTS').text),
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 0bce7937f..7e47960ab 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -84,4 +84,4 @@ class CBSNewsIE(InfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 90a3dddb9..97feb6704 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -92,7 +92,7 @@ class CeskaTelevizeIE(InfoExtractor):
         req.add_header('Referer', url)
 
         playlist = self._download_xml(req, video_id)
-        
+
         formats = []
         for i in playlist.find('smilRoot/body'):
             if 'AD' not in i.attrib['id']:
diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 16d800512..82a48870a 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -5,6 +5,7 @@ import re
 from .common import InfoExtractor
 from ..utils import ExtractorError
 
+
 class Channel9IE(InfoExtractor):
     '''
     Common extractor for channel9.msdn.com.
@@ -31,7 +32,7 @@ class Channel9IE(InfoExtractor):
                 'session_code': 'KOS002',
                 'session_day': 'Day 1',
                 'session_room': 'Arena 1A',
-                'session_speakers': [ 'Ed Blankenship', 'Andrew Coates', 'Brady Gaster', 'Patrick Klug', 'Mads Kristensen' ],
+                'session_speakers': ['Ed Blankenship', 'Andrew Coates', 'Brady Gaster', 'Patrick Klug', 'Mads Kristensen'],
             },
         },
         {
@@ -44,7 +45,7 @@ class Channel9IE(InfoExtractor):
                 'description': 'md5:d1e6ecaafa7fb52a2cacdf9599829f5b',
                 'duration': 1540,
                 'thumbnail': 'http://video.ch9.ms/ch9/87e1/0300391f-a455-4c72-bec3-4422f19287e1/selfservicenuk_512.jpg',
-                'authors': [ 'Mike Wilmot' ],
+                'authors': ['Mike Wilmot'],
             },
         }
     ]
@@ -83,7 +84,7 @@ class Channel9IE(InfoExtractor):
             'format_id': x.group('quality'),
             'format_note': x.group('note'),
             'format': '%s (%s)' % (x.group('quality'), x.group('note')),
-            'filesize': self._restore_bytes(x.group('filesize')), # File size is approximate
+            'filesize': self._restore_bytes(x.group('filesize')),  # File size is approximate
             'preference': self._known_formats.index(x.group('quality')),
             'vcodec': 'none' if x.group('note') == 'Audio only' else None,
         } for x in list(re.finditer(FORMAT_REGEX, html)) if x.group('quality') in self._known_formats]
@@ -202,17 +203,17 @@ class Channel9IE(InfoExtractor):
 
         if slides is not None:
             d = common.copy()
-            d.update({ 'title': title + '-Slides', 'url': slides })
+            d.update({'title': title + '-Slides', 'url': slides})
             result.append(d)
 
         if zip_ is not None:
             d = common.copy()
-            d.update({ 'title': title + '-Zip', 'url': zip_ })
+            d.update({'title': title + '-Zip', 'url': zip_})
             result.append(d)
 
         if len(formats) > 0:
             d = common.copy()
-            d.update({ 'title': title, 'formats': formats })
+            d.update({'title': title, 'formats': formats})
             result.append(d)
 
         return result
@@ -270,5 +271,5 @@ class Channel9IE(InfoExtractor):
             else:
                 raise ExtractorError('Unexpected WT.entryid %s' % page_type, expected=True)
 
-        else: # Assuming list
+        else:  # Assuming list
             return self._extract_list(content_path)
diff --git a/youtube_dl/extractor/clipsyndicate.py b/youtube_dl/extractor/clipsyndicate.py
index 02a1667fa..d07d544ea 100644
--- a/youtube_dl/extractor/clipsyndicate.py
+++ b/youtube_dl/extractor/clipsyndicate.py
@@ -39,6 +39,7 @@ class ClipsyndicateIE(InfoExtractor):
             transform_source=fix_xml_ampersands)
 
         track_doc = pdoc.find('trackList/track')
+
         def find_param(name):
             node = find_xpath_attr(track_doc, './/param', 'name', name)
             if node is not None:
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 93a5a3d57..f0489ede4 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -423,17 +423,18 @@ class InfoExtractor(object):
         """Report attempt to log in."""
         self.to_screen('Logging in')
 
-    #Methods for following #608
+    # Methods for following #608
     @staticmethod
     def url_result(url, ie=None, video_id=None):
         """Returns a url that points to a page that should be processed"""
-        #TODO: ie should be the class used for getting the info
+        # TODO: ie should be the class used for getting the info
         video_info = {'_type': 'url',
                       'url': url,
                       'ie_key': ie}
         if video_id is not None:
             video_info['id'] = video_id
         return video_info
+
     @staticmethod
     def playlist_result(entries, playlist_id=None, playlist_title=None):
         """Returns a playlist"""
@@ -517,7 +518,7 @@ class InfoExtractor(object):
                     raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
             except (IOError, netrc.NetrcParseError) as err:
                 self._downloader.report_warning('parsing .netrc: %s' % compat_str(err))
-        
+
         return (username, password)
 
     def _get_tfa_info(self):
diff --git a/youtube_dl/extractor/cracked.py b/youtube_dl/extractor/cracked.py
index 74b880ffc..cf763ee7e 100644
--- a/youtube_dl/extractor/cracked.py
+++ b/youtube_dl/extractor/cracked.py
@@ -54,7 +54,7 @@ class CrackedIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'url':video_url,
+            'url': video_url,
             'title': title,
             'description': description,
             'timestamp': timestamp,
@@ -62,4 +62,4 @@ class CrackedIE(InfoExtractor):
             'comment_count': comment_count,
             'height': height,
             'width': width,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index fe1324fe3..c45858160 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -69,11 +69,9 @@ class CrunchyrollIE(SubtitlesInfoExtractor):
         login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         self._download_webpage(login_request, None, False, 'Wrong login info')
 
-
     def _real_initialize(self):
         self._login()
 
-
     def _decrypt_subtitles(self, data, iv, id):
         data = bytes_to_intlist(data)
         iv = bytes_to_intlist(iv)
@@ -99,8 +97,10 @@ class CrunchyrollIE(SubtitlesInfoExtractor):
             return shaHash + [0] * 12
 
         key = obfuscate_key(id)
+
         class Counter:
             __value = iv
+
             def next_value(self):
                 temp = self.__value
                 self.__value = inc(self.__value)
@@ -183,7 +183,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
         return output
 
-    def _real_extract(self,url):
+    def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('video_id')
 
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index fd4bc75b2..22cdcdfa5 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -1,4 +1,4 @@
-#coding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -18,6 +18,7 @@ from ..utils import (
     unescapeHTML,
 )
 
+
 class DailymotionBaseInfoExtractor(InfoExtractor):
     @staticmethod
     def _build_request(url):
@@ -27,6 +28,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
         request.add_header('Cookie', 'ff=off')
         return request
 
+
 class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
     """Information Extractor for Dailymotion"""
 
diff --git a/youtube_dl/extractor/defense.py b/youtube_dl/extractor/defense.py
index c5529f8d4..3ffed3d44 100644
--- a/youtube_dl/extractor/defense.py
+++ b/youtube_dl/extractor/defense.py
@@ -26,13 +26,13 @@ class DefenseGouvFrIE(InfoExtractor):
         video_id = self._search_regex(
             r"flashvars.pvg_id=\"(\d+)\";",
             webpage, 'ID')
-        
+
         json_url = ('http://static.videos.gouv.fr/brightcovehub/export/json/'
             + video_id)
         info = self._download_webpage(json_url, title,
                                                   'Downloading JSON config')
         video_url = json.loads(info)['renditions'][0]['url']
-        
+
         return {'id': video_id,
                 'ext': 'mp4',
                 'url': video_url,
diff --git a/youtube_dl/extractor/dotsub.py b/youtube_dl/extractor/dotsub.py
index 5ae0ad5b6..70e368021 100644
--- a/youtube_dl/extractor/dotsub.py
+++ b/youtube_dl/extractor/dotsub.py
@@ -27,7 +27,7 @@ class DotsubIE(InfoExtractor):
         video_id = mobj.group('id')
         info_url = "https://dotsub.com/api/media/%s/metadata" % video_id
         info = self._download_json(info_url, video_id)
-        date = time.gmtime(info['dateCreated']/1000) # The timestamp is in miliseconds
+        date = time.gmtime(info['dateCreated']/1000)  # The timestamp is in miliseconds
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index c663a0f81..6f5d23559 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -40,7 +40,7 @@ class FC2IE(InfoExtractor):
 
         info_url = (
             "http://video.fc2.com/ginfo.php?mimi={1:s}&href={2:s}&v={0:s}&fversion=WIN%2011%2C6%2C602%2C180&from=2&otag=0&upid={0:s}&tk=null&".
-            format(video_id, mimi, compat_urllib_request.quote(refer, safe='').replace('.','%2E')))
+            format(video_id, mimi, compat_urllib_request.quote(refer, safe='').replace('.', '%2E')))
 
         info_webpage = self._download_webpage(
             info_url, video_id, note='Downloading info page')
diff --git a/youtube_dl/extractor/firsttv.py b/youtube_dl/extractor/firsttv.py
index c2e987ff7..3410daa98 100644
--- a/youtube_dl/extractor/firsttv.py
+++ b/youtube_dl/extractor/firsttv.py
@@ -57,4 +57,4 @@ class FirstTVIE(InfoExtractor):
             'duration': int_or_none(duration),
             'like_count': int_or_none(like_count),
             'dislike_count': int_or_none(dislike_count),
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index e09982e88..5b0bc9d21 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -17,8 +17,8 @@ class FlickrIE(InfoExtractor):
         'info_dict': {
             'id': '5645318632',
             'ext': 'mp4',
-            "description": "Waterfalls in the Springtime at Dark Hollow Waterfalls. These are located just off of Skyline Drive in Virginia. They are only about 6/10 of a mile hike but it is a pretty steep hill and a good climb back up.", 
-            "uploader_id": "forestwander-nature-pictures", 
+            "description": "Waterfalls in the Springtime at Dark Hollow Waterfalls. These are located just off of Skyline Drive in Virginia. They are only about 6/10 of a mile hike but it is a pretty steep hill and a good climb back up.",
+            "uploader_id": "forestwander-nature-pictures",
             "title": "Dark Hollow Waterfalls"
         }
     }
diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index 7d56b9be9..24d4e9754 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -92,4 +92,4 @@ class FourTubeIE(InfoExtractor):
             'duration': duration,
             'age_limit': 18,
             'webpage_url': webpage_url,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c7a824c29..2dc801ad9 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -733,7 +733,7 @@ class GenericIE(InfoExtractor):
                 'title': video_title,
                 'id': video_id,
             }
-            
+
         match = re.search(r'(?:id=["\']wistia_|data-wistia-?id=["\']|Wistia\.embed\(["\'])(?P<id>[^"\']+)', webpage)
         if match:
             return {
@@ -1025,4 +1025,3 @@ class GenericIE(InfoExtractor):
                 '_type': 'playlist',
                 'entries': entries,
             }
-
diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 66ca37918..6949a57c7 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -397,4 +397,4 @@ class GloboIE(InfoExtractor):
             'uploader_id': uploader_id,
             'like_count': like_count,
             'formats': formats
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index e21e57510..65b153417 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -69,7 +69,7 @@ class GorillaVidIE(InfoExtractor):
             (?:id="[^"]+"\s+)?
             value="([^"]*)"
             ''', webpage))
-        
+
         if fields['op'] == 'download1':
             post = compat_urllib_parse.urlencode(fields)
 
diff --git a/youtube_dl/extractor/hornbunny.py b/youtube_dl/extractor/hornbunny.py
index 7e7714438..5b6efb27e 100644
--- a/youtube_dl/extractor/hornbunny.py
+++ b/youtube_dl/extractor/hornbunny.py
@@ -37,7 +37,7 @@ class HornBunnyIE(InfoExtractor):
         webpage2 = self._download_webpage(redirect_url, video_id)
         video_url = self._html_search_regex(
             r'flvMask:(.*?);', webpage2, 'video_url')
-        
+
         duration = parse_duration(self._search_regex(
             r'<strong>Runtime:</strong>\s*([0-9:]+)</div>',
             webpage, 'duration', fatal=False))
diff --git a/youtube_dl/extractor/howcast.py b/youtube_dl/extractor/howcast.py
index 6ae04782c..4ddf06409 100644
--- a/youtube_dl/extractor/howcast.py
+++ b/youtube_dl/extractor/howcast.py
@@ -13,7 +13,7 @@ class HowcastIE(InfoExtractor):
         'info_dict': {
             'id': '390161',
             'ext': 'mp4',
-            'description': 'The square knot, also known as the reef knot, is one of the oldest, most basic knots to tie, and can be used in many different ways. Here\'s the proper way to tie a square knot.', 
+            'description': 'The square knot, also known as the reef knot, is one of the oldest, most basic knots to tie, and can be used in many different ways. Here\'s the proper way to tie a square knot.',
             'title': 'How to Tie a Square Knot Properly',
         }
     }
diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index 6108ed552..f2c1c10f5 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -71,7 +71,7 @@ class ImdbListIE(InfoExtractor):
         },
         'playlist_count': 7,
     }
-    
+
     def _real_extract(self, url):
         list_id = self._match_id(url)
         webpage = self._download_webpage(url, list_id)
diff --git a/youtube_dl/extractor/internetvideoarchive.py b/youtube_dl/extractor/internetvideoarchive.py
index 53f9a5f75..c137f4a5d 100644
--- a/youtube_dl/extractor/internetvideoarchive.py
+++ b/youtube_dl/extractor/internetvideoarchive.py
@@ -32,7 +32,7 @@ class InternetVideoArchiveIE(InfoExtractor):
     def _clean_query(query):
         NEEDED_ARGS = ['publishedid', 'customerid']
         query_dic = compat_urlparse.parse_qs(query)
-        cleaned_dic = dict((k,v[0]) for (k,v) in query_dic.items() if k in NEEDED_ARGS)
+        cleaned_dic = dict((k, v[0]) for (k, v) in query_dic.items() if k in NEEDED_ARGS)
         # Other player ids return m3u8 urls
         cleaned_dic['playerid'] = '247'
         cleaned_dic['videokbrate'] = '100000'
diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index 75b543b7c..06af73417 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -102,7 +102,7 @@ class IviIE(InfoExtractor):
         compilation = result['compilation']
         title = result['title']
 
-        title = '%s - %s' % (compilation, title) if compilation is not None else title  
+        title = '%s - %s' % (compilation, title) if compilation is not None else title
 
         previews = result['preview']
         previews.sort(key=lambda fmt: self._known_thumbnails.index(fmt['content_format']))
@@ -152,17 +152,17 @@ class IviCompilationIE(InfoExtractor):
         compilation_id = mobj.group('compilationid')
         season_id = mobj.group('seasonid')
 
-        if season_id is not None: # Season link
+        if season_id is not None:  # Season link
             season_page = self._download_webpage(url, compilation_id, 'Downloading season %s web page' % season_id)
             playlist_id = '%s/season%s' % (compilation_id, season_id)
             playlist_title = self._html_search_meta('title', season_page, 'title')
             entries = self._extract_entries(season_page, compilation_id)
-        else: # Compilation link            
+        else:  # Compilation link
             compilation_page = self._download_webpage(url, compilation_id, 'Downloading compilation web page')
             playlist_id = compilation_id
             playlist_title = self._html_search_meta('title', compilation_page, 'title')
             seasons = re.findall(r'<a href="/watch/%s/season(\d+)">[^<]+</a>' % compilation_id, compilation_page)
-            if len(seasons) == 0: # No seasons in this compilation
+            if len(seasons) == 0:  # No seasons in this compilation
                 entries = self._extract_entries(compilation_page, compilation_id)
             else:
                 entries = []
@@ -172,4 +172,4 @@ class IviCompilationIE(InfoExtractor):
                         compilation_id, 'Downloading season %s web page' % season_id)
                     entries.extend(self._extract_entries(season_page, compilation_id))
 
-        return self.playlist_result(entries, playlist_id, playlist_title)
\ No newline at end of file
+        return self.playlist_result(entries, playlist_id, playlist_title)
diff --git a/youtube_dl/extractor/jadorecettepub.py b/youtube_dl/extractor/jadorecettepub.py
index ace08769b..063e86de4 100644
--- a/youtube_dl/extractor/jadorecettepub.py
+++ b/youtube_dl/extractor/jadorecettepub.py
@@ -45,4 +45,3 @@ class JadoreCettePubIE(InfoExtractor):
             'title': title,
             'description': description,
         }
-
diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py
index 188165966..8094cc2e4 100644
--- a/youtube_dl/extractor/jeuxvideo.py
+++ b/youtube_dl/extractor/jeuxvideo.py
@@ -29,7 +29,7 @@ class JeuxVideoIE(InfoExtractor):
         xml_link = self._html_search_regex(
             r'<param name="flashvars" value="config=(.*?)" />',
             webpage, 'config URL')
-        
+
         video_id = self._search_regex(
             r'http://www\.jeuxvideo\.com/config/\w+/\d+/(.*?)/\d+_player\.xml',
             xml_link, 'video ID')
@@ -38,7 +38,7 @@ class JeuxVideoIE(InfoExtractor):
             xml_link, title, 'Downloading XML config')
         info_json = config.find('format.json').text
         info = json.loads(info_json)['versions'][0]
-        
+
         video_url = 'http://video720.jeuxvideo.com/' + info['file']
 
         return {
diff --git a/youtube_dl/extractor/kankan.py b/youtube_dl/extractor/kankan.py
index 23103b163..dbfe4cc03 100644
--- a/youtube_dl/extractor/kankan.py
+++ b/youtube_dl/extractor/kankan.py
@@ -10,7 +10,7 @@ _md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
 
 class KankanIE(InfoExtractor):
     _VALID_URL = r'https?://(?:.*?\.)?kankan\.com/.+?/(?P<id>\d+)\.shtml'
-    
+
     _TEST = {
         'url': 'http://yinyue.kankan.com/vod/48/48863.shtml',
         'file': '48863.flv',
diff --git a/youtube_dl/extractor/kontrtube.py b/youtube_dl/extractor/kontrtube.py
index 8a73ecfa0..41fd62009 100644
--- a/youtube_dl/extractor/kontrtube.py
+++ b/youtube_dl/extractor/kontrtube.py
@@ -63,4 +63,4 @@ class KontrTubeIE(InfoExtractor):
             'duration': duration,
             'view_count': int_or_none(view_count),
             'comment_count': int_or_none(comment_count),
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/ku6.py b/youtube_dl/extractor/ku6.py
index 89013e522..a602980a1 100644
--- a/youtube_dl/extractor/ku6.py
+++ b/youtube_dl/extractor/ku6.py
@@ -30,4 +30,3 @@ class Ku6IE(InfoExtractor):
             'title': title,
             'url': downloadUrl
         }
-
diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index 102e29f7a..2fd3b4699 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -75,4 +75,3 @@ class Laola1TvIE(InfoExtractor):
             'categories': categories,
             'ext': 'mp4',
         }
-
diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 8d9491f23..1f1e23dc3 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -52,7 +52,7 @@ class LifeNewsIE(InfoExtractor):
             r'<div class=\'comments\'>\s*<span class=\'counter\'>(\d+)</span>', webpage, 'comment count', fatal=False)
 
         upload_date = self._html_search_regex(
-            r'<time datetime=\'([^\']+)\'>', webpage, 'upload date',fatal=False)
+            r'<time datetime=\'([^\']+)\'>', webpage, 'upload date', fatal=False)
         if upload_date is not None:
             upload_date = unified_strdate(upload_date)
 
@@ -71,4 +71,4 @@ class LifeNewsIE(InfoExtractor):
         if len(videos) == 1:
             return make_entry(video_id, videos[0])
         else:
-            return [make_entry(video_id, media, video_number+1) for video_number, media in enumerate(videos)]
\ No newline at end of file
+            return [make_entry(video_id, media, video_number+1) for video_number, media in enumerate(videos)]
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 33f34f4e9..97ca4337b 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -109,7 +109,7 @@ class LyndaIE(SubtitlesInfoExtractor):
             'password': password,
             'remember': 'false',
             'stayPut': 'false'
-        }        
+        }
         request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
         login_page = self._download_webpage(request, None, 'Logging in as %s' % username)
 
@@ -117,7 +117,7 @@ class LyndaIE(SubtitlesInfoExtractor):
         m = re.search(r'loginResultJson = \'(?P<json>[^\']+)\';', login_page)
         if m is not None:
             response = m.group('json')
-            response_json = json.loads(response)            
+            response_json = json.loads(response)
             state = response_json['state']
 
             if state == 'notlogged':
@@ -187,7 +187,7 @@ class LyndaCourseIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         course_path = mobj.group('coursepath')
         course_id = mobj.group('courseid')
-        
+
         page = self._download_webpage('http://www.lynda.com/ajax/player?courseId=%s&type=course' % course_id,
                                       course_id, 'Downloading course JSON')
         course_json = json.loads(page)
@@ -221,4 +221,4 @@ class LyndaCourseIE(InfoExtractor):
 
         course_title = course_json['Title']
 
-        return self.playlist_result(entries, course_id, course_title)
\ No newline at end of file
+        return self.playlist_result(entries, course_id, course_title)
diff --git a/youtube_dl/extractor/m6.py b/youtube_dl/extractor/m6.py
index 1a26b5d57..3d806323a 100644
--- a/youtube_dl/extractor/m6.py
+++ b/youtube_dl/extractor/m6.py
@@ -53,4 +53,4 @@ class M6IE(InfoExtractor):
             'duration': duration,
             'view_count': view_count,
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/malemotion.py b/youtube_dl/extractor/malemotion.py
index 8c1966ab2..1abf6e4f8 100644
--- a/youtube_dl/extractor/malemotion.py
+++ b/youtube_dl/extractor/malemotion.py
@@ -7,6 +7,7 @@ from ..utils import (
     compat_urllib_parse,
 )
 
+
 class MalemotionIE(InfoExtractor):
     _VALID_URL = r'^(?:https?://)?malemotion\.com/video/(.+?)\.(?P<id>.+?)(#|$)'
     _TEST = {
diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py
index 1b8c4a32e..5fdd19027 100644
--- a/youtube_dl/extractor/mdr.py
+++ b/youtube_dl/extractor/mdr.py
@@ -7,7 +7,7 @@ from .common import InfoExtractor
 
 class MDRIE(InfoExtractor):
     _VALID_URL = r'^(?P<domain>https?://(?:www\.)?mdr\.de)/(?:.*)/(?P<type>video|audio)(?P<video_id>[^/_]+)(?:_|\.html)'
-    
+
     # No tests, MDR regularily deletes its videos
     _TEST = {
         'url': 'http://www.mdr.de/fakt/video189002.html',
diff --git a/youtube_dl/extractor/mojvideo.py b/youtube_dl/extractor/mojvideo.py
index 90b460d65..0ba435dc5 100644
--- a/youtube_dl/extractor/mojvideo.py
+++ b/youtube_dl/extractor/mojvideo.py
@@ -55,4 +55,4 @@ class MojvideoIE(InfoExtractor):
             'title': title,
             'thumbnail': thumbnail,
             'duration': duration,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 79bb2ca59..1c4f589cc 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -54,7 +54,7 @@ class MonikerIE(InfoExtractor):
 
         title = os.path.splitext(data['fname'])[0]
 
-        #Could be several links with different quality
+        # Could be several links with different quality
         links = re.findall(r'"file" : "?(.+?)",', webpage)
         # Assume the links are ordered in quality
         formats = [{
diff --git a/youtube_dl/extractor/mooshare.py b/youtube_dl/extractor/mooshare.py
index 7d21ea18f..f50e432a3 100644
--- a/youtube_dl/extractor/mooshare.py
+++ b/youtube_dl/extractor/mooshare.py
@@ -111,4 +111,4 @@ class MooshareIE(InfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 3621ff99e..97d5da626 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -72,7 +72,7 @@ class MotherlessIE(InfoExtractor):
         like_count = str_to_int(self._html_search_regex(
             r'<strong>Favorited</strong>\s+([^<]+)<',
             webpage, 'like count', fatal=False))
- 
+
         upload_date = self._html_search_regex(
             r'<strong>Uploaded</strong>\s+([^<]+)<', webpage, 'upload date')
         if 'Ago' in upload_date:
diff --git a/youtube_dl/extractor/moviezine.py b/youtube_dl/extractor/moviezine.py
index 43146180a..f130b75c4 100644
--- a/youtube_dl/extractor/moviezine.py
+++ b/youtube_dl/extractor/moviezine.py
@@ -27,7 +27,7 @@ class MoviezineIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         jsplayer = self._download_webpage('http://www.moviezine.se/api/player.js?video=%s' % video_id, video_id, 'Downloading js api player')
 
-        formats =[{
+        formats = [{
             'format_id': 'sd',
             'url': self._html_search_regex(r'file: "(.+?)",', jsplayer, 'file'),
             'quality': 0,
diff --git a/youtube_dl/extractor/movshare.py b/youtube_dl/extractor/movshare.py
index 4191cf7a0..6101063f2 100644
--- a/youtube_dl/extractor/movshare.py
+++ b/youtube_dl/extractor/movshare.py
@@ -24,4 +24,4 @@ class MovShareIE(NovaMovIE):
             'title': 'dissapeared image',
             'description': 'optical illusion  dissapeared image  magic illusion',
         }
-    }
\ No newline at end of file
+    }
diff --git a/youtube_dl/extractor/mpora.py b/youtube_dl/extractor/mpora.py
index 387935d4d..88c9501cd 100644
--- a/youtube_dl/extractor/mpora.py
+++ b/youtube_dl/extractor/mpora.py
@@ -44,7 +44,7 @@ class MporaIE(InfoExtractor):
                     r'_([0-9]+)\.[a-zA-Z0-9]+$', src['src'],
                     False, default=None)
                 vcodec = src['type'].partition('/')[2]
-                
+
                 formats.append({
                     'format_id': encoding_id + '-' + vcodec,
                     'url': src['src'],
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 5f0f476b6..b2f37f692 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -60,7 +60,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         url = response.geturl()
         # Transform the url to get the best quality:
         url = re.sub(r'.+pxE=mp4', 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=0+_pxK=18639+_pxE=mp4', url, 1)
-        return [{'url': url,'ext': 'mp4'}]
+        return [{'url': url, 'ext': 'mp4'}]
 
     def _extract_video_formats(self, mdoc, mtvn_id):
         if re.match(r'.*/(error_country_block\.swf|geoblock\.mp4)$', mdoc.find('.//src').text) is not None:
@@ -240,7 +240,7 @@ class MTVIE(MTVServicesInfoExtractor):
         uri = mobj.groupdict().get('mgid')
         if uri is None:
             webpage = self._download_webpage(url, video_id)
-    
+
             # Some videos come from Vevo.com
             m_vevo = re.search(r'isVevoVideo = true;.*?vevoVideoId = "(.*?)";',
                                webpage, re.DOTALL)
@@ -248,7 +248,7 @@ class MTVIE(MTVServicesInfoExtractor):
                 vevo_id = m_vevo.group(1);
                 self.to_screen('Vevo video detected: %s' % vevo_id)
                 return self.url_result('vevo:%s' % vevo_id, ie='Vevo')
-    
+
             uri = self._html_search_regex(r'/uri/(.*?)\?', webpage, 'uri')
         return self._get_videos_info(uri)
 
diff --git a/youtube_dl/extractor/muenchentv.py b/youtube_dl/extractor/muenchentv.py
index c7f6beb9c..b4e8ad17e 100644
--- a/youtube_dl/extractor/muenchentv.py
+++ b/youtube_dl/extractor/muenchentv.py
@@ -73,4 +73,3 @@ class MuenchenTVIE(InfoExtractor):
             'is_live': True,
             'thumbnail': thumbnail,
         }
-
diff --git a/youtube_dl/extractor/musicplayon.py b/youtube_dl/extractor/musicplayon.py
index 42d7a82a5..50d92b50a 100644
--- a/youtube_dl/extractor/musicplayon.py
+++ b/youtube_dl/extractor/musicplayon.py
@@ -72,4 +72,4 @@ class MusicPlayOnIE(InfoExtractor):
             'duration': int_or_none(duration),
             'view_count': int_or_none(view_count),
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/muzu.py b/youtube_dl/extractor/muzu.py
index 1772b7f9a..97edd3ef6 100644
--- a/youtube_dl/extractor/muzu.py
+++ b/youtube_dl/extractor/muzu.py
@@ -37,7 +37,7 @@ class MuzuTVIE(InfoExtractor):
         player_info_page = self._download_webpage('http://player.muzu.tv/player/playerInit?ai=%s' % video_id,
                                                   video_id, u'Downloading player info')
         video_info = json.loads(player_info_page)['videos'][0]
-        for quality in ['1080' , '720', '480', '360']:
+        for quality in ['1080', '720', '480', '360']:
             if video_info.get('v%s' % quality):
                 break
 
diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
index a89153985..956cf8b86 100644
--- a/youtube_dl/extractor/myvideo.py
+++ b/youtube_dl/extractor/myvideo.py
@@ -33,7 +33,7 @@ class MyVideoIE(InfoExtractor):
     # Original Code from: https://github.com/dersphere/plugin.video.myvideo_de.git
     # Released into the Public Domain by Tristan Fischer on 2013-05-19
     # https://github.com/rg3/youtube-dl/pull/842
-    def __rc4crypt(self,data, key):
+    def __rc4crypt(self, data, key):
         x = 0
         box = list(range(256))
         for i in list(range(256)):
@@ -49,10 +49,10 @@ class MyVideoIE(InfoExtractor):
             out += chr(compat_ord(char) ^ box[(box[x] + box[y]) % 256])
         return out
 
-    def __md5(self,s):
+    def __md5(self, s):
         return hashlib.md5(s).hexdigest().encode()
 
-    def _real_extract(self,url):
+    def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
 
@@ -173,4 +173,3 @@ class MyVideoIE(InfoExtractor):
             'play_path': video_playpath,
             'player_url': video_swfobj,
         }
-
diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index cb1af9ecc..5ce35dbf5 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -40,7 +40,7 @@ class NaverIE(InfoExtractor):
             raise ExtractorError('couldn\'t extract vid and key')
         vid = m_id.group(1)
         key = m_id.group(2)
-        query = compat_urllib_parse.urlencode({'vid': vid, 'inKey': key,})
+        query = compat_urllib_parse.urlencode({'vid': vid, 'inKey': key, })
         query_urls = compat_urllib_parse.urlencode({
             'masterVid': vid,
             'protocol': 'p2p',
@@ -65,7 +65,7 @@ class NaverIE(InfoExtractor):
             if domain.startswith('rtmp'):
                 f.update({
                     'ext': 'flv',
-                    'rtmp_protocol': '1', # rtmpt
+                    'rtmp_protocol': '1',  # rtmpt
                 })
             formats.append(f)
         self._sort_formats(formats)
diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 78e650b2d..f69fe0925 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -39,7 +39,6 @@ class NBAIE(InfoExtractor):
         duration = parse_duration(
             self._html_search_meta('duration', webpage, 'duration', fatal=False))
 
-
         return {
             'id': shortened_video_id,
             'url': video_url,
diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index c31ff8160..f49c66690 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -91,4 +91,4 @@ class NDRIE(InfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py
index 2e72e8915..cd117b04e 100644
--- a/youtube_dl/extractor/newgrounds.py
+++ b/youtube_dl/extractor/newgrounds.py
@@ -23,12 +23,12 @@ class NewgroundsIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         music_id = mobj.group('id')
         webpage = self._download_webpage(url, music_id)
-        
+
         title = self._html_search_regex(
             r',"name":"([^"]+)",', webpage, 'music title')
         uploader = self._html_search_regex(
             r',"artist":"([^"]+)",', webpage, 'music uploader')
-        
+
         music_url_json_string = self._html_search_regex(
             r'({"url":"[^"]+"),', webpage, 'music url') + '}'
         music_url_json = json.loads(music_url_json_string)
diff --git a/youtube_dl/extractor/newstube.py b/youtube_dl/extractor/newstube.py
index 551bd4d7a..85fcad06b 100644
--- a/youtube_dl/extractor/newstube.py
+++ b/youtube_dl/extractor/newstube.py
@@ -89,4 +89,4 @@ class NewstubeIE(InfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/nfb.py b/youtube_dl/extractor/nfb.py
index ba7b77a46..09dcc8c84 100644
--- a/youtube_dl/extractor/nfb.py
+++ b/youtube_dl/extractor/nfb.py
@@ -93,4 +93,4 @@ class NFBIE(InfoExtractor):
             'uploader': uploader,
             'uploader_id': uploader_id,
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index 82af6e330..31813eb68 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -31,7 +31,7 @@ class NHLBaseInfoExtractor(InfoExtractor):
                 path_url, video_id, 'Downloading final video url')
             video_url = path_doc.find('path').text
         else:
-           video_url = initial_video_url
+            video_url = initial_video_url
 
         join = compat_urlparse.urljoin
         return {
diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 7f1bc6377..18c869b39 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -163,4 +163,4 @@ class NocoIE(InfoExtractor):
             'uploader_id': uploader_id,
             'duration': duration,
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 2e7ab1e4f..38d05e466 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -66,4 +66,4 @@ class NovaMovIE(InfoExtractor):
             'url': video_url,
             'title': title,
             'description': description
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/nowvideo.py b/youtube_dl/extractor/nowvideo.py
index bfba18418..ecb38de2d 100644
--- a/youtube_dl/extractor/nowvideo.py
+++ b/youtube_dl/extractor/nowvideo.py
@@ -25,4 +25,4 @@ class NowVideoIE(NovaMovIE):
             'title': 'youtubedl test video _BaW_jenozKc.mp4',
             'description': 'Description',
         }
-    }
\ No newline at end of file
+    }
diff --git a/youtube_dl/extractor/ntv.py b/youtube_dl/extractor/ntv.py
index ed60314ec..13c8d79cd 100644
--- a/youtube_dl/extractor/ntv.py
+++ b/youtube_dl/extractor/ntv.py
@@ -145,4 +145,4 @@ class NTVIE(InfoExtractor):
             'duration': duration,
             'view_count': view_count,
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/nuvid.py b/youtube_dl/extractor/nuvid.py
index 58ec81f91..449c8a6a3 100644
--- a/youtube_dl/extractor/nuvid.py
+++ b/youtube_dl/extractor/nuvid.py
@@ -71,4 +71,4 @@ class NuvidIE(InfoExtractor):
             'upload_date': upload_date,
             'age_limit': 18,
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index 7bf105d38..56e1cad3b 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -74,4 +74,4 @@ class NYTimesIE(InfoExtractor):
             'duration': duration,
             'formats': formats,
             'thumbnails': thumbnails,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 2044e107e..30d80fa6b 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -97,4 +97,3 @@ class OoyalaIE(InfoExtractor):
             }
         else:
             return self._extract_result(videos_info[0], videos_more_info)
-        
diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 011e6be13..572a234ad 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -178,4 +178,4 @@ class ORFFM4IE(InfoExtractor):
             'title': data['title'],
             'description': data['subtitle'],
             'entries': entries
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/podomatic.py b/youtube_dl/extractor/podomatic.py
index ffafd2380..f20946a2b 100644
--- a/youtube_dl/extractor/podomatic.py
+++ b/youtube_dl/extractor/podomatic.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..utils import int_or_none
 
+
 class PodomaticIE(InfoExtractor):
     IE_NAME = 'podomatic'
     _VALID_URL = r'^(?P<proto>https?)://(?P<channel>[^.]+)\.podomatic\.com/entry/(?P<id>[^?]+)'
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 618e8f5dd..2ca15b717 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -56,7 +56,7 @@ class PornHubIE(InfoExtractor):
         comment_count = self._extract_count(
             r'All comments \(<var class="videoCommentCount">([\d,\.]+)</var>', webpage, 'comment')
 
-        video_urls = list(map(compat_urllib_parse.unquote , re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
+        video_urls = list(map(compat_urllib_parse.unquote, re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
         if webpage.find('"encrypted":true') != -1:
             password = compat_urllib_parse.unquote_plus(self._html_search_regex(r'"video_title":"([^"]+)', webpage, 'password'))
             video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls))
diff --git a/youtube_dl/extractor/pornotube.py b/youtube_dl/extractor/pornotube.py
index 04bd3d979..5253aa3d3 100644
--- a/youtube_dl/extractor/pornotube.py
+++ b/youtube_dl/extractor/pornotube.py
@@ -38,7 +38,7 @@ class PornotubeIE(InfoExtractor):
         video_url = self._search_regex(VIDEO_URL_RE, webpage, 'video url')
         video_url = compat_urllib_parse.unquote(video_url)
 
-        #Get the uploaded date
+        # Get the uploaded date
         VIDEO_UPLOADED_RE = r'<div class="video_added_by">Added (?P<date>[0-9\/]+) by'
         upload_date = self._html_search_regex(VIDEO_UPLOADED_RE, webpage, 'upload date', fatal=False)
         if upload_date:
diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 619496de7..32d747ede 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -280,4 +280,4 @@ class ProSiebenSat1IE(InfoExtractor):
             'upload_date': upload_date,
             'duration': duration,
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index ba3dd707f..2d39ecfe4 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -119,4 +119,4 @@ class RaiIE(SubtitlesInfoExtractor):
             if captions.endswith(STL_EXT):
                 captions = captions[:-len(STL_EXT)] + SRT_EXT
             subtitles['it'] = 'http://www.rai.tv%s' % compat_urllib_parse.quote(captions)
-        return subtitles
\ No newline at end of file
+        return subtitles
diff --git a/youtube_dl/extractor/ringtv.py b/youtube_dl/extractor/ringtv.py
index 9fbdb9fcb..59dc137cc 100644
--- a/youtube_dl/extractor/ringtv.py
+++ b/youtube_dl/extractor/ringtv.py
@@ -41,4 +41,3 @@ class RingTVIE(InfoExtractor):
             'thumbnail': thumbnail_url,
             'description': description,
         }
-
diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py
index 1a41cbe40..285c3c4be 100644
--- a/youtube_dl/extractor/rtlnow.py
+++ b/youtube_dl/extractor/rtlnow.py
@@ -122,7 +122,7 @@ class RTLnowIE(InfoExtractor):
         playerdata = self._download_xml(playerdata_url, video_id, 'Downloading player data XML')
 
         videoinfo = playerdata.find('./playlist/videoinfo')
-        
+
         formats = []
         for filename in videoinfo.findall('filename'):
             mobj = re.search(r'(?P<url>rtmpe://(?:[^/]+/){2})(?P<play_path>.+)', filename.text)
@@ -153,4 +153,4 @@ class RTLnowIE(InfoExtractor):
             'upload_date': upload_date,
             'duration': duration,
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 4dd35a47b..0ce22d60c 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -54,7 +54,6 @@ def _decrypt_url(png):
     return url
 
 
-
 class RTVEALaCartaIE(InfoExtractor):
     IE_NAME = 'rtve.es:alacarta'
     IE_DESC = 'RTVE a la carta'
diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index f737b4e5f..a73e6f331 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -191,4 +191,4 @@ class RUTVIE(InfoExtractor):
             'view_count': view_count,
             'duration': duration,
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/scivee.py b/youtube_dl/extractor/scivee.py
index 55a481cc0..3bf93c870 100644
--- a/youtube_dl/extractor/scivee.py
+++ b/youtube_dl/extractor/scivee.py
@@ -53,4 +53,4 @@ class SciVeeIE(InfoExtractor):
             'description': description,
             'thumbnail': 'http://www.scivee.tv/assets/videothumb/%s' % video_id,
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/servingsys.py b/youtube_dl/extractor/servingsys.py
index 1dc551d5c..16dc3736b 100644
--- a/youtube_dl/extractor/servingsys.py
+++ b/youtube_dl/extractor/servingsys.py
@@ -67,5 +67,3 @@ class ServingSysIE(InfoExtractor):
             'title': title,
             'entries': entries,
         }
-
- 
\ No newline at end of file
diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index badba2ac6..fdc31603a 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -54,4 +54,4 @@ class SharedIE(InfoExtractor):
             'filesize': filesize,
             'title': title,
             'thumbnail': thumbnail,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 6c1d82dd7..9d2f8d40b 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -212,7 +212,7 @@ class SmotriCommunityIE(InfoExtractor):
         },
         'playlist_mincount': 4,
     }
-    
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         community_id = mobj.group('communityid')
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 54256e1a2..c830d9da7 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -229,9 +229,9 @@ class SoundcloudIE(InfoExtractor):
             full_title = resolve_title = '%s/%s' % (uploader, slug_title)
             if token:
                 resolve_title += '/%s' % token
-    
+
             self.report_resolve(full_title)
-    
+
             url = 'http://soundcloud.com/%s' % resolve_title
             info_json_url = self._resolv_url(url)
         info = self._download_json(info_json_url, full_title, 'Downloading info JSON')
diff --git a/youtube_dl/extractor/spiegeltv.py b/youtube_dl/extractor/spiegeltv.py
index ef5e7c08b..98cf92d89 100644
--- a/youtube_dl/extractor/spiegeltv.py
+++ b/youtube_dl/extractor/spiegeltv.py
@@ -77,4 +77,4 @@ class SpiegeltvIE(InfoExtractor):
             'description': description,
             'duration': duration,
             'thumbnails': thumbnails
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/sport5.py b/youtube_dl/extractor/sport5.py
index 3f680bfc6..dfe50ed45 100644
--- a/youtube_dl/extractor/sport5.py
+++ b/youtube_dl/extractor/sport5.py
@@ -89,4 +89,4 @@ class Sport5IE(InfoExtractor):
             'duration': duration,
             'categories': categories,
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/sportdeutschland.py b/youtube_dl/extractor/sportdeutschland.py
index abb827783..057ef5251 100644
--- a/youtube_dl/extractor/sportdeutschland.py
+++ b/youtube_dl/extractor/sportdeutschland.py
@@ -93,4 +93,3 @@ class SportDeutschlandIE(InfoExtractor):
             'rtmp_live': asset.get('live'),
             'timestamp': parse_iso8601(asset.get('date')),
         }
-
diff --git a/youtube_dl/extractor/subtitles.py b/youtube_dl/extractor/subtitles.py
index db33745c1..41d740e87 100644
--- a/youtube_dl/extractor/subtitles.py
+++ b/youtube_dl/extractor/subtitles.py
@@ -50,7 +50,7 @@ class SubtitlesInfoExtractor(InfoExtractor):
 
             sub_lang_list = {}
             for sub_lang in requested_langs:
-                if not sub_lang in available_subs_list:
+                if sub_lang not in available_subs_list:
                     self._downloader.report_warning(u'no closed captions found in the specified language "%s"' % sub_lang)
                     continue
                 sub_lang_list[sub_lang] = available_subs_list[sub_lang]
diff --git a/youtube_dl/extractor/teachertube.py b/youtube_dl/extractor/teachertube.py
index 8a95fd656..6c3445d79 100644
--- a/youtube_dl/extractor/teachertube.py
+++ b/youtube_dl/extractor/teachertube.py
@@ -121,7 +121,7 @@ class TeacherTubeUserIE(InfoExtractor):
         urls = []
         webpage = self._download_webpage(url, user_id)
         urls.extend(re.findall(self._MEDIA_RE, webpage))
-        
+
         pages = re.findall(r'/ajax-user/user-videos/%s\?page=([0-9]+)' % user_id, webpage)[:-1]
         for p in pages:
             more = 'http://www.teachertube.com/ajax-user/user-videos/%s?page=%s' % (user_id, p)
diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index fa796ce72..d5dda34f6 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -33,7 +33,7 @@ class TeamcocoIE(InfoExtractor):
 
         display_id = mobj.group('display_id')
         webpage = self._download_webpage(url, display_id)
-        
+
         video_id = mobj.group("video_id")
         if not video_id:
             video_id = self._html_search_regex(
diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index db9788c18..2a2fff5e1 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -1,4 +1,4 @@
-#coding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .mitele import MiTeleIE
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index a04925633..b83d15b90 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -35,11 +35,12 @@ class ThePlatformIE(InfoExtractor):
             'skip_download': True,
         },
     }
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         if mobj.group('config'):
-            config_url = url+ '&form=json'
+            config_url = url + '&form=json'
             config_url = config_url.replace('swf/', 'config/')
             config_url = config_url.replace('onsite/', 'onsite/config/')
             config = self._download_json(config_url, video_id, 'Downloading config')
@@ -48,7 +49,6 @@ class ThePlatformIE(InfoExtractor):
             smil_url = ('http://link.theplatform.com/s/dJ5BDC/{0}/meta.smil?'
                 'format=smil&mbr=true'.format(video_id))
 
-
         meta = self._download_xml(smil_url, video_id)
         try:
             error_msg = next(
diff --git a/youtube_dl/extractor/thisav.py b/youtube_dl/extractor/thisav.py
index bfb9d2fc9..350a5cdb5 100644
--- a/youtube_dl/extractor/thisav.py
+++ b/youtube_dl/extractor/thisav.py
@@ -1,4 +1,4 @@
-#coding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -36,7 +36,7 @@ class ThisAVIE(InfoExtractor):
             r': <a href="http://www.thisav.com/user/[0-9]+/([^"]+)">(?:[^<]+)</a>',
             webpage, 'uploader id', fatal=False)
         ext = determine_ext(video_url)
-        
+
         return {
             'id':          video_id,
             'url':         video_url,
diff --git a/youtube_dl/extractor/tinypic.py b/youtube_dl/extractor/tinypic.py
index a4aa25f66..6ba5dc5f1 100644
--- a/youtube_dl/extractor/tinypic.py
+++ b/youtube_dl/extractor/tinypic.py
@@ -26,7 +26,7 @@ class TinyPicIE(InfoExtractor):
         video_id = mobj.group('id')
 
         webpage = self._download_webpage(url, video_id, 'Downloading page')
-        
+
         mobj = re.search(r'(?m)fo\.addVariable\("file",\s"(?P<fileid>[\da-z]+)"\);\n'
             '\s+fo\.addVariable\("s",\s"(?P<serverid>\d+)"\);', webpage)
         if mobj is None:
@@ -47,4 +47,4 @@ class TinyPicIE(InfoExtractor):
             'url': video_url,
             'thumbnail': thumbnail,
             'title': title
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 4956f8577..0ecd695f8 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -71,7 +71,7 @@ class TNAFlixIE(InfoExtractor):
                 fmt['height'] = int(m.group(1))
             formats.append(fmt)
         self._sort_formats(formats)
-        
+
         return {
             'id': video_id,
             'display_id': display_id,
diff --git a/youtube_dl/extractor/traileraddict.py b/youtube_dl/extractor/traileraddict.py
index 11407428b..b1a440e79 100644
--- a/youtube_dl/extractor/traileraddict.py
+++ b/youtube_dl/extractor/traileraddict.py
@@ -25,7 +25,7 @@ class TrailerAddictIE(InfoExtractor):
         webpage = self._download_webpage(url, name)
 
         title = self._search_regex(r'<title>(.+?)</title>',
-                webpage, 'video title').replace(' - Trailer Addict','')
+                webpage, 'video title').replace(' - Trailer Addict', '')
         view_count_str = self._search_regex(
             r'<span class="views_n">([0-9,.]+)</span>',
             webpage, 'view count', fatal=False)
@@ -43,10 +43,10 @@ class TrailerAddictIE(InfoExtractor):
             fvar = "fvar"
 
         info_url = "http://www.traileraddict.com/%s.php?tid=%s" % (fvar, str(video_id))
-        info_webpage = self._download_webpage(info_url, video_id , "Downloading the info webpage")
+        info_webpage = self._download_webpage(info_url, video_id, "Downloading the info webpage")
 
         final_url = self._search_regex(r'&fileurl=(.+)',
-                info_webpage, 'Download url').replace('%3F','?')
+                info_webpage, 'Download url').replace('%3F', '?')
         thumbnail_url = self._search_regex(r'&image=(.+?)&',
                 info_webpage, 'thumbnail url')
 
diff --git a/youtube_dl/extractor/trilulilu.py b/youtube_dl/extractor/trilulilu.py
index d64aaa41f..2f2c8aeda 100644
--- a/youtube_dl/extractor/trilulilu.py
+++ b/youtube_dl/extractor/trilulilu.py
@@ -39,7 +39,7 @@ class TriluliluIE(InfoExtractor):
             format_url, video_id,
             note=u'Downloading formats',
             errnote=u'Error while downloading formats')
- 
+
         video_url_template = (
             u'http://fs%(server)s.trilulilu.ro/stream.php?type=video'
             u'&source=site&hash=%(hash)s&username=%(userid)s&'
@@ -63,4 +63,3 @@ class TriluliluIE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
         }
-
diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index dcd823d08..c8780790a 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -41,7 +41,7 @@ class TudouIE(InfoExtractor):
         if quality:
             info_url += '&hd' + quality
         webpage = self._download_webpage(info_url, id, "Opening the info webpage")
-        final_url = self._html_search_regex('>(.+?)</f>',webpage, 'video url')
+        final_url = self._html_search_regex('>(.+?)</f>', webpage, 'video url')
         return final_url
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index 27962b5fe..d81d1d1a6 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -84,4 +84,4 @@ class TvigleIE(InfoExtractor):
             'duration': duration,
             'age_limit': age_limit,
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 054f42725..7df11fc19 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -161,4 +161,4 @@ class UdemyCourseIE(UdemyIE):
             for asset in response if asset.get('assetType') == 'Video'
         ]
 
-        return self.playlist_result(entries, course_id, course_title)
\ No newline at end of file
+        return self.playlist_result(entries, course_id, course_title)
diff --git a/youtube_dl/extractor/vesti.py b/youtube_dl/extractor/vesti.py
index 27f9acb67..5aebcecd7 100644
--- a/youtube_dl/extractor/vesti.py
+++ b/youtube_dl/extractor/vesti.py
@@ -118,4 +118,4 @@ class VestiIE(InfoExtractor):
         if rutv_url:
             return self.url_result(rutv_url, 'RUTV')
 
-        raise ExtractorError('No video found', expected=True)
\ No newline at end of file
+        raise ExtractorError('No video found', expected=True)
diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index d3fa70e0e..70578a4cc 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -114,4 +114,4 @@ class VGTVIE(InfoExtractor):
             'duration': float_or_none(data['duration'], 1000),
             'view_count': data['displays'],
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py
index f11ca8217..71f520fb5 100644
--- a/youtube_dl/extractor/vice.py
+++ b/youtube_dl/extractor/vice.py
@@ -35,4 +35,3 @@ class ViceIE(InfoExtractor):
         except ExtractorError:
             raise ExtractorError('The page doesn\'t contain a video', expected=True)
         return self.url_result(ooyala_url, ie='Ooyala')
-
diff --git a/youtube_dl/extractor/videobam.py b/youtube_dl/extractor/videobam.py
index fed95ef71..0eb3d9414 100644
--- a/youtube_dl/extractor/videobam.py
+++ b/youtube_dl/extractor/videobam.py
@@ -78,4 +78,4 @@ class VideoBamIE(InfoExtractor):
             'view_count': view_count,
             'formats': formats,
             'age_limit': 18,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/videofyme.py b/youtube_dl/extractor/videofyme.py
index f75169041..2dd51d866 100644
--- a/youtube_dl/extractor/videofyme.py
+++ b/youtube_dl/extractor/videofyme.py
@@ -6,6 +6,7 @@ from ..utils import (
     determine_ext,
 )
 
+
 class VideofyMeIE(InfoExtractor):
     _VALID_URL = r'https?://(www\.videofy\.me/.+?|p\.videofy\.me/v)/(?P<id>\d+)(&|#|$)'
     IE_NAME = u'videofy.me'
@@ -20,7 +21,7 @@ class VideofyMeIE(InfoExtractor):
             u'uploader': u'VideofyMe',
             u'uploader_id': u'thisisvideofyme',
         },
-        
+
     }
 
     def _real_extract(self, url):
@@ -30,7 +31,7 @@ class VideofyMeIE(InfoExtractor):
                                             video_id)
         video = config.find('video')
         sources = video.find('sources')
-        url_node = next(node for node in [find_xpath_attr(sources, 'source', 'id', 'HQ %s' % key) 
+        url_node = next(node for node in [find_xpath_attr(sources, 'source', 'id', 'HQ %s' % key)
             for key in ['on', 'av', 'off']] if node is not None)
         video_url = url_node.find('url').text
 
diff --git a/youtube_dl/extractor/videott.py b/youtube_dl/extractor/videott.py
index a647807d0..1f938838c 100644
--- a/youtube_dl/extractor/videott.py
+++ b/youtube_dl/extractor/videott.py
@@ -58,4 +58,4 @@ class VideoTtIE(InfoExtractor):
             'like_count': int_or_none(video['liked']),
             'dislike_count': int_or_none(video['disliked']),
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/videoweed.py b/youtube_dl/extractor/videoweed.py
index 4a08ddd43..ca2e50935 100644
--- a/youtube_dl/extractor/videoweed.py
+++ b/youtube_dl/extractor/videoweed.py
@@ -23,4 +23,4 @@ class VideoWeedIE(NovaMovIE):
             'title': 'optical illusion  dissapeared image magic illusion',
             'description': ''
         },
-    }
\ No newline at end of file
+    }
diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
index 669979e13..08a5a7b8d 100644
--- a/youtube_dl/extractor/vidzi.py
+++ b/youtube_dl/extractor/vidzi.py
@@ -1,4 +1,4 @@
-#coding: utf-8
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
@@ -18,16 +18,15 @@ class VidziIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        
+
         webpage = self._download_webpage(url, video_id)
         video_url = self._html_search_regex(
             r'{\s*file\s*:\s*"([^"]+)"\s*}', webpage, 'video url')
         title = self._html_search_regex(
             r'(?s)<h2 class="video-title">(.*?)</h2>', webpage, 'title')
-        
+
         return {
             'id': video_id,
             'title': title,
             'url': video_url,
         }
-        
\ No newline at end of file
diff --git a/youtube_dl/extractor/vrt.py b/youtube_dl/extractor/vrt.py
index 57ef8dc30..bbd3bbf7b 100644
--- a/youtube_dl/extractor/vrt.py
+++ b/youtube_dl/extractor/vrt.py
@@ -36,7 +36,7 @@ class VRTIE(InfoExtractor):
                 'timestamp': 1413835980.560,
                 'upload_date': '20141020',
                 'duration': 3238,
-            }  
+            }
         },
         # cobra.be
         {
@@ -92,4 +92,4 @@ class VRTIE(InfoExtractor):
             'timestamp': timestamp,
             'duration': duration,
             'formats': formats,
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 54d37da61..93a6e6454 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -224,4 +224,4 @@ class WDRMausIE(InfoExtractor):
             'upload_date': upload_date,
         }
 
-# TODO test _1
\ No newline at end of file
+# TODO test _1
diff --git a/youtube_dl/extractor/worldstarhiphop.py b/youtube_dl/extractor/worldstarhiphop.py
index bda3870db..d5c26a032 100644
--- a/youtube_dl/extractor/worldstarhiphop.py
+++ b/youtube_dl/extractor/worldstarhiphop.py
@@ -51,4 +51,3 @@ class WorldStarHipHopIE(InfoExtractor):
             'title': video_title,
             'thumbnail': thumbnail,
         }
-
diff --git a/youtube_dl/extractor/xbef.py b/youtube_dl/extractor/xbef.py
index 71bd7c463..1b4e88365 100644
--- a/youtube_dl/extractor/xbef.py
+++ b/youtube_dl/extractor/xbef.py
@@ -47,4 +47,3 @@ class XBefIE(InfoExtractor):
             'thumbnail': thumbnail,
             'age_limit': 18,
         }
-
diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 4e8fbde8d..1e6c94623 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -42,7 +42,7 @@ class XHamsterIE(InfoExtractor):
         }
     ]
 
-    def _real_extract(self,url):
+    def _real_extract(self, url):
         def extract_video_url(webpage):
             mp4 = re.search(r'<video\s+.*?file="([^"]+)".*?>', webpage)
             if mp4 is None:
diff --git a/youtube_dl/extractor/ynet.py b/youtube_dl/extractor/ynet.py
index 9cd7989cc..7b621a9e3 100644
--- a/youtube_dl/extractor/ynet.py
+++ b/youtube_dl/extractor/ynet.py
@@ -47,4 +47,4 @@ class YnetIE(InfoExtractor):
             'title': title,
             'formats': self._extract_f4m_formats(f4m_url, video_id),
             'thumbnail': self._og_search_thumbnail(webpage),
-        }
\ No newline at end of file
+        }
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 48d47a245..e155b3a94 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -35,10 +35,10 @@ class YoukuIE(InfoExtractor):
 
     def _gen_sid(self):
         nowTime = int(time.time() * 1000)
-        random1 = random.randint(1000,1998)
-        random2 = random.randint(1000,9999)
+        random1 = random.randint(1000, 1998)
+        random2 = random.randint(1000, 9999)
 
-        return "%d%d%d" %(nowTime,random1,random2)
+        return "%d%d%d" % (nowTime, random1, random2)
 
     def _get_file_ID_mix_string(self, seed):
         mixed = []
@@ -49,7 +49,7 @@ class YoukuIE(InfoExtractor):
             index  =  math.floor(seed / 65536 * len(source))
             mixed.append(source[int(index)])
             source.remove(source[int(index)])
-        #return ''.join(mixed)
+        # return ''.join(mixed)
         return mixed
 
     def _get_file_id(self, fileId, seed):
@@ -100,12 +100,12 @@ class YoukuIE(InfoExtractor):
         keys = [s['k'] for s in config['data'][0]['segs'][format]]
         # segs is usually a dictionary, but an empty *list* if an error occured.
 
-        files_info=[]
+        files_info = []
         sid = self._gen_sid()
         fileid = self._get_file_id(fileid, seed)
 
-        #column 8,9 of fileid represent the segment number
-        #fileid[7:9] should be changed
+        # column 8,9 of fileid represent the segment number
+        # fileid[7:9] should be changed
         for index, key in enumerate(keys):
             temp_fileid = '%s%02X%s' % (fileid[0:8], index, fileid[10:])
             download_url = 'http://k.youku.com/player/getFlvPath/sid/%s_%02X/st/flv/fileid/%s?k=%s' % (sid, index, temp_fileid, key)
diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 7bfda45e7..ee61e2381 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -73,7 +73,7 @@ class YouPornIE(InfoExtractor):
         for encrypted_link in encrypted_links:
             link = aes_decrypt_text(encrypted_link, video_title, 32).decode('utf-8')
             links.append(link)
-        
+
         formats = []
         for link in links:
             # A link looks like this:
@@ -104,7 +104,7 @@ class YouPornIE(InfoExtractor):
 
         if not formats:
             raise ExtractorError(u'ERROR: no known formats available for video')
-        
+
         return {
             'id': video_id,
             'uploader': video_uploader,
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0cb837afc..98cac7c17 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -33,6 +33,7 @@ from ..utils import (
     uppercase_escape,
 )
 
+
 class YoutubeBaseInfoExtractor(InfoExtractor):
     """Provide base functions for Youtube extractors"""
     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
@@ -99,7 +100,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
         # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
         # chokes on unicode
-        login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in login_form_strs.items())
+        login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in login_form_strs.items())
         login_data = compat_urllib_parse.urlencode(login_form).encode('ascii')
 
         req = compat_urllib_request.Request(self._LOGIN_URL, login_data)
@@ -149,7 +150,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                 'service': 'youtube',
                 'hl': 'en_US',
             }
-            tfa_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in tfa_form_strs.items())
+            tfa_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k, v in tfa_form_strs.items())
             tfa_data = compat_urllib_parse.urlencode(tfa_form).encode('ascii')
 
             tfa_req = compat_urllib_request.Request(self._TWOFACTOR_URL, tfa_data)
@@ -618,7 +619,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             list_url = caption_url + '&' + list_params
             caption_list = self._download_xml(list_url, video_id)
             original_lang_node = caption_list.find('track')
-            if original_lang_node is None or original_lang_node.attrib.get('kind') != 'asr' :
+            if original_lang_node is None or original_lang_node.attrib.get('kind') != 'asr':
                 self._downloader.report_warning('Video doesn\'t have automatic captions')
                 return {}
             original_lang = original_lang_node.attrib['lang_code']
@@ -651,6 +652,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
     def _extract_from_m3u8(self, manifest_url, video_id):
         url_map = {}
+
         def _get_urls(_manifest):
             lines = _manifest.split('\n')
             urls = filter(lambda l: l and not l.startswith('#'),
@@ -850,7 +852,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         # annotations
         video_annotations = None
         if self._downloader.params.get('writeannotations', False):
-                video_annotations = self._extract_annotations(video_id)
+            video_annotations = self._extract_annotations(video_id)
 
         # Decide which formats to download
         try:
@@ -900,7 +902,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'player_url': player_url,
             }]
         elif len(video_info.get('url_encoded_fmt_stream_map', [])) >= 1 or len(video_info.get('adaptive_fmts', [])) >= 1:
-            encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts',[''])[0]
+            encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
             if 'rtmpe%3Dyes' in encoded_url_map:
                 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
             url_map = {}
@@ -974,6 +976,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                     dash_manifest_url = video_info.get('dashmpd')[0]
                 else:
                     dash_manifest_url = ytplayer_config['args']['dashmpd']
+
                 def decrypt_sig(mobj):
                     s = mobj.group(1)
                     dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
@@ -1033,6 +1036,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             'formats':      formats,
         }
 
+
 class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
     IE_DESC = 'YouTube.com playlists'
     _VALID_URL = r"""(?x)(?:
@@ -1046,7 +1050,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
                         )
                         (
                             (?:PL|LL|EC|UU|FL|RD)?[0-9A-Za-z-_]{10,}
-                            # Top tracks, they can also include dots 
+                            # Top tracks, they can also include dots
                             |(?:MC)[\w\.]*
                         )
                         .*
@@ -1230,7 +1234,7 @@ class YoutubeTopListIE(YoutubePlaylistIE):
                 <span[^>]*>.*?%s.*?</span>''' % re.escape(query),
             channel_page, 'list')
         url = compat_urlparse.urljoin('https://www.youtube.com/', link)
-        
+
         video_re = r'data-index="\d+".*?data-video-id="([0-9A-Za-z_-]{11})"'
         ids = []
         # sometimes the webpage doesn't contain the videos
@@ -1298,7 +1302,7 @@ class YoutubeChannelIE(InfoExtractor):
 
                 ids_in_page = self.extract_videos_from_page(page['content_html'])
                 video_ids.extend(ids_in_page)
-    
+
                 if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
                     break
 
@@ -1333,8 +1337,10 @@ class YoutubeUserIE(InfoExtractor):
         # Don't return True if the url can be extracted with other youtube
         # extractor, the regex would is too permissive and it would match.
         other_ies = iter(klass for (name, klass) in globals().items() if name.endswith('IE') and klass is not cls)
-        if any(ie.suitable(url) for ie in other_ies): return False
-        else: return super(YoutubeUserIE, cls).suitable(url)
+        if any(ie.suitable(url) for ie in other_ies):
+            return False
+        else:
+            return super(YoutubeUserIE, cls).suitable(url)
 
     def _real_extract(self, url):
         # Extract username
@@ -1557,12 +1563,14 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
             paging = mobj.group('paging')
         return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE)
 
+
 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
     IE_DESC = 'YouTube.com recommended videos, "ytrec" keyword (requires authentication)'
     _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
     _FEED_NAME = 'recommended'
     _PLAYLIST_TITLE = 'Youtube Recommended videos'
 
+
 class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor):
     IE_DESC = 'Youtube watch later list, "ytwatchlater" keyword (requires authentication)'
     _VALID_URL = r'https?://www\.youtube\.com/feed/watch_later|:ytwatchlater'
@@ -1570,6 +1578,7 @@ class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor):
     _PLAYLIST_TITLE = 'Youtube Watch Later'
     _PERSONAL_FEED = True
 
+
 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
     IE_DESC = 'Youtube watch history, "ythistory" keyword (requires authentication)'
     _VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
@@ -1577,6 +1586,7 @@ class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
     _PERSONAL_FEED = True
     _PLAYLIST_TITLE = 'Youtube Watch History'
 
+
 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
     IE_NAME = 'youtube:favorites'
     IE_DESC = 'YouTube.com favourite videos, "ytfav" keyword (requires authentication)'
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 16aa6ffca..38177c1d7 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -621,7 +621,7 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--exec',
         metavar='CMD', dest='exec_cmd',
-        help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'' )
+        help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'')
 
     parser.add_option_group(general)
     parser.add_option_group(selection)
diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py
index baf1b1945..09db43611 100644
--- a/youtube_dl/postprocessor/execafterdownload.py
+++ b/youtube_dl/postprocessor/execafterdownload.py
@@ -26,4 +26,3 @@ class ExecAfterDownloadPP(PostProcessor):
                 'Command returned error code %d' % retCode)
 
         return None, information  # by default, keep file and do nothing
-
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index f3f2743c0..dfae735f3 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -202,7 +202,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 extension = 'wav'
                 more_opts += ['-f', 'wav']
 
-        prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups
+        prefix, sep, ext = path.rpartition(u'.')  # not os.path.splitext, since the latter does not work on unicode in all setups
         new_path = prefix + sep + extension
 
         # If we download foo.mp3 and convert it to... foo.mp3, then don't delete foo.mp3, silly.
@@ -216,7 +216,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 self._downloader.to_screen(u'[' + self._executable + '] Destination: ' + new_path)
                 self.run_ffmpeg(path, new_path, acodec, more_opts)
         except:
-            etype,e,tb = sys.exc_info()
+            etype, e, tb = sys.exc_info()
             if isinstance(e, AudioConversionError):
                 msg = u'audio conversion failed: ' + e.msg
             else:
@@ -231,13 +231,13 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 self._downloader.report_warning(u'Cannot update utime of audio file')
 
         information['filepath'] = new_path
-        return self._nopostoverwrites,information
+        return self._nopostoverwrites, information
 
 
 class FFmpegVideoConvertor(FFmpegPostProcessor):
-    def __init__(self, downloader=None,preferedformat=None):
+    def __init__(self, downloader=None, preferedformat=None):
         super(FFmpegVideoConvertor, self).__init__(downloader)
-        self._preferedformat=preferedformat
+        self._preferedformat = preferedformat
 
     def run(self, information):
         path = information['filepath']
@@ -245,13 +245,13 @@ class FFmpegVideoConvertor(FFmpegPostProcessor):
         outpath = prefix + sep + self._preferedformat
         if information['ext'] == self._preferedformat:
             self._downloader.to_screen(u'[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
-            return True,information
-        self._downloader.to_screen(u'['+'ffmpeg'+'] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) +outpath)
+            return True, information
+        self._downloader.to_screen(u'['+'ffmpeg'+'] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath)
         self.run_ffmpeg(path, outpath, [])
         information['filepath'] = outpath
         information['format'] = self._preferedformat
         information['ext'] = self._preferedformat
-        return False,information
+        return False, information
 
 
 class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
@@ -457,7 +457,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
             self._downloader.to_screen(u'[ffmpeg] Subtitles can only be embedded in mp4 files')
             return True, information
         if not information.get('subtitles'):
-            self._downloader.to_screen(u'[ffmpeg] There aren\'t any subtitles to embed') 
+            self._downloader.to_screen(u'[ffmpeg] There aren\'t any subtitles to embed')
             return True, information
 
         sub_langs = [key for key in information['subtitles']]
diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index b5cae41c8..d1991c4f8 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -108,4 +108,3 @@ class XAttrMetadataPP(PostProcessor):
         except (subprocess.CalledProcessError, OSError):
             self._downloader.report_error("This filesystem doesn't support extended attributes. (You may have to enable them in your /etc/fstab)")
             return False, info
-
diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index 99220940f..2bd264b30 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -827,4 +827,3 @@ class SWFInterpreter(object):
 
         avm_class.method_pyfunctions[func_name] = resfunc
         return resfunc
-
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 273083761..d32a86708 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -13,13 +13,17 @@ from .utils import (
 )
 from .version import __version__
 
+
 def rsa_verify(message, signature, key):
     from struct import pack
     from hashlib import sha256
     from sys import version_info
+
     def b(x):
-        if version_info[0] == 2: return x
-        else: return x.encode('latin1')
+        if version_info[0] == 2:
+            return x
+        else:
+            return x.encode('latin1')
     assert(type(message) == type(b('')))
     block_size = 0
     n = key[0]
@@ -32,13 +36,17 @@ def rsa_verify(message, signature, key):
         raw_bytes.insert(0, pack("B", signature & 0xFF))
         signature >>= 8
     signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
-    if signature[0:2] != b('\x00\x01'): return False
+    if signature[0:2] != b('\x00\x01'):
+        return False
     signature = signature[2:]
-    if not b('\x00') in signature: return False
+    if not b('\x00') in signature:
+        return False
     signature = signature[signature.index(b('\x00'))+1:]
-    if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False
+    if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')):
+        return False
     signature = signature[19:]
-    if signature != sha256(message).digest(): return False
+    if signature != sha256(message).digest():
+        return False
     return True
 
 
@@ -58,7 +66,8 @@ def update_self(to_screen, verbose):
     try:
         newversion = compat_urllib_request.urlopen(VERSION_URL).read().decode('utf-8').strip()
     except:
-        if verbose: to_screen(compat_str(traceback.format_exc()))
+        if verbose:
+            to_screen(compat_str(traceback.format_exc()))
         to_screen(u'ERROR: can\'t find the current version. Please try again later.')
         return
     if newversion == __version__:
@@ -70,7 +79,8 @@ def update_self(to_screen, verbose):
         versions_info = compat_urllib_request.urlopen(JSON_URL).read().decode('utf-8')
         versions_info = json.loads(versions_info)
     except:
-        if verbose: to_screen(compat_str(traceback.format_exc()))
+        if verbose:
+            to_screen(compat_str(traceback.format_exc()))
         to_screen(u'ERROR: can\'t obtain versions info. Please try again later.')
         return
     if not 'signature' in versions_info:
@@ -118,7 +128,8 @@ def update_self(to_screen, verbose):
             newcontent = urlh.read()
             urlh.close()
         except (IOError, OSError):
-            if verbose: to_screen(compat_str(traceback.format_exc()))
+            if verbose:
+                to_screen(compat_str(traceback.format_exc()))
             to_screen(u'ERROR: unable to download latest version')
             return
 
@@ -131,7 +142,8 @@ def update_self(to_screen, verbose):
             with open(exe + '.new', 'wb') as outf:
                 outf.write(newcontent)
         except (IOError, OSError):
-            if verbose: to_screen(compat_str(traceback.format_exc()))
+            if verbose:
+                to_screen(compat_str(traceback.format_exc()))
             to_screen(u'ERROR: unable to write the new version')
             return
 
@@ -150,7 +162,8 @@ start /b "" cmd /c del "%%~f0"&exit /b"
             subprocess.Popen([bat])  # Continues to run in the background
             return  # Do not show premature success messages
         except (IOError, OSError):
-            if verbose: to_screen(compat_str(traceback.format_exc()))
+            if verbose:
+                to_screen(compat_str(traceback.format_exc()))
             to_screen(u'ERROR: unable to overwrite current version')
             return
 
@@ -161,7 +174,8 @@ start /b "" cmd /c del "%%~f0"&exit /b"
             newcontent = urlh.read()
             urlh.close()
         except (IOError, OSError):
-            if verbose: to_screen(compat_str(traceback.format_exc()))
+            if verbose:
+                to_screen(compat_str(traceback.format_exc()))
             to_screen(u'ERROR: unable to download latest version')
             return
 
@@ -174,19 +188,22 @@ start /b "" cmd /c del "%%~f0"&exit /b"
             with open(filename, 'wb') as outf:
                 outf.write(newcontent)
         except (IOError, OSError):
-            if verbose: to_screen(compat_str(traceback.format_exc()))
+            if verbose:
+                to_screen(compat_str(traceback.format_exc()))
             to_screen(u'ERROR: unable to overwrite current version')
             return
 
     to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
 
+
 def get_notes(versions, fromVersion):
     notes = []
-    for v,vdata in sorted(versions.items()):
+    for v, vdata in sorted(versions.items()):
         if v > fromVersion:
             notes.extend(vdata.get('notes', []))
     return notes
 
+
 def print_notes(to_screen, versions, fromVersion=__version__):
     notes = get_notes(versions, fromVersion)
     if notes:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c3d8bf8e9..9ad0952d5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -56,6 +56,7 @@ std_headers = {
     'Accept-Language': 'en-us,en;q=0.5',
 }
 
+
 def preferredencoding():
     """Get preferred encoding.
 
@@ -146,6 +147,8 @@ else:
 
 # On python2.6 the xml.etree.ElementTree.Element methods don't support
 # the namespace parameter
+
+
 def xpath_with_ns(path, ns_map):
     components = [c.split(':') for c in path.split('/')]
     replaced = []
@@ -256,6 +259,7 @@ def timeconvert(timestr):
         timestamp = email.utils.mktime_tz(timetuple)
     return timestamp
 
+
 def sanitize_filename(s, restricted=False, is_id=False):
     """Sanitizes a string so it could be used as part of a filename.
     If restricted is set, use a stricter subset of allowed characters.
@@ -288,6 +292,7 @@ def sanitize_filename(s, restricted=False, is_id=False):
             result = '_'
     return result
 
+
 def orderedSet(iterable):
     """ Remove all duplicates from the input iterable """
     res = []
@@ -372,6 +377,7 @@ def decodeOption(optval):
     assert isinstance(optval, compat_str)
     return optval
 
+
 def formatSeconds(secs):
     if secs > 3600:
         return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
@@ -424,6 +430,7 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
 
 class ExtractorError(Exception):
     """Error during info extraction."""
+
     def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
         """ tb, if given, is the original traceback (so that it can be printed out).
         If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
@@ -468,6 +475,7 @@ class DownloadError(Exception):
     configured to continue on errors. They will contain the appropriate
     error message.
     """
+
     def __init__(self, msg, exc_info=None):
         """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
         super(DownloadError, self).__init__(msg)
@@ -489,9 +497,11 @@ class PostProcessingError(Exception):
     This exception may be raised by PostProcessor's .run() method to
     indicate an error in the postprocessing task.
     """
+
     def __init__(self, msg):
         self.msg = msg
 
+
 class MaxDownloadsReached(Exception):
     """ --max-downloads limit has been reached. """
     pass
@@ -521,6 +531,7 @@ class ContentTooShortError(Exception):
         self.downloaded = downloaded
         self.expected = expected
 
+
 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     """Handler for HTTP requests and responses.
 
@@ -640,7 +651,7 @@ def unified_strdate(date_str):
         return None
 
     upload_date = None
-    #Replace commas
+    # Replace commas
     date_str = date_str.replace(',', ' ')
     # %z (UTC offset) is only supported in python>=3.2
     date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
@@ -681,6 +692,7 @@ def unified_strdate(date_str):
             upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
     return upload_date
 
+
 def determine_ext(url, default_ext='unknown_video'):
     if url is None:
         return default_ext
@@ -690,9 +702,11 @@ def determine_ext(url, default_ext='unknown_video'):
     else:
         return default_ext
 
+
 def subtitles_filename(filename, sub_lang, sub_format):
     return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
 
+
 def date_from_str(date_str):
     """
     Return a datetime object from a string in the format YYYYMMDD or
@@ -707,7 +721,7 @@ def date_from_str(date_str):
         if sign == '-':
             time = -time
         unit = match.group('unit')
-        #A bad aproximation?
+        # A bad aproximation?
         if unit == 'month':
             unit = 'day'
             time *= 30
@@ -718,7 +732,8 @@ def date_from_str(date_str):
         delta = datetime.timedelta(**{unit: time})
         return today + delta
     return datetime.datetime.strptime(date_str, "%Y%m%d").date()
-    
+
+
 def hyphenate_date(date_str):
     """
     Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
@@ -728,8 +743,10 @@ def hyphenate_date(date_str):
     else:
         return date_str
 
+
 class DateRange(object):
     """Represents a time interval between two dates"""
+
     def __init__(self, start=None, end=None):
         """start and end must be strings in the format accepted by date"""
         if start is not None:
@@ -742,17 +759,20 @@ class DateRange(object):
             self.end = datetime.datetime.max.date()
         if self.start > self.end:
             raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
+
     @classmethod
     def day(cls, day):
         """Returns a range that only contains the given day"""
-        return cls(day,day)
+        return cls(day, day)
+
     def __contains__(self, date):
         """Check if the date is in the range"""
         if not isinstance(date, datetime.date):
             date = date_from_str(date)
         return self.start <= date <= self.end
+
     def __str__(self):
-        return '%s - %s' % ( self.start.isoformat(), self.end.isoformat())
+        return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
 
 
 def platform_name():
@@ -1150,7 +1170,7 @@ def parse_duration(s):
 
 
 def prepend_extension(filename, ext):
-    name, real_ext = os.path.splitext(filename) 
+    name, real_ext = os.path.splitext(filename)
     return '{0}.{1}{2}'.format(name, ext, real_ext)
 
 

From 8bcc875676b56c062a4fdd81763a6adb0fb1390c Mon Sep 17 00:00:00 2001
From: Jouke Waleson <jouke.waleson@mendix.com>
Date: Sun, 23 Nov 2014 21:20:46 +0100
Subject: [PATCH 0579/1937] PEP8: more applied

---
 devscripts/gh-pages/generate-download.py      |  3 -
 .../transition_helper_exe/youtube-dl.py       |  2 +-
 test/test_utils.py                            |  1 -
 test/test_youtube_lists.py                    |  4 -
 youtube_dl/YoutubeDL.py                       |  1 -
 youtube_dl/__init__.py                        | 18 ++--
 youtube_dl/aes.py                             | 30 +++----
 youtube_dl/downloader/f4m.py                  |  4 +-
 youtube_dl/downloader/rtmp.py                 | 10 +--
 youtube_dl/extractor/__init__.py              |  2 +-
 youtube_dl/extractor/bbccouk.py               |  2 +-
 youtube_dl/extractor/cinemassacre.py          |  2 +-
 youtube_dl/extractor/crunchyroll.py           | 20 ++---
 youtube_dl/extractor/dotsub.py                |  2 +-
 youtube_dl/extractor/generic.py               |  2 +-
 youtube_dl/extractor/iprima.py                |  2 +-
 youtube_dl/extractor/lifenews.py              |  2 +-
 youtube_dl/extractor/liveleak.py              |  6 +-
 youtube_dl/extractor/metacafe.py              |  2 +-
 youtube_dl/extractor/mtv.py                   |  2 +-
 youtube_dl/extractor/sohu.py                  |  2 +-
 youtube_dl/extractor/soundcloud.py            |  2 +-
 youtube_dl/extractor/swrmediathek.py          |  2 +-
 youtube_dl/extractor/theplatform.py           |  2 +-
 youtube_dl/extractor/thisav.py                | 10 +--
 youtube_dl/extractor/tudou.py                 |  2 +-
 youtube_dl/extractor/videofyme.py             |  2 +-
 youtube_dl/extractor/videopremium.py          | 14 +--
 youtube_dl/extractor/xtube.py                 |  2 +-
 youtube_dl/extractor/yahoo.py                 |  2 +-
 youtube_dl/extractor/youku.py                 |  4 +-
 youtube_dl/extractor/youtube.py               | 88 ++++++++++---------
 youtube_dl/postprocessor/ffmpeg.py            |  4 +-
 youtube_dl/update.py                          |  2 +-
 34 files changed, 123 insertions(+), 132 deletions(-)

diff --git a/devscripts/gh-pages/generate-download.py b/devscripts/gh-pages/generate-download.py
index 55912e12c..e90c787fd 100755
--- a/devscripts/gh-pages/generate-download.py
+++ b/devscripts/gh-pages/generate-download.py
@@ -1,8 +1,5 @@
 #!/usr/bin/env python3
 import hashlib
-import shutil
-import subprocess
-import tempfile
 import urllib.request
 import json
 
diff --git a/devscripts/transition_helper_exe/youtube-dl.py b/devscripts/transition_helper_exe/youtube-dl.py
index edb449fb3..d75316cfa 100644
--- a/devscripts/transition_helper_exe/youtube-dl.py
+++ b/devscripts/transition_helper_exe/youtube-dl.py
@@ -32,7 +32,7 @@ def rsa_verify(message, signature, key):
     signature = signature[2:]
     if not b('\x00') in signature:
         return False
-    signature = signature[signature.index(b('\x00'))+1:]
+    signature = signature[signature.index(b('\x00')) +1:]
     if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')):
         return False
     signature = signature[19:]
diff --git a/test/test_utils.py b/test/test_utils.py
index 380d1059d..8307599b3 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -45,7 +45,6 @@ from youtube_dl.utils import (
     escape_rfc3986,
     escape_url,
     js_to_json,
-    get_filesystem_encoding,
     intlist_to_bytes,
     args_to_str,
 )
diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index 452d342ae..4ceba64ae 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -12,10 +12,6 @@ from test.helper import FakeYDL
 from youtube_dl.extractor import (
     YoutubePlaylistIE,
     YoutubeIE,
-    YoutubeChannelIE,
-    YoutubeShowIE,
-    YoutubeTopListIE,
-    YoutubeSearchURLIE,
 )
 
 
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index c1a529f13..d9e93f5d2 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -29,7 +29,6 @@ from .compat import (
     compat_str,
     compat_urllib_error,
     compat_urllib_request,
-    shlex_quote,
 )
 from .utils import (
     escape_url,
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ee3067134..427b6ad27 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -76,10 +76,10 @@ def _real_main(argv=None):
     if opts.headers is not None:
         for h in opts.headers:
             if h.find(':', 1) < 0:
-                parser.error('wrong header formatting, it should be key:value, not "%s"'%h)
+                parser.error('wrong header formatting, it should be key:value, not "%s"' % h)
             key, value = h.split(':', 2)
             if opts.verbose:
-                write_string('[debug] Adding header from command line option %s:%s\n'%(key, value))
+                write_string('[debug] Adding header from command line option %s:%s\n' % (key, value))
             std_headers[key] = value
 
     # Dump user agent
@@ -197,13 +197,13 @@ def _real_main(argv=None):
         if opts.outtmpl is not None:
             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
     outtmpl = ((opts.outtmpl is not None and opts.outtmpl)
-            or (opts.format == '-1' and opts.usetitle and '%(title)s-%(id)s-%(format)s.%(ext)s')
-            or (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s')
-            or (opts.usetitle and opts.autonumber and '%(autonumber)s-%(title)s-%(id)s.%(ext)s')
-            or (opts.usetitle and '%(title)s-%(id)s.%(ext)s')
-            or (opts.useid and '%(id)s.%(ext)s')
-            or (opts.autonumber and '%(autonumber)s-%(id)s.%(ext)s')
-            or DEFAULT_OUTTMPL)
+               or (opts.format == '-1' and opts.usetitle and '%(title)s-%(id)s-%(format)s.%(ext)s')
+               or (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s')
+               or (opts.usetitle and opts.autonumber and '%(autonumber)s-%(title)s-%(id)s.%(ext)s')
+               or (opts.usetitle and '%(title)s-%(id)s.%(ext)s')
+               or (opts.useid and '%(id)s.%(ext)s')
+               or (opts.autonumber and '%(autonumber)s-%(id)s.%(ext)s')
+               or DEFAULT_OUTTMPL)
     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
         parser.error('Cannot download a video and extract audio into the same'
                      ' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py
index ccfd73770..2db8ab3a1 100644
--- a/youtube_dl/aes.py
+++ b/youtube_dl/aes.py
@@ -24,8 +24,8 @@ def aes_ctr_decrypt(data, key, counter):
     decrypted_data = []
     for i in range(block_count):
         counter_block = counter.next_value()
-        block = data[i*BLOCK_SIZE_BYTES: (i+1)*BLOCK_SIZE_BYTES]
-        block += [0]*(BLOCK_SIZE_BYTES - len(block))
+        block = data[i *BLOCK_SIZE_BYTES: (i +1) *BLOCK_SIZE_BYTES]
+        block += [0] *(BLOCK_SIZE_BYTES - len(block))
 
         cipher_counter_block = aes_encrypt(counter_block, expanded_key)
         decrypted_data += xor(block, cipher_counter_block)
@@ -49,8 +49,8 @@ def aes_cbc_decrypt(data, key, iv):
     decrypted_data = []
     previous_cipher_block = iv
     for i in range(block_count):
-        block = data[i*BLOCK_SIZE_BYTES: (i+1)*BLOCK_SIZE_BYTES]
-        block += [0]*(BLOCK_SIZE_BYTES - len(block))
+        block = data[i *BLOCK_SIZE_BYTES: (i +1) *BLOCK_SIZE_BYTES]
+        block += [0] *(BLOCK_SIZE_BYTES - len(block))
 
         decrypted_block = aes_decrypt(block, expanded_key)
         decrypted_data += xor(decrypted_block, previous_cipher_block)
@@ -76,20 +76,20 @@ def key_expansion(data):
         temp = data[-4:]
         temp = key_schedule_core(temp, rcon_iteration)
         rcon_iteration += 1
-        data += xor(temp, data[-key_size_bytes: 4-key_size_bytes])
+        data += xor(temp, data[-key_size_bytes: 4 -key_size_bytes])
 
         for _ in range(3):
             temp = data[-4:]
-            data += xor(temp, data[-key_size_bytes: 4-key_size_bytes])
+            data += xor(temp, data[-key_size_bytes: 4 -key_size_bytes])
 
         if key_size_bytes == 32:
             temp = data[-4:]
             temp = sub_bytes(temp)
-            data += xor(temp, data[-key_size_bytes: 4-key_size_bytes])
+            data += xor(temp, data[-key_size_bytes: 4 -key_size_bytes])
 
         for _ in range(3 if key_size_bytes == 32  else 2 if key_size_bytes == 24 else 0):
             temp = data[-4:]
-            data += xor(temp, data[-key_size_bytes: 4-key_size_bytes])
+            data += xor(temp, data[-key_size_bytes: 4 -key_size_bytes])
     data = data[:expanded_key_size_bytes]
 
     return data
@@ -106,12 +106,12 @@ def aes_encrypt(data, expanded_key):
     rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1
 
     data = xor(data, expanded_key[:BLOCK_SIZE_BYTES])
-    for i in range(1, rounds+1):
+    for i in range(1, rounds +1):
         data = sub_bytes(data)
         data = shift_rows(data)
         if i != rounds:
             data = mix_columns(data)
-        data = xor(data, expanded_key[i*BLOCK_SIZE_BYTES: (i+1)*BLOCK_SIZE_BYTES])
+        data = xor(data, expanded_key[i *BLOCK_SIZE_BYTES: (i +1) *BLOCK_SIZE_BYTES])
 
     return data
 
@@ -127,7 +127,7 @@ def aes_decrypt(data, expanded_key):
     rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1
 
     for i in range(rounds, 0, -1):
-        data = xor(data, expanded_key[i*BLOCK_SIZE_BYTES: (i+1)*BLOCK_SIZE_BYTES])
+        data = xor(data, expanded_key[i *BLOCK_SIZE_BYTES: (i +1) *BLOCK_SIZE_BYTES])
         if i != rounds:
             data = mix_columns_inv(data)
         data = shift_rows_inv(data)
@@ -155,14 +155,14 @@ def aes_decrypt_text(data, password, key_size_bytes):
     data = bytes_to_intlist(base64.b64decode(data))
     password = bytes_to_intlist(password.encode('utf-8'))
 
-    key = password[:key_size_bytes] + [0]*(key_size_bytes - len(password))
+    key = password[:key_size_bytes] + [0] *(key_size_bytes - len(password))
     key = aes_encrypt(key[:BLOCK_SIZE_BYTES], key_expansion(key)) * (key_size_bytes // BLOCK_SIZE_BYTES)
 
     nonce = data[:NONCE_LENGTH_BYTES]
     cipher = data[NONCE_LENGTH_BYTES:]
 
     class Counter:
-        __value = nonce + [0]*(BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES)
+        __value = nonce + [0] *(BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES)
 
         def next_value(self):
             temp = self.__value
@@ -293,7 +293,7 @@ def mix_column(data, matrix):
 def mix_columns(data, matrix=MIX_COLUMN_MATRIX):
     data_mixed = []
     for i in range(4):
-        column = data[i*4: (i+1)*4]
+        column = data[i *4: (i +1) *4]
         data_mixed += mix_column(column, matrix)
     return data_mixed
 
@@ -320,7 +320,7 @@ def shift_rows_inv(data):
 
 def inc(data):
     data = data[:]  # copy
-    for i in range(len(data)-1, -1, -1):
+    for i in range(len(data) -1, -1, -1):
         if data[i] == 255:
             data[i] = 0
         else:
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index b607f6485..ed8df16d2 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -55,7 +55,7 @@ class FlvReader(io.BytesIO):
         if size == 1:
             real_size = self.read_unsigned_long_long()
             header_end = 16
-        return real_size, box_type, self.read(real_size-header_end)
+        return real_size, box_type, self.read(real_size -header_end)
 
     def read_asrt(self):
         # version
@@ -180,7 +180,7 @@ def build_fragments_list(boot_info):
     n_frags = segment_run_entry[1]
     fragment_run_entry_table = boot_info['fragments'][0]['fragments']
     first_frag_number = fragment_run_entry_table[0]['first']
-    for (i, frag_number) in zip(range(1, n_frags+1), itertools.count(first_frag_number)):
+    for (i, frag_number) in zip(range(1, n_frags +1), itertools.count(first_frag_number)):
         res.append((1, frag_number))
     return res
 
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index b8e104c59..642f13b9a 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -46,13 +46,13 @@ class RtmpFD(FileDownloader):
                     continue
                 mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec \(([0-9]{1,2}\.[0-9])%\)', line)
                 if mobj:
-                    downloaded_data_len = int(float(mobj.group(1))*1024)
+                    downloaded_data_len = int(float(mobj.group(1)) *1024)
                     percent = float(mobj.group(2))
                     if not resume_percent:
                         resume_percent = percent
                         resume_downloaded_data_len = downloaded_data_len
-                    eta = self.calc_eta(start, time.time(), 100-resume_percent, percent-resume_percent)
-                    speed = self.calc_speed(start, time.time(), downloaded_data_len-resume_downloaded_data_len)
+                    eta = self.calc_eta(start, time.time(), 100 -resume_percent, percent -resume_percent)
+                    speed = self.calc_speed(start, time.time(), downloaded_data_len -resume_downloaded_data_len)
                     data_len = None
                     if percent > 0:
                         data_len = int(downloaded_data_len * 100 / percent)
@@ -72,7 +72,7 @@ class RtmpFD(FileDownloader):
                     # no percent for live streams
                     mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec', line)
                     if mobj:
-                        downloaded_data_len = int(float(mobj.group(1))*1024)
+                        downloaded_data_len = int(float(mobj.group(1)) *1024)
                         time_now = time.time()
                         speed = self.calc_speed(start, time_now, downloaded_data_len)
                         self.report_progress_live_stream(downloaded_data_len, speed, time_now - start)
@@ -88,7 +88,7 @@ class RtmpFD(FileDownloader):
                         if not cursor_in_new_line:
                             self.to_screen('')
                         cursor_in_new_line = True
-                        self.to_screen('[rtmpdump] '+line)
+                        self.to_screen('[rtmpdump] ' +line)
             proc.wait()
             if not cursor_in_new_line:
                 self.to_screen('')
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fcb75af34..43185345e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -529,4 +529,4 @@ def gen_extractors():
 
 def get_info_extractor(ie_name):
     """Returns the info extractor class with the given ie_name"""
-    return globals()[ie_name+'IE']
+    return globals()[ie_name + 'IE']
diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 89476f032..fa15bf19c 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -195,7 +195,7 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
             duration = int(item.get('duration'))
 
             media_selection = self._download_xml(
-                'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s'  % programme_id,
+                'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s' % programme_id,
                 programme_id, 'Downloading media selection XML')
 
             for media in self._extract_medias(media_selection):
diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index 31fe906b4..bdc84f1f5 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -77,7 +77,7 @@ class CinemassacreIE(InfoExtractor):
         if videolist_url:
             videolist = self._download_xml(videolist_url, video_id, 'Downloading videolist XML')
             formats = []
-            baseurl = vidurl[:vidurl.rfind('/')+1]
+            baseurl = vidurl[:vidurl.rfind('/') +1]
             for video in videolist.findall('.//video'):
                 src = video.get('src')
                 if not src:
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index c45858160..ab03c8602 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -226,10 +226,10 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         formats = []
         for fmt in re.findall(r'\?p([0-9]{3,4})=1', webpage):
             stream_quality, stream_format = self._FORMAT_IDS[fmt]
-            video_format = fmt+'p'
+            video_format = fmt +'p'
             streamdata_req = compat_urllib_request.Request('http://www.crunchyroll.com/xml/')
             # urlencode doesn't work!
-            streamdata_req.data = 'req=RpcApiVideoEncode%5FGetStreamInfo&video%5Fencode%5Fquality='+stream_quality+'&media%5Fid='+stream_id+'&video%5Fformat='+stream_format
+            streamdata_req.data = 'req=RpcApiVideoEncode%5FGetStreamInfo&video%5Fencode%5Fquality=' +stream_quality +'&media%5Fid=' +stream_id +'&video%5Fformat=' +stream_format
             streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
             streamdata_req.add_header('Content-Length', str(len(streamdata_req.data)))
             streamdata = self._download_xml(
@@ -248,8 +248,8 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         subtitles = {}
         sub_format = self._downloader.params.get('subtitlesformat', 'srt')
         for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
-            sub_page = self._download_webpage('http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id='+sub_id,\
-                                              video_id, note='Downloading subtitles for '+sub_name)
+            sub_page = self._download_webpage('http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' +sub_id,\
+                                              video_id, note='Downloading subtitles for ' +sub_name)
             id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False)
             iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False)
             data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False)
@@ -274,14 +274,14 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
             return
 
         return {
-            'id':          video_id,
-            'title':       video_title,
+            'id': video_id,
+            'title': video_title,
             'description': video_description,
-            'thumbnail':   video_thumbnail,
-            'uploader':    video_uploader,
+            'thumbnail': video_thumbnail,
+            'uploader': video_uploader,
             'upload_date': video_upload_date,
-            'subtitles':   subtitles,
-            'formats':     formats,
+            'subtitles': subtitles,
+            'formats': formats,
         }
 
 
diff --git a/youtube_dl/extractor/dotsub.py b/youtube_dl/extractor/dotsub.py
index 70e368021..b30d70e7b 100644
--- a/youtube_dl/extractor/dotsub.py
+++ b/youtube_dl/extractor/dotsub.py
@@ -27,7 +27,7 @@ class DotsubIE(InfoExtractor):
         video_id = mobj.group('id')
         info_url = "https://dotsub.com/api/media/%s/metadata" % video_id
         info = self._download_json(info_url, video_id)
-        date = time.gmtime(info['dateCreated']/1000)  # The timestamp is in miliseconds
+        date = time.gmtime(info['dateCreated'] /1000)  # The timestamp is in miliseconds
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2dc801ad9..a40ff6b64 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -748,7 +748,7 @@ class GenericIE(InfoExtractor):
         # Look for embedded blip.tv player
         mobj = re.search(r'<meta\s[^>]*https?://api\.blip\.tv/\w+/redirect/\w+/(\d+)', webpage)
         if mobj:
-            return self.url_result('http://blip.tv/a/a-'+mobj.group(1), 'BlipTV')
+            return self.url_result('http://blip.tv/a/a-' +mobj.group(1), 'BlipTV')
         mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*(https?://(?:\w+\.)?blip\.tv/(?:play/|api\.swf#)[a-zA-Z0-9_]+)', webpage)
         if mobj:
             return self.url_result(mobj.group(1), 'BlipTV')
diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index d1defd363..90b4c966d 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -54,7 +54,7 @@ class IPrimaIE(InfoExtractor):
 
         player_url = (
             'http://embed.livebox.cz/iprimaplay/player-embed-v2.js?__tok%s__=%s' %
-            (floor(random()*1073741824), floor(random()*1073741824))
+            (floor(random() *1073741824), floor(random() *1073741824))
         )
 
         req = compat_urllib_request.Request(player_url)
diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index 1f1e23dc3..e7ee3bba8 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -71,4 +71,4 @@ class LifeNewsIE(InfoExtractor):
         if len(videos) == 1:
             return make_entry(video_id, videos[0])
         else:
-            return [make_entry(video_id, media, video_number+1) for video_number, media in enumerate(videos)]
+            return [make_entry(video_id, media, video_number +1) for video_number, media in enumerate(videos)]
diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index 8e50e8f79..b04be1e8c 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -19,8 +19,7 @@ class LiveLeakIE(InfoExtractor):
             'uploader': 'ljfriel2',
             'title': 'Most unlucky car accident'
         }
-    },
-    {
+    }, {
         'url': 'http://www.liveleak.com/view?i=f93_1390833151',
         'md5': 'd3f1367d14cc3c15bf24fbfbe04b9abf',
         'info_dict': {
@@ -30,8 +29,7 @@ class LiveLeakIE(InfoExtractor):
             'uploader': 'ARD_Stinkt',
             'title': 'German Television does first Edward Snowden Interview (ENGLISH)',
         }
-    },
-    {
+    }, {
         'url': 'http://www.liveleak.com/view?i=4f7_1392687779',
         'md5': '42c6d97d54f1db107958760788c5f48f',
         'info_dict': {
diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 1a896b536..f68add6c0 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -22,7 +22,7 @@ class MetacafeIE(InfoExtractor):
         # Youtube video
         {
             'add_ie': ['Youtube'],
-            'url':  'http://metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/',
+            'url': 'http://metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/',
             'info_dict': {
                 'id': '_aUehQsCQtM',
                 'ext': 'mp4',
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index b2f37f692..b6755ff01 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -245,7 +245,7 @@ class MTVIE(MTVServicesInfoExtractor):
             m_vevo = re.search(r'isVevoVideo = true;.*?vevoVideoId = "(.*?)";',
                                webpage, re.DOTALL)
             if m_vevo:
-                vevo_id = m_vevo.group(1);
+                vevo_id = m_vevo.group(1)
                 self.to_screen('Vevo video detected: %s' % vevo_id)
                 return self.url_result('vevo:%s' % vevo_id, ie='Vevo')
 
diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index bebcafb62..875d09faa 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -69,7 +69,7 @@ class SohuIE(InfoExtractor):
                         (allot, prot, clipsURL[i], su[i]))
             part_str = self._download_webpage(
                 part_url, video_id,
-                note=u'Downloading part %d of %d' % (i+1, part_count))
+                note=u'Downloading part %d of %d' % (i +1, part_count))
 
             part_info = part_str.split('|')
             video_url = '%s%s?key=%s' % (part_info[0], su[i], part_info[3])
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index c830d9da7..f92f7fa24 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -224,7 +224,7 @@ class SoundcloudIE(InfoExtractor):
             # extract uploader (which is in the url)
             uploader = mobj.group('uploader')
             # extract simple title (uploader + slug of song title)
-            slug_title =  mobj.group('title')
+            slug_title = mobj.group('title')
             token = mobj.group('token')
             full_title = resolve_title = '%s/%s' % (uploader, slug_title)
             if token:
diff --git a/youtube_dl/extractor/swrmediathek.py b/youtube_dl/extractor/swrmediathek.py
index bf430d870..4132b6428 100644
--- a/youtube_dl/extractor/swrmediathek.py
+++ b/youtube_dl/extractor/swrmediathek.py
@@ -80,7 +80,7 @@ class SWRMediathekIE(InfoExtractor):
 
             if media_type == 'Video':
                 fmt.update({
-                    'format_note': ['144p', '288p', '544p', '720p'][quality-1],
+                    'format_note': ['144p', '288p', '544p', '720p'][quality -1],
                     'vcodec': codec,
                 })
             elif media_type == 'Audio':
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index b83d15b90..40b22677b 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -118,5 +118,5 @@ class ThePlatformIE(InfoExtractor):
             'formats': formats,
             'description': info['description'],
             'thumbnail': info['defaultThumbnailUrl'],
-            'duration': info['duration']//1000,
+            'duration': info['duration'] //1000,
         }
diff --git a/youtube_dl/extractor/thisav.py b/youtube_dl/extractor/thisav.py
index 350a5cdb5..7f323c938 100644
--- a/youtube_dl/extractor/thisav.py
+++ b/youtube_dl/extractor/thisav.py
@@ -38,10 +38,10 @@ class ThisAVIE(InfoExtractor):
         ext = determine_ext(video_url)
 
         return {
-            'id':          video_id,
-            'url':         video_url,
-            'uploader':    uploader,
+            'id': video_id,
+            'url': video_url,
+            'uploader': uploader,
             'uploader_id': uploader_id,
-            'title':       title,
-            'ext':         ext,
+            'title': title,
+            'ext': ext,
         }
diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index c8780790a..3007b136f 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -37,7 +37,7 @@ class TudouIE(InfoExtractor):
     }]
 
     def _url_for_id(self, id, quality = None):
-        info_url = "http://v2.tudou.com/f?id="+str(id)
+        info_url = "http://v2.tudou.com/f?id=" +str(id)
         if quality:
             info_url += '&hd' + quality
         webpage = self._download_webpage(info_url, id, "Opening the info webpage")
diff --git a/youtube_dl/extractor/videofyme.py b/youtube_dl/extractor/videofyme.py
index 2dd51d866..d69fe1e77 100644
--- a/youtube_dl/extractor/videofyme.py
+++ b/youtube_dl/extractor/videofyme.py
@@ -13,7 +13,7 @@ class VideofyMeIE(InfoExtractor):
 
     _TEST = {
         u'url': u'http://www.videofy.me/thisisvideofyme/1100701',
-        u'file':  u'1100701.mp4',
+        u'file': u'1100701.mp4',
         u'md5': u'c77d700bdc16ae2e9f3c26019bd96143',
         u'info_dict': {
             u'title': u'This is VideofyMe',
diff --git a/youtube_dl/extractor/videopremium.py b/youtube_dl/extractor/videopremium.py
index 65463c733..9f2cdb6ef 100644
--- a/youtube_dl/extractor/videopremium.py
+++ b/youtube_dl/extractor/videopremium.py
@@ -35,11 +35,11 @@ class VideoPremiumIE(InfoExtractor):
             r'<h2(?:.*?)>\s*(.+?)\s*<', webpage, u'video title')
 
         return {
-            'id':          video_id,
-            'url':         "rtmp://e%d.md.iplay.md/play" % random.randint(1, 16),
-            'play_path':   "mp4:%s.f4v" % video_id,
-            'page_url':    "http://videopremium.tv/" + video_id,
-            'player_url':  "http://videopremium.tv/uplayer/uppod.swf",
-            'ext':         'f4v',
-            'title':       video_title,
+            'id': video_id,
+            'url': "rtmp://e%d.md.iplay.md/play" % random.randint(1, 16),
+            'play_path': "mp4:%s.f4v" % video_id,
+            'page_url': "http://videopremium.tv/" + video_id,
+            'player_url': "http://videopremium.tv/uplayer/uppod.swf",
+            'ext': 'f4v',
+            'title': video_title,
         }
diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index c3bb9b2cf..26a51340f 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -97,7 +97,7 @@ class XTubeUserIE(InfoExtractor):
             url, username, note='Retrieving profile page')
 
         video_count = int(self._search_regex(
-            r'<strong>%s\'s Videos \(([0-9]+)\)</strong>'%username, profile_page,
+            r'<strong>%s\'s Videos \(([0-9]+)\)</strong>' %username, profile_page,
             'video count'))
 
         PAGE_SIZE = 25
diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 117f0856a..514c16127 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -229,7 +229,7 @@ class YahooSearchIE(SearchInfoExtractor):
         for pagenum in itertools.count(0):
             result_url = 'http://video.search.yahoo.com/search/?p=%s&fr=screen&o=js&gs=0&b=%d' % (compat_urllib_parse.quote_plus(query), pagenum * 30)
             info = self._download_json(result_url, query,
-                note='Downloading results page '+str(pagenum+1))
+                note='Downloading results page ' +str(pagenum +1))
             m = info['m']
             results = info['results']
 
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index e155b3a94..830ae6cd9 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -45,8 +45,8 @@ class YoukuIE(InfoExtractor):
         source = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890")
         seed = float(seed)
         for i in range(len(source)):
-            seed  =  (seed * 211 + 30031) % 65536
-            index  =  math.floor(seed / 65536 * len(source))
+            seed = (seed * 211 + 30031) % 65536
+            index = math.floor(seed / 65536 * len(source))
             mixed.append(source[int(index)])
             source.remove(source[int(index)])
         # return ''.join(mixed)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 98cac7c17..8711b06d4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -77,25 +77,25 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
         # Log in
         login_form_strs = {
-                'continue': 'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
-                'Email': username,
-                'GALX': galx,
-                'Passwd': password,
+            'continue': 'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
+            'Email': username,
+            'GALX': galx,
+            'Passwd': password,
 
-                'PersistentCookie': 'yes',
-                '_utf8': '霱',
-                'bgresponse': 'js_disabled',
-                'checkConnection': '',
-                'checkedDomains': 'youtube',
-                'dnConn': '',
-                'pstMsg': '0',
-                'rmShown': '1',
-                'secTok': '',
-                'signIn': 'Sign in',
-                'timeStmp': '',
-                'service': 'youtube',
-                'uilel': '3',
-                'hl': 'en_US',
+            'PersistentCookie': 'yes',
+            '_utf8': '霱',
+            'bgresponse': 'js_disabled',
+            'checkConnection': '',
+            'checkedDomains': 'youtube',
+            'dnConn': '',
+            'pstMsg': '0',
+            'rmShown': '1',
+            'secTok': '',
+            'signIn': 'Sign in',
+            'timeStmp': '',
+            'service': 'youtube',
+            'uilel': '3',
+            'hl': 'en_US',
         }
 
         # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
@@ -181,8 +181,10 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'next_url': '/',
             'action_confirm': 'Confirm',
         }
-        req = compat_urllib_request.Request(self._AGE_URL,
-            compat_urllib_parse.urlencode(age_form).encode('ascii'))
+        req = compat_urllib_request.Request(
+            self._AGE_URL,
+            compat_urllib_parse.urlencode(age_form).encode('ascii')
+        )
 
         self._download_webpage(
             req, None,
@@ -492,7 +494,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         def gen_sig_code(idxs):
             def _genslice(start, end, step):
                 starts = '' if start == 0 else str(start)
-                ends = (':%d' % (end+step)) if end + step >= 0 else ':'
+                ends = (':%d' % (end + step)) if end + step >= 0 else ':'
                 steps = '' if step == 1 else (':%d' % step)
                 return 's[%s%s%s]' % (starts, ends, steps)
 
@@ -530,7 +532,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
     def _parse_sig_js(self, jscode):
         funcname = self._search_regex(
             r'\.sig\|\|([a-zA-Z0-9]+)\(', jscode,
-             'Initial JS player signature function name')
+            'Initial JS player signature function name')
 
         jsi = JSInterpreter(jscode)
         initial_function = jsi.extract_function(funcname)
@@ -656,7 +658,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         def _get_urls(_manifest):
             lines = _manifest.split('\n')
             urls = filter(lambda l: l and not l.startswith('#'),
-                            lines)
+                          lines)
             return urls
         manifest = self._download_webpage(manifest_url, video_id, 'Downloading formats manifest')
         formats_urls = _get_urls(manifest)
@@ -723,10 +725,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             age_gate = False
             for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
                 video_info_url = (proto + '://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
-                        % (video_id, el_type))
+                                  % (video_id, el_type))
                 video_info_webpage = self._download_webpage(video_info_url, video_id,
-                                        note=False,
-                                        errnote='unable to download video info webpage')
+                                                            note=False,
+                                                            errnote='unable to download video info webpage')
                 video_info = compat_parse_qs(video_info_webpage)
                 if 'token' in video_info:
                     break
@@ -1017,23 +1019,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id':           video_id,
-            'uploader':     video_uploader,
-            'uploader_id':  video_uploader_id,
-            'upload_date':  upload_date,
-            'title':        video_title,
-            'thumbnail':    video_thumbnail,
-            'description':  video_description,
-            'categories':   video_categories,
-            'subtitles':    video_subtitles,
-            'duration':     video_duration,
-            'age_limit':    18 if age_gate else 0,
-            'annotations':  video_annotations,
+            'id': video_id,
+            'uploader': video_uploader,
+            'uploader_id': video_uploader_id,
+            'upload_date': upload_date,
+            'title': video_title,
+            'thumbnail': video_thumbnail,
+            'description': video_description,
+            'categories': video_categories,
+            'subtitles': video_subtitles,
+            'duration': video_duration,
+            'age_limit': 18 if age_gate else 0,
+            'annotations': video_annotations,
             'webpage_url': proto + '://www.youtube.com/watch?v=%s' % video_id,
-            'view_count':   view_count,
+            'view_count': view_count,
             'like_count': like_count,
             'dislike_count': dislike_count,
-            'formats':      formats,
+            'formats': formats,
         }
 
 
@@ -1167,7 +1169,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
             return self._extract_mix(playlist_id)
         if playlist_id.startswith('TL'):
             raise ExtractorError('For downloading YouTube.com top lists, use '
-                'the "yttoplist" keyword, for example "youtube-dl \'yttoplist:music:Top Tracks\'"', expected=True)
+                                 'the "yttoplist" keyword, for example "youtube-dl \'yttoplist:music:Top Tracks\'"', expected=True)
 
         url = self._TEMPLATE_URL % playlist_id
         page = self._download_webpage(url, playlist_id)
@@ -1546,8 +1548,8 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
         paging = 0
         for i in itertools.count(1):
             info = self._download_json(self._FEED_TEMPLATE % paging,
-                                          '%s feed' % self._FEED_NAME,
-                                          'Downloading page %s' % i)
+                                       '%s feed' % self._FEED_NAME,
+                                       'Downloading page %s' % i)
             feed_html = info.get('feed_html') or info.get('content_html')
             load_more_widget_html = info.get('load_more_widget_html') or feed_html
             m_ids = re.finditer(r'"/watch\?v=(.*?)["&]', feed_html)
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index dfae735f3..d9f1cceb9 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -246,7 +246,7 @@ class FFmpegVideoConvertor(FFmpegPostProcessor):
         if information['ext'] == self._preferedformat:
             self._downloader.to_screen(u'[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
             return True, information
-        self._downloader.to_screen(u'['+'ffmpeg'+'] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath)
+        self._downloader.to_screen(u'[' +'ffmpeg' +'] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath)
         self.run_ffmpeg(path, outpath, [])
         information['filepath'] = outpath
         information['format'] = self._preferedformat
@@ -466,7 +466,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
 
         opts = ['-map', '0:0', '-map', '0:1', '-c:v', 'copy', '-c:a', 'copy']
         for (i, lang) in enumerate(sub_langs):
-            opts.extend(['-map', '%d:0' % (i+1), '-c:s:%d' % i, 'mov_text'])
+            opts.extend(['-map', '%d:0' % (i +1), '-c:s:%d' % i, 'mov_text'])
             lang_code = self._conver_lang_code(lang)
             if lang_code is not None:
                 opts.extend(['-metadata:s:s:%d' % i, 'language=%s' % lang_code])
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index d32a86708..23457e07a 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -41,7 +41,7 @@ def rsa_verify(message, signature, key):
     signature = signature[2:]
     if not b('\x00') in signature:
         return False
-    signature = signature[signature.index(b('\x00'))+1:]
+    signature = signature[signature.index(b('\x00')) +1:]
     if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')):
         return False
     signature = signature[19:]

From 2514d2635e0c8ff0fb72d00a093a849001df2bdd Mon Sep 17 00:00:00 2001
From: Jouke Waleson <jouke.waleson@mendix.com>
Date: Sun, 23 Nov 2014 21:23:05 +0100
Subject: [PATCH 0580/1937] PEP8: E225,E227

---
 .../transition_helper_exe/youtube-dl.py       |  2 +-
 youtube_dl/aes.py                             | 32 +++++++++----------
 youtube_dl/downloader/f4m.py                  |  4 +--
 youtube_dl/downloader/rtmp.py                 | 10 +++---
 youtube_dl/extractor/cinemassacre.py          |  2 +-
 youtube_dl/extractor/crunchyroll.py           |  8 ++---
 youtube_dl/extractor/dotsub.py                |  2 +-
 youtube_dl/extractor/generic.py               |  2 +-
 youtube_dl/extractor/iprima.py                |  2 +-
 youtube_dl/extractor/lifenews.py              |  2 +-
 youtube_dl/extractor/smotri.py                |  2 +-
 youtube_dl/extractor/sohu.py                  |  2 +-
 youtube_dl/extractor/swrmediathek.py          |  2 +-
 youtube_dl/extractor/theplatform.py           |  2 +-
 youtube_dl/extractor/tudou.py                 |  2 +-
 youtube_dl/extractor/xtube.py                 |  2 +-
 youtube_dl/extractor/yahoo.py                 |  2 +-
 youtube_dl/postprocessor/ffmpeg.py            |  4 +--
 youtube_dl/update.py                          |  2 +-
 19 files changed, 43 insertions(+), 43 deletions(-)

diff --git a/devscripts/transition_helper_exe/youtube-dl.py b/devscripts/transition_helper_exe/youtube-dl.py
index d75316cfa..ab59d9376 100644
--- a/devscripts/transition_helper_exe/youtube-dl.py
+++ b/devscripts/transition_helper_exe/youtube-dl.py
@@ -32,7 +32,7 @@ def rsa_verify(message, signature, key):
     signature = signature[2:]
     if not b('\x00') in signature:
         return False
-    signature = signature[signature.index(b('\x00')) +1:]
+    signature = signature[signature.index(b('\x00')) + 1:]
     if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')):
         return False
     signature = signature[19:]
diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py
index 2db8ab3a1..662d49011 100644
--- a/youtube_dl/aes.py
+++ b/youtube_dl/aes.py
@@ -24,8 +24,8 @@ def aes_ctr_decrypt(data, key, counter):
     decrypted_data = []
     for i in range(block_count):
         counter_block = counter.next_value()
-        block = data[i *BLOCK_SIZE_BYTES: (i +1) *BLOCK_SIZE_BYTES]
-        block += [0] *(BLOCK_SIZE_BYTES - len(block))
+        block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
+        block += [0] * (BLOCK_SIZE_BYTES - len(block))
 
         cipher_counter_block = aes_encrypt(counter_block, expanded_key)
         decrypted_data += xor(block, cipher_counter_block)
@@ -49,8 +49,8 @@ def aes_cbc_decrypt(data, key, iv):
     decrypted_data = []
     previous_cipher_block = iv
     for i in range(block_count):
-        block = data[i *BLOCK_SIZE_BYTES: (i +1) *BLOCK_SIZE_BYTES]
-        block += [0] *(BLOCK_SIZE_BYTES - len(block))
+        block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
+        block += [0] * (BLOCK_SIZE_BYTES - len(block))
 
         decrypted_block = aes_decrypt(block, expanded_key)
         decrypted_data += xor(decrypted_block, previous_cipher_block)
@@ -76,20 +76,20 @@ def key_expansion(data):
         temp = data[-4:]
         temp = key_schedule_core(temp, rcon_iteration)
         rcon_iteration += 1
-        data += xor(temp, data[-key_size_bytes: 4 -key_size_bytes])
+        data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
 
         for _ in range(3):
             temp = data[-4:]
-            data += xor(temp, data[-key_size_bytes: 4 -key_size_bytes])
+            data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
 
         if key_size_bytes == 32:
             temp = data[-4:]
             temp = sub_bytes(temp)
-            data += xor(temp, data[-key_size_bytes: 4 -key_size_bytes])
+            data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
 
         for _ in range(3 if key_size_bytes == 32  else 2 if key_size_bytes == 24 else 0):
             temp = data[-4:]
-            data += xor(temp, data[-key_size_bytes: 4 -key_size_bytes])
+            data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
     data = data[:expanded_key_size_bytes]
 
     return data
@@ -106,12 +106,12 @@ def aes_encrypt(data, expanded_key):
     rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1
 
     data = xor(data, expanded_key[:BLOCK_SIZE_BYTES])
-    for i in range(1, rounds +1):
+    for i in range(1, rounds + 1):
         data = sub_bytes(data)
         data = shift_rows(data)
         if i != rounds:
             data = mix_columns(data)
-        data = xor(data, expanded_key[i *BLOCK_SIZE_BYTES: (i +1) *BLOCK_SIZE_BYTES])
+        data = xor(data, expanded_key[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES])
 
     return data
 
@@ -127,7 +127,7 @@ def aes_decrypt(data, expanded_key):
     rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1
 
     for i in range(rounds, 0, -1):
-        data = xor(data, expanded_key[i *BLOCK_SIZE_BYTES: (i +1) *BLOCK_SIZE_BYTES])
+        data = xor(data, expanded_key[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES])
         if i != rounds:
             data = mix_columns_inv(data)
         data = shift_rows_inv(data)
@@ -155,14 +155,14 @@ def aes_decrypt_text(data, password, key_size_bytes):
     data = bytes_to_intlist(base64.b64decode(data))
     password = bytes_to_intlist(password.encode('utf-8'))
 
-    key = password[:key_size_bytes] + [0] *(key_size_bytes - len(password))
+    key = password[:key_size_bytes] + [0] * (key_size_bytes - len(password))
     key = aes_encrypt(key[:BLOCK_SIZE_BYTES], key_expansion(key)) * (key_size_bytes // BLOCK_SIZE_BYTES)
 
     nonce = data[:NONCE_LENGTH_BYTES]
     cipher = data[NONCE_LENGTH_BYTES:]
 
     class Counter:
-        __value = nonce + [0] *(BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES)
+        __value = nonce + [0] * (BLOCK_SIZE_BYTES - NONCE_LENGTH_BYTES)
 
         def next_value(self):
             temp = self.__value
@@ -270,7 +270,7 @@ def key_schedule_core(data, rcon_iteration):
 
 
 def xor(data1, data2):
-    return [x^y for x, y in zip(data1, data2)]
+    return [x ^ y for x, y in zip(data1, data2)]
 
 
 def rijndael_mul(a, b):
@@ -293,7 +293,7 @@ def mix_column(data, matrix):
 def mix_columns(data, matrix=MIX_COLUMN_MATRIX):
     data_mixed = []
     for i in range(4):
-        column = data[i *4: (i +1) *4]
+        column = data[i * 4: (i + 1) * 4]
         data_mixed += mix_column(column, matrix)
     return data_mixed
 
@@ -320,7 +320,7 @@ def shift_rows_inv(data):
 
 def inc(data):
     data = data[:]  # copy
-    for i in range(len(data) -1, -1, -1):
+    for i in range(len(data) - 1, -1, -1):
         if data[i] == 255:
             data[i] = 0
         else:
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index ed8df16d2..c752e8e24 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -55,7 +55,7 @@ class FlvReader(io.BytesIO):
         if size == 1:
             real_size = self.read_unsigned_long_long()
             header_end = 16
-        return real_size, box_type, self.read(real_size -header_end)
+        return real_size, box_type, self.read(real_size - header_end)
 
     def read_asrt(self):
         # version
@@ -180,7 +180,7 @@ def build_fragments_list(boot_info):
     n_frags = segment_run_entry[1]
     fragment_run_entry_table = boot_info['fragments'][0]['fragments']
     first_frag_number = fragment_run_entry_table[0]['first']
-    for (i, frag_number) in zip(range(1, n_frags +1), itertools.count(first_frag_number)):
+    for (i, frag_number) in zip(range(1, n_frags + 1), itertools.count(first_frag_number)):
         res.append((1, frag_number))
     return res
 
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 642f13b9a..58ae2005c 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -46,13 +46,13 @@ class RtmpFD(FileDownloader):
                     continue
                 mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec \(([0-9]{1,2}\.[0-9])%\)', line)
                 if mobj:
-                    downloaded_data_len = int(float(mobj.group(1)) *1024)
+                    downloaded_data_len = int(float(mobj.group(1)) * 1024)
                     percent = float(mobj.group(2))
                     if not resume_percent:
                         resume_percent = percent
                         resume_downloaded_data_len = downloaded_data_len
-                    eta = self.calc_eta(start, time.time(), 100 -resume_percent, percent -resume_percent)
-                    speed = self.calc_speed(start, time.time(), downloaded_data_len -resume_downloaded_data_len)
+                    eta = self.calc_eta(start, time.time(), 100 - resume_percent, percent - resume_percent)
+                    speed = self.calc_speed(start, time.time(), downloaded_data_len - resume_downloaded_data_len)
                     data_len = None
                     if percent > 0:
                         data_len = int(downloaded_data_len * 100 / percent)
@@ -72,7 +72,7 @@ class RtmpFD(FileDownloader):
                     # no percent for live streams
                     mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec', line)
                     if mobj:
-                        downloaded_data_len = int(float(mobj.group(1)) *1024)
+                        downloaded_data_len = int(float(mobj.group(1)) * 1024)
                         time_now = time.time()
                         speed = self.calc_speed(start, time_now, downloaded_data_len)
                         self.report_progress_live_stream(downloaded_data_len, speed, time_now - start)
@@ -88,7 +88,7 @@ class RtmpFD(FileDownloader):
                         if not cursor_in_new_line:
                             self.to_screen('')
                         cursor_in_new_line = True
-                        self.to_screen('[rtmpdump] ' +line)
+                        self.to_screen('[rtmpdump] ' + line)
             proc.wait()
             if not cursor_in_new_line:
                 self.to_screen('')
diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index bdc84f1f5..b7fa73c3b 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -77,7 +77,7 @@ class CinemassacreIE(InfoExtractor):
         if videolist_url:
             videolist = self._download_xml(videolist_url, video_id, 'Downloading videolist XML')
             formats = []
-            baseurl = vidurl[:vidurl.rfind('/') +1]
+            baseurl = vidurl[:vidurl.rfind('/') + 1]
             for video in videolist.findall('.//video'):
                 src = video.get('src')
                 if not src:
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index ab03c8602..c3c4d114a 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -226,10 +226,10 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         formats = []
         for fmt in re.findall(r'\?p([0-9]{3,4})=1', webpage):
             stream_quality, stream_format = self._FORMAT_IDS[fmt]
-            video_format = fmt +'p'
+            video_format = fmt + 'p'
             streamdata_req = compat_urllib_request.Request('http://www.crunchyroll.com/xml/')
             # urlencode doesn't work!
-            streamdata_req.data = 'req=RpcApiVideoEncode%5FGetStreamInfo&video%5Fencode%5Fquality=' +stream_quality +'&media%5Fid=' +stream_id +'&video%5Fformat=' +stream_format
+            streamdata_req.data = 'req=RpcApiVideoEncode%5FGetStreamInfo&video%5Fencode%5Fquality=' + stream_quality + '&media%5Fid=' + stream_id + '&video%5Fformat=' + stream_format
             streamdata_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
             streamdata_req.add_header('Content-Length', str(len(streamdata_req.data)))
             streamdata = self._download_xml(
@@ -248,8 +248,8 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         subtitles = {}
         sub_format = self._downloader.params.get('subtitlesformat', 'srt')
         for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
-            sub_page = self._download_webpage('http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' +sub_id,\
-                                              video_id, note='Downloading subtitles for ' +sub_name)
+            sub_page = self._download_webpage('http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id,\
+                                              video_id, note='Downloading subtitles for ' + sub_name)
             id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False)
             iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False)
             data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False)
diff --git a/youtube_dl/extractor/dotsub.py b/youtube_dl/extractor/dotsub.py
index b30d70e7b..638bb33cd 100644
--- a/youtube_dl/extractor/dotsub.py
+++ b/youtube_dl/extractor/dotsub.py
@@ -27,7 +27,7 @@ class DotsubIE(InfoExtractor):
         video_id = mobj.group('id')
         info_url = "https://dotsub.com/api/media/%s/metadata" % video_id
         info = self._download_json(info_url, video_id)
-        date = time.gmtime(info['dateCreated'] /1000)  # The timestamp is in miliseconds
+        date = time.gmtime(info['dateCreated'] / 1000)  # The timestamp is in miliseconds
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a40ff6b64..109dd20db 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -748,7 +748,7 @@ class GenericIE(InfoExtractor):
         # Look for embedded blip.tv player
         mobj = re.search(r'<meta\s[^>]*https?://api\.blip\.tv/\w+/redirect/\w+/(\d+)', webpage)
         if mobj:
-            return self.url_result('http://blip.tv/a/a-' +mobj.group(1), 'BlipTV')
+            return self.url_result('http://blip.tv/a/a-' + mobj.group(1), 'BlipTV')
         mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*(https?://(?:\w+\.)?blip\.tv/(?:play/|api\.swf#)[a-zA-Z0-9_]+)', webpage)
         if mobj:
             return self.url_result(mobj.group(1), 'BlipTV')
diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 90b4c966d..4247d6391 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -54,7 +54,7 @@ class IPrimaIE(InfoExtractor):
 
         player_url = (
             'http://embed.livebox.cz/iprimaplay/player-embed-v2.js?__tok%s__=%s' %
-            (floor(random() *1073741824), floor(random() *1073741824))
+            (floor(random() * 1073741824), floor(random() * 1073741824))
         )
 
         req = compat_urllib_request.Request(player_url)
diff --git a/youtube_dl/extractor/lifenews.py b/youtube_dl/extractor/lifenews.py
index e7ee3bba8..1dfe7f77f 100644
--- a/youtube_dl/extractor/lifenews.py
+++ b/youtube_dl/extractor/lifenews.py
@@ -71,4 +71,4 @@ class LifeNewsIE(InfoExtractor):
         if len(videos) == 1:
             return make_entry(video_id, videos[0])
         else:
-            return [make_entry(video_id, media, video_number +1) for video_number, media in enumerate(videos)]
+            return [make_entry(video_id, media, video_number + 1) for video_number, media in enumerate(videos)]
diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 9d2f8d40b..b6a71305f 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -184,7 +184,7 @@ class SmotriIE(InfoExtractor):
 
         view_count = self._html_search_regex(
             'Общее количество просмотров.*?<span class="Number">(\\d+)</span>',
-            webpage, 'view count', fatal=False, flags=re.MULTILINE|re.DOTALL)
+            webpage, 'view count', fatal=False, flags=re.MULTILINE | re.DOTALL)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 875d09faa..0348e7460 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -69,7 +69,7 @@ class SohuIE(InfoExtractor):
                         (allot, prot, clipsURL[i], su[i]))
             part_str = self._download_webpage(
                 part_url, video_id,
-                note=u'Downloading part %d of %d' % (i +1, part_count))
+                note=u'Downloading part %d of %d' % (i + 1, part_count))
 
             part_info = part_str.split('|')
             video_url = '%s%s?key=%s' % (part_info[0], su[i], part_info[3])
diff --git a/youtube_dl/extractor/swrmediathek.py b/youtube_dl/extractor/swrmediathek.py
index 4132b6428..58073eefe 100644
--- a/youtube_dl/extractor/swrmediathek.py
+++ b/youtube_dl/extractor/swrmediathek.py
@@ -80,7 +80,7 @@ class SWRMediathekIE(InfoExtractor):
 
             if media_type == 'Video':
                 fmt.update({
-                    'format_note': ['144p', '288p', '544p', '720p'][quality -1],
+                    'format_note': ['144p', '288p', '544p', '720p'][quality - 1],
                     'vcodec': codec,
                 })
             elif media_type == 'Audio':
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 40b22677b..522a095a2 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -118,5 +118,5 @@ class ThePlatformIE(InfoExtractor):
             'formats': formats,
             'description': info['description'],
             'thumbnail': info['defaultThumbnailUrl'],
-            'duration': info['duration'] //1000,
+            'duration': info['duration'] // 1000,
         }
diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 3007b136f..b6e4a432b 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -37,7 +37,7 @@ class TudouIE(InfoExtractor):
     }]
 
     def _url_for_id(self, id, quality = None):
-        info_url = "http://v2.tudou.com/f?id=" +str(id)
+        info_url = "http://v2.tudou.com/f?id=" + str(id)
         if quality:
             info_url += '&hd' + quality
         webpage = self._download_webpage(info_url, id, "Opening the info webpage")
diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 26a51340f..38448e7c0 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -97,7 +97,7 @@ class XTubeUserIE(InfoExtractor):
             url, username, note='Retrieving profile page')
 
         video_count = int(self._search_regex(
-            r'<strong>%s\'s Videos \(([0-9]+)\)</strong>' %username, profile_page,
+            r'<strong>%s\'s Videos \(([0-9]+)\)</strong>' % username, profile_page,
             'video count'))
 
         PAGE_SIZE = 25
diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 514c16127..39caf60f2 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -229,7 +229,7 @@ class YahooSearchIE(SearchInfoExtractor):
         for pagenum in itertools.count(0):
             result_url = 'http://video.search.yahoo.com/search/?p=%s&fr=screen&o=js&gs=0&b=%d' % (compat_urllib_parse.quote_plus(query), pagenum * 30)
             info = self._download_json(result_url, query,
-                note='Downloading results page ' +str(pagenum +1))
+                note='Downloading results page ' + str(pagenum + 1))
             m = info['m']
             results = info['results']
 
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index d9f1cceb9..002a35c67 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -246,7 +246,7 @@ class FFmpegVideoConvertor(FFmpegPostProcessor):
         if information['ext'] == self._preferedformat:
             self._downloader.to_screen(u'[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
             return True, information
-        self._downloader.to_screen(u'[' +'ffmpeg' +'] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath)
+        self._downloader.to_screen(u'[' + 'ffmpeg' + '] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath)
         self.run_ffmpeg(path, outpath, [])
         information['filepath'] = outpath
         information['format'] = self._preferedformat
@@ -466,7 +466,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
 
         opts = ['-map', '0:0', '-map', '0:1', '-c:v', 'copy', '-c:a', 'copy']
         for (i, lang) in enumerate(sub_langs):
-            opts.extend(['-map', '%d:0' % (i +1), '-c:s:%d' % i, 'mov_text'])
+            opts.extend(['-map', '%d:0' % (i + 1), '-c:s:%d' % i, 'mov_text'])
             lang_code = self._conver_lang_code(lang)
             if lang_code is not None:
                 opts.extend(['-metadata:s:s:%d' % i, 'language=%s' % lang_code])
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 23457e07a..448ad229c 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -41,7 +41,7 @@ def rsa_verify(message, signature, key):
     signature = signature[2:]
     if not b('\x00') in signature:
         return False
-    signature = signature[signature.index(b('\x00')) +1:]
+    signature = signature[signature.index(b('\x00')) + 1:]
     if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')):
         return False
     signature = signature[19:]

From c66bdc48692c516c86481e115a0c84ab49630f62 Mon Sep 17 00:00:00 2001
From: Dao Hoang Son <daohoangson@gmail.com>
Date: Mon, 24 Nov 2014 03:25:47 +0700
Subject: [PATCH 0581/1937] [zingmp3] Added support for songs and albums

---
 youtube_dl/extractor/__init__.py |   4 ++
 youtube_dl/extractor/zingmp3.py  | 103 +++++++++++++++++++++++++++++++
 2 files changed, 107 insertions(+)
 create mode 100644 youtube_dl/extractor/zingmp3.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fcb75af34..3a520908b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -511,6 +511,10 @@ from .youtube import (
     YoutubeWatchLaterIE,
 )
 from .zdf import ZDFIE
+from .zingmp3 import (
+    ZingMp3SongIE,
+    ZingMp3AlbumIE,
+)
 
 _ALL_CLASSES = [
     klass
diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
new file mode 100644
index 000000000..2c2073adf
--- /dev/null
+++ b/youtube_dl/extractor/zingmp3.py
@@ -0,0 +1,103 @@
+# coding=utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class ZingMp3BaseInfoExtractor(InfoExtractor):
+
+    @staticmethod
+    def _extract_item(item):
+        title = item.find('./title').text.strip()
+        source = item.find('./source').text
+        extension = item.attrib['type']
+        thumbnail = item.find('./backimage').text
+
+        return {
+            'title': title,
+            'url': source,
+            'ext': extension,
+            'thumbnail': thumbnail,
+        }
+
+    def _extract_player_xml(self, player_xml_url, id, playlist_title=None):
+        player_xml = self._download_xml(player_xml_url, id, 'Downloading Player XML')
+
+        self.report_extraction(id)
+        items = player_xml.findall('./item')
+
+        if len(items) == 1:
+            # one single song
+            data = self._extract_item(items[0])
+            data['id'] = id
+
+            return data
+        else:
+            # playlist of songs
+            entries = []
+
+            for i, item in enumerate(items, 1):
+                entry = self._extract_item(item)
+                entry['id'] = '%s-%d' % (id, i)
+                entries.append(entry)
+
+            return {
+                '_type': 'playlist',
+                'id': id,
+                'title': playlist_title,
+                'entries': entries,
+            }
+
+
+class ZingMp3SongIE(ZingMp3BaseInfoExtractor):
+    _VALID_URL = r'https?://mp3\.zing\.vn/bai-hat/(?P<slug>[^/]+)/(?P<song_id>\w+)\.html'
+    _TESTS = [{
+        'url': 'http://mp3.zing.vn/bai-hat/Xa-Mai-Xa-Bao-Thy/ZWZB9WAB.html',
+        'info_dict': {
+            'id': 'ZWZB9WAB',
+            'title': u'Xa Mãi Xa',
+            'ext': 'mp3',
+        },
+        'md5': 'ead7ae13693b3205cbc89536a077daed',
+    }]
+    IE_NAME = 'zingmp3:song'
+    IE_DESC = 'mp3.zing.vn songs'
+
+    def _real_extract(self, url):
+        matched = re.match(self._VALID_URL, url)
+        slug = matched.group('slug')
+        song_id = matched.group('song_id')
+
+        webpage = self._download_webpage('http://mp3.zing.vn/bai-hat/%s/%s.html' % (slug, song_id), song_id)
+
+        player_xml_url = self._search_regex(r'&amp;xmlURL=(?P<xml_url>[^&]+)&', webpage, 'player xml url')
+
+        return self._extract_player_xml(player_xml_url, song_id)
+
+
+class ZingMp3AlbumIE(ZingMp3BaseInfoExtractor):
+    _VALID_URL = r'https?://mp3\.zing\.vn/album/(?P<slug>[^/]+)/(?P<album_id>\w+)\.html'
+    _TESTS = [{
+        'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': 'ZWZBWDAF',
+            'title': u'Lâu Đài Tình Ái - Bằng Kiều ft. Minh Tuyết | Album 320 lossless',
+        },
+        'playlist_count': 10,
+    }]
+    IE_NAME = 'zingmp3:album'
+    IE_DESC = 'mp3.zing.vn albums'
+
+    def _real_extract(self, url):
+        matched = re.match(self._VALID_URL, url)
+        slug = matched.group('slug')
+        album_id = matched.group('album_id')
+
+        webpage = self._download_webpage('http://mp3.zing.vn/album/%s/%s.html' % (slug, album_id), album_id)
+
+        player_xml_url = self._search_regex(r'&amp;xmlURL=(?P<xml_url>[^&]+)&', webpage, 'player xml url')
+
+        return self._extract_player_xml(player_xml_url, album_id, playlist_title=self._og_search_title(webpage))
\ No newline at end of file

From 9e1a5b845586a0a5431fb72467142046d8571e6f Mon Sep 17 00:00:00 2001
From: Jouke Waleson <jouke.waleson@mendix.com>
Date: Sun, 23 Nov 2014 21:39:15 +0100
Subject: [PATCH 0582/1937] PEP8: applied even more rules

---
 devscripts/buildserver.py                    |  2 +-
 devscripts/fish-completion.py                |  2 +-
 setup.py                                     |  1 -
 test/helper.py                               |  8 +-
 youtube_dl/YoutubeDL.py                      | 22 +++---
 youtube_dl/compat.py                         |  6 +-
 youtube_dl/downloader/f4m.py                 | 12 +--
 youtube_dl/extractor/appletrailers.py        |  2 +-
 youtube_dl/extractor/bambuser.py             |  6 +-
 youtube_dl/extractor/bbccouk.py              |  4 +-
 youtube_dl/extractor/cnn.py                  |  3 +-
 youtube_dl/extractor/collegehumor.py         | 79 ++++++++++----------
 youtube_dl/extractor/common.py               |  4 +-
 youtube_dl/extractor/dailymotion.py          |  4 +-
 youtube_dl/extractor/defense.py              |  6 +-
 youtube_dl/extractor/discovery.py            |  8 +-
 youtube_dl/extractor/dropbox.py              | 24 +++---
 youtube_dl/extractor/ehow.py                 |  2 +-
 youtube_dl/extractor/facebook.py             |  8 +-
 youtube_dl/extractor/firsttv.py              |  4 +-
 youtube_dl/extractor/fivemin.py              |  2 +-
 youtube_dl/extractor/fktv.py                 |  4 +-
 youtube_dl/extractor/flickr.py               |  2 +-
 youtube_dl/extractor/fourtube.py             |  2 +-
 youtube_dl/extractor/francetv.py             |  2 +-
 youtube_dl/extractor/generic.py              |  2 +-
 youtube_dl/extractor/howcast.py              |  4 +-
 youtube_dl/extractor/ign.py                  |  2 +-
 youtube_dl/extractor/instagram.py            |  4 +-
 youtube_dl/extractor/internetvideoarchive.py | 18 +++--
 youtube_dl/extractor/jukebox.py              |  6 +-
 youtube_dl/extractor/kickstarter.py          |  6 +-
 youtube_dl/extractor/lynda.py                |  2 +-
 youtube_dl/extractor/m6.py                   |  2 +-
 youtube_dl/extractor/metacafe.py             |  4 +-
 youtube_dl/extractor/metacritic.py           |  4 +-
 youtube_dl/extractor/mtv.py                  | 12 +--
 youtube_dl/extractor/myspace.py              |  4 +-
 youtube_dl/extractor/myvideo.py              |  4 +-
 youtube_dl/extractor/naver.py                |  2 +-
 youtube_dl/extractor/nfb.py                  |  6 +-
 youtube_dl/extractor/nhl.py                  |  2 +-
 youtube_dl/extractor/niconico.py             |  6 +-
 youtube_dl/extractor/ninegag.py              |  3 +-
 youtube_dl/extractor/normalboots.py          |  4 +-
 youtube_dl/extractor/ooyala.py               |  2 +-
 youtube_dl/extractor/photobucket.py          |  2 +-
 youtube_dl/extractor/rbmaradio.py            |  2 +-
 youtube_dl/extractor/sbs.py                  |  3 +-
 youtube_dl/extractor/screencast.py           |  2 +-
 youtube_dl/extractor/sina.py                 |  2 +-
 youtube_dl/extractor/slutload.py             |  2 +-
 youtube_dl/extractor/smotri.py               |  2 +-
 youtube_dl/extractor/soundcloud.py           |  2 +-
 youtube_dl/extractor/stanfordoc.py           |  2 +-
 youtube_dl/extractor/teamcoco.py             | 33 ++++----
 youtube_dl/extractor/ted.py                  | 10 +--
 youtube_dl/extractor/tf1.py                  |  2 +-
 youtube_dl/extractor/theplatform.py          |  2 +-
 youtube_dl/extractor/tinypic.py              |  2 +-
 youtube_dl/extractor/traileraddict.py        |  6 +-
 youtube_dl/extractor/tumblr.py               |  2 +-
 youtube_dl/extractor/udemy.py                |  2 +-
 youtube_dl/extractor/ustream.py              |  6 +-
 youtube_dl/extractor/vbox7.py                |  6 +-
 youtube_dl/extractor/veehd.py                |  6 +-
 youtube_dl/extractor/vesti.py                |  2 +-
 youtube_dl/extractor/videofyme.py            |  4 +-
 youtube_dl/extractor/vimeo.py                |  2 +-
 youtube_dl/extractor/vk.py                   |  4 +-
 youtube_dl/extractor/weibo.py                |  2 +-
 youtube_dl/extractor/xhamster.py             |  6 +-
 youtube_dl/extractor/xnxx.py                 |  6 +-
 youtube_dl/extractor/yahoo.py                |  2 +-
 youtube_dl/extractor/youku.py                |  2 +-
 youtube_dl/extractor/youporn.py              |  2 +-
 youtube_dl/extractor/youtube.py              |  4 +-
 77 files changed, 227 insertions(+), 223 deletions(-)

diff --git a/devscripts/buildserver.py b/devscripts/buildserver.py
index 42ee2b5cb..7c2f49f8b 100644
--- a/devscripts/buildserver.py
+++ b/devscripts/buildserver.py
@@ -142,7 +142,7 @@ def win_service_set_status(handle, status_code):
 
 def win_service_main(service_name, real_main, argc, argv_raw):
     try:
-        #args = [argv_raw[i].value for i in range(argc)]
+        # args = [argv_raw[i].value for i in range(argc)]
         stop_event = threading.Event()
         handler = HandlerEx(functools.partial(stop_event, win_service_handler))
         h = advapi32.RegisterServiceCtrlHandlerExW(service_name, handler, None)
diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py
index 2185d5522..f2445984f 100755
--- a/devscripts/fish-completion.py
+++ b/devscripts/fish-completion.py
@@ -30,7 +30,7 @@ def build_completion(opt_parser):
     for group in opt_parser.option_groups:
         for option in group.option_list:
             long_option = option.get_opt_string().strip('-')
-            help_msg = shell_quote([option.help])
+            shell_quote([option.help])
             complete_cmd = ['complete', '--command', 'youtube-dl', '--long-option', long_option]
             if option._short_opts:
                 complete_cmd += ['--short-option', option._short_opts[0].strip('-')]
diff --git a/setup.py b/setup.py
index cf6b92b0f..5a0d084cc 100644
--- a/setup.py
+++ b/setup.py
@@ -4,7 +4,6 @@
 from __future__ import print_function
 
 import os.path
-import pkg_resources
 import warnings
 import sys
 
diff --git a/test/helper.py b/test/helper.py
index 91822935f..2e320e2cf 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -116,14 +116,14 @@ def expect_info_dict(self, expected_dict, got_dict):
         elif isinstance(expected, type):
             got = got_dict.get(info_field)
             self.assertTrue(isinstance(got, expected),
-                'Expected type %r for field %s, but got value %r of type %r' % (expected, info_field, got, type(got)))
+                            'Expected type %r for field %s, but got value %r of type %r' % (expected, info_field, got, type(got)))
         else:
             if isinstance(expected, compat_str) and expected.startswith('md5:'):
                 got = 'md5:' + md5(got_dict.get(info_field))
             else:
                 got = got_dict.get(info_field)
             self.assertEqual(expected, got,
-                'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
+                             'invalid value for field %s, expected %r, got %r' % (info_field, expected, got))
 
     # Check for the presence of mandatory fields
     if got_dict.get('_type') != 'playlist':
@@ -135,8 +135,8 @@ def expect_info_dict(self, expected_dict, got_dict):
 
     # Are checkable fields missing from the test case definition?
     test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value))
-        for key, value in got_dict.items()
-        if value and key in ('title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location'))
+                          for key, value in got_dict.items()
+                          if value and key in ('title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location'))
     missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys())
     if missing_keys:
         def _repr(v):
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d9e93f5d2..21c7c298a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -314,7 +314,7 @@ class YoutubeDL(object):
         self._output_process.stdin.write((message + '\n').encode('utf-8'))
         self._output_process.stdin.flush()
         res = ''.join(self._output_channel.readline().decode('utf-8')
-                       for _ in range(line_count))
+                      for _ in range(line_count))
         return res[:-len('\n')]
 
     def to_screen(self, message, skip_eol=False):
@@ -701,13 +701,15 @@ class YoutubeDL(object):
                 'It needs to be updated.' % ie_result.get('extractor'))
 
             def _fixup(r):
-                self.add_extra_info(r,
+                self.add_extra_info(
+                    r,
                     {
                         'extractor': ie_result['extractor'],
                         'webpage_url': ie_result['webpage_url'],
                         'webpage_url_basename': url_basename(ie_result['webpage_url']),
                         'extractor_key': ie_result['extractor_key'],
-                    })
+                    }
+                )
                 return r
             ie_result['entries'] = [
                 self.process_ie_result(_fixup(r), download, extra_info)
@@ -857,14 +859,14 @@ class YoutubeDL(object):
                         # Two formats have been requested like '137+139'
                         format_1, format_2 = rf.split('+')
                         formats_info = (self.select_format(format_1, formats),
-                            self.select_format(format_2, formats))
+                                        self.select_format(format_2, formats))
                         if all(formats_info):
                             # The first format must contain the video and the
                             # second the audio
                             if formats_info[0].get('vcodec') == 'none':
                                 self.report_error('The first format must '
-                                    'contain the video, try using '
-                                    '"-f %s+%s"' % (format_2, format_1))
+                                                  'contain the video, try using '
+                                                  '"-f %s+%s"' % (format_2, format_1))
                                 return
                             selected_format = {
                                 'requested_formats': formats_info,
@@ -1042,10 +1044,10 @@ class YoutubeDL(object):
                         with open(thumb_filename, 'wb') as thumbf:
                             shutil.copyfileobj(uf, thumbf)
                         self.to_screen('[%s] %s: Writing thumbnail to: %s' %
-                            (info_dict['extractor'], info_dict['id'], thumb_filename))
+                                       (info_dict['extractor'], info_dict['id'], thumb_filename))
                     except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
                         self.report_warning('Unable to download thumbnail "%s": %s' %
-                            (info_dict['thumbnail'], compat_str(err)))
+                                            (info_dict['thumbnail'], compat_str(err)))
 
         if not self.params.get('skip_download', False):
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
@@ -1066,8 +1068,8 @@ class YoutubeDL(object):
                         if not merger._executable:
                             postprocessors = []
                             self.report_warning('You have requested multiple '
-                                'formats but ffmpeg or avconv are not installed.'
-                                ' The formats won\'t be merged')
+                                                'formats but ffmpeg or avconv are not installed.'
+                                                ' The formats won\'t be merged')
                         else:
                             postprocessors = [merger]
                         for f in info_dict['requested_formats']:
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9087b4f85..2983501ae 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -116,7 +116,7 @@ except ImportError:  # Python 2
     # Python 2's version is apparently totally broken
 
     def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
-                encoding='utf-8', errors='replace'):
+                   encoding='utf-8', errors='replace'):
         qs, _coerce_result = qs, unicode
         pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
         r = []
@@ -145,10 +145,10 @@ except ImportError:  # Python 2
         return r
 
     def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
-                encoding='utf-8', errors='replace'):
+                        encoding='utf-8', errors='replace'):
         parsed_result = {}
         pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
-                        encoding=encoding, errors=errors)
+                           encoding=encoding, errors=errors)
         for name, value in pairs:
             if name in parsed_result:
                 parsed_result[name].append(value)
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index c752e8e24..7cd22c504 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -225,13 +225,15 @@ class F4mFD(FileDownloader):
         self.to_screen('[download] Downloading f4m manifest')
         manifest = self.ydl.urlopen(man_url).read()
         self.report_destination(filename)
-        http_dl = HttpQuietDownloader(self.ydl,
+        http_dl = HttpQuietDownloader(
+            self.ydl,
             {
                 'continuedl': True,
                 'quiet': True,
                 'noprogress': True,
                 'test': self.params.get('test', False),
-            })
+            }
+        )
 
         doc = etree.fromstring(manifest)
         formats = [(int(f.attrib.get('bitrate', -1)), f) for f in doc.findall(_add_ns('media'))]
@@ -277,7 +279,7 @@ class F4mFD(FileDownloader):
         def frag_progress_hook(status):
             frag_total_bytes = status.get('total_bytes', 0)
             estimated_size = (state['downloaded_bytes'] +
-                (total_frags - state['frag_counter']) * frag_total_bytes)
+                              (total_frags - state['frag_counter']) * frag_total_bytes)
             if status['status'] == 'finished':
                 state['downloaded_bytes'] += frag_total_bytes
                 state['frag_counter'] += 1
@@ -287,13 +289,13 @@ class F4mFD(FileDownloader):
                 frag_downloaded_bytes = status['downloaded_bytes']
                 byte_counter = state['downloaded_bytes'] + frag_downloaded_bytes
                 frag_progress = self.calc_percent(frag_downloaded_bytes,
-                    frag_total_bytes)
+                                                  frag_total_bytes)
                 progress = self.calc_percent(state['frag_counter'], total_frags)
                 progress += frag_progress / float(total_frags)
 
             eta = self.calc_eta(start, time.time(), estimated_size, byte_counter)
             self.report_progress(progress, format_bytes(estimated_size),
-                status.get('speed'), eta)
+                                 status.get('speed'), eta)
         http_dl.add_progress_hook(frag_progress_hook)
 
         frags_filenames = []
diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 567a76cf0..87580147d 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -88,7 +88,7 @@ class AppleTrailersIE(InfoExtractor):
         for li in doc.findall('./div/ul/li'):
             on_click = li.find('.//a').attrib['onClick']
             trailer_info_json = self._search_regex(self._JSON_RE,
-                on_click, 'trailer info')
+                                                   on_click, 'trailer info')
             trailer_info = json.loads(trailer_info_json)
             title = trailer_info['title']
             video_id = movie + '-' + re.sub(r'[^a-zA-Z0-9]', '', title).lower()
diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index eab99faaa..928ea61a3 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -38,7 +38,7 @@ class BambuserIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         info_url = ('http://player-c.api.bambuser.com/getVideo.json?'
-            '&api_key=%s&vid=%s' % (self._API_KEY, video_id))
+                    '&api_key=%s&vid=%s' % (self._API_KEY, video_id))
         info_json = self._download_webpage(info_url, video_id)
         info = json.loads(info_json)['result']
 
@@ -74,8 +74,8 @@ class BambuserChannelIE(InfoExtractor):
         last_id = ''
         for i in itertools.count(1):
             req_url = ('http://bambuser.com/xhr-api/index.php?username={user}'
-                '&sort=created&access_mode=0%2C1%2C2&limit={count}'
-                '&method=broadcast&format=json&vid_older_than={last}'
+                       '&sort=created&access_mode=0%2C1%2C2&limit={count}'
+                       '&method=broadcast&format=json&vid_older_than={last}'
                 ).format(user=user, count=self._STEP, last=last_id)
             req = compat_urllib_request.Request(req_url)
             # Without setting this header, we wouldn't get any result
diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index fa15bf19c..6a507e113 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -165,10 +165,10 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
         webpage = self._download_webpage(url, group_id, 'Downloading video page')
         if re.search(r'id="emp-error" class="notinuk">', webpage):
             raise ExtractorError('Currently BBC iPlayer TV programmes are available to play in the UK only',
-                expected=True)
+                                 expected=True)
 
         playlist = self._download_xml('http://www.bbc.co.uk/iplayer/playlist/%s' % group_id, group_id,
-            'Downloading playlist XML')
+                                      'Downloading playlist XML')
 
         no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
         if no_items is not None:
diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 3826ce7e1..81142ee41 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -25,8 +25,7 @@ class CNNIE(InfoExtractor):
             'duration': 135,
             'upload_date': '20130609',
         },
-    },
-    {
+    }, {
         "url": "http://edition.cnn.com/video/?/video/us/2013/08/21/sot-student-gives-epic-speech.georgia-institute-of-technology&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+rss%2Fcnn_topstories+%28RSS%3A+Top+Stories%29",
         "md5": "b5cc60c60a3477d185af8f19a2a26f4e",
         "info_dict": {
diff --git a/youtube_dl/extractor/collegehumor.py b/youtube_dl/extractor/collegehumor.py
index 6f866e7fc..002b24037 100644
--- a/youtube_dl/extractor/collegehumor.py
+++ b/youtube_dl/extractor/collegehumor.py
@@ -10,47 +10,46 @@ from ..utils import int_or_none
 class CollegeHumorIE(InfoExtractor):
     _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/(video|embed|e)/(?P<videoid>[0-9]+)/?(?P<shorttitle>.*)$'
 
-    _TESTS = [{
-        'url': 'http://www.collegehumor.com/video/6902724/comic-con-cosplay-catastrophe',
-        'md5': 'dcc0f5c1c8be98dc33889a191f4c26bd',
-        'info_dict': {
-            'id': '6902724',
-            'ext': 'mp4',
-            'title': 'Comic-Con Cosplay Catastrophe',
-            'description': "Fans get creative this year at San Diego.  Too creative.  And yes, that's really Joss Whedon.",
-            'age_limit': 13,
-            'duration': 187,
+    _TESTS = [
+        {
+            'url': 'http://www.collegehumor.com/video/6902724/comic-con-cosplay-catastrophe',
+            'md5': 'dcc0f5c1c8be98dc33889a191f4c26bd',
+            'info_dict': {
+                'id': '6902724',
+                'ext': 'mp4',
+                'title': 'Comic-Con Cosplay Catastrophe',
+                'description': "Fans get creative this year at San Diego.  Too creative.  And yes, that's really Joss Whedon.",
+                'age_limit': 13,
+                'duration': 187,
+            },
+        }, {
+            'url': 'http://www.collegehumor.com/video/3505939/font-conference',
+            'md5': '72fa701d8ef38664a4dbb9e2ab721816',
+            'info_dict': {
+                'id': '3505939',
+                'ext': 'mp4',
+                'title': 'Font Conference',
+                'description': "This video wasn't long enough, so we made it double-spaced.",
+                'age_limit': 10,
+                'duration': 179,
+            },
+        }, {
+            # embedded youtube video
+            'url': 'http://www.collegehumor.com/embed/6950306',
+            'info_dict': {
+                'id': 'Z-bao9fg6Yc',
+                'ext': 'mp4',
+                'title': 'Young Americans Think President John F. Kennedy Died THIS MORNING IN A CAR ACCIDENT!!!',
+                'uploader': 'Mark Dice',
+                'uploader_id': 'MarkDice',
+                'description': 'md5:62c3dab9351fac7bb44b53b69511d87f',
+                'upload_date': '20140127',
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'add_ie': ['Youtube'],
         },
-    },
-    {
-        'url': 'http://www.collegehumor.com/video/3505939/font-conference',
-        'md5': '72fa701d8ef38664a4dbb9e2ab721816',
-        'info_dict': {
-            'id': '3505939',
-            'ext': 'mp4',
-            'title': 'Font Conference',
-            'description': "This video wasn't long enough, so we made it double-spaced.",
-            'age_limit': 10,
-            'duration': 179,
-        },
-    },
-    # embedded youtube video
-    {
-        'url': 'http://www.collegehumor.com/embed/6950306',
-        'info_dict': {
-            'id': 'Z-bao9fg6Yc',
-            'ext': 'mp4',
-            'title': 'Young Americans Think President John F. Kennedy Died THIS MORNING IN A CAR ACCIDENT!!!',
-            'uploader': 'Mark Dice',
-            'uploader_id': 'MarkDice',
-            'description': 'md5:62c3dab9351fac7bb44b53b69511d87f',
-            'upload_date': '20140127',
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'add_ie': ['Youtube'],
-    },
     ]
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f0489ede4..3c2d46dd5 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -478,7 +478,7 @@ class InfoExtractor(object):
             raise RegexNotFoundError('Unable to extract %s' % _name)
         else:
             self._downloader.report_warning('unable to extract %s; '
-                'please report this issue on http://yt-dl.org/bug' % _name)
+                                            'please report this issue on http://yt-dl.org/bug' % _name)
             return None
 
     def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
@@ -612,7 +612,7 @@ class InfoExtractor(object):
 
     def _twitter_search_player(self, html):
         return self._html_search_meta('twitter:player', html,
-            'twitter card player')
+                                      'twitter card player')
 
     def _sort_formats(self, formats):
         if not formats:
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 22cdcdfa5..936c13cd6 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -114,7 +114,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
         embed_page = self._download_webpage(embed_url, video_id,
                                             'Downloading embed page')
         info = self._search_regex(r'var info = ({.*?}),$', embed_page,
-            'video info', flags=re.MULTILINE)
+                                  'video info', flags=re.MULTILINE)
         info = json.loads(info)
         if info.get('error') is not None:
             msg = 'Couldn\'t get video, Dailymotion says: %s' % info['error']['title']
@@ -208,7 +208,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
             if re.search(self._MORE_PAGES_INDICATOR, webpage) is None:
                 break
         return [self.url_result('http://www.dailymotion.com/video/%s' % video_id, 'Dailymotion')
-                   for video_id in orderedSet(video_ids)]
+                for video_id in orderedSet(video_ids)]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
diff --git a/youtube_dl/extractor/defense.py b/youtube_dl/extractor/defense.py
index 3ffed3d44..5e50c63d9 100644
--- a/youtube_dl/extractor/defense.py
+++ b/youtube_dl/extractor/defense.py
@@ -9,7 +9,7 @@ from .common import InfoExtractor
 class DefenseGouvFrIE(InfoExtractor):
     IE_NAME = 'defense.gouv.fr'
     _VALID_URL = (r'http://.*?\.defense\.gouv\.fr/layout/set/'
-        r'ligthboxvideo/base-de-medias/webtv/(.*)')
+                  r'ligthboxvideo/base-de-medias/webtv/(.*)')
 
     _TEST = {
         'url': 'http://www.defense.gouv.fr/layout/set/ligthboxvideo/base-de-medias/webtv/attaque-chimique-syrienne-du-21-aout-2013-1',
@@ -28,9 +28,9 @@ class DefenseGouvFrIE(InfoExtractor):
             webpage, 'ID')
 
         json_url = ('http://static.videos.gouv.fr/brightcovehub/export/json/'
-            + video_id)
+                    + video_id)
         info = self._download_webpage(json_url, title,
-                                                  'Downloading JSON config')
+                                      'Downloading JSON config')
         video_url = json.loads(info)['renditions'][0]['url']
 
         return {'id': video_id,
diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index 554df6735..52c2d7ddf 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -16,9 +16,9 @@ class DiscoveryIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'MythBusters: Mission Impossible Outtakes',
             'description': ('Watch Jamie Hyneman and Adam Savage practice being'
-                ' each other -- to the point of confusing Jamie\'s dog -- and '
-                'don\'t miss Adam moon-walking as Jamie ... behind Jamie\'s'
-                ' back.'),
+                            ' each other -- to the point of confusing Jamie\'s dog -- and '
+                            'don\'t miss Adam moon-walking as Jamie ... behind Jamie\'s'
+                            ' back.'),
             'duration': 156,
         },
     }
@@ -29,7 +29,7 @@ class DiscoveryIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         video_list_json = self._search_regex(r'var videoListJSON = ({.*?});',
-            webpage, 'video list', flags=re.DOTALL)
+                                             webpage, 'video list', flags=re.DOTALL)
         video_list = json.loads(video_list_json)
         info = video_list['clips'][0]
         formats = []
diff --git a/youtube_dl/extractor/dropbox.py b/youtube_dl/extractor/dropbox.py
index aefca848a..14b6c00b0 100644
--- a/youtube_dl/extractor/dropbox.py
+++ b/youtube_dl/extractor/dropbox.py
@@ -11,18 +11,18 @@ from ..utils import url_basename
 
 class DropboxIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dropbox[.]com/sh?/(?P<id>[a-zA-Z0-9]{15})/.*'
-    _TESTS = [{
-        'url': 'https://www.dropbox.com/s/nelirfsxnmcfbfh/youtube-dl%20test%20video%20%27%C3%A4%22BaW_jenozKc.mp4?dl=0',
-        'info_dict': {
-            'id': 'nelirfsxnmcfbfh',
-            'ext': 'mp4',
-            'title': 'youtube-dl test video \'ä"BaW_jenozKc'
-        }
-    },
-    {
-        'url': 'https://www.dropbox.com/sh/662glsejgzoj9sr/AAByil3FGH9KFNZ13e08eSa1a/Pregame%20Ceremony%20Program%20PA%2020140518.m4v',
-        'only_matching': True,
-    },
+    _TESTS = [
+        {
+            'url': 'https://www.dropbox.com/s/nelirfsxnmcfbfh/youtube-dl%20test%20video%20%27%C3%A4%22BaW_jenozKc.mp4?dl=0',
+            'info_dict': {
+                'id': 'nelirfsxnmcfbfh',
+                'ext': 'mp4',
+                'title': 'youtube-dl test video \'ä"BaW_jenozKc'
+            }
+        }, {
+            'url': 'https://www.dropbox.com/sh/662glsejgzoj9sr/AAByil3FGH9KFNZ13e08eSa1a/Pregame%20Ceremony%20Program%20PA%2020140518.m4v',
+            'only_matching': True,
+        },
     ]
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/ehow.py b/youtube_dl/extractor/ehow.py
index f8f49a013..b766e17f2 100644
--- a/youtube_dl/extractor/ehow.py
+++ b/youtube_dl/extractor/ehow.py
@@ -28,7 +28,7 @@ class EHowIE(InfoExtractor):
         video_id = mobj.group('id')
         webpage = self._download_webpage(url, video_id)
         video_url = self._search_regex(r'(?:file|source)=(http[^\'"&]*)',
-            webpage, 'video URL')
+                                       webpage, 'video URL')
         final_url = compat_urllib_parse.unquote(video_url)
         uploader = self._html_search_meta('uploader', webpage)
         title = self._og_search_title(webpage).replace(' | eHow', '')
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 104803563..c989879ba 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -60,8 +60,8 @@ class FacebookIE(InfoExtractor):
         login_page_req = compat_urllib_request.Request(self._LOGIN_URL)
         login_page_req.add_header('Cookie', 'locale=en_US')
         login_page = self._download_webpage(login_page_req, None,
-            note='Downloading login page',
-            errnote='Unable to download login page')
+                                            note='Downloading login page',
+                                            errnote='Unable to download login page')
         lsd = self._search_regex(
             r'<input type="hidden" name="lsd" value="([^"]*)"',
             login_page, 'lsd')
@@ -82,7 +82,7 @@ class FacebookIE(InfoExtractor):
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         try:
             login_results = self._download_webpage(request, None,
-                note='Logging in', errnote='unable to fetch login page')
+                                                   note='Logging in', errnote='unable to fetch login page')
             if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
                 self._downloader.report_warning('unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
                 return
@@ -96,7 +96,7 @@ class FacebookIE(InfoExtractor):
             check_req = compat_urllib_request.Request(self._CHECKPOINT_URL, urlencode_postdata(check_form))
             check_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
             check_response = self._download_webpage(check_req, None,
-                note='Confirming login')
+                                                    note='Confirming login')
             if re.search(r'id="checkpointSubmitButton"', check_response) is not None:
                 self._downloader.report_warning('Unable to confirm login, you have to login in your brower and authorize the login.')
         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
diff --git a/youtube_dl/extractor/firsttv.py b/youtube_dl/extractor/firsttv.py
index 3410daa98..08ceee4ed 100644
--- a/youtube_dl/extractor/firsttv.py
+++ b/youtube_dl/extractor/firsttv.py
@@ -44,9 +44,9 @@ class FirstTVIE(InfoExtractor):
         duration = self._og_search_property('video:duration', webpage, 'video duration', fatal=False)
 
         like_count = self._html_search_regex(r'title="Понравилось".*?/></label> \[(\d+)\]',
-            webpage, 'like count', fatal=False)
+                                             webpage, 'like count', fatal=False)
         dislike_count = self._html_search_regex(r'title="Не понравилось".*?/></label> \[(\d+)\]',
-            webpage, 'dislike count', fatal=False)
+                                                webpage, 'dislike count', fatal=False)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/fivemin.py b/youtube_dl/extractor/fivemin.py
index 3a50bab5c..f9c127ce6 100644
--- a/youtube_dl/extractor/fivemin.py
+++ b/youtube_dl/extractor/fivemin.py
@@ -50,7 +50,7 @@ class FiveMinIE(InfoExtractor):
         video_id = mobj.group('id')
         embed_url = 'https://embed.5min.com/playerseed/?playList=%s' % video_id
         embed_page = self._download_webpage(embed_url, video_id,
-            'Downloading embed page')
+                                            'Downloading embed page')
         sid = self._search_regex(r'sid=(\d+)', embed_page, 'sid')
         query = compat_urllib_parse.urlencode({
             'func': 'GetResults',
diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index 21b89142c..d09d1c13a 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -32,9 +32,9 @@ class FKTVIE(InfoExtractor):
         server = random.randint(2, 4)
         video_thumbnail = 'http://fernsehkritik.tv/images/magazin/folge%d.jpg' % episode
         start_webpage = self._download_webpage('http://fernsehkritik.tv/folge-%d/Start' % episode,
-            episode)
+                                               episode)
         playlist = self._search_regex(r'playlist = (\[.*?\]);', start_webpage,
-            'playlist', flags=re.DOTALL)
+                                      'playlist', flags=re.DOTALL)
         files = json.loads(re.sub('{[^{}]*?}', '{}', playlist))
         # TODO: return a single multipart video
         videos = []
diff --git a/youtube_dl/extractor/flickr.py b/youtube_dl/extractor/flickr.py
index 5b0bc9d21..0c858b654 100644
--- a/youtube_dl/extractor/flickr.py
+++ b/youtube_dl/extractor/flickr.py
@@ -37,7 +37,7 @@ class FlickrIE(InfoExtractor):
         first_xml = self._download_webpage(first_url, video_id, 'Downloading first data webpage')
 
         node_id = self._html_search_regex(r'<Item id="id">(\d+-\d+)</Item>',
-            first_xml, 'node_id')
+                                          first_xml, 'node_id')
 
         second_url = 'https://secure.flickr.com/video_playlist.gne?node_id=' + node_id + '&tech=flash&mode=playlist&bitrate=700&secret=' + secret + '&rd=video.yahoo.com&noad=1'
         second_xml = self._download_webpage(second_url, video_id, 'Downloading second data webpage')
diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index 24d4e9754..701241bb0 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -55,7 +55,7 @@ class FourTubeIE(InfoExtractor):
         description = self._html_search_meta('description', webpage, 'description')
         if description:
             upload_date = self._search_regex(r'Published Date: (\d{2} [a-zA-Z]{3} \d{4})', description, 'upload date',
-                fatal=False)
+                                             fatal=False)
             if upload_date:
                 upload_date = unified_strdate(upload_date)
             view_count = self._search_regex(r'Views: ([\d,\.]+)', description, 'view count', fatal=False)
diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 35d7d15e1..d7e9aef90 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -234,7 +234,7 @@ class GenerationQuoiIE(InfoExtractor):
         info_json = self._download_webpage(info_url, name)
         info = json.loads(info_json)
         return self.url_result('http://www.dailymotion.com/video/%s' % info['id'],
-            ie='Dailymotion')
+                               ie='Dailymotion')
 
 
 class CultureboxIE(FranceTVBaseInfoExtractor):
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 109dd20db..d224aa8e1 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -784,7 +784,7 @@ class GenericIE(InfoExtractor):
 
         # Look for Ooyala videos
         mobj = (re.search(r'player.ooyala.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
-             re.search(r'OO.Player.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage))
+                re.search(r'OO.Player.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage))
         if mobj is not None:
             return OoyalaIE._build_url_result(mobj.group('ec'))
 
diff --git a/youtube_dl/extractor/howcast.py b/youtube_dl/extractor/howcast.py
index 4ddf06409..3f7d6666c 100644
--- a/youtube_dl/extractor/howcast.py
+++ b/youtube_dl/extractor/howcast.py
@@ -27,10 +27,10 @@ class HowcastIE(InfoExtractor):
         self.report_extraction(video_id)
 
         video_url = self._search_regex(r'\'?file\'?: "(http://mobile-media\.howcast\.com/[0-9]+\.mp4)',
-            webpage, 'video URL')
+                                       webpage, 'video URL')
 
         video_description = self._html_search_regex(r'<meta content=(?:"([^"]+)"|\'([^\']+)\') name=\'description\'',
-            webpage, 'description', fatal=False)
+                                                    webpage, 'description', fatal=False)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index c80185b53..3555f98a5 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -99,7 +99,7 @@ class IGNIE(InfoExtractor):
         video_id = self._find_video_id(webpage)
         result = self._get_video_info(video_id)
         description = self._html_search_regex(self._DESCRIPTION_RE,
-            webpage, 'video description', flags=re.DOTALL)
+                                              webpage, 'video description', flags=re.DOTALL)
         result['description'] = description
         return result
 
diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 5109f26ce..b020e2621 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -27,9 +27,9 @@ class InstagramIE(InfoExtractor):
         video_id = mobj.group('id')
         webpage = self._download_webpage(url, video_id)
         uploader_id = self._search_regex(r'"owner":{"username":"(.+?)"',
-            webpage, 'uploader id', fatal=False)
+                                         webpage, 'uploader id', fatal=False)
         desc = self._search_regex(r'"caption":"(.*?)"', webpage, 'description',
-            fatal=False)
+                                  fatal=False)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/internetvideoarchive.py b/youtube_dl/extractor/internetvideoarchive.py
index c137f4a5d..1e4799187 100644
--- a/youtube_dl/extractor/internetvideoarchive.py
+++ b/youtube_dl/extractor/internetvideoarchive.py
@@ -45,22 +45,26 @@ class InternetVideoArchiveIE(InfoExtractor):
         url = self._build_url(query)
 
         flashconfiguration = self._download_xml(url, video_id,
-            'Downloading flash configuration')
+                                                'Downloading flash configuration')
         file_url = flashconfiguration.find('file').text
         file_url = file_url.replace('/playlist.aspx', '/mrssplaylist.aspx')
         # Replace some of the parameters in the query to get the best quality
         # and http links (no m3u8 manifests)
         file_url = re.sub(r'(?<=\?)(.+)$',
-            lambda m: self._clean_query(m.group()),
-            file_url)
+                          lambda m: self._clean_query(m.group()),
+                          file_url)
         info = self._download_xml(file_url, video_id,
-            'Downloading video info')
+                                  'Downloading video info')
         item = info.find('channel/item')
 
         def _bp(p):
-            return xpath_with_ns(p,
-                {'media': 'http://search.yahoo.com/mrss/',
-                'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats'})
+            return xpath_with_ns(
+                p,
+                {
+                    'media': 'http://search.yahoo.com/mrss/',
+                    'jwplayer': 'http://developer.longtailvideo.com/trac/wiki/FlashFormats',
+                }
+            )
         formats = []
         for content in item.findall(_bp('media:group/media:content')):
             attr = content.attrib
diff --git a/youtube_dl/extractor/jukebox.py b/youtube_dl/extractor/jukebox.py
index 5aa32bf09..da8068efc 100644
--- a/youtube_dl/extractor/jukebox.py
+++ b/youtube_dl/extractor/jukebox.py
@@ -36,7 +36,7 @@ class JukeboxIE(InfoExtractor):
 
         try:
             video_url = self._search_regex(r'"config":{"file":"(?P<video_url>http:[^"]+\?mdtk=[0-9]+)"',
-                iframe_html, 'video url')
+                                           iframe_html, 'video url')
             video_url = unescapeHTML(video_url).replace('\/', '/')
         except RegexNotFoundError:
             youtube_url = self._search_regex(
@@ -47,9 +47,9 @@ class JukeboxIE(InfoExtractor):
             return self.url_result(youtube_url, ie='Youtube')
 
         title = self._html_search_regex(r'<h1 class="inline">([^<]+)</h1>',
-            html, 'title')
+                                        html, 'title')
         artist = self._html_search_regex(r'<span id="infos_article_artist">([^<]+)</span>',
-            html, 'artist')
+                                         html, 'artist')
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/kickstarter.py b/youtube_dl/extractor/kickstarter.py
index 827091e60..7d4b57056 100644
--- a/youtube_dl/extractor/kickstarter.py
+++ b/youtube_dl/extractor/kickstarter.py
@@ -13,8 +13,10 @@ class KickStarterIE(InfoExtractor):
             'id': '1404461844',
             'ext': 'mp4',
             'title': 'Intersection: The Story of Josh Grant by Kyle Cowling',
-            'description': 'A unique motocross documentary that examines the '
-                'life and mind of one of sports most elite athletes: Josh Grant.',
+            'description': (
+                'A unique motocross documentary that examines the '
+                'life and mind of one of sports most elite athletes: Josh Grant.'
+            ),
         },
     }, {
         'note': 'Embedded video (not using the native kickstarter video service)',
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 97ca4337b..2160d6cb0 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -45,7 +45,7 @@ class LyndaIE(SubtitlesInfoExtractor):
         video_id = mobj.group(1)
 
         page = self._download_webpage('http://www.lynda.com/ajax/player?videoId=%s&type=video' % video_id, video_id,
-            'Downloading video JSON')
+                                      'Downloading video JSON')
         video_json = json.loads(page)
 
         if 'Status' in video_json:
diff --git a/youtube_dl/extractor/m6.py b/youtube_dl/extractor/m6.py
index 3d806323a..7e025831b 100644
--- a/youtube_dl/extractor/m6.py
+++ b/youtube_dl/extractor/m6.py
@@ -27,7 +27,7 @@ class M6IE(InfoExtractor):
         video_id = mobj.group('id')
 
         rss = self._download_xml('http://ws.m6.fr/v1/video/info/m6/bonus/%s' % video_id, video_id,
-            'Downloading video RSS')
+                                 'Downloading video RSS')
 
         title = rss.find('./channel/item/title').text
         description = rss.find('./channel/item/description').text
diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index f68add6c0..858c1c0c3 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -219,8 +219,8 @@ class MetacafeIE(InfoExtractor):
         description = self._og_search_description(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
         video_uploader = self._html_search_regex(
-                r'submitter=(.*?);|googletag\.pubads\(\)\.setTargeting\("(?:channel|submiter)","([^"]+)"\);',
-                webpage, 'uploader nickname', fatal=False)
+            r'submitter=(.*?);|googletag\.pubads\(\)\.setTargeting\("(?:channel|submiter)","([^"]+)"\);',
+            webpage, 'uploader nickname', fatal=False)
         duration = int_or_none(
             self._html_search_meta('video:duration', webpage))
 
diff --git a/youtube_dl/extractor/metacritic.py b/youtube_dl/extractor/metacritic.py
index 07f072924..e30320569 100644
--- a/youtube_dl/extractor/metacritic.py
+++ b/youtube_dl/extractor/metacritic.py
@@ -28,7 +28,7 @@ class MetacriticIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         # The xml is not well formatted, there are raw '&'
         info = self._download_xml('http://www.metacritic.com/video_data?video=' + video_id,
-            video_id, 'Downloading info xml', transform_source=fix_xml_ampersands)
+                                  video_id, 'Downloading info xml', transform_source=fix_xml_ampersands)
 
         clip = next(c for c in info.findall('playList/clip') if c.find('id').text == video_id)
         formats = []
@@ -44,7 +44,7 @@ class MetacriticIE(InfoExtractor):
         self._sort_formats(formats)
 
         description = self._html_search_regex(r'<b>Description:</b>(.*?)</p>',
-            webpage, 'description', flags=re.DOTALL)
+                                              webpage, 'description', flags=re.DOTALL)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index b6755ff01..506d2d5a0 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -53,7 +53,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         # Otherwise we get a webpage that would execute some javascript
         req.add_header('Youtubedl-user-agent', 'curl/7')
         webpage = self._download_webpage(req, mtvn_id,
-            'Downloading mobile page')
+                                         'Downloading mobile page')
         metrics_url = unescapeHTML(self._search_regex(r'<a href="(http://metrics.+?)"', webpage, 'url'))
         req = HEADRequest(metrics_url)
         response = self._request_webpage(req, mtvn_id, 'Resolving url')
@@ -66,10 +66,10 @@ class MTVServicesInfoExtractor(InfoExtractor):
         if re.match(r'.*/(error_country_block\.swf|geoblock\.mp4)$', mdoc.find('.//src').text) is not None:
             if mtvn_id is not None and self._MOBILE_TEMPLATE is not None:
                 self.to_screen('The normal version is not available from your '
-                    'country, trying with the mobile version')
+                               'country, trying with the mobile version')
                 return self._extract_mobile_video_formats(mtvn_id)
             raise ExtractorError('This video is not available from your country.',
-                expected=True)
+                                 expected=True)
 
         formats = []
         for rendition in mdoc.findall('.//rendition'):
@@ -98,7 +98,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
             mediagen_url += '&acceptMethods=fms'
 
         mediagen_doc = self._download_xml(mediagen_url, video_id,
-            'Downloading video urls')
+                                          'Downloading video urls')
 
         description_node = itemdoc.find('description')
         if description_node is not None:
@@ -126,7 +126,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         # This a short id that's used in the webpage urls
         mtvn_id = None
         mtvn_id_node = find_xpath_attr(itemdoc, './/{http://search.yahoo.com/mrss/}category',
-                'scheme', 'urn:mtvn:id')
+                                       'scheme', 'urn:mtvn:id')
         if mtvn_id_node is not None:
             mtvn_id = mtvn_id_node.text
 
@@ -188,7 +188,7 @@ class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):
         video_id = self._id_from_uri(uri)
         site_id = uri.replace(video_id, '')
         config_url = ('http://media.mtvnservices.com/pmt/e1/players/{0}/'
-            'context4/context5/config.xml'.format(site_id))
+                      'context4/context5/config.xml'.format(site_id))
         config_doc = self._download_xml(config_url, video_id)
         feed_node = config_doc.find('.//feed')
         feed_url = feed_node.text.strip().split('?')[0]
diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index c16939f54..553ded56d 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -53,7 +53,7 @@ class MySpaceIE(InfoExtractor):
             # songs don't store any useful info in the 'context' variable
             def search_data(name):
                 return self._search_regex(r'data-%s="(.*?)"' % name, webpage,
-                    name)
+                                          name)
             streamUrl = search_data('stream-url')
             info = {
                 'id': video_id,
@@ -63,7 +63,7 @@ class MySpaceIE(InfoExtractor):
             }
         else:
             context = json.loads(self._search_regex(r'context = ({.*?});', webpage,
-                u'context'))
+                                                    u'context'))
             video = context['video']
             streamUrl = video['streamUrl']
             info = {
diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
index 956cf8b86..9195e53bd 100644
--- a/youtube_dl/extractor/myvideo.py
+++ b/youtube_dl/extractor/myvideo.py
@@ -72,7 +72,7 @@ class MyVideoIE(InfoExtractor):
             video_url = mobj.group(1) + '.flv'
 
             video_title = self._html_search_regex('<title>([^<]+)</title>',
-                webpage, 'title')
+                                                  webpage, 'title')
 
             return {
                 'id': video_id,
@@ -162,7 +162,7 @@ class MyVideoIE(InfoExtractor):
         video_swfobj = compat_urllib_parse.unquote(video_swfobj)
 
         video_title = self._html_search_regex("<h1(?: class='globalHd')?>(.*?)</h1>",
-            webpage, 'title')
+                                              webpage, 'title')
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index 5ce35dbf5..fbe34defd 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -30,7 +30,7 @@ class NaverIE(InfoExtractor):
         video_id = mobj.group(1)
         webpage = self._download_webpage(url, video_id)
         m_id = re.search(r'var rmcPlayer = new nhn.rmcnmv.RMCVideoPlayer\("(.+?)", "(.+?)"',
-            webpage)
+                         webpage)
         if m_id is None:
             m_error = re.search(
                 r'(?s)<div class="nation_error">\s*(?:<!--.*?-->)?\s*<p class="[^"]+">(?P<msg>.+?)</p>\s*</div>',
diff --git a/youtube_dl/extractor/nfb.py b/youtube_dl/extractor/nfb.py
index 09dcc8c84..7ce1d481d 100644
--- a/youtube_dl/extractor/nfb.py
+++ b/youtube_dl/extractor/nfb.py
@@ -38,12 +38,12 @@ class NFBIE(InfoExtractor):
         page = self._download_webpage('https://www.nfb.ca/film/%s' % video_id, video_id, 'Downloading film page')
 
         uploader_id = self._html_search_regex(r'<a class="director-link" href="/explore-all-directors/([^/]+)/"',
-            page, 'director id', fatal=False)
+                                              page, 'director id', fatal=False)
         uploader = self._html_search_regex(r'<em class="director-name" itemprop="name">([^<]+)</em>',
-            page, 'director name', fatal=False)
+                                           page, 'director name', fatal=False)
 
         request = compat_urllib_request.Request('https://www.nfb.ca/film/%s/player_config' % video_id,
-            compat_urllib_parse.urlencode({'getConfig': 'true'}).encode('ascii'))
+                                                compat_urllib_parse.urlencode({'getConfig': 'true'}).encode('ascii'))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         request.add_header('X-NFB-Referer', 'http://www.nfb.ca/medias/flash/NFBVideoPlayer.swf')
 
diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index 31813eb68..719eb51a4 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -125,7 +125,7 @@ class NHLVideocenterIE(NHLBaseInfoExtractor):
             self._downloader.report_warning(u'Got an empty reponse, trying '
                                             'adding the "newvideos" parameter')
             response = self._download_webpage(request_url + '&newvideos=true',
-                playlist_title)
+                                              playlist_title)
             response = self._fix_json(response)
         videos = json.loads(response)
 
diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 3b5784e8f..1d9c1a096 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -111,7 +111,7 @@ class NiconicoIE(InfoExtractor):
 
         if 'deleted=' in flv_info_webpage:
             raise ExtractorError('The video has been deleted.',
-                expected=True)
+                                 expected=True)
         video_real_url = compat_urlparse.parse_qs(flv_info_webpage)['url'][0]
 
         # Start extracting information
@@ -170,13 +170,13 @@ class NiconicoPlaylistIE(InfoExtractor):
         webpage = self._download_webpage(url, list_id)
 
         entries_json = self._search_regex(r'Mylist\.preload\(\d+, (\[.*\])\);',
-            webpage, 'entries')
+                                          webpage, 'entries')
         entries = json.loads(entries_json)
         entries = [{
             '_type': 'url',
             'ie_key': NiconicoIE.ie_key(),
             'url': ('http://www.nicovideo.jp/watch/%s' %
-                entry['item_data']['video_id']),
+                    entry['item_data']['video_id']),
         } for entry in entries]
 
         return {
diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 33daa0dec..16a02ad79 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -27,8 +27,7 @@ class NineGagIE(InfoExtractor):
             "thumbnail": "re:^https?://",
         },
         'add_ie': ['Youtube']
-    },
-    {
+    }, {
         'url': 'http://9gag.tv/p/KklwM/alternate-banned-opening-scene-of-gravity?ref=fsidebar',
         'info_dict': {
             'id': 'KklwM',
diff --git a/youtube_dl/extractor/normalboots.py b/youtube_dl/extractor/normalboots.py
index 25e71a56e..3d35b11ac 100644
--- a/youtube_dl/extractor/normalboots.py
+++ b/youtube_dl/extractor/normalboots.py
@@ -31,9 +31,9 @@ class NormalbootsIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
         video_uploader = self._html_search_regex(r'Posted\sby\s<a\shref="[A-Za-z0-9/]*">(?P<uploader>[A-Za-z]*)\s</a>',
-            webpage, 'uploader')
+                                                 webpage, 'uploader')
         raw_upload_date = self._html_search_regex('<span style="text-transform:uppercase; font-size:inherit;">[A-Za-z]+, (?P<date>.*)</span>',
-            webpage, 'date')
+                                                  webpage, 'date')
         video_upload_date = unified_strdate(raw_upload_date)
 
         player_url = self._html_search_regex(r'<iframe\swidth="[0-9]+"\sheight="[0-9]+"\ssrc="(?P<url>[\S]+)"', webpage, 'url')
diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index 30d80fa6b..f17a52858 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -43,7 +43,7 @@ class OoyalaIE(InfoExtractor):
     @classmethod
     def _build_url_result(cls, embed_code):
         return cls.url_result(cls._url_for_embed_code(embed_code),
-            ie=cls.ie_key())
+                              ie=cls.ie_key())
 
     def _extract_result(self, info, more_info):
         return {
diff --git a/youtube_dl/extractor/photobucket.py b/youtube_dl/extractor/photobucket.py
index 8aa69c46e..b4389e0b6 100644
--- a/youtube_dl/extractor/photobucket.py
+++ b/youtube_dl/extractor/photobucket.py
@@ -31,7 +31,7 @@ class PhotobucketIE(InfoExtractor):
         # Extract URL, uploader, and title from webpage
         self.report_extraction(video_id)
         info_json = self._search_regex(r'Pb\.Data\.Shared\.put\(Pb\.Data\.Shared\.MEDIA, (.*?)\);',
-            webpage, 'info json')
+                                       webpage, 'info json')
         info = json.loads(info_json)
         url = compat_urllib_parse.unquote(self._html_search_regex(r'file=(.+\.mp4)', info['linkcodes']['html'], 'url'))
         return {
diff --git a/youtube_dl/extractor/rbmaradio.py b/youtube_dl/extractor/rbmaradio.py
index 2c53ed2e1..0f8f3ebde 100644
--- a/youtube_dl/extractor/rbmaradio.py
+++ b/youtube_dl/extractor/rbmaradio.py
@@ -33,7 +33,7 @@ class RBMARadioIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         json_data = self._search_regex(r'window\.gon.*?gon\.show=(.+?);$',
-            webpage, 'json data', flags=re.MULTILINE)
+                                       webpage, 'json data', flags=re.MULTILINE)
 
         try:
             data = json.loads(json_data)
diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index 409f8540a..b8775c2f9 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -27,8 +27,7 @@ class SBSIE(InfoExtractor):
             'thumbnail': 're:http://.*\.jpg',
         },
         'add_ies': ['generic'],
-    },
-    {
+    }, {
         'url': 'http://www.sbs.com.au/ondemand/video/320403011771/Dingo-Conservation-The-Feed',
         'only_matching': True,
     }]
diff --git a/youtube_dl/extractor/screencast.py b/youtube_dl/extractor/screencast.py
index 306869e6a..c145f6fc7 100644
--- a/youtube_dl/extractor/screencast.py
+++ b/youtube_dl/extractor/screencast.py
@@ -96,7 +96,7 @@ class ScreencastIE(InfoExtractor):
         if title is None:
             title = self._html_search_regex(
                 [r'<b>Title:</b> ([^<]*)</div>',
-                r'class="tabSeperator">></span><span class="tabText">(.*?)<'],
+                 r'class="tabSeperator">></span><span class="tabText">(.*?)<'],
                 webpage, 'title')
         thumbnail = self._og_search_thumbnail(webpage)
         description = self._og_search_description(webpage, default=None)
diff --git a/youtube_dl/extractor/sina.py b/youtube_dl/extractor/sina.py
index 2909ef18b..5eadbb7ea 100644
--- a/youtube_dl/extractor/sina.py
+++ b/youtube_dl/extractor/sina.py
@@ -46,7 +46,7 @@ class SinaIE(InfoExtractor):
     def _extract_video(self, video_id):
         data = compat_urllib_parse.urlencode({'vid': video_id})
         url_doc = self._download_xml('http://v.iask.com/v_play.php?%s' % data,
-            video_id, 'Downloading video url')
+                                     video_id, 'Downloading video url')
         image_page = self._download_webpage(
             'http://interface.video.sina.com.cn/interface/common/getVideoImage.php?%s' % data,
             video_id, 'Downloading thumbnail info')
diff --git a/youtube_dl/extractor/slutload.py b/youtube_dl/extractor/slutload.py
index e6e7d0865..3df71304d 100644
--- a/youtube_dl/extractor/slutload.py
+++ b/youtube_dl/extractor/slutload.py
@@ -26,7 +26,7 @@ class SlutloadIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         video_title = self._html_search_regex(r'<h1><strong>([^<]+)</strong>',
-            webpage, 'title').strip()
+                                              webpage, 'title').strip()
 
         video_url = self._html_search_regex(
             r'(?s)<div id="vidPlayer"\s+data-url="([^"]+)"',
diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index b6a71305f..0751efc61 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -282,7 +282,7 @@ class SmotriBroadcastIE(InfoExtractor):
             (username, password) = self._get_login_info()
             if username is None:
                 raise ExtractorError('Erotic broadcasts allowed only for registered users, '
-                    'use --username and --password options to provide account credentials.', expected=True)
+                                     'use --username and --password options to provide account credentials.', expected=True)
 
             login_form = {
                 'login-hint53': '1',
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index f92f7fa24..410477d74 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -159,7 +159,7 @@ class SoundcloudIE(InfoExtractor):
 
         # We have to retrieve the url
         streams_url = ('http://api.soundcloud.com/i1/tracks/{0}/streams?'
-            'client_id={1}&secret_token={2}'.format(track_id, self._IPHONE_CLIENT_ID, secret_token))
+                       'client_id={1}&secret_token={2}'.format(track_id, self._IPHONE_CLIENT_ID, secret_token))
         format_dict = self._download_json(
             streams_url,
             track_id, 'Downloading track url')
diff --git a/youtube_dl/extractor/stanfordoc.py b/youtube_dl/extractor/stanfordoc.py
index 5feb4ff83..4a3d8bb8f 100644
--- a/youtube_dl/extractor/stanfordoc.py
+++ b/youtube_dl/extractor/stanfordoc.py
@@ -82,7 +82,7 @@ class StanfordOpenClassroomIE(InfoExtractor):
 
             rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
             rootpage = self._download_webpage(rootURL, info['id'],
-                errnote='Unable to download course info page')
+                                              errnote='Unable to download course info page')
 
             links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
             info['entries'] = [self.url_result(
diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index d5dda34f6..5fa67eb8d 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -8,24 +8,23 @@ from .common import InfoExtractor
 class TeamcocoIE(InfoExtractor):
     _VALID_URL = r'http://teamcoco\.com/video/(?P<video_id>[0-9]+)?/?(?P<display_id>.*)'
     _TESTS = [
-    {
-        'url': 'http://teamcoco.com/video/80187/conan-becomes-a-mary-kay-beauty-consultant',
-        'file': '80187.mp4',
-        'md5': '3f7746aa0dc86de18df7539903d399ea',
-        'info_dict': {
-            'title': 'Conan Becomes A Mary Kay Beauty Consultant',
-            'description': 'Mary Kay is perhaps the most trusted name in female beauty, so of course Conan is a natural choice to sell their products.'
+        {
+            'url': 'http://teamcoco.com/video/80187/conan-becomes-a-mary-kay-beauty-consultant',
+            'file': '80187.mp4',
+            'md5': '3f7746aa0dc86de18df7539903d399ea',
+            'info_dict': {
+                'title': 'Conan Becomes A Mary Kay Beauty Consultant',
+                'description': 'Mary Kay is perhaps the most trusted name in female beauty, so of course Conan is a natural choice to sell their products.'
+            }
+        }, {
+            'url': 'http://teamcoco.com/video/louis-ck-interview-george-w-bush',
+            'file': '19705.mp4',
+            'md5': 'cde9ba0fa3506f5f017ce11ead928f9a',
+            'info_dict': {
+                "description": "Louis C.K. got starstruck by George W. Bush, so what? Part one.",
+                "title": "Louis C.K. Interview Pt. 1 11/3/11"
+            }
         }
-    },
-    {
-        'url': 'http://teamcoco.com/video/louis-ck-interview-george-w-bush',
-        'file': '19705.mp4',
-        'md5': 'cde9ba0fa3506f5f017ce11ead928f9a',
-        'info_dict': {
-            "description": "Louis C.K. got starstruck by George W. Bush, so what? Part one.",
-            "title": "Louis C.K. Interview Pt. 1 11/3/11"
-        }
-    }
     ]
 
     def _real_extract(self, url):
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 855038077..f8a87afda 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -33,9 +33,9 @@ class TEDIE(SubtitlesInfoExtractor):
             'ext': 'mp4',
             'title': 'The illusion of consciousness',
             'description': ('Philosopher Dan Dennett makes a compelling '
-                'argument that not only don\'t we understand our own '
-                'consciousness, but that half the time our brains are '
-                'actively fooling us.'),
+                            'argument that not only don\'t we understand our own '
+                            'consciousness, but that half the time our brains are '
+                            'actively fooling us.'),
             'uploader': 'Dan Dennett',
             'width': 854,
             'duration': 1308,
@@ -93,7 +93,7 @@ class TEDIE(SubtitlesInfoExtractor):
 
     def _extract_info(self, webpage):
         info_json = self._search_regex(r'q\("\w+.init",({.+})\)</script>',
-            webpage, 'info json')
+                                       webpage, 'info json')
         return json.loads(info_json)
 
     def _real_extract(self, url):
@@ -113,7 +113,7 @@ class TEDIE(SubtitlesInfoExtractor):
         '''Returns the videos of the playlist'''
 
         webpage = self._download_webpage(url, name,
-            'Downloading playlist webpage')
+                                         'Downloading playlist webpage')
         info = self._extract_info(webpage)
         playlist_info = info['playlist']
 
diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index fdae17b1b..6e61cc9e2 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -30,7 +30,7 @@ class TF1IE(InfoExtractor):
         embed_url = self._html_search_regex(
             r'"(https://www.wat.tv/embedframe/.*?)"', webpage, 'embed url')
         embed_page = self._download_webpage(embed_url, video_id,
-            'Downloading embed player page')
+                                            'Downloading embed player page')
         wat_id = self._search_regex(r'UVID=(.*?)&', embed_page, 'wat id')
         wat_info = self._download_json(
             'http://www.wat.tv/interface/contentv3/%s' % wat_id, video_id)
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 522a095a2..e2653d62d 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -47,7 +47,7 @@ class ThePlatformIE(InfoExtractor):
             smil_url = config['releaseUrl'] + '&format=SMIL&formats=MPEG4&manifest=f4m'
         else:
             smil_url = ('http://link.theplatform.com/s/dJ5BDC/{0}/meta.smil?'
-                'format=smil&mbr=true'.format(video_id))
+                        'format=smil&mbr=true'.format(video_id))
 
         meta = self._download_xml(smil_url, video_id)
         try:
diff --git a/youtube_dl/extractor/tinypic.py b/youtube_dl/extractor/tinypic.py
index 6ba5dc5f1..4fe89dbe5 100644
--- a/youtube_dl/extractor/tinypic.py
+++ b/youtube_dl/extractor/tinypic.py
@@ -28,7 +28,7 @@ class TinyPicIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id, 'Downloading page')
 
         mobj = re.search(r'(?m)fo\.addVariable\("file",\s"(?P<fileid>[\da-z]+)"\);\n'
-            '\s+fo\.addVariable\("s",\s"(?P<serverid>\d+)"\);', webpage)
+                         '\s+fo\.addVariable\("s",\s"(?P<serverid>\d+)"\);', webpage)
         if mobj is None:
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
diff --git a/youtube_dl/extractor/traileraddict.py b/youtube_dl/extractor/traileraddict.py
index b1a440e79..1c53a3fd0 100644
--- a/youtube_dl/extractor/traileraddict.py
+++ b/youtube_dl/extractor/traileraddict.py
@@ -25,7 +25,7 @@ class TrailerAddictIE(InfoExtractor):
         webpage = self._download_webpage(url, name)
 
         title = self._search_regex(r'<title>(.+?)</title>',
-                webpage, 'video title').replace(' - Trailer Addict', '')
+                                   webpage, 'video title').replace(' - Trailer Addict', '')
         view_count_str = self._search_regex(
             r'<span class="views_n">([0-9,.]+)</span>',
             webpage, 'view count', fatal=False)
@@ -46,9 +46,9 @@ class TrailerAddictIE(InfoExtractor):
         info_webpage = self._download_webpage(info_url, video_id, "Downloading the info webpage")
 
         final_url = self._search_regex(r'&fileurl=(.+)',
-                info_webpage, 'Download url').replace('%3F', '?')
+                                       info_webpage, 'Download url').replace('%3F', '?')
         thumbnail_url = self._search_regex(r'&image=(.+?)&',
-                info_webpage, 'thumbnail url')
+                                           info_webpage, 'thumbnail url')
 
         description = self._html_search_regex(
             r'(?s)<div class="synopsis">.*?<div class="movie_label_info"[^>]*>(.*?)</div>',
diff --git a/youtube_dl/extractor/tumblr.py b/youtube_dl/extractor/tumblr.py
index 40c53ff17..2a1ae5a71 100644
--- a/youtube_dl/extractor/tumblr.py
+++ b/youtube_dl/extractor/tumblr.py
@@ -43,7 +43,7 @@ class TumblrIE(InfoExtractor):
             webpage, 'iframe url')
         iframe = self._download_webpage(iframe_url, video_id)
         video_url = self._search_regex(r'<source src="([^"]+)"',
-            iframe, 'video url')
+                                       iframe, 'video url')
 
         # The only place where you can get a title, it's not complete,
         # but searching in other places doesn't work for all videos
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 7df11fc19..1a7d01c67 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -154,7 +154,7 @@ class UdemyCourseIE(UdemyIE):
             self.to_screen('%s: Already enrolled in' % course_id)
 
         response = self._download_json('https://www.udemy.com/api-1.1/courses/%s/curriculum' % course_id,
-            course_id, 'Downloading course curriculum')
+                                       course_id, 'Downloading course curriculum')
 
         entries = [
             self.url_result('https://www.udemy.com/%s/#/lecture/%s' % (course_path, asset['id']), 'Udemy')
diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 875450908..53dc3a496 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -45,13 +45,13 @@ class UstreamIE(InfoExtractor):
         self.report_extraction(video_id)
 
         video_title = self._html_search_regex(r'data-title="(?P<title>.+)"',
-            webpage, 'title')
+                                              webpage, 'title')
 
         uploader = self._html_search_regex(r'data-content-type="channel".*?>(?P<uploader>.*?)</a>',
-            webpage, 'uploader', fatal=False, flags=re.DOTALL)
+                                           webpage, 'uploader', fatal=False, flags=re.DOTALL)
 
         thumbnail = self._html_search_regex(r'<link rel="image_src" href="(?P<thumb>.*?)"',
-            webpage, 'thumbnail', fatal=False)
+                                            webpage, 'thumbnail', fatal=False)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index ebd64f0f5..455b6d9da 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -30,13 +30,13 @@ class Vbox7IE(InfoExtractor):
 
         redirect_page, urlh = self._download_webpage_handle(url, video_id)
         new_location = self._search_regex(r'window\.location = \'(.*)\';',
-            redirect_page, 'redirect location')
+                                          redirect_page, 'redirect location')
         redirect_url = urlh.geturl() + new_location
         webpage = self._download_webpage(redirect_url, video_id,
-            'Downloading redirect page')
+                                         'Downloading redirect page')
 
         title = self._html_search_regex(r'<title>(.*)</title>',
-            webpage, 'title').split('/')[0].strip()
+                                        webpage, 'title').split('/')[0].strip()
 
         info_url = "http://vbox7.com/play/magare.do"
         data = compat_urllib_parse.urlencode({'as3': '1', 'vid': video_id})
diff --git a/youtube_dl/extractor/veehd.py b/youtube_dl/extractor/veehd.py
index 77b1f91ce..94647d1c8 100644
--- a/youtube_dl/extractor/veehd.py
+++ b/youtube_dl/extractor/veehd.py
@@ -48,11 +48,11 @@ class VeeHDIE(InfoExtractor):
         video_url = compat_urlparse.unquote(config['clip']['url'])
         title = clean_html(get_element_by_id('videoName', webpage).rpartition('|')[0])
         uploader_id = self._html_search_regex(r'<a href="/profile/\d+">(.+?)</a>',
-            webpage, 'uploader')
+                                              webpage, 'uploader')
         thumbnail = self._search_regex(r'<img id="veehdpreview" src="(.+?)"',
-            webpage, 'thumbnail')
+                                       webpage, 'thumbnail')
         description = self._html_search_regex(r'<td class="infodropdown".*?<div>(.*?)<ul',
-            webpage, 'description', flags=re.DOTALL)
+                                              webpage, 'description', flags=re.DOTALL)
 
         return {
             '_type': 'video',
diff --git a/youtube_dl/extractor/vesti.py b/youtube_dl/extractor/vesti.py
index 5aebcecd7..a0c59a2e0 100644
--- a/youtube_dl/extractor/vesti.py
+++ b/youtube_dl/extractor/vesti.py
@@ -112,7 +112,7 @@ class VestiIE(InfoExtractor):
         if mobj:
             video_id = mobj.group('id')
             page = self._download_webpage('http://www.vesti.ru/only_video.html?vid=%s' % video_id, video_id,
-                'Downloading video page')
+                                          'Downloading video page')
 
         rutv_url = RUTVIE._extract_url(page)
         if rutv_url:
diff --git a/youtube_dl/extractor/videofyme.py b/youtube_dl/extractor/videofyme.py
index d69fe1e77..d764e60fb 100644
--- a/youtube_dl/extractor/videofyme.py
+++ b/youtube_dl/extractor/videofyme.py
@@ -28,11 +28,11 @@ class VideofyMeIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         config = self._download_xml('http://sunshine.videofy.me/?videoId=%s' % video_id,
-                                            video_id)
+                                    video_id)
         video = config.find('video')
         sources = video.find('sources')
         url_node = next(node for node in [find_xpath_attr(sources, 'source', 'id', 'HQ %s' % key)
-            for key in ['on', 'av', 'off']] if node is not None)
+                                          for key in ['on', 'av', 'off']] if node is not None)
         video_url = url_node.find('url').text
 
         return {'id': video_id,
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index c744d4f04..06b0bed41 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -260,7 +260,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
                 else:
                     config_re = [r' = {config:({.+?}),assets:', r'(?:[abc])=({.+?});']
                 config = self._search_regex(config_re, webpage, 'info section',
-                    flags=re.DOTALL)
+                                            flags=re.DOTALL)
                 config = json.loads(config)
         except Exception as e:
             if re.search('The creator of this video has not given you permission to embed it on this domain.', webpage):
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 9d6ec4382..d9acafd70 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -121,7 +121,7 @@ class VKIE(InfoExtractor):
         }
 
         request = compat_urllib_request.Request('https://login.vk.com/?act=login',
-            compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+                                                compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         login_page = self._download_webpage(request, None, note='Logging in as %s' % username)
 
         if re.search(r'onLoginFailed', login_page):
@@ -175,7 +175,7 @@ class VKIE(InfoExtractor):
         upload_date = None
         mobj = re.search(r'id="mv_date_wrap".*?Added ([a-zA-Z]+ [0-9]+), ([0-9]+) at', info_page)
         if mobj is not None:
-            x = mobj.group(1) + ' ' + mobj.group(2)
+            mobj.group(1) + ' ' + mobj.group(2)
             upload_date = unified_strdate(mobj.group(1) + ' ' + mobj.group(2))
 
         formats = [{
diff --git a/youtube_dl/extractor/weibo.py b/youtube_dl/extractor/weibo.py
index b24297a40..20bb039d3 100644
--- a/youtube_dl/extractor/weibo.py
+++ b/youtube_dl/extractor/weibo.py
@@ -41,7 +41,7 @@ class WeiboIE(InfoExtractor):
         videos_urls = sorted(videos_urls, key=lambda u: 'video.sina.com' in u)
         player_url = videos_urls[-1]
         m_sina = re.match(r'https?://video\.sina\.com\.cn/v/b/(\d+)-\d+\.html',
-            player_url)
+                          player_url)
         if m_sina is not None:
             self.to_screen('Sina video detected')
             sina_id = m_sina.group(1)
diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 1e6c94623..6b37bcbc9 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -67,17 +67,17 @@ class XHamsterIE(InfoExtractor):
         description = mobj.group(1) if mobj else None
 
         upload_date = self._html_search_regex(r'hint=\'(\d{4}-\d{2}-\d{2}) \d{2}:\d{2}:\d{2} [A-Z]{3,4}\'',
-            webpage, 'upload date', fatal=False)
+                                              webpage, 'upload date', fatal=False)
         if upload_date:
             upload_date = unified_strdate(upload_date)
 
         uploader_id = self._html_search_regex(r'<a href=\'/user/[^>]+>(?P<uploader_id>[^<]+)',
-            webpage, 'uploader id', default='anonymous')
+                                              webpage, 'uploader id', default='anonymous')
 
         thumbnail = self._html_search_regex(r'<video\s+.*?poster="([^"]+)".*?>', webpage, 'thumbnail', fatal=False)
 
         duration = parse_duration(self._html_search_regex(r'<span>Runtime:</span> (\d+:\d+)</div>',
-            webpage, 'duration', fatal=False))
+                                                          webpage, 'duration', fatal=False))
 
         view_count = self._html_search_regex(r'<span>Views:</span> ([^<]+)</div>', webpage, 'view count', fatal=False)
         if view_count:
diff --git a/youtube_dl/extractor/xnxx.py b/youtube_dl/extractor/xnxx.py
index 7a73b2430..53ed7ef5a 100644
--- a/youtube_dl/extractor/xnxx.py
+++ b/youtube_dl/extractor/xnxx.py
@@ -30,14 +30,14 @@ class XNXXIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         video_url = self._search_regex(r'flv_url=(.*?)&amp;',
-            webpage, 'video URL')
+                                       webpage, 'video URL')
         video_url = compat_urllib_parse.unquote(video_url)
 
         video_title = self._html_search_regex(r'<title>(.*?)\s+-\s+XNXX.COM',
-            webpage, 'title')
+                                              webpage, 'title')
 
         video_thumbnail = self._search_regex(r'url_bigthumb=(.*?)&amp;',
-            webpage, 'thumbnail', fatal=False)
+                                             webpage, 'thumbnail', fatal=False)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 39caf60f2..0fdb12243 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -229,7 +229,7 @@ class YahooSearchIE(SearchInfoExtractor):
         for pagenum in itertools.count(0):
             result_url = 'http://video.search.yahoo.com/search/?p=%s&fr=screen&o=js&gs=0&b=%d' % (compat_urllib_parse.quote_plus(query), pagenum * 30)
             info = self._download_json(result_url, query,
-                note='Downloading results page ' + str(pagenum + 1))
+                                       note='Downloading results page ' + str(pagenum + 1))
             m = info['m']
             results = info['results']
 
diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 830ae6cd9..97b98bbe8 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -74,7 +74,7 @@ class YoukuIE(InfoExtractor):
             # -8 means blocked outside China.
             error = config['data'][0].get('error')  # Chinese and English, separated by newline.
             raise ExtractorError(error or 'Server reported error %i' % error_code,
-                expected=True)
+                                 expected=True)
 
         video_title = config['data'][0]['title']
         seed = config['data'][0]['seed']
diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index ee61e2381..97ef9c17e 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -64,7 +64,7 @@ class YouPornIE(InfoExtractor):
         # Get all of the links from the page
         DOWNLOAD_LIST_RE = r'(?s)<ul class="downloadList">(?P<download_list>.*?)</ul>'
         download_list_html = self._search_regex(DOWNLOAD_LIST_RE,
-            webpage, 'download list').strip()
+                                                webpage, 'download list').strip()
         LINK_RE = r'<a href="([^"]+)">'
         links = re.findall(LINK_RE, download_list_html)
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8711b06d4..fa3ead95b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -950,7 +950,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
                         parts_sizes = self._signature_cache_id(encrypted_sig)
                         self.to_screen('{%s} signature length %s, %s' %
-                            (format_id, parts_sizes, player_desc))
+                                       (format_id, parts_sizes, player_desc))
 
                     signature = self._decrypt_signature(
                         encrypted_sig, video_id, player_url, age_gate)
@@ -1214,7 +1214,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
 class YoutubeTopListIE(YoutubePlaylistIE):
     IE_NAME = 'youtube:toplist'
     IE_DESC = ('YouTube.com top lists, "yttoplist:{channel}:{list title}"'
-        ' (Example: "yttoplist:music:Top Tracks")')
+               ' (Example: "yttoplist:music:Top Tracks")')
     _VALID_URL = r'yttoplist:(?P<chann>.*?):(?P<title>.*?)$'
     _TESTS = [{
         'url': 'yttoplist:music:Trending',

From 4ffc31033e5fd2bfc5a9882f14eca01382341682 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Nov 2014 22:00:25 +0100
Subject: [PATCH 0583/1937] [zingmp3] Simplify and PEP8 (#4288)

---
 youtube_dl/extractor/zingmp3.py | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index 2c2073adf..1afbe68ed 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -24,8 +24,6 @@ class ZingMp3BaseInfoExtractor(InfoExtractor):
 
     def _extract_player_xml(self, player_xml_url, id, playlist_title=None):
         player_xml = self._download_xml(player_xml_url, id, 'Downloading Player XML')
-
-        self.report_extraction(id)
         items = player_xml.findall('./item')
 
         if len(items) == 1:
@@ -55,12 +53,13 @@ class ZingMp3SongIE(ZingMp3BaseInfoExtractor):
     _VALID_URL = r'https?://mp3\.zing\.vn/bai-hat/(?P<slug>[^/]+)/(?P<song_id>\w+)\.html'
     _TESTS = [{
         'url': 'http://mp3.zing.vn/bai-hat/Xa-Mai-Xa-Bao-Thy/ZWZB9WAB.html',
+        'md5': 'ead7ae13693b3205cbc89536a077daed',
         'info_dict': {
             'id': 'ZWZB9WAB',
-            'title': u'Xa Mãi Xa',
+            'title': 'Xa Mãi Xa',
             'ext': 'mp3',
+            'thumbnail': 're:^https?://.*\.jpg$',
         },
-        'md5': 'ead7ae13693b3205cbc89536a077daed',
     }]
     IE_NAME = 'zingmp3:song'
     IE_DESC = 'mp3.zing.vn songs'
@@ -70,9 +69,11 @@ class ZingMp3SongIE(ZingMp3BaseInfoExtractor):
         slug = matched.group('slug')
         song_id = matched.group('song_id')
 
-        webpage = self._download_webpage('http://mp3.zing.vn/bai-hat/%s/%s.html' % (slug, song_id), song_id)
+        webpage = self._download_webpage(
+            'http://mp3.zing.vn/bai-hat/%s/%s.html' % (slug, song_id), song_id)
 
-        player_xml_url = self._search_regex(r'&amp;xmlURL=(?P<xml_url>[^&]+)&', webpage, 'player xml url')
+        player_xml_url = self._search_regex(
+            r'&amp;xmlURL=(?P<xml_url>[^&]+)&', webpage, 'player xml url')
 
         return self._extract_player_xml(player_xml_url, song_id)
 
@@ -84,7 +85,7 @@ class ZingMp3AlbumIE(ZingMp3BaseInfoExtractor):
         'info_dict': {
             '_type': 'playlist',
             'id': 'ZWZBWDAF',
-            'title': u'Lâu Đài Tình Ái - Bằng Kiều ft. Minh Tuyết | Album 320 lossless',
+            'title': 'Lâu Đài Tình Ái - Bằng Kiều ft. Minh Tuyết | Album 320 lossless',
         },
         'playlist_count': 10,
     }]
@@ -96,8 +97,11 @@ class ZingMp3AlbumIE(ZingMp3BaseInfoExtractor):
         slug = matched.group('slug')
         album_id = matched.group('album_id')
 
-        webpage = self._download_webpage('http://mp3.zing.vn/album/%s/%s.html' % (slug, album_id), album_id)
+        webpage = self._download_webpage(
+            'http://mp3.zing.vn/album/%s/%s.html' % (slug, album_id), album_id)
+        player_xml_url = self._search_regex(
+            r'&amp;xmlURL=(?P<xml_url>[^&]+)&', webpage, 'player xml url')
 
-        player_xml_url = self._search_regex(r'&amp;xmlURL=(?P<xml_url>[^&]+)&', webpage, 'player xml url')
-
-        return self._extract_player_xml(player_xml_url, album_id, playlist_title=self._og_search_title(webpage))
\ No newline at end of file
+        return self._extract_player_xml(
+            player_xml_url, album_id,
+            playlist_title=self._og_search_title(webpage))

From fab6d4c048f56aa1c36897396a7fb590ae415d66 Mon Sep 17 00:00:00 2001
From: Jouke Waleson <jouke.waleson@mendix.com>
Date: Sun, 23 Nov 2014 22:00:35 +0100
Subject: [PATCH 0584/1937] remove useless line, the result is never used

---
 devscripts/fish-completion.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py
index f2445984f..c2f238798 100755
--- a/devscripts/fish-completion.py
+++ b/devscripts/fish-completion.py
@@ -30,7 +30,6 @@ def build_completion(opt_parser):
     for group in opt_parser.option_groups:
         for option in group.option_list:
             long_option = option.get_opt_string().strip('-')
-            shell_quote([option.help])
             complete_cmd = ['complete', '--command', 'youtube-dl', '--long-option', long_option]
             if option._short_opts:
                 complete_cmd += ['--short-option', option._short_opts[0].strip('-')]

From e311b6389a7582eef6f3f8ca4b41edd4efd6cd55 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Nov 2014 22:01:08 +0100
Subject: [PATCH 0585/1937] Credit @daohoangson for zingmp3 (#4288)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 5ec2f1dc7..c8d60282c 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -84,3 +84,4 @@ xantares
 Jan Matějka
 Mauroy Sébastien
 William Sewell
+Dao Hoang Son

From b66e699877937b4520db885c82958714304bf74a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Nov 2014 22:12:18 +0100
Subject: [PATCH 0586/1937] [myspace] pep8 and modernization

---
 youtube_dl/extractor/myspace.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index 553ded56d..c3c1a27c3 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -4,7 +4,7 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
 )
 
@@ -52,8 +52,8 @@ class MySpaceIE(InfoExtractor):
         if mobj.group('mediatype').startswith('music/song'):
             # songs don't store any useful info in the 'context' variable
             def search_data(name):
-                return self._search_regex(r'data-%s="(.*?)"' % name, webpage,
-                                          name)
+                return self._search_regex(
+                    r'data-%s="(.*?)"' % name, webpage, name)
             streamUrl = search_data('stream-url')
             info = {
                 'id': video_id,
@@ -62,8 +62,8 @@ class MySpaceIE(InfoExtractor):
                 'thumbnail': self._og_search_thumbnail(webpage),
             }
         else:
-            context = json.loads(self._search_regex(r'context = ({.*?});', webpage,
-                                                    u'context'))
+            context = json.loads(self._search_regex(
+                r'context = ({.*?});', webpage, 'context'))
             video = context['video']
             streamUrl = video['streamUrl']
             info = {

From 380b8220038d69c701cbaba04d1b049628d237b4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Nov 2014 22:13:03 +0100
Subject: [PATCH 0587/1937] Remove outdated transition helper scripts

---
 devscripts/transition_helper.py               |  40 -------
 devscripts/transition_helper_exe/setup.py     |  12 --
 .../transition_helper_exe/youtube-dl.py       | 110 ------------------
 3 files changed, 162 deletions(-)
 delete mode 100644 devscripts/transition_helper.py
 delete mode 100644 devscripts/transition_helper_exe/setup.py
 delete mode 100644 devscripts/transition_helper_exe/youtube-dl.py

diff --git a/devscripts/transition_helper.py b/devscripts/transition_helper.py
deleted file mode 100644
index cfd21919b..000000000
--- a/devscripts/transition_helper.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python
-
-import sys, os
-
-try:
-    import urllib.request as compat_urllib_request
-except ImportError:  # Python 2
-    import urllib2 as compat_urllib_request
-
-sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')
-sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n')
-sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n')
-
-try:
-    raw_input()
-except NameError:  # Python 3
-    input()
-
-filename = sys.argv[0]
-
-API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads"
-BIN_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl"
-
-if not os.access(filename, os.W_OK):
-    sys.exit('ERROR: no write permissions on %s' % filename)
-
-try:
-    urlh = compat_urllib_request.urlopen(BIN_URL)
-    newcontent = urlh.read()
-    urlh.close()
-except (IOError, OSError) as err:
-    sys.exit('ERROR: unable to download latest version')
-
-try:
-    with open(filename, 'wb') as outf:
-        outf.write(newcontent)
-except (IOError, OSError) as err:
-    sys.exit('ERROR: unable to overwrite current version')
-
-sys.stderr.write(u'Done! Now you can run youtube-dl.\n')
diff --git a/devscripts/transition_helper_exe/setup.py b/devscripts/transition_helper_exe/setup.py
deleted file mode 100644
index bb3264af0..000000000
--- a/devscripts/transition_helper_exe/setup.py
+++ /dev/null
@@ -1,12 +0,0 @@
-from distutils.core import setup
-import py2exe
-
-py2exe_options = {
-    "bundle_files": 1,
-    "compressed": 1,
-    "optimize": 2,
-    "dist_dir": '.',
-    "dll_excludes": ['w9xpopen.exe']
-}
-
-setup(console=['youtube-dl.py'], options={"py2exe": py2exe_options}, zipfile=None)
diff --git a/devscripts/transition_helper_exe/youtube-dl.py b/devscripts/transition_helper_exe/youtube-dl.py
deleted file mode 100644
index ab59d9376..000000000
--- a/devscripts/transition_helper_exe/youtube-dl.py
+++ /dev/null
@@ -1,110 +0,0 @@
-#!/usr/bin/env python
-
-import sys, os
-import urllib2
-import json, hashlib
-
-
-def rsa_verify(message, signature, key):
-    from struct import pack
-    from hashlib import sha256
-    from sys import version_info
-
-    def b(x):
-        if version_info[0] == 2:
-            return x
-        else:
-            return x.encode('latin1')
-    assert(type(message) == type(b('')))
-    block_size = 0
-    n = key[0]
-    while n:
-        block_size += 1
-        n >>= 8
-    signature = pow(int(signature, 16), key[1], key[0])
-    raw_bytes = []
-    while signature:
-        raw_bytes.insert(0, pack("B", signature & 0xFF))
-        signature >>= 8
-    signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
-    if signature[0:2] != b('\x00\x01'):
-        return False
-    signature = signature[2:]
-    if not b('\x00') in signature:
-        return False
-    signature = signature[signature.index(b('\x00')) + 1:]
-    if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')):
-        return False
-    signature = signature[19:]
-    if signature != sha256(message).digest():
-        return False
-    return True
-
-sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n')
-sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n')
-sys.stderr.write(u'From now on, get the binaries from http://rg3.github.com/youtube-dl/download.html, not from the git repository.\n\n')
-
-raw_input()
-
-filename = sys.argv[0]
-
-UPDATE_URL = "http://rg3.github.io/youtube-dl/update/"
-VERSION_URL = UPDATE_URL + 'LATEST_VERSION'
-JSON_URL = UPDATE_URL + 'versions.json'
-UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
-
-if not os.access(filename, os.W_OK):
-    sys.exit('ERROR: no write permissions on %s' % filename)
-
-exe = os.path.abspath(filename)
-directory = os.path.dirname(exe)
-if not os.access(directory, os.W_OK):
-    sys.exit('ERROR: no write permissions on %s' % directory)
-
-try:
-    versions_info = urllib2.urlopen(JSON_URL).read().decode('utf-8')
-    versions_info = json.loads(versions_info)
-except:
-    sys.exit(u'ERROR: can\'t obtain versions info. Please try again later.')
-if not 'signature' in versions_info:
-    sys.exit(u'ERROR: the versions file is not signed or corrupted. Aborting.')
-signature = versions_info['signature']
-del versions_info['signature']
-if not rsa_verify(json.dumps(versions_info, sort_keys=True), signature, UPDATES_RSA_KEY):
-    sys.exit(u'ERROR: the versions file signature is invalid. Aborting.')
-
-version = versions_info['versions'][versions_info['latest']]
-
-try:
-    urlh = urllib2.urlopen(version['exe'][0])
-    newcontent = urlh.read()
-    urlh.close()
-except (IOError, OSError) as err:
-    sys.exit('ERROR: unable to download latest version')
-
-newcontent_hash = hashlib.sha256(newcontent).hexdigest()
-if newcontent_hash != version['exe'][1]:
-    sys.exit(u'ERROR: the downloaded file hash does not match. Aborting.')
-
-try:
-    with open(exe + '.new', 'wb') as outf:
-        outf.write(newcontent)
-except (IOError, OSError) as err:
-    sys.exit(u'ERROR: unable to write the new version')
-
-try:
-    bat = os.path.join(directory, 'youtube-dl-updater.bat')
-    b = open(bat, 'w')
-    b.write("""
-echo Updating youtube-dl...
-ping 127.0.0.1 -n 5 -w 1000 > NUL
-move /Y "%s.new" "%s"
-del "%s"
-    \n""" % (exe, exe, bat))
-    b.close()
-
-    os.startfile(bat)
-except (IOError, OSError) as err:
-    sys.exit('ERROR: unable to overwrite current version')
-
-sys.stderr.write(u'Done! Now you can run youtube-dl.\n')

From 3d36cea4aca303f6bde5b5453f4e3aeec8ee41e1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Nov 2014 22:14:27 +0100
Subject: [PATCH 0588/1937] [vk] PEP8

---
 youtube_dl/extractor/vk.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index d9acafd70..ca6b0d5b3 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -142,12 +142,14 @@ class VKIE(InfoExtractor):
 
         ERRORS = {
             r'>Видеозапись .*? была изъята из публичного доступа в связи с обращением правообладателя.<':
-                'Video %s has been removed from public access due to rightholder complaint.',
+            'Video %s has been removed from public access due to rightholder complaint.',
+
             r'<!>Please log in or <':
-                'Video %s is only available for registered users, '
-                'use --username and --password options to provide account credentials.',
-            '<!>Unknown error':
-                'Video %s does not exist.'
+            'Video %s is only available for registered users, '
+            'use --username and --password options to provide account credentials.',
+
+            r'<!>Unknown error':
+            'Video %s does not exist.'
         }
 
         for error_re, error_msg in ERRORS.items():

From b74e86f48aa6d007b681e2723ff28fe82a49fa9d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Nov 2014 22:21:46 +0100
Subject: [PATCH 0589/1937] Fix all PEP8 issues except E501

---
 devscripts/gh-pages/update-sites.py |  2 +-
 youtube_dl/__init__.py              |  2 +-
 youtube_dl/aes.py                   |  2 +-
 youtube_dl/extractor/bambuser.py    |  9 +++++----
 youtube_dl/extractor/channel9.py    | 21 +++++++++++----------
 youtube_dl/extractor/crunchyroll.py |  5 +++--
 youtube_dl/extractor/facebook.py    |  2 +-
 youtube_dl/extractor/fourtube.py    |  8 ++++----
 youtube_dl/extractor/generic.py     |  6 +++---
 youtube_dl/extractor/ign.py         |  6 ++++--
 youtube_dl/extractor/ivi.py         |  2 +-
 youtube_dl/extractor/myvideo.py     |  6 +++---
 youtube_dl/extractor/soundcloud.py  |  2 +-
 youtube_dl/extractor/tlc.py         |  5 +++--
 youtube_dl/extractor/tudou.py       |  2 +-
 youtube_dl/options.py               |  5 +++--
 youtube_dl/utils.py                 |  6 +++---
 17 files changed, 49 insertions(+), 42 deletions(-)

diff --git a/devscripts/gh-pages/update-sites.py b/devscripts/gh-pages/update-sites.py
index 0d526784d..4a6bb5e35 100755
--- a/devscripts/gh-pages/update-sites.py
+++ b/devscripts/gh-pages/update-sites.py
@@ -22,7 +22,7 @@ def main():
             continue
         elif ie_desc is not None:
             ie_html += ': {}'.format(ie.IE_DESC)
-        if ie.working() == False:
+        if not ie.working():
             ie_html += ' (Currently broken)'
         ie_htmls.append('<li>{}</li>'.format(ie_html))
 
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 427b6ad27..77b3384a0 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -189,7 +189,7 @@ def _real_main(argv=None):
 
     # --all-sub automatically sets --write-sub if --write-auto-sub is not given
     # this was the old behaviour if only --all-sub was given.
-    if opts.allsubtitles and (opts.writeautomaticsub == False):
+    if opts.allsubtitles and not opts.writeautomaticsub:
         opts.writesubtitles = True
 
     if sys.version_info < (3,):
diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py
index 662d49011..2eeb96a5a 100644
--- a/youtube_dl/aes.py
+++ b/youtube_dl/aes.py
@@ -87,7 +87,7 @@ def key_expansion(data):
             temp = sub_bytes(temp)
             data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
 
-        for _ in range(3 if key_size_bytes == 32  else 2 if key_size_bytes == 24 else 0):
+        for _ in range(3 if key_size_bytes == 32 else 2 if key_size_bytes == 24 else 0):
             temp = data[-4:]
             data += xor(temp, data[-key_size_bytes: 4 - key_size_bytes])
     data = data[:expanded_key_size_bytes]
diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index 928ea61a3..ea2ba0fe2 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -73,10 +73,11 @@ class BambuserChannelIE(InfoExtractor):
         urls = []
         last_id = ''
         for i in itertools.count(1):
-            req_url = ('http://bambuser.com/xhr-api/index.php?username={user}'
-                       '&sort=created&access_mode=0%2C1%2C2&limit={count}'
-                       '&method=broadcast&format=json&vid_older_than={last}'
-                ).format(user=user, count=self._STEP, last=last_id)
+            req_url = (
+                'http://bambuser.com/xhr-api/index.php?username={user}'
+                '&sort=created&access_mode=0%2C1%2C2&limit={count}'
+                '&method=broadcast&format=json&vid_older_than={last}'
+            ).format(user=user, count=self._STEP, last=last_id)
             req = compat_urllib_request.Request(req_url)
             # Without setting this header, we wouldn't get any result
             req.add_header('Referer', 'http://bambuser.com/channel/%s' % user)
diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 82a48870a..2a05813f8 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -188,16 +188,17 @@ class Channel9IE(InfoExtractor):
         view_count = self._extract_view_count(html)
         comment_count = self._extract_comment_count(html)
 
-        common = {'_type': 'video',
-                  'id': content_path,
-                  'description': description,
-                  'thumbnail': thumbnail,
-                  'duration': duration,
-                  'avg_rating': avg_rating,
-                  'rating_count': rating_count,
-                  'view_count': view_count,
-                  'comment_count': comment_count,
-                }
+        common = {
+            '_type': 'video',
+            'id': content_path,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'avg_rating': avg_rating,
+            'rating_count': rating_count,
+            'view_count': view_count,
+            'comment_count': comment_count,
+        }
 
         result = []
 
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index c3c4d114a..d7e2b841e 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -248,8 +248,9 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         subtitles = {}
         sub_format = self._downloader.params.get('subtitlesformat', 'srt')
         for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
-            sub_page = self._download_webpage('http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id,\
-                                              video_id, note='Downloading subtitles for ' + sub_name)
+            sub_page = self._download_webpage(
+                'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id,
+                video_id, note='Downloading subtitles for ' + sub_name)
             id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False)
             iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False)
             data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False)
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index c989879ba..2139f68aa 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -77,7 +77,7 @@ class FacebookIE(InfoExtractor):
             'legacy_return': '1',
             'timezone': '-60',
             'trynum': '1',
-            }
+        }
         request = compat_urllib_request.Request(self._LOGIN_URL, urlencode_postdata(login_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         try:
diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index 701241bb0..b22ce2acb 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -65,9 +65,9 @@ class FourTubeIE(InfoExtractor):
 
         token_url = "http://tkn.4tube.com/{0}/desktop/{1}".format(media_id, "+".join(sources))
         headers = {
-                b'Content-Type': b'application/x-www-form-urlencoded',
-                b'Origin': b'http://www.4tube.com',
-                }
+            b'Content-Type': b'application/x-www-form-urlencoded',
+            b'Origin': b'http://www.4tube.com',
+        }
         token_req = compat_urllib_request.Request(token_url, b'{}', headers)
         tokens = self._download_json(token_req, video_id)
 
@@ -76,7 +76,7 @@ class FourTubeIE(InfoExtractor):
             'format_id': format + 'p',
             'resolution': format + 'p',
             'quality': int(format),
-            } for format in sources]
+        } for format in sources]
 
         self._sort_formats(formats)
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index d224aa8e1..128715caf 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -537,9 +537,9 @@ class GenericIE(InfoExtractor):
 
             if default_search in ('error', 'fixup_error'):
                 raise ExtractorError(
-                    ('%r is not a valid URL. '
-                     'Set --default-search "ytsearch" (or run  youtube-dl "ytsearch:%s" ) to search YouTube'
-                    ) % (url, url), expected=True)
+                    '%r is not a valid URL. '
+                    'Set --default-search "ytsearch" (or run  youtube-dl "ytsearch:%s" ) to search YouTube'
+                    % (url, url), expected=True)
             else:
                 if ':' not in default_search:
                     default_search += ':'
diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index 3555f98a5..3db668cd0 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -63,8 +63,10 @@ class IGNIE(InfoExtractor):
                 'id': '078fdd005f6d3c02f63d795faa1b984f',
                 'ext': 'mp4',
                 'title': 'Rewind Theater - Wild Trailer Gamescom 2014',
-                'description': 'Giant skeletons, bloody hunts, and captivating'
-                    ' natural beauty take our breath away.',
+                'description': (
+                    'Giant skeletons, bloody hunts, and captivating'
+                    ' natural beauty take our breath away.'
+                ),
             },
         },
     ]
diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index 06af73417..f0fba1adb 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -43,7 +43,7 @@ class IviIE(InfoExtractor):
                 'thumbnail': 'http://thumbs.ivi.ru/f15.vcp.digitalaccess.ru/contents/8/4/0068dc0677041f3336b7c2baad8fc0.jpg',
             },
             'skip': 'Only works from Russia',
-         }
+        }
     ]
 
     # Sorted by quality
diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py
index 9195e53bd..5e754fcff 100644
--- a/youtube_dl/extractor/myvideo.py
+++ b/youtube_dl/extractor/myvideo.py
@@ -57,9 +57,9 @@ class MyVideoIE(InfoExtractor):
         video_id = mobj.group('id')
 
         GK = (
-          b'WXpnME1EZGhNRGhpTTJNM01XVmhOREU0WldNNVpHTTJOakpt'
-          b'TW1FMU5tVTBNR05pWkRaa05XRXhNVFJoWVRVd1ptSXhaVEV3'
-          b'TnpsbA0KTVRkbU1tSTRNdz09'
+            b'WXpnME1EZGhNRGhpTTJNM01XVmhOREU0WldNNVpHTTJOakpt'
+            b'TW1FMU5tVTBNR05pWkRaa05XRXhNVFJoWVRVd1ptSXhaVEV3'
+            b'TnpsbA0KTVRkbU1tSTRNdz09'
         )
 
         # Get video webpage
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 410477d74..3c1d058db 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -371,7 +371,7 @@ class SoundcloudPlaylistIE(SoundcloudIE):
 
         entries = [
             self._extract_info_dict(t, quiet=True, secret_token=token)
-                for t in data['tracks']]
+            for t in data['tracks']]
 
         return {
             '_type': 'playlist',
diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/tlc.py
index d848ee186..66d159e99 100644
--- a/youtube_dl/extractor/tlc.py
+++ b/youtube_dl/extractor/tlc.py
@@ -36,9 +36,10 @@ class TlcDeIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Breaking Amish: Die Welt da draußen',
             'uploader': 'Discovery Networks - Germany',
-            'description': 'Vier Amische und eine Mennonitin wagen in New York'
+            'description': (
+                'Vier Amische und eine Mennonitin wagen in New York'
                 '  den Sprung in ein komplett anderes Leben. Begleitet sie auf'
-                ' ihrem spannenden Weg.',
+                ' ihrem spannenden Weg.'),
         },
     }
 
diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index b6e4a432b..be51f4f87 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -36,7 +36,7 @@ class TudouIE(InfoExtractor):
         'skip': 'Only works from China'
     }]
 
-    def _url_for_id(self, id, quality = None):
+    def _url_for_id(self, id, quality=None):
         info_url = "http://v2.tudou.com/f?id=" + str(id)
         if quality:
             info_url += '&hd' + quality
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 38177c1d7..da02fb6bb 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -262,7 +262,8 @@ def parseOpts(overrideArguments=None):
     video_format.add_option(
         '-f', '--format',
         action='store', dest='format', metavar='FORMAT', default=None,
-        help='video format code, specify the order of preference using'
+        help=(
+            'video format code, specify the order of preference using'
             ' slashes: -f 22/17/18 .  -f mp4 , -f m4a and  -f flv  are also'
             ' supported. You can also use the special names "best",'
             ' "bestvideo", "bestaudio", "worst", "worstvideo" and'
@@ -271,7 +272,7 @@ def parseOpts(overrideArguments=None):
             ' -f  136/137/mp4/bestvideo,140/m4a/bestaudio.'
             ' You can merge the video and audio of two formats into a single'
             ' file using -f <video-format>+<audio-format> (requires ffmpeg or'
-            ' avconv), for example -f bestvideo+bestaudio.')
+            ' avconv), for example -f bestvideo+bestaudio.'))
     video_format.add_option(
         '--all-formats',
         action='store_const', dest='format', const='all',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 9ad0952d5..dd1023eef 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -240,9 +240,9 @@ def sanitize_open(filename, open_mode):
 
         # In case of error, try to remove win32 forbidden chars
         alt_filename = os.path.join(
-                        re.sub('[/<>:"\\|\\\\?\\*]', '#', path_part)
-                        for path_part in os.path.split(filename)
-                       )
+            re.sub('[/<>:"\\|\\\\?\\*]', '#', path_part)
+            for path_part in os.path.split(filename)
+        )
         if alt_filename == filename:
             raise
         else:

From 14d4e90eb155d29e9dc448350f45f0acab5d68d3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 23 Nov 2014 22:25:12 +0100
Subject: [PATCH 0590/1937] [downloader/__init__] Define proper __all__

---
 youtube_dl/downloader/__init__.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 3f941596e..31e28df58 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -30,3 +30,8 @@ def get_suitable_downloader(info_dict):
         return F4mFD
     else:
         return HttpFD
+
+__all__ = [
+    'get_suitable_downloader',
+    'FileDownloader',
+]

From 00e9d396ab4f772c7c76752e7ab096076dfbfd2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 24 Nov 2014 19:37:20 +0100
Subject: [PATCH 0591/1937] [francetv] Use the m3u8 manifest for georestricted
 videos (closes #3963)

Generating the correct urls for the f4m segments seems to require a lot of work.
Also raise an error if the video is not available from your location.
---
 youtube_dl/extractor/francetv.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index d7e9aef90..9bc959a9b 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -26,6 +26,21 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
         if info.get('status') == 'NOK':
             raise ExtractorError(
                 '%s returned error: %s' % (self.IE_NAME, info['message']), expected=True)
+        allowed_countries = info['videos'][0].get('geoblocage')
+        if allowed_countries:
+            georestricted = True
+            geo_info = self._download_json(
+                'http://geo.francetv.fr/ws/edgescape.json', video_id,
+                'Downloading geo restriction info')
+            country = geo_info['reponse']['geo_info']['country_code']
+            if country not in allowed_countries:
+                raise ExtractorError(
+                    'The video is not available from your location',
+                    expected=True)
+        else:
+            georestricted = False
+
+
 
         formats = []
         for video in info['videos']:
@@ -36,6 +51,10 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
                 continue
             format_id = video['format']
             if video_url.endswith('.f4m'):
+                if georestricted:
+                    # See https://github.com/rg3/youtube-dl/issues/3963
+                    # m3u8 urls work fine
+                    continue
                 video_url_parsed = compat_urllib_parse_urlparse(video_url)
                 f4m_url = self._download_webpage(
                     'http://hdfauth.francetv.fr/esi/urltokengen2.html?url=%s' % video_url_parsed.path,

From 355682be016e6916ea0b3565f7e5fb496f3a660a Mon Sep 17 00:00:00 2001
From: Oskar Jauch <oskar.jauch@gmail.com>
Date: Mon, 24 Nov 2014 20:02:00 +0100
Subject: [PATCH 0592/1937] bpb Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/bpb.py      | 41 ++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 youtube_dl/extractor/bpb.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fb5e6ac77..a038f298a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -32,6 +32,7 @@ from .bilibili import BiliBiliIE
 from .blinkx import BlinkxIE
 from .bliptv import BlipTVIE, BlipTVUserIE
 from .bloomberg import BloombergIE
+from .bpb import BpbIE
 from .br import BRIE
 from .breakcom import BreakIE
 from .brightcove import BrightcoveIE
diff --git a/youtube_dl/extractor/bpb.py b/youtube_dl/extractor/bpb.py
new file mode 100644
index 000000000..560688589
--- /dev/null
+++ b/youtube_dl/extractor/bpb.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+class BpbIE(InfoExtractor):
+	IE_NAME = 'Bundeszentrale für politische Bildung'
+	_VALID_URL = r'http://www\.bpb\.de/mediathek/.*'
+	
+	_TEST = {
+		'url': 'http://www.bpb.de/mediathek/297/joachim-gauck-zu-1989-und-die-erinnerung-an-die-ddr',
+		'md5': '0792086e8e2bfbac9cdf27835d5f2093',
+		'info_dict': {
+			'id': '12490',
+			'ext': 'mp4',
+			'title': 'Joachim Gauck zu 1989 und die Erinnerung an die DDR',
+			'description': 'Joachim Gauck, erster Beauftragter für die Stasi-Unterlagen, spricht auf dem Geschichtsforum über die friedliche Revolution 1989 und eine "gewisse Traurigkeit" im Umgang mit der DDR-Vergangenheit.'
+		}
+	}
+	
+	def _real_extract(self, url):
+		webpage = self._download_webpage(url, '')
+		
+		title = self._html_search_regex(r'<h2 class="white">(.*?)</h2>', webpage, 'title')
+		
+		video_id = self._html_search_regex(r'http://film\.bpb\.de/player/dokument_(?P<video_id>[0-9]+)\.mp4', webpage, 'video_id')
+		
+		url = 'http://film.bpb.de/player/dokument_' + video_id + '.mp4'
+		
+		description = self._og_search_description(webpage)
+		
+		return {
+			'id': video_id,
+			'url': url,
+			'title': title,
+			'description': description,
+			'ext': 'mp4'
+		}

From 2c25a2bd29236ef744e3b9032230dc74c1029df5 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 24 Nov 2014 23:15:33 +0200
Subject: [PATCH 0593/1937] [tunein] Add new extractor (Closes #4097)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/tunein.py   | 101 +++++++++++++++++++++++++++++++
 2 files changed, 102 insertions(+)
 create mode 100644 youtube_dl/extractor/tunein.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fb5e6ac77..cdcda1fa9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -405,6 +405,7 @@ from .trutube import TruTubeIE
 from .tube8 import Tube8IE
 from .tudou import TudouIE
 from .tumblr import TumblrIE
+from .tunein import TuneInIE
 from .turbo import TurboIE
 from .tutv import TutvIE
 from .tvigle import TvigleIE
diff --git a/youtube_dl/extractor/tunein.py b/youtube_dl/extractor/tunein.py
new file mode 100644
index 000000000..8c29f1634
--- /dev/null
+++ b/youtube_dl/extractor/tunein.py
@@ -0,0 +1,101 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class TuneInIE(InfoExtractor):
+    _VALID_URL = r'''(?x)https?://(?:www\.)?
+    (?:
+        tunein\.com/
+        (?:
+            radio/.*?-s|
+            station/.*?StationId\=
+        )(?P<id>[0-9]+)
+        |tun\.in/(?P<redirect_id>[A-Za-z0-9]+)
+    )
+    '''
+
+    _INFO_DICT = {
+        'id': '34682',
+        'title': 'Jazz 24 on 88.5 Jazz24 - KPLU-HD2',
+        'ext': 'AAC',
+        'thumbnail': 're:^https?://.*\.png$',
+        'location': 'Tacoma, WA',
+    }
+    _TESTS = [
+        {
+            'url': 'http://tunein.com/radio/Jazz24-885-s34682/',
+            'info_dict': _INFO_DICT,
+            'params': {
+                'skip_download': True,  # live stream
+            },
+        },
+        {  # test redirection
+            'url': 'http://tun.in/ser7s',
+            'info_dict': _INFO_DICT,
+            'params': {
+                'skip_download': True,  # live stream
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        redirect_id = mobj.group('redirect_id')
+        if redirect_id:
+            # The server doesn't support HEAD requests
+            urlh = self._request_webpage(
+                url, redirect_id, note='Downloading redirect page')
+            url = urlh.geturl()
+            self.to_screen('Following redirect: %s' % url)
+            mobj = re.match(self._VALID_URL, url)
+        station_id = mobj.group('id')
+
+        webpage = self._download_webpage(
+            url, station_id, note='Downloading station webpage')
+
+        payload = self._html_search_regex(
+            r'(?m)TuneIn\.payload\s*=\s*(\{[^$]+?)$', webpage, 'JSON data')
+        json_data = json.loads(payload)
+        station_info = json_data['Station']['broadcast']
+        title = station_info['Title']
+        thumbnail = station_info.get('Logo')
+        location = station_info.get('Location')
+        streams_url = station_info.get('StreamUrl')
+        if not streams_url:
+            raise ExtractorError('No downloadable streams found',
+                                 expected=True)
+        stream_data = self._download_webpage(
+            streams_url, station_id, note='Downloading stream data')
+        streams = json.loads(self._search_regex(
+            r'\((.*)\);', stream_data, 'stream info'))['Streams']
+
+        is_live = None
+        formats = []
+        for stream in streams:
+            if stream.get('Type') == 'Live':
+                is_live = True
+            formats.append({
+                'abr': stream.get('Bandwidth'),
+                'ext': stream.get('MediaType'),
+                'acodec': stream.get('MediaType'),
+                'vcodec': 'none',
+                'url': stream.get('Url'),
+                # Sometimes streams with the highest quality do not exist
+                'preference': stream.get('Reliability'),
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': station_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'location': location,
+            'is_live': is_live,
+        }

From 3d960a22fad1a820f5d0fe9e6f3fde2e0a5a927c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 24 Nov 2014 22:47:23 +0100
Subject: [PATCH 0594/1937] [bpb] Simplify (#4298)

---
 youtube_dl/extractor/bpb.py | 66 +++++++++++++++++--------------------
 1 file changed, 31 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/bpb.py b/youtube_dl/extractor/bpb.py
index 560688589..510813f76 100644
--- a/youtube_dl/extractor/bpb.py
+++ b/youtube_dl/extractor/bpb.py
@@ -1,41 +1,37 @@
 # coding: utf-8
-
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
+
 class BpbIE(InfoExtractor):
-	IE_NAME = 'Bundeszentrale für politische Bildung'
-	_VALID_URL = r'http://www\.bpb\.de/mediathek/.*'
-	
-	_TEST = {
-		'url': 'http://www.bpb.de/mediathek/297/joachim-gauck-zu-1989-und-die-erinnerung-an-die-ddr',
-		'md5': '0792086e8e2bfbac9cdf27835d5f2093',
-		'info_dict': {
-			'id': '12490',
-			'ext': 'mp4',
-			'title': 'Joachim Gauck zu 1989 und die Erinnerung an die DDR',
-			'description': 'Joachim Gauck, erster Beauftragter für die Stasi-Unterlagen, spricht auf dem Geschichtsforum über die friedliche Revolution 1989 und eine "gewisse Traurigkeit" im Umgang mit der DDR-Vergangenheit.'
-		}
-	}
-	
-	def _real_extract(self, url):
-		webpage = self._download_webpage(url, '')
-		
-		title = self._html_search_regex(r'<h2 class="white">(.*?)</h2>', webpage, 'title')
-		
-		video_id = self._html_search_regex(r'http://film\.bpb\.de/player/dokument_(?P<video_id>[0-9]+)\.mp4', webpage, 'video_id')
-		
-		url = 'http://film.bpb.de/player/dokument_' + video_id + '.mp4'
-		
-		description = self._og_search_description(webpage)
-		
-		return {
-			'id': video_id,
-			'url': url,
-			'title': title,
-			'description': description,
-			'ext': 'mp4'
-		}
+    IE_DESC = 'Bundeszentrale für politische Bildung'
+    _VALID_URL = r'http://www\.bpb\.de/mediathek/(?P<id>[0-9]+)/'
+
+    _TEST = {
+        'url': 'http://www.bpb.de/mediathek/297/joachim-gauck-zu-1989-und-die-erinnerung-an-die-ddr',
+        'md5': '0792086e8e2bfbac9cdf27835d5f2093',
+        'info_dict': {
+            'id': '297',
+            'ext': 'mp4',
+            'title': 'Joachim Gauck zu 1989 und die Erinnerung an die DDR',
+            'description': 'Joachim Gauck, erster Beauftragter für die Stasi-Unterlagen, spricht auf dem Geschichtsforum über die friedliche Revolution 1989 und eine "gewisse Traurigkeit" im Umgang mit der DDR-Vergangenheit.'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(
+            r'<h2 class="white">(.*?)</h2>', webpage, 'title')
+        video_url = self._html_search_regex(
+            r'(http://film\.bpb\.de/player/dokument_[0-9]+\.mp4)',
+            webpage, 'video URL')
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'description': self._og_search_description(webpage),
+        }

From fab89c67c5e623a6b1a8e518b78a70fc39d5a006 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 24 Nov 2014 22:47:44 +0100
Subject: [PATCH 0595/1937] Credit @ossi96 for bpb (#4298)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index c8d60282c..24358d548 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -85,3 +85,4 @@ Jan Matějka
 Mauroy Sébastien
 William Sewell
 Dao Hoang Son
+Oskar Jauch

From 93129d9442cf96b71a6f0d191ed0e80de38382d1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 24 Nov 2014 22:56:43 +0100
Subject: [PATCH 0596/1937] release 2014.11.24

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6fff4fcc5..788b8acd8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.23.1'
+__version__ = '2014.11.24'

From 1cdedfee105f09e974d196b7f385cfcce99545f1 Mon Sep 17 00:00:00 2001
From: t0mm0 <dev@onairsoftware.co.uk>
Date: Tue, 25 Nov 2014 03:25:28 +0000
Subject: [PATCH 0597/1937] [XMinus] Added new extractor.

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/xminus.py   | 53 ++++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)
 create mode 100644 youtube_dl/extractor/xminus.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a680973de..45edb6a24 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -483,6 +483,7 @@ from .wrzuta import WrzutaIE
 from .xbef import XBefIE
 from .xboxclips import XboxClipsIE
 from .xhamster import XHamsterIE
+from .xminus import XMinusIE
 from .xnxx import XNXXIE
 from .xvideos import XVideosIE
 from .xtube import XTubeUserIE, XTubeIE
diff --git a/youtube_dl/extractor/xminus.py b/youtube_dl/extractor/xminus.py
new file mode 100644
index 000000000..c92c8451a
--- /dev/null
+++ b/youtube_dl/extractor/xminus.py
@@ -0,0 +1,53 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class XMinusIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?x-minus\.org/track/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://x-minus.org/track/4542/%D0%BF%D0%B5%D1%81%D0%B5%D0%BD%D0%BA%D0%B0-%D1%88%D0%BE%D1%84%D0%B5%D1%80%D0%B0.html',
+        'md5': '401a15f2d2dcf6d592cb95528d72a2a8',
+        'info_dict': {
+            'id': '4542',
+            'ext': 'mp3',
+            'title': 'Леонид Агутин-Песенка шофера',
+            'duration': 156,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        # TODO more code goes here, for example ...
+        webpage = self._download_webpage(url, video_id)
+        artist = self._html_search_regex(
+            r'minus_track.artist="(.+?)"', webpage, 'artist')
+        title = artist + '-' + self._html_search_regex(
+            r'minus_track.title="(.+?)"', webpage, 'title')
+        duration = int_or_none(self._html_search_regex(
+            r'minus_track.dur_sec=\'([0-9]+?)\'', webpage, 'duration'))
+        enc_token = self._html_search_regex(
+            r'data-mt="(.*?)"', webpage, 'enc_token')
+        token = self._decode_token(enc_token)
+        url = 'http://x-minus.org/dwlf/{}/{}.mp3'.format(video_id, token)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': url,
+            'duration': duration,
+        }
+
+    def _decode_token(self, enc_token):
+        token = ''
+        pos = 0
+        for c in reversed(enc_token):
+            if pos != 3:
+                token += chr(ord(c) - 1)
+            else:
+                token += c
+            pos += 1
+        return token

From f0c3d729d7acb3af64f21b6042d20127ce7149ae Mon Sep 17 00:00:00 2001
From: Matthew Rayfield <matthewrayfield@gmail.com>
Date: Tue, 25 Nov 2014 02:54:13 -0500
Subject: [PATCH 0598/1937] [tmz] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tmz.py      | 39 ++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 youtube_dl/extractor/tmz.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a680973de..7e91db601 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -393,6 +393,7 @@ from .thesixtyone import TheSixtyOneIE
 from .thisav import ThisAVIE
 from .tinypic import TinyPicIE
 from .tlc import TlcIE, TlcDeIE
+from .tmz import TMZIE
 from .tnaflix import TNAFlixIE
 from .thvideo import (
     THVideoIE,
diff --git a/youtube_dl/extractor/tmz.py b/youtube_dl/extractor/tmz.py
new file mode 100644
index 000000000..b82dc8233
--- /dev/null
+++ b/youtube_dl/extractor/tmz.py
@@ -0,0 +1,39 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class TMZIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?tmz\.com/videos/(?P<id>[^/]+)/?'
+    _TEST = {
+        'url': 'http://www.tmz.com/videos/0_okj015ty/',
+        'md5': '791204e3bf790b1426cb2db0706184c0',
+        'info_dict': {
+            'id': '0_okj015ty',
+            'url': 'http://tmz.vo.llnwd.net/o28/2014-03/13/0_okj015ty_0_rt8ro3si_2.mp4',
+            'ext': 'mp4',
+            'title': 'Kim Kardashian\'s Boobs Unlock a Mystery!',
+            'description': 'Did Kim Kardasain try to one-up Khloe by one-upping Kylie???  Or is she just showing off her amazing boobs?',
+            'thumbnail': 'http://cdnbakmi.kaltura.com/p/591531/sp/59153100/thumbnail/entry_id/0_okj015ty/version/100002/acv/182/width/640',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        url = self._html_search_regex(r'<meta name="VideoURL" content="(.+)"', webpage, 'url')
+        title = self._html_search_regex(r'<meta property="og:title" content="(.+)"', webpage, 'title')
+        description = self._html_search_regex(r'<meta property="og:description" content="(.+)"', webpage, 'description')
+        thumbnail_url = self._html_search_regex(r'<meta name="ThumbURL" content="(.+)"', webpage, 'thumbnail url')
+
+        return {
+            'id': video_id,
+            'url': url,
+            'ext': 'mp4',
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail_url,
+        }

From c553fe5d293dd88bffa8b04a1f83244bb30f44b1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 25 Nov 2014 09:16:40 +0100
Subject: [PATCH 0599/1937] [tmz] Simplify (#4304)

---
 youtube_dl/extractor/tmz.py | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/tmz.py b/youtube_dl/extractor/tmz.py
index b82dc8233..827aa08a4 100644
--- a/youtube_dl/extractor/tmz.py
+++ b/youtube_dl/extractor/tmz.py
@@ -21,19 +21,12 @@ class TMZIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         webpage = self._download_webpage(url, video_id)
 
-        url = self._html_search_regex(r'<meta name="VideoURL" content="(.+)"', webpage, 'url')
-        title = self._html_search_regex(r'<meta property="og:title" content="(.+)"', webpage, 'title')
-        description = self._html_search_regex(r'<meta property="og:description" content="(.+)"', webpage, 'description')
-        thumbnail_url = self._html_search_regex(r'<meta name="ThumbURL" content="(.+)"', webpage, 'thumbnail url')
-
         return {
             'id': video_id,
-            'url': url,
-            'ext': 'mp4',
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail_url,
+            'url': self._html_search_meta('VideoURL', webpage, fatal=True),
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._html_search_meta('ThumbURL', webpage),
         }

From 70d68eb46f80cbf30471f943c5dd7bd4ea0dba95 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 25 Nov 2014 09:17:59 +0100
Subject: [PATCH 0600/1937] Credit @MatthewRayfield for tmz (#4304)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 24358d548..cc129896b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -86,3 +86,4 @@ Mauroy Sébastien
 William Sewell
 Dao Hoang Son
 Oskar Jauch
+Matthew Rayfield

From c920d7f00d333e498edbe15b7e29cc2e97b54d5e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 25 Nov 2014 09:23:44 +0100
Subject: [PATCH 0601/1937] [README] Adapt code to new style

Next to every IE will download the webpage first anyways.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c284db495..6e572813c 100644
--- a/README.md
+++ b/README.md
@@ -492,9 +492,9 @@ If you want to add support for a new site, you can follow this quick list (assum
 
         def _real_extract(self, url):
             video_id = self._match_id(url)
+            webpage = self._download_webpage(url, video_id)
 
             # TODO more code goes here, for example ...
-            webpage = self._download_webpage(url, video_id)
             title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
 
             return {

From c3e74731c2acd2878a38c2a2daa2ed0c11352762 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 25 Nov 2014 09:36:25 +0100
Subject: [PATCH 0602/1937] [README] Mention _og_search_description (#4304)

Lots of sites do have this meta tag, so just add it to the example.
---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 6e572813c..cce2819fd 100644
--- a/README.md
+++ b/README.md
@@ -500,6 +500,7 @@ If you want to add support for a new site, you can follow this quick list (assum
             return {
                 'id': video_id,
                 'title': title,
+                'description': self._og_search_description(webpage),
                 # TODO more properties (see youtube_dl/extractor/common.py)
             }
     ```

From be64b5b098e3563d563bcf091f6f74edf22d7764 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 25 Nov 2014 09:54:54 +0100
Subject: [PATCH 0603/1937] [xminus] Simplify and extend (#4302)

---
 test/test_utils.py             | 10 +++++++
 youtube_dl/extractor/xminus.py | 54 +++++++++++++++++++++-------------
 youtube_dl/utils.py            | 51 ++++++++++++++++++++++++++++++++
 3 files changed, 95 insertions(+), 20 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 8307599b3..9dc879e0d 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -47,6 +47,7 @@ from youtube_dl.utils import (
     js_to_json,
     intlist_to_bytes,
     args_to_str,
+    parse_filesize,
 )
 
 
@@ -367,5 +368,14 @@ class TestUtil(unittest.TestCase):
             'foo ba/r -baz \'2 be\' \'\''
         )
 
+    def test_parse_filesize(self):
+        self.assertEqual(parse_filesize(None), None)
+        self.assertEqual(parse_filesize(''), None)
+        self.assertEqual(parse_filesize('91 B'), 91)
+        self.assertEqual(parse_filesize('foobar'), None)
+        self.assertEqual(parse_filesize('2 MiB'), 2097152)
+        self.assertEqual(parse_filesize('5 GB'), 5000000000)
+        self.assertEqual(parse_filesize('1.2Tb'), 1200000000000)
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/xminus.py b/youtube_dl/extractor/xminus.py
index c92c8451a..a3fd8f48c 100644
--- a/youtube_dl/extractor/xminus.py
+++ b/youtube_dl/extractor/xminus.py
@@ -2,7 +2,14 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..compat import (
+    compat_chr,
+    compat_ord,
+)
+from ..utils import (
+    int_or_none,
+    parse_filesize,
+)
 
 
 class XMinusIE(InfoExtractor):
@@ -15,39 +22,46 @@ class XMinusIE(InfoExtractor):
             'ext': 'mp3',
             'title': 'Леонид Агутин-Песенка шофера',
             'duration': 156,
+            'tbr': 320,
+            'filesize_approx': 5900000,
+            'view_count': int,
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
-        # TODO more code goes here, for example ...
         webpage = self._download_webpage(url, video_id)
+
         artist = self._html_search_regex(
-            r'minus_track.artist="(.+?)"', webpage, 'artist')
+            r'minus_track\.artist="(.+?)"', webpage, 'artist')
         title = artist + '-' + self._html_search_regex(
-            r'minus_track.title="(.+?)"', webpage, 'title')
+            r'minus_track\.title="(.+?)"', webpage, 'title')
         duration = int_or_none(self._html_search_regex(
-            r'minus_track.dur_sec=\'([0-9]+?)\'', webpage, 'duration'))
+            r'minus_track\.dur_sec=\'([0-9]*?)\'',
+            webpage, 'duration', fatal=False))
+        filesize_approx = parse_filesize(self._html_search_regex(
+            r'<div class="filesize[^"]*"></div>\s*([0-9.]+\s*[a-zA-Z][bB])',
+            webpage, 'approximate filesize', fatal=False))
+        tbr = int_or_none(self._html_search_regex(
+            r'<div class="quality[^"]*"></div>\s*([0-9]+)\s*kbps',
+            webpage, 'bitrate', fatal=False))
+        view_count = int_or_none(self._html_search_regex(
+            r'<div class="quality.*?► ([0-9]+)',
+            webpage, 'view count', fatal=False))
+
         enc_token = self._html_search_regex(
             r'data-mt="(.*?)"', webpage, 'enc_token')
-        token = self._decode_token(enc_token)
-        url = 'http://x-minus.org/dwlf/{}/{}.mp3'.format(video_id, token)
+        token = ''.join(
+            c if pos == 3 else compat_chr(compat_ord(c) - 1)
+            for pos, c in enumerate(reversed(enc_token)))
+        video_url = 'http://x-minus.org/dwlf/%s/%s.mp3' % (video_id, token)
 
         return {
             'id': video_id,
             'title': title,
-            'url': url,
+            'url': video_url,
             'duration': duration,
+            'filesize_approx': filesize_approx,
+            'tbr': tbr,
+            'view_count': view_count,
         }
-
-    def _decode_token(self, enc_token):
-        token = ''
-        pos = 0
-        for c in reversed(enc_token):
-            if pos != 3:
-                token += chr(ord(c) - 1)
-            else:
-                token += c
-            pos += 1
-        return token
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index dd1023eef..f9b5f9867 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1046,6 +1046,57 @@ def format_bytes(bytes):
     return '%.2f%s' % (converted, suffix)
 
 
+def parse_filesize(s):
+    if s is None:
+        return None
+
+    # The lower-case forms are of course incorrect and inofficial,
+    # but we support those too
+    _UNIT_TABLE = {
+        'B': 1,
+        'b': 1,
+        'KiB': 1024,
+        'KB': 1000,
+        'kB': 1024,
+        'Kb': 1000,
+        'MiB': 1024 ** 2,
+        'MB': 1000 ** 2,
+        'mB': 1024 ** 2,
+        'Mb': 1000 ** 2,
+        'GiB': 1024 ** 3,
+        'GB': 1000 ** 3,
+        'gB': 1024 ** 3,
+        'Gb': 1000 ** 3,
+        'TiB': 1024 ** 4,
+        'TB': 1000 ** 4,
+        'tB': 1024 ** 4,
+        'Tb': 1000 ** 4,
+        'PiB': 1024 ** 5,
+        'PB': 1000 ** 5,
+        'pB': 1024 ** 5,
+        'Pb': 1000 ** 5,
+        'EiB': 1024 ** 6,
+        'EB': 1000 ** 6,
+        'eB': 1024 ** 6,
+        'Eb': 1000 ** 6,
+        'ZiB': 1024 ** 7,
+        'ZB': 1000 ** 7,
+        'zB': 1024 ** 7,
+        'Zb': 1000 ** 7,
+        'YiB': 1024 ** 8,
+        'YB': 1000 ** 8,
+        'yB': 1024 ** 8,
+        'Yb': 1000 ** 8,
+    }
+
+    units_re = '|'.join(re.escape(u) for u in _UNIT_TABLE)
+    m = re.match(r'(?P<num>[0-9]+(?:\.[0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
+    if not m:
+        return None
+
+    return int(float(m.group('num')) * _UNIT_TABLE[m.group('unit')])
+
+
 def get_term_width():
     columns = compat_getenv('COLUMNS', None)
     if columns:

From d9836a591714aceda843d33ba7786353a6c8fa1b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 25 Nov 2014 09:56:52 +0100
Subject: [PATCH 0604/1937] release 2014.11.25

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 788b8acd8..c3c79d520 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.24'
+__version__ = '2014.11.25'

From 3004ae2c3a2465ce83f7339a0a81584abd9dd84f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 25 Nov 2014 12:16:48 +0100
Subject: [PATCH 0605/1937] Credit @t0mm0 for xminus (#4302)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index cc129896b..d4b0b75fa 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -87,3 +87,4 @@ William Sewell
 Dao Hoang Son
 Oskar Jauch
 Matthew Rayfield
+t0mm0

From 65f3a228b16c55fee959eee055767a796479270f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 25 Nov 2014 14:34:19 +0100
Subject: [PATCH 0606/1937] [generic] Add support for LazyYT embeds (Fixes
 #4306)

---
 youtube_dl/extractor/generic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 128715caf..6a95e2952 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -445,6 +445,14 @@ class GenericIE(InfoExtractor):
                 'title': 'Rosetta #CometLanding webcast HL 10',
             }
         },
+        # LazyYT
+        {
+            'url': 'http://discourse.ubuntu.com/t/unity-8-desktop-mode-windows-on-mir/1986',
+            'info_dict': {
+                'title': 'Unity 8 desktop-mode windows on Mir! - Ubuntu Discourse',
+            },
+            'playlist_mincount': 2,
+        }
     ]
 
     def report_following_redirect(self, new_url):
@@ -702,6 +710,12 @@ class GenericIE(InfoExtractor):
             return _playlist_from_matches(
                 matches, lambda m: unescapeHTML(m[1]))
 
+        # Look for lazyYT YouTube embed
+        matches = re.findall(
+            r'class="lazyYT" data-youtube-id="([^"]+)"', webpage)
+        if matches:
+            return _playlist_from_matches(matches, lambda m: unescapeHTML(m))
+
         # Look for embedded Dailymotion player
         matches = re.findall(
             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/embed/video/.+?)\1', webpage)

From 63c9b2c1d916ec3e3d2ab5e5247ac8a372c7957e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 25 Nov 2014 14:34:29 +0100
Subject: [PATCH 0607/1937] release 2014.11.25.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c3c79d520..65163c16b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.25'
+__version__ = '2014.11.25.1'

From b9ed3af34368e110e6ab2262ce908d2e8572f025 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 25 Nov 2014 22:24:33 +0600
Subject: [PATCH 0608/1937] [tass] Add extractor (Closes #4296)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tass.py     | 62 ++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+)
 create mode 100644 youtube_dl/extractor/tass.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3e2133f5c..5c096542b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -373,6 +373,7 @@ from .syfy import SyfyIE
 from .sztvhu import SztvHuIE
 from .tagesschau import TagesschauIE
 from .tapely import TapelyIE
+from .tass import TassIE
 from .teachertube import (
     TeacherTubeIE,
     TeacherTubeUserIE,
diff --git a/youtube_dl/extractor/tass.py b/youtube_dl/extractor/tass.py
new file mode 100644
index 000000000..c4ef70778
--- /dev/null
+++ b/youtube_dl/extractor/tass.py
@@ -0,0 +1,62 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    js_to_json,
+    qualities,
+)
+
+
+class TassIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:tass\.ru|itar-tass\.com)/[^/]+/(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'http://tass.ru/obschestvo/1586870',
+            'md5': '3b4cdd011bc59174596b6145cda474a4',
+            'info_dict': {
+                'id': '1586870',
+                'ext': 'mp4',
+                'title': 'Посетителям московского зоопарка показали красную панду',
+                'description': 'Приехавшую из Дублина Зейну можно увидеть в павильоне "Кошки тропиков"',
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+        },
+        {
+            'url': 'http://itar-tass.com/obschestvo/1600009',
+            'only_matching': True,
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        sources = json.loads(js_to_json(self._search_regex(
+            r'(?s)sources\s*:\s*(\[.+?\])', webpage, 'sources')))
+
+        quality = qualities(['sd', 'hd'])
+
+        formats = []
+        for source in sources:
+            video_url = source.get('file')
+            if not video_url or not video_url.startswith('http') or not video_url.endswith('.mp4'):
+                continue
+            label = source.get('label')
+            formats.append({
+                'url': video_url,
+                'format_id': label,
+                'quality': quality(label),
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'formats': formats,
+        }

From 4e262a8838d487362a85eb8b8693d2fa84899f17 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 10:44:39 +0100
Subject: [PATCH 0609/1937] [generic] Detect direct video links (Fixes #4149,
 #4313)

---
 youtube_dl/extractor/common.py  |  4 +++-
 youtube_dl/extractor/generic.py | 42 +++++++++++++++++++++++++++++----
 2 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 3c2d46dd5..3e84d0e62 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -296,9 +296,11 @@ class InfoExtractor(object):
         content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal)
         return (content, urlh)
 
-    def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True):
+    def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None):
         content_type = urlh.headers.get('Content-Type', '')
         webpage_bytes = urlh.read()
+        if prefix is not None:
+            webpage_bytes = prefix + webpage_bytes
         m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
         if m:
             encoding = m.group(1)
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6a95e2952..328301de3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -452,7 +452,23 @@ class GenericIE(InfoExtractor):
                 'title': 'Unity 8 desktop-mode windows on Mir! - Ubuntu Discourse',
             },
             'playlist_mincount': 2,
+        },
+        # Direct link with incorrect MIME type
+        {
+            'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
+            'md5': '4ccbebe5f36706d85221f204d7eb5913',
+            'info_dict': {
+                'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
+                'id': '5_Lennart_Poettering_-_Systemd',
+                'ext': 'webm',
+                'title': '5_Lennart_Poettering_-_Systemd',
+                'upload_date': '20141120',
+            },
+            'expected_warnings': [
+                'URL could be a direct video link, returning it as such.'
+            ]
         }
+
     ]
 
     def report_following_redirect(self, new_url):
@@ -606,10 +622,28 @@ class GenericIE(InfoExtractor):
         if not self._downloader.params.get('test', False) and not is_intentional:
             self._downloader.report_warning('Falling back on generic information extractor.')
 
-        if full_response:
-            webpage = self._webpage_read_content(full_response, url, video_id)
-        else:
-            webpage = self._download_webpage(url, video_id)
+        if not full_response:
+            full_response = self._request_webpage(url, video_id)
+
+        # Maybe it's a direct link to a video?
+        # Be careful not to download the whole thing!
+        first_bytes = full_response.read(512)
+        if not re.match(r'^\s*<', first_bytes.decode('utf-8', 'replace')):
+            self._downloader.report_warning(
+                'URL could be a direct video link, returning it as such.')
+            upload_date = unified_strdate(
+                head_response.headers.get('Last-Modified'))
+            return {
+                'id': video_id,
+                'title': os.path.splitext(url_basename(url))[0],
+                'direct': True,
+                'url': url,
+                'upload_date': upload_date,
+            }
+
+        webpage = self._webpage_read_content(
+            full_response, url, video_id, prefix=first_bytes)
+
         self.report_extraction(video_id)
 
         # Is it an RSS feed?

From ddfd0f2727797f42c807cfc55c6ffa2b2561343f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 10:46:12 +0100
Subject: [PATCH 0610/1937] release 2014.11.26

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 65163c16b..82c204592 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2014.11.25.1'
+__version__ = '2014.11.26'

From acda92f6bc76a81e6457448fdf371f9d04e0065d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 10:51:03 +0100
Subject: [PATCH 0611/1937] Clarify --no-playlist documentation (Closes #4309)

---
 README.md               | 3 ++-
 youtube_dl/YoutubeDL.py | 2 ++
 youtube_dl/options.py   | 2 +-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index cce2819fd..6669c1d7c 100644
--- a/README.md
+++ b/README.md
@@ -93,7 +93,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      COUNT views
     --max-views COUNT                Do not download any videos with more than
                                      COUNT views
-    --no-playlist                    download only the currently playing video
+    --no-playlist                    If the URL refers to a video and a
+                                     playlist, download only the video.
     --age-limit YEARS                download only videos suitable for the given
                                      age
     --download-archive FILE          Download only videos not listed in the
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 21c7c298a..6342ae767 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -174,6 +174,8 @@ class YoutubeDL(object):
     extract_flat:      Do not resolve URLs, return the immediate result.
                        Pass in 'in_playlist' to only show this behavior for
                        playlist items.
+    no_playlist:       If the URL contains both a playlist and a video ID,
+                       download the video, not the playlist.
 
     The following parameters are not used by YoutubeDL itself, they are used by
     the FileDownloader:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index da02fb6bb..2e8c71508 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -222,7 +222,7 @@ def parseOpts(overrideArguments=None):
     selection.add_option(
         '--no-playlist',
         action='store_true', dest='noplaylist', default=False,
-        help='download only the currently playing video')
+        help='If the URL refers to a video and a playlist, download only the video.')
     selection.add_option(
         '--age-limit',
         metavar='YEARS', dest='age_limit', default=None, type=int,

From 4aae94f9d080980212856e220d254a27140af0a7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 11:25:43 +0100
Subject: [PATCH 0612/1937] [YoutubeDL] Remove incorrect documentation

---
 youtube_dl/YoutubeDL.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6342ae767..21c7c298a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -174,8 +174,6 @@ class YoutubeDL(object):
     extract_flat:      Do not resolve URLs, return the immediate result.
                        Pass in 'in_playlist' to only show this behavior for
                        playlist items.
-    no_playlist:       If the URL contains both a playlist and a video ID,
-                       download the video, not the playlist.
 
     The following parameters are not used by YoutubeDL itself, they are used by
     the FileDownloader:

From b061ea6e9f49ef2f80e8dac6413088b1b83cc92d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 11:48:09 +0100
Subject: [PATCH 0613/1937] [compat] Beautify assertion

---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 2983501ae..27596687d 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -270,7 +270,7 @@ if sys.version_info < (3, 0):
         print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
 else:
     def compat_print(s):
-        assert type(s) == type(u'')
+        assert isinstance(s, compat_str)
         print(s)
 
 

From ab4ee31eb15842a2f100bf618579422d3604f03c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 11:50:22 +0100
Subject: [PATCH 0614/1937] [utils] remove useless u prefix

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f9b5f9867..4d3cbac74 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -131,7 +131,7 @@ if sys.version_info >= (2, 7):
         """ Find the xpath xpath[@key=val] """
         assert re.match(r'^[a-zA-Z-]+$', key)
         assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
-        expr = xpath + u"[@%s='%s']" % (key, val)
+        expr = xpath + "[@%s='%s']" % (key, val)
         return node.find(expr)
 else:
     def find_xpath_attr(node, xpath, key, val):

From 15938ab67a61ec7558311d964fb65ccd4e05a782 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 12:24:57 +0100
Subject: [PATCH 0615/1937] [update] Modernize

---
 youtube_dl/update.py | 67 +++++++++++++++++++++-----------------------
 1 file changed, 32 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 448ad229c..941db3c8f 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import io
 import json
 import traceback
@@ -7,7 +9,7 @@ import subprocess
 import sys
 from zipimport import zipimporter
 
-from .utils import (
+from .compat import (
     compat_str,
     compat_urllib_request,
 )
@@ -19,12 +21,7 @@ def rsa_verify(message, signature, key):
     from hashlib import sha256
     from sys import version_info
 
-    def b(x):
-        if version_info[0] == 2:
-            return x
-        else:
-            return x.encode('latin1')
-    assert(type(message) == type(b('')))
+    assert isinstance(message, bytes)
     block_size = 0
     n = key[0]
     while n:
@@ -35,14 +32,14 @@ def rsa_verify(message, signature, key):
     while signature:
         raw_bytes.insert(0, pack("B", signature & 0xFF))
         signature >>= 8
-    signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes)
-    if signature[0:2] != b('\x00\x01'):
+    signature = (block_size - len(raw_bytes)) * b'\x00' + b''.join(raw_bytes)
+    if signature[0:2] != b'\x00\x01':
         return False
     signature = signature[2:]
-    if not b('\x00') in signature:
+    if b'\x00' not in signature:
         return False
-    signature = signature[signature.index(b('\x00')) + 1:]
-    if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')):
+    signature = signature[signature.index(b'\x00') + 1:]
+    if not signature.startswith(b'\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20'):
         return False
     signature = signature[19:]
     if signature != sha256(message).digest():
@@ -59,7 +56,7 @@ def update_self(to_screen, verbose):
     UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537)
 
     if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, "frozen"):
-        to_screen(u'It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
+        to_screen('It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
         return
 
     # Check if there is a new version
@@ -68,10 +65,10 @@ def update_self(to_screen, verbose):
     except:
         if verbose:
             to_screen(compat_str(traceback.format_exc()))
-        to_screen(u'ERROR: can\'t find the current version. Please try again later.')
+        to_screen('ERROR: can\'t find the current version. Please try again later.')
         return
     if newversion == __version__:
-        to_screen(u'youtube-dl is up-to-date (' + __version__ + ')')
+        to_screen('youtube-dl is up-to-date (' + __version__ + ')')
         return
 
     # Download and check versions info
@@ -81,15 +78,15 @@ def update_self(to_screen, verbose):
     except:
         if verbose:
             to_screen(compat_str(traceback.format_exc()))
-        to_screen(u'ERROR: can\'t obtain versions info. Please try again later.')
+        to_screen('ERROR: can\'t obtain versions info. Please try again later.')
         return
     if not 'signature' in versions_info:
-        to_screen(u'ERROR: the versions file is not signed or corrupted. Aborting.')
+        to_screen('ERROR: the versions file is not signed or corrupted. Aborting.')
         return
     signature = versions_info['signature']
     del versions_info['signature']
     if not rsa_verify(json.dumps(versions_info, sort_keys=True).encode('utf-8'), signature, UPDATES_RSA_KEY):
-        to_screen(u'ERROR: the versions file signature is invalid. Aborting.')
+        to_screen('ERROR: the versions file signature is invalid. Aborting.')
         return
 
     version_id = versions_info['latest']
@@ -97,10 +94,10 @@ def update_self(to_screen, verbose):
     def version_tuple(version_str):
         return tuple(map(int, version_str.split('.')))
     if version_tuple(__version__) >= version_tuple(version_id):
-        to_screen(u'youtube-dl is up to date (%s)' % __version__)
+        to_screen('youtube-dl is up to date (%s)' % __version__)
         return
 
-    to_screen(u'Updating to version ' + version_id + ' ...')
+    to_screen('Updating to version ' + version_id + ' ...')
     version = versions_info['versions'][version_id]
 
     print_notes(to_screen, versions_info['versions'])
@@ -108,11 +105,11 @@ def update_self(to_screen, verbose):
     filename = sys.argv[0]
     # Py2EXE: Filename could be different
     if hasattr(sys, "frozen") and not os.path.isfile(filename):
-        if os.path.isfile(filename + u'.exe'):
-            filename += u'.exe'
+        if os.path.isfile(filename + '.exe'):
+            filename += '.exe'
 
     if not os.access(filename, os.W_OK):
-        to_screen(u'ERROR: no write permissions on %s' % filename)
+        to_screen('ERROR: no write permissions on %s' % filename)
         return
 
     # Py2EXE
@@ -120,7 +117,7 @@ def update_self(to_screen, verbose):
         exe = os.path.abspath(filename)
         directory = os.path.dirname(exe)
         if not os.access(directory, os.W_OK):
-            to_screen(u'ERROR: no write permissions on %s' % directory)
+            to_screen('ERROR: no write permissions on %s' % directory)
             return
 
         try:
@@ -130,12 +127,12 @@ def update_self(to_screen, verbose):
         except (IOError, OSError):
             if verbose:
                 to_screen(compat_str(traceback.format_exc()))
-            to_screen(u'ERROR: unable to download latest version')
+            to_screen('ERROR: unable to download latest version')
             return
 
         newcontent_hash = hashlib.sha256(newcontent).hexdigest()
         if newcontent_hash != version['exe'][1]:
-            to_screen(u'ERROR: the downloaded file hash does not match. Aborting.')
+            to_screen('ERROR: the downloaded file hash does not match. Aborting.')
             return
 
         try:
@@ -144,27 +141,27 @@ def update_self(to_screen, verbose):
         except (IOError, OSError):
             if verbose:
                 to_screen(compat_str(traceback.format_exc()))
-            to_screen(u'ERROR: unable to write the new version')
+            to_screen('ERROR: unable to write the new version')
             return
 
         try:
             bat = os.path.join(directory, 'youtube-dl-updater.bat')
             with io.open(bat, 'w') as batfile:
-                batfile.write(u"""
+                batfile.write('''
 @echo off
 echo Waiting for file handle to be closed ...
 ping 127.0.0.1 -n 5 -w 1000 > NUL
 move /Y "%s.new" "%s" > NUL
 echo Updated youtube-dl to version %s.
 start /b "" cmd /c del "%%~f0"&exit /b"
-                \n""" % (exe, exe, version_id))
+                \n''' % (exe, exe, version_id))
 
             subprocess.Popen([bat])  # Continues to run in the background
             return  # Do not show premature success messages
         except (IOError, OSError):
             if verbose:
                 to_screen(compat_str(traceback.format_exc()))
-            to_screen(u'ERROR: unable to overwrite current version')
+            to_screen('ERROR: unable to overwrite current version')
             return
 
     # Zip unix package
@@ -176,12 +173,12 @@ start /b "" cmd /c del "%%~f0"&exit /b"
         except (IOError, OSError):
             if verbose:
                 to_screen(compat_str(traceback.format_exc()))
-            to_screen(u'ERROR: unable to download latest version')
+            to_screen('ERROR: unable to download latest version')
             return
 
         newcontent_hash = hashlib.sha256(newcontent).hexdigest()
         if newcontent_hash != version['bin'][1]:
-            to_screen(u'ERROR: the downloaded file hash does not match. Aborting.')
+            to_screen('ERROR: the downloaded file hash does not match. Aborting.')
             return
 
         try:
@@ -190,10 +187,10 @@ start /b "" cmd /c del "%%~f0"&exit /b"
         except (IOError, OSError):
             if verbose:
                 to_screen(compat_str(traceback.format_exc()))
-            to_screen(u'ERROR: unable to overwrite current version')
+            to_screen('ERROR: unable to overwrite current version')
             return
 
-    to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.')
+    to_screen('Updated youtube-dl. Restart youtube-dl to use the new version.')
 
 
 def get_notes(versions, fromVersion):
@@ -207,6 +204,6 @@ def get_notes(versions, fromVersion):
 def print_notes(to_screen, versions, fromVersion=__version__):
     notes = get_notes(versions, fromVersion)
     if notes:
-        to_screen(u'PLEASE NOTE:')
+        to_screen('PLEASE NOTE:')
         for note in notes:
             to_screen(note)

From 0e44f90e18e01c7e64b7914cd0f5d64883a88264 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 12:26:21 +0100
Subject: [PATCH 0616/1937] [hls] Remove useless u porefixes

---
 youtube_dl/downloader/hls.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 0c33d39ee..954beffd5 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -28,14 +28,14 @@ class HlsFD(FileDownloader):
             if check_executable(program, ['-version']):
                 break
         else:
-            self.report_error(u'm3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
+            self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
             return False
         cmd = [program] + args
 
         retval = subprocess.call(cmd)
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
-            self.to_screen(u'\r[%s] %s bytes' % (cmd[0], fsize))
+            self.to_screen('\r[%s] %s bytes' % (cmd[0], fsize))
             self.try_rename(tmpfilename, filename)
             self._hook_progress({
                 'downloaded_bytes': fsize,
@@ -45,8 +45,8 @@ class HlsFD(FileDownloader):
             })
             return True
         else:
-            self.to_stderr(u"\n")
-            self.report_error(u'%s exited with code %d' % (program, retval))
+            self.to_stderr('\n')
+            self.report_error('%s exited with code %d' % (program, retval))
             return False
 
 

From 4d2d638df4d8c8866be5b83f419999dfb11ab91e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 12:27:36 +0100
Subject: [PATCH 0617/1937] [http] Modernize

---
 youtube_dl/downloader/http.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index f62555ce0..8491cee8a 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import os
 import time
 
@@ -106,7 +108,7 @@ class HttpFD(FileDownloader):
                 self.report_retry(count, retries)
 
         if count > retries:
-            self.report_error(u'giving up after %s retries' % retries)
+            self.report_error('giving up after %s retries' % retries)
             return False
 
         data_len = data.info().get('Content-length', None)
@@ -124,10 +126,10 @@ class HttpFD(FileDownloader):
             min_data_len = self.params.get("min_filesize", None)
             max_data_len = self.params.get("max_filesize", None)
             if min_data_len is not None and data_len < min_data_len:
-                self.to_screen(u'\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
+                self.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len))
                 return False
             if max_data_len is not None and data_len > max_data_len:
-                self.to_screen(u'\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
+                self.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
                 return False
 
         data_len_str = format_bytes(data_len)
@@ -151,13 +153,13 @@ class HttpFD(FileDownloader):
                     filename = self.undo_temp_name(tmpfilename)
                     self.report_destination(filename)
                 except (OSError, IOError) as err:
-                    self.report_error(u'unable to open for writing: %s' % str(err))
+                    self.report_error('unable to open for writing: %s' % str(err))
                     return False
             try:
                 stream.write(data_block)
             except (IOError, OSError) as err:
-                self.to_stderr(u"\n")
-                self.report_error(u'unable to write data: %s' % str(err))
+                self.to_stderr('\n')
+                self.report_error('unable to write data: %s' % str(err))
                 return False
             if not self.params.get('noresizebuffer', False):
                 block_size = self.best_block_size(after - before, len(data_block))
@@ -188,10 +190,10 @@ class HttpFD(FileDownloader):
             self.slow_down(start, byte_counter - resume_len)
 
         if stream is None:
-            self.to_stderr(u"\n")
-            self.report_error(u'Did not get any data blocks')
+            self.to_stderr('\n')
+            self.report_error('Did not get any data blocks')
             return False
-        if tmpfilename != u'-':
+        if tmpfilename != '-':
             stream.close()
         self.report_finish(data_len_str, (time.time() - start))
         if data_len is not None and byte_counter != data_len:

From 80310134e0ae4eb3b2fcb7ce9abc9f1139c51345 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 12:34:52 +0100
Subject: [PATCH 0618/1937] [mplayer] Modernize

---
 youtube_dl/downloader/mplayer.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/downloader/mplayer.py b/youtube_dl/downloader/mplayer.py
index 4de7f15f4..c53195da0 100644
--- a/youtube_dl/downloader/mplayer.py
+++ b/youtube_dl/downloader/mplayer.py
@@ -1,7 +1,10 @@
+from __future__ import unicode_literals
+
 import os
 import subprocess
 
 from .common import FileDownloader
+from ..compat import compat_subprocess_get_DEVNULL
 from ..utils import (
     encodeFilename,
 )
@@ -13,19 +16,23 @@ class MplayerFD(FileDownloader):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
-        args = ['mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy', '-dumpstream', '-dumpfile', tmpfilename, url]
+        args = [
+            'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
+            '-dumpstream', '-dumpfile', tmpfilename, url]
         # Check for mplayer first
         try:
-            subprocess.call(['mplayer', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
+            subprocess.call(
+                ['mplayer', '-h'],
+                stdout=compat_subprocess_get_DEVNULL(), stderr=subprocess.STDOUT)
         except (OSError, IOError):
-            self.report_error(u'MMS or RTSP download detected but "%s" could not be run' % args[0])
+            self.report_error('MMS or RTSP download detected but "%s" could not be run' % args[0])
             return False
 
         # Download using mplayer.
         retval = subprocess.call(args)
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
-            self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize))
+            self.to_screen('\r[%s] %s bytes' % (args[0], fsize))
             self.try_rename(tmpfilename, filename)
             self._hook_progress({
                 'downloaded_bytes': fsize,
@@ -35,6 +42,6 @@ class MplayerFD(FileDownloader):
             })
             return True
         else:
-            self.to_stderr(u"\n")
-            self.report_error(u'mplayer exited with code %d' % retval)
+            self.to_stderr('\n')
+            self.report_error('mplayer exited with code %d' % retval)
             return False

From f8aace93cd6450f19cdd4a1d540ec82a480724b4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 12:35:57 +0100
Subject: [PATCH 0619/1937] [academicearth] Modernize

---
 youtube_dl/extractor/academicearth.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/academicearth.py b/youtube_dl/extractor/academicearth.py
index c983ef0f5..47313fba8 100644
--- a/youtube_dl/extractor/academicearth.py
+++ b/youtube_dl/extractor/academicearth.py
@@ -1,4 +1,5 @@
 from __future__ import unicode_literals
+
 import re
 
 from .common import InfoExtractor
@@ -18,15 +19,14 @@ class AcademicEarthCourseIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        playlist_id = m.group('id')
+        playlist_id = self._match_id(url)
 
         webpage = self._download_webpage(url, playlist_id)
         title = self._html_search_regex(
-            r'<h1 class="playlist-name"[^>]*?>(.*?)</h1>', webpage, u'title')
+            r'<h1 class="playlist-name"[^>]*?>(.*?)</h1>', webpage, 'title')
         description = self._html_search_regex(
             r'<p class="excerpt"[^>]*?>(.*?)</p>',
-            webpage, u'description', fatal=False)
+            webpage, 'description', fatal=False)
         urls = re.findall(
             r'<li class="lecture-preview">\s*?<a target="_blank" href="([^"]+)">',
             webpage)

From 673cf0e773eb60dd29d1ef1a8cf24bfbdcbb8a5b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 12:37:45 +0100
Subject: [PATCH 0620/1937] [update] Remove useless import

---
 youtube_dl/update.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 941db3c8f..4c07a558e 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -19,7 +19,6 @@ from .version import __version__
 def rsa_verify(message, signature, key):
     from struct import pack
     from hashlib import sha256
-    from sys import version_info
 
     assert isinstance(message, bytes)
     block_size = 0

From b72823a0a4bb78ed52845e505736bd15db1f3553 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 12:38:20 +0100
Subject: [PATCH 0621/1937] [francetv] PEP8

---
 youtube_dl/extractor/francetv.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 9bc959a9b..e0420a48f 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -40,8 +40,6 @@ class FranceTVBaseInfoExtractor(InfoExtractor):
         else:
             georestricted = False
 
-
-
         formats = []
         for video in info['videos']:
             if video['statut'] != 'ONLINE':

From e0938e773170938eb93c8f159b7c41b7a046b1c8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 12:40:05 +0100
Subject: [PATCH 0622/1937] [addanime] Modernize

---
 youtube_dl/extractor/addanime.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/addanime.py b/youtube_dl/extractor/addanime.py
index 11f149f9e..203936e54 100644
--- a/youtube_dl/extractor/addanime.py
+++ b/youtube_dl/extractor/addanime.py
@@ -15,8 +15,7 @@ from ..utils import (
 
 
 class AddAnimeIE(InfoExtractor):
-
-    _VALID_URL = r'^http://(?:\w+\.)?add-anime\.net/watch_video\.php\?(?:.*?)v=(?P<video_id>[\w_]+)(?:.*)'
+    _VALID_URL = r'^http://(?:\w+\.)?add-anime\.net/watch_video\.php\?(?:.*?)v=(?P<id>[\w_]+)(?:.*)'
     _TEST = {
         'url': 'http://www.add-anime.net/watch_video.php?v=24MR3YO5SAS9',
         'md5': '72954ea10bc979ab5e2eb288b21425a0',
@@ -29,9 +28,9 @@ class AddAnimeIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
+        video_id = self._match_id(url)
+
         try:
-            mobj = re.match(self._VALID_URL, url)
-            video_id = mobj.group('video_id')
             webpage = self._download_webpage(url, video_id)
         except ExtractorError as ee:
             if not isinstance(ee.cause, compat_HTTPError) or \
@@ -49,7 +48,7 @@ class AddAnimeIE(InfoExtractor):
                 r'a\.value = ([0-9]+)[+]([0-9]+)[*]([0-9]+);',
                 redir_webpage)
             if av is None:
-                raise ExtractorError(u'Cannot find redirect math task')
+                raise ExtractorError('Cannot find redirect math task')
             av_res = int(av.group(1)) + int(av.group(2)) * int(av.group(3))
 
             parsed_url = compat_urllib_parse_urlparse(url)

From 27e1400f5541ff46ee11b63219baff8e1b255a92 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 12:40:51 +0100
Subject: [PATCH 0623/1937] [aparat] Modernize

---
 youtube_dl/extractor/aparat.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/aparat.py b/youtube_dl/extractor/aparat.py
index 56775289b..15006336f 100644
--- a/youtube_dl/extractor/aparat.py
+++ b/youtube_dl/extractor/aparat.py
@@ -1,5 +1,4 @@
 # coding: utf-8
-
 from __future__ import unicode_literals
 
 import re
@@ -26,8 +25,7 @@ class AparatIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('id')
+        video_id = self._match_id(url)
 
         # Note: There is an easier-to-parse configuration at
         # http://www.aparat.com/video/video/config/videohash/%video_id
@@ -40,15 +38,15 @@ class AparatIE(InfoExtractor):
         for i, video_url in enumerate(video_urls):
             req = HEADRequest(video_url)
             res = self._request_webpage(
-                req, video_id, note=u'Testing video URL %d' % i, errnote=False)
+                req, video_id, note='Testing video URL %d' % i, errnote=False)
             if res:
                 break
         else:
-            raise ExtractorError(u'No working video URLs found')
+            raise ExtractorError('No working video URLs found')
 
-        title = self._search_regex(r'\s+title:\s*"([^"]+)"', webpage, u'title')
+        title = self._search_regex(r'\s+title:\s*"([^"]+)"', webpage, 'title')
         thumbnail = self._search_regex(
-            r'\s+image:\s*"([^"]+)"', webpage, u'thumbnail', fatal=False)
+            r'\s+image:\s*"([^"]+)"', webpage, 'thumbnail', fatal=False)
 
         return {
             'id': video_id,

From e91cdcae1a8e33cd63e68e81c14abb163b9b4c64 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 12:41:24 +0100
Subject: [PATCH 0624/1937] [appletrailers] Modernize

---
 youtube_dl/extractor/appletrailers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 87580147d..0c01fa1a1 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -80,7 +80,7 @@ class AppleTrailersIE(InfoExtractor):
             def _clean_json(m):
                 return 'iTunes.playURL(%s);' % m.group(1).replace('\'', '&#39;')
             s = re.sub(self._JSON_RE, _clean_json, s)
-            s = '<html>' + s + u'</html>'
+            s = '<html>%s</html>' % s
             return s
         doc = self._download_xml(playlist_url, movie, transform_source=fix_html)
 

From 0792d5634ebc607bcd38f6e9333240d4cfa001c2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 12:41:53 +0100
Subject: [PATCH 0625/1937] [youtube] Remove useless u prefixes

---
 youtube_dl/extractor/youtube.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4ebc7b712..115fc6840 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -609,9 +609,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             return {}
         player_config = json.loads(mobj.group(1))
         try:
-            args = player_config[u'args']
-            caption_url = args[u'ttsurl']
-            timestamp = args[u'timestamp']
+            args = player_config['args']
+            caption_url = args['ttsurl']
+            timestamp = args['timestamp']
             # We get the available subtitles
             list_params = compat_urllib_parse.urlencode({
                 'type': 'list',

From 855dc479c2b02b0cd3a68af3df0857d17d485be6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 12:43:06 +0100
Subject: [PATCH 0626/1937] [subtitles] Modernize

---
 youtube_dl/extractor/subtitles.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/subtitles.py b/youtube_dl/extractor/subtitles.py
index 41d740e87..59a51268d 100644
--- a/youtube_dl/extractor/subtitles.py
+++ b/youtube_dl/extractor/subtitles.py
@@ -1,7 +1,8 @@
+from __future__ import unicode_literals
 from .common import InfoExtractor
 
+from ..compat import compat_str
 from ..utils import (
-    compat_str,
     ExtractorError,
 )
 
@@ -17,10 +18,10 @@ class SubtitlesInfoExtractor(InfoExtractor):
         sub_lang_list = self._get_available_subtitles(video_id, webpage)
         auto_captions_list = self._get_available_automatic_caption(video_id, webpage)
         sub_lang = ",".join(list(sub_lang_list.keys()))
-        self.to_screen(u'%s: Available subtitles for video: %s' %
+        self.to_screen('%s: Available subtitles for video: %s' %
                        (video_id, sub_lang))
         auto_lang = ",".join(auto_captions_list.keys())
-        self.to_screen(u'%s: Available automatic captions for video: %s' %
+        self.to_screen('%s: Available automatic captions for video: %s' %
                        (video_id, auto_lang))
 
     def extract_subtitles(self, video_id, webpage):
@@ -51,7 +52,7 @@ class SubtitlesInfoExtractor(InfoExtractor):
             sub_lang_list = {}
             for sub_lang in requested_langs:
                 if sub_lang not in available_subs_list:
-                    self._downloader.report_warning(u'no closed captions found in the specified language "%s"' % sub_lang)
+                    self._downloader.report_warning('no closed captions found in the specified language "%s"' % sub_lang)
                     continue
                 sub_lang_list[sub_lang] = available_subs_list[sub_lang]
 
@@ -70,10 +71,10 @@ class SubtitlesInfoExtractor(InfoExtractor):
         try:
             sub = self._download_subtitle_url(sub_lang, url)
         except ExtractorError as err:
-            self._downloader.report_warning(u'unable to download video subtitles for %s: %s' % (sub_lang, compat_str(err)))
+            self._downloader.report_warning('unable to download video subtitles for %s: %s' % (sub_lang, compat_str(err)))
             return
         if not sub:
-            self._downloader.report_warning(u'Did not fetch video subtitles')
+            self._downloader.report_warning('Did not fetch video subtitles')
             return
         return sub
 
@@ -94,5 +95,5 @@ class SubtitlesInfoExtractor(InfoExtractor):
         Must be redefined by the subclasses that support automatic captions,
         otherwise it will return {}
         """
-        self._downloader.report_warning(u'Automatic Captions not supported by this server')
+        self._downloader.report_warning('Automatic Captions not supported by this server')
         return {}

From 5c40bba82f9666450cf25336208667fc0c2364c3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 12:45:40 +0100
Subject: [PATCH 0627/1937] [hotnewhiphop] Modernize

---
 youtube_dl/extractor/hotnewhiphop.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/hotnewhiphop.py b/youtube_dl/extractor/hotnewhiphop.py
index 80b48b1b3..651784b73 100644
--- a/youtube_dl/extractor/hotnewhiphop.py
+++ b/youtube_dl/extractor/hotnewhiphop.py
@@ -1,12 +1,13 @@
 from __future__ import unicode_literals
 
-import re
 import base64
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
     HEADRequest,
 )
@@ -16,25 +17,24 @@ class HotNewHipHopIE(InfoExtractor):
     _VALID_URL = r'http://www\.hotnewhiphop\.com/.*\.(?P<id>.*)\.html'
     _TEST = {
         'url': 'http://www.hotnewhiphop.com/freddie-gibbs-lay-it-down-song.1435540.html',
-        'file': '1435540.mp3',
         'md5': '2c2cd2f76ef11a9b3b581e8b232f3d96',
         'info_dict': {
+            'id': '1435540',
+            'ext': 'mp3',
             'title': 'Freddie Gibbs - Lay It Down'
         }
     }
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('id')
-
-        webpage_src = self._download_webpage(url, video_id)
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
 
         video_url_base64 = self._search_regex(
-            r'data-path="(.*?)"', webpage_src, u'video URL', fatal=False)
+            r'data-path="(.*?)"', webpage, 'video URL', default=None)
 
         if video_url_base64 is None:
             video_url = self._search_regex(
-                r'"contentUrl" content="(.*?)"', webpage_src, u'video URL')
+                r'"contentUrl" content="(.*?)"', webpage, 'content URL')
             return self.url_result(video_url, ie='Youtube')
 
         reqdata = compat_urllib_parse.urlencode([
@@ -59,11 +59,11 @@ class HotNewHipHopIE(InfoExtractor):
         if video_url.endswith('.html'):
             raise ExtractorError('Redirect failed')
 
-        video_title = self._og_search_title(webpage_src).strip()
+        video_title = self._og_search_title(webpage).strip()
 
         return {
             'id': video_id,
             'url': video_url,
             'title': video_title,
-            'thumbnail': self._og_search_thumbnail(webpage_src),
+            'thumbnail': self._og_search_thumbnail(webpage),
         }

From 535a66ef664baa5c9e11205947406653fa35263c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 12:50:37 +0100
Subject: [PATCH 0628/1937] [muzu] Modernize

---
 youtube_dl/extractor/muzu.py | 85 ++++++++++++++++++------------------
 1 file changed, 43 insertions(+), 42 deletions(-)

diff --git a/youtube_dl/extractor/muzu.py b/youtube_dl/extractor/muzu.py
index 97edd3ef6..1e9cf8de9 100644
--- a/youtube_dl/extractor/muzu.py
+++ b/youtube_dl/extractor/muzu.py
@@ -1,64 +1,65 @@
-import re
-import json
+from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
-    determine_ext,
 )
 
 
 class MuzuTVIE(InfoExtractor):
     _VALID_URL = r'https?://www\.muzu\.tv/(.+?)/(.+?)/(?P<id>\d+)'
-    IE_NAME = u'muzu.tv'
+    IE_NAME = 'muzu.tv'
 
     _TEST = {
-        u'url': u'http://www.muzu.tv/defected/marcashken-featuring-sos-cat-walk-original-mix-music-video/1981454/',
-        u'file': u'1981454.mp4',
-        u'md5': u'98f8b2c7bc50578d6a0364fff2bfb000',
-        u'info_dict': {
-            u'title': u'Cat Walk (Original Mix)',
-            u'description': u'md5:90e868994de201b2570e4e5854e19420',
-            u'uploader': u'MarcAshken featuring SOS',
+        'url': 'http://www.muzu.tv/defected/marcashken-featuring-sos-cat-walk-original-mix-music-video/1981454/',
+        'md5': '98f8b2c7bc50578d6a0364fff2bfb000',
+        'info_dict': {
+            'id': '1981454',
+            'ext': 'mp4',
+            'title': 'Cat Walk (Original Mix)',
+            'description': 'md5:90e868994de201b2570e4e5854e19420',
+            'uploader': 'MarcAshken featuring SOS',
         },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
-        info_data = compat_urllib_parse.urlencode({'format': 'json',
-                                                   'url': url,
-                                                   })
-        video_info_page = self._download_webpage('http://www.muzu.tv/api/oembed/?%s' % info_data,
-                                                 video_id, u'Downloading video info')
-        info = json.loads(video_info_page)
+        info_data = compat_urllib_parse.urlencode({
+            'format': 'json',
+            'url': url,
+        })
+        info = self._download_json(
+            'http://www.muzu.tv/api/oembed/?%s' % info_data,
+            video_id, 'Downloading video info')
 
-        player_info_page = self._download_webpage('http://player.muzu.tv/player/playerInit?ai=%s' % video_id,
-                                                  video_id, u'Downloading player info')
-        video_info = json.loads(player_info_page)['videos'][0]
+        player_info = self._download_json(
+            'http://player.muzu.tv/player/playerInit?ai=%s' % video_id,
+            video_id, 'Downloading player info')
+        video_info = player_info['videos'][0]
         for quality in ['1080', '720', '480', '360']:
             if video_info.get('v%s' % quality):
                 break
 
-        data = compat_urllib_parse.urlencode({'ai': video_id,
-                                              # Even if each time you watch a video the hash changes,
-                                              # it seems to work for different videos, and it will work
-                                              # even if you use any non empty string as a hash
-                                              'viewhash': 'VBNff6djeV4HV5TRPW5kOHub2k',
-                                              'device': 'web',
-                                              'qv': quality,
-                                              })
-        video_url_page = self._download_webpage('http://player.muzu.tv/player/requestVideo?%s' % data,
-                                                video_id, u'Downloading video url')
-        video_url_info = json.loads(video_url_page)
+        data = compat_urllib_parse.urlencode({
+            'ai': video_id,
+            # Even if each time you watch a video the hash changes,
+            # it seems to work for different videos, and it will work
+            # even if you use any non empty string as a hash
+            'viewhash': 'VBNff6djeV4HV5TRPW5kOHub2k',
+            'device': 'web',
+            'qv': quality,
+        })
+        video_url_info = self._download_json(
+            'http://player.muzu.tv/player/requestVideo?%s' % data,
+            video_id, 'Downloading video url')
         video_url = video_url_info['url']
 
-        return {'id': video_id,
-                'title': info['title'],
-                'url': video_url,
-                'ext': determine_ext(video_url),
-                'thumbnail': info['thumbnail_url'],
-                'description': info['description'],
-                'uploader': info['author_name'],
-                }
+        return {
+            'id': video_id,
+            'title': info['title'],
+            'url': video_url,
+            'thumbnail': info['thumbnail_url'],
+            'description': info['description'],
+            'uploader': info['author_name'],
+        }

From faf34948946dc09ab4593da663ac6a4e4164d21b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 12:52:45 +0100
Subject: [PATCH 0629/1937] [redtube] Modernize

---
 youtube_dl/extractor/redtube.py | 23 ++++++-----------------
 1 file changed, 6 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py
index d1e12dd8d..846b76c81 100644
--- a/youtube_dl/extractor/redtube.py
+++ b/youtube_dl/extractor/redtube.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -9,32 +7,23 @@ class RedTubeIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?redtube\.com/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.redtube.com/66418',
-        'file': '66418.mp4',
-        # md5 varies from time to time, as in
-        # https://travis-ci.org/rg3/youtube-dl/jobs/14052463#L295
-        #'md5': u'7b8c22b5e7098a3e1c09709df1126d2d',
         'info_dict': {
+            'id': '66418',
+            'ext': 'mp4',
             "title": "Sucked on a toilet",
             "age_limit": 18,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-
-        video_id = mobj.group('id')
-        video_extension = 'mp4'
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        self.report_extraction(video_id)
-
         video_url = self._html_search_regex(
-            r'<source src="(.+?)" type="video/mp4">', webpage, u'video URL')
-
+            r'<source src="(.+?)" type="video/mp4">', webpage, 'video URL')
         video_title = self._html_search_regex(
             r'<h1 class="videoTitle[^"]*">(.+?)</h1>',
-            webpage, u'title')
-
+            webpage, 'title')
         video_thumbnail = self._og_search_thumbnail(webpage)
 
         # No self-labeling, but they describe themselves as
@@ -44,7 +33,7 @@ class RedTubeIE(InfoExtractor):
         return {
             'id': video_id,
             'url': video_url,
-            'ext': video_extension,
+            'ext': 'mp4',
             'title': video_title,
             'thumbnail': video_thumbnail,
             'age_limit': age_limit,

From 4c6d2ff8dca6454fdb700a9b959559a6c7776c88 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 12:53:55 +0100
Subject: [PATCH 0630/1937] [sohu] Modernize

---
 youtube_dl/extractor/sohu.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 0348e7460..331b345dc 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -1,4 +1,5 @@
 # encoding: utf-8
+from __future__ import unicode_literals
 
 import json
 import re
@@ -11,13 +12,14 @@ class SohuIE(InfoExtractor):
     _VALID_URL = r'https?://(?P<mytv>my\.)?tv\.sohu\.com/.+?/(?(mytv)|n)(?P<id>\d+)\.shtml.*?'
 
     _TEST = {
-        u'url': u'http://tv.sohu.com/20130724/n382479172.shtml#super',
-        u'file': u'382479172.mp4',
-        u'md5': u'bde8d9a6ffd82c63a1eefaef4eeefec7',
-        u'info_dict': {
-            u'title': u'MV：Far East Movement《The Illest》',
+        'url': 'http://tv.sohu.com/20130724/n382479172.shtml#super',
+        'md5': 'bde8d9a6ffd82c63a1eefaef4eeefec7',
+        'info_dict': {
+            'id': '382479172',
+            'ext': 'mp4',
+            'title': 'MV：Far East Movement《The Illest》',
         },
-        u'skip': u'Only available from China',
+        'skip': 'Only available from China',
     }
 
     def _real_extract(self, url):
@@ -26,7 +28,7 @@ class SohuIE(InfoExtractor):
             if mytv:
                 base_data_url = 'http://my.tv.sohu.com/play/videonew.do?vid='
             else:
-                base_data_url = u'http://hot.vrs.sohu.com/vrs_flash.action?vid='
+                base_data_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid='
             data_url = base_data_url + str(vid_id)
             data_json = self._download_webpage(
                 data_url, video_id,
@@ -39,11 +41,11 @@ class SohuIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
         raw_title = self._html_search_regex(r'(?s)<title>(.+?)</title>',
-                                            webpage, u'video title')
+                                            webpage, 'video title')
         title = raw_title.partition('-')[0].strip()
 
         vid = self._html_search_regex(r'var vid ?= ?["\'](\d+)["\']', webpage,
-                                      u'video path')
+                                      'video path')
         data = _fetch_data(vid, mytv)
 
         QUALITIES = ('ori', 'super', 'high', 'nor')

From b3034f9df78b17fc496339db7cfb0d1c59f509a8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 12:56:28 +0100
Subject: [PATCH 0631/1937] [trilulilu] Modernize

---
 youtube_dl/extractor/trilulilu.py | 40 +++++++++++++++----------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/trilulilu.py b/youtube_dl/extractor/trilulilu.py
index 2f2c8aeda..220a05b7b 100644
--- a/youtube_dl/extractor/trilulilu.py
+++ b/youtube_dl/extractor/trilulilu.py
@@ -1,28 +1,28 @@
+from __future__ import unicode_literals
+
 import json
-import re
 
 from .common import InfoExtractor
 
 
 class TriluliluIE(InfoExtractor):
-    _VALID_URL = r'(?x)(?:https?://)?(?:www\.)?trilulilu\.ro/video-(?P<category>[^/]+)/(?P<video_id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?trilulilu\.ro/video-[^/]+/(?P<id>[^/]+)'
     _TEST = {
-        u"url": u"http://www.trilulilu.ro/video-animatie/big-buck-bunny-1",
-        u'file': u"big-buck-bunny-1.mp4",
-        u'info_dict': {
-            u"title": u"Big Buck Bunny",
-            u"description": u":) pentru copilul din noi",
+        'url': 'http://www.trilulilu.ro/video-animatie/big-buck-bunny-1',
+        'info_dict': {
+            'id': 'big-buck-bunny-1',
+            'ext': 'mp4',
+            'title': 'Big Buck Bunny',
+            'description': ':) pentru copilul din noi',
         },
         # Server ignores Range headers (--test)
-        u"params": {
-            u"skip_download": True
+        'params': {
+            'skip_download': True
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         title = self._og_search_title(webpage)
@@ -30,20 +30,20 @@ class TriluliluIE(InfoExtractor):
         description = self._og_search_description(webpage)
 
         log_str = self._search_regex(
-            r'block_flash_vars[ ]=[ ]({[^}]+})', webpage, u'log info')
+            r'block_flash_vars[ ]=[ ]({[^}]+})', webpage, 'log info')
         log = json.loads(log_str)
 
-        format_url = (u'http://fs%(server)s.trilulilu.ro/%(hash)s/'
-                      u'video-formats2' % log)
+        format_url = ('http://fs%(server)s.trilulilu.ro/%(hash)s/'
+                      'video-formats2' % log)
         format_doc = self._download_xml(
             format_url, video_id,
-            note=u'Downloading formats',
-            errnote=u'Error while downloading formats')
+            note='Downloading formats',
+            errnote='Error while downloading formats')
 
         video_url_template = (
-            u'http://fs%(server)s.trilulilu.ro/stream.php?type=video'
-            u'&source=site&hash=%(hash)s&username=%(userid)s&'
-            u'key=ministhebest&format=%%s&sig=&exp=' %
+            'http://fs%(server)s.trilulilu.ro/stream.php?type=video'
+            '&source=site&hash=%(hash)s&username=%(userid)s&'
+            'key=ministhebest&format=%%s&sig=&exp=' %
             log)
         formats = [
             {

From 24144e3b8d465cb1d142bb1ff0450669ecceb2a0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 12:58:53 +0100
Subject: [PATCH 0632/1937] [tvp] Modernize

---
 youtube_dl/extractor/tvp.py | 33 ++++++++++++++-------------------
 1 file changed, 14 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index bfed9dd04..a64580005 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -1,40 +1,35 @@
-import json
-import re
+from __future__ import unicode_literals
 
 from .common import InfoExtractor
 
 
 class TvpIE(InfoExtractor):
-    IE_NAME = u'tvp.pl'
+    IE_NAME = 'tvp.pl'
     _VALID_URL = r'https?://www\.tvp\.pl/.*?wideo/(?P<date>\d+)/(?P<id>\d+)'
 
     _TEST = {
-        u'url': u'http://www.tvp.pl/warszawa/magazyny/campusnews/wideo/31102013/12878238',
-        u'md5': u'148408967a6a468953c0a75cbdaf0d7a',
-        u'file': u'12878238.wmv',
-        u'info_dict': {
-            u'title': u'31.10.2013 - Odcinek 2',
-            u'description': u'31.10.2013 - Odcinek 2',
+        'url': 'http://www.tvp.pl/warszawa/magazyny/campusnews/wideo/31102013/12878238',
+        'md5': '148408967a6a468953c0a75cbdaf0d7a',
+        'info_dict': {
+            'id': '12878238',
+            'ext': 'wmv',
+            'title': '31.10.2013 - Odcinek 2',
+            'description': '31.10.2013 - Odcinek 2',
         },
-        u'skip': u'Download has to use same server IP as extraction. Therefore, a good (load-balancing) DNS resolver will make the download fail.'
+        'skip': 'Download has to use same server IP as extraction. Therefore, a good (load-balancing) DNS resolver will make the download fail.'
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         json_url = 'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s' % video_id
-        json_params = self._download_webpage(
-            json_url, video_id, u"Downloading video metadata")
-
-        params = json.loads(json_params)
-        self.report_extraction(video_id)
+        params = self._download_json(
+            json_url, video_id, "Downloading video metadata")
         video_url = params['video_url']
 
-        title = self._og_search_title(webpage, fatal=True)
         return {
             'id': video_id,
-            'title': title,
+            'title': self._og_search_title(webpage),
             'ext': 'wmv',
             'url': video_url,
             'description': self._og_search_description(webpage),

From 5c32110114a6083637802388463d59a42fe9242f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 13:01:39 +0100
Subject: [PATCH 0633/1937] [videofyme] Modernize

---
 youtube_dl/extractor/videofyme.py | 51 ++++++++++++++++---------------
 1 file changed, 27 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/videofyme.py b/youtube_dl/extractor/videofyme.py
index d764e60fb..94f9e9be9 100644
--- a/youtube_dl/extractor/videofyme.py
+++ b/youtube_dl/extractor/videofyme.py
@@ -1,32 +1,33 @@
-import re
+from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
     find_xpath_attr,
-    determine_ext,
+    int_or_none,
 )
 
 
 class VideofyMeIE(InfoExtractor):
-    _VALID_URL = r'https?://(www\.videofy\.me/.+?|p\.videofy\.me/v)/(?P<id>\d+)(&|#|$)'
-    IE_NAME = u'videofy.me'
+    _VALID_URL = r'https?://(?:www\.videofy\.me/.+?|p\.videofy\.me/v)/(?P<id>\d+)(&|#|$)'
+    IE_NAME = 'videofy.me'
 
     _TEST = {
-        u'url': u'http://www.videofy.me/thisisvideofyme/1100701',
-        u'file': u'1100701.mp4',
-        u'md5': u'c77d700bdc16ae2e9f3c26019bd96143',
-        u'info_dict': {
-            u'title': u'This is VideofyMe',
-            u'description': None,
-            u'uploader': u'VideofyMe',
-            u'uploader_id': u'thisisvideofyme',
+        'url': 'http://www.videofy.me/thisisvideofyme/1100701',
+        'md5': 'c77d700bdc16ae2e9f3c26019bd96143',
+        'info_dict': {
+            'id': '1100701',
+            'ext': 'mp4',
+            'title': 'This is VideofyMe',
+            'description': None,
+            'uploader': 'VideofyMe',
+            'uploader_id': 'thisisvideofyme',
+            'view_count': int,
         },
 
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         config = self._download_xml('http://sunshine.videofy.me/?videoId=%s' % video_id,
                                     video_id)
         video = config.find('video')
@@ -34,14 +35,16 @@ class VideofyMeIE(InfoExtractor):
         url_node = next(node for node in [find_xpath_attr(sources, 'source', 'id', 'HQ %s' % key)
                                           for key in ['on', 'av', 'off']] if node is not None)
         video_url = url_node.find('url').text
+        view_count = int_or_none(self._search_regex(
+            r'([0-9]+)', video.find('views').text, 'view count', fatal=False))
 
-        return {'id': video_id,
-                'title': video.find('title').text,
-                'url': video_url,
-                'ext': determine_ext(video_url),
-                'thumbnail': video.find('thumb').text,
-                'description': video.find('description').text,
-                'uploader': config.find('blog/name').text,
-                'uploader_id': video.find('identifier').text,
-                'view_count': re.search(r'\d+', video.find('views').text).group(),
-                }
+        return {
+            'id': video_id,
+            'title': video.find('title').text,
+            'url': video_url,
+            'thumbnail': video.find('thumb').text,
+            'description': video.find('description').text,
+            'uploader': config.find('blog/name').text,
+            'uploader_id': video.find('identifier').text,
+            'view_count': view_count,
+        }

From d3b5101a91aab997f3a7bc4ccd48f14f52c0e607 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 13:03:22 +0100
Subject: [PATCH 0634/1937] [videopremium] Modernize

---
 youtube_dl/extractor/videopremium.py | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/videopremium.py b/youtube_dl/extractor/videopremium.py
index 9f2cdb6ef..3176e3b9d 100644
--- a/youtube_dl/extractor/videopremium.py
+++ b/youtube_dl/extractor/videopremium.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import re
 import random
 
@@ -5,23 +7,22 @@ from .common import InfoExtractor
 
 
 class VideoPremiumIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://)?(?:www\.)?videopremium\.(?:tv|me)/(?P<id>\w+)(?:/.*)?'
+    _VALID_URL = r'https?://(?:www\.)?videopremium\.(?:tv|me)/(?P<id>\w+)(?:/.*)?'
     _TEST = {
-        u'url': u'http://videopremium.tv/4w7oadjsf156',
-        u'file': u'4w7oadjsf156.f4v',
-        u'info_dict': {
-            u"title": u"youtube-dl_test_video____a_________-BaW_jenozKc.mp4.mp4"
+        'url': 'http://videopremium.tv/4w7oadjsf156',
+        'info_dict': {
+            'id': '4w7oadjsf156',
+            'ext': 'f4v',
+            'title': 'youtube-dl_test_video____a_________-BaW_jenozKc.mp4.mp4'
         },
-        u'params': {
-            u'skip_download': True,
+        'params': {
+            'skip_download': True,
         },
-        u'skip': u'Test file has been deleted.',
+        'skip': 'Test file has been deleted.',
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage_url = 'http://videopremium.tv/' + video_id
         webpage = self._download_webpage(webpage_url, video_id)
 
@@ -29,10 +30,10 @@ class VideoPremiumIE(InfoExtractor):
             # Download again, we need a cookie
             webpage = self._download_webpage(
                 webpage_url, video_id,
-                note=u'Downloading webpage again (with cookie)')
+                note='Downloading webpage again (with cookie)')
 
         video_title = self._html_search_regex(
-            r'<h2(?:.*?)>\s*(.+?)\s*<', webpage, u'video title')
+            r'<h2(?:.*?)>\s*(.+?)\s*<', webpage, 'video title')
 
         return {
             'id': video_id,

From 3aa578cad2eae6bb5461b1b05e5471847546ff83 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 13:05:11 +0100
Subject: [PATCH 0635/1937] [ffmpeg] Modernize

---
 youtube_dl/postprocessor/ffmpeg.py | 54 ++++++++++++++++--------------
 1 file changed, 28 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 002a35c67..9303b8378 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import os
 import subprocess
 import sys
@@ -33,12 +35,12 @@ class FFmpegPostProcessor(PostProcessor):
 
     def check_version(self):
         if not self._executable:
-            raise FFmpegPostProcessorError(u'ffmpeg or avconv not found. Please install one.')
+            raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')
 
         REQUIRED_VERSION = '1.0'
         if is_outdated_version(
                 self._versions[self._executable], REQUIRED_VERSION):
-            warning = u'Your copy of %s is outdated, update %s to version %s or newer if you encounter any errors.' % (
+            warning = 'Your copy of %s is outdated, update %s to version %s or newer if you encounter any errors.' % (
                 self._executable, self._executable, REQUIRED_VERSION)
             if self._downloader:
                 self._downloader.report_warning(warning)
@@ -84,7 +86,7 @@ class FFmpegPostProcessor(PostProcessor):
                [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])
 
         if self._downloader.params.get('verbose', False):
-            self._downloader.to_screen(u'[debug] ffmpeg command line: %s' % shell_quote(cmd))
+            self._downloader.to_screen('[debug] ffmpeg command line: %s' % shell_quote(cmd))
         p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         stdout, stderr = p.communicate()
         if p.returncode != 0:
@@ -100,8 +102,8 @@ class FFmpegPostProcessor(PostProcessor):
 
     def _ffmpeg_filename_argument(self, fn):
         # ffmpeg broke --, see https://ffmpeg.org/trac/ffmpeg/ticket/2127 for details
-        if fn.startswith(u'-'):
-            return u'./' + fn
+        if fn.startswith('-'):
+            return './' + fn
         return fn
 
 
@@ -117,7 +119,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
     def get_audio_codec(self, path):
 
         if not self._probe_executable:
-            raise PostProcessingError(u'ffprobe or avprobe not found. Please install one.')
+            raise PostProcessingError('ffprobe or avprobe not found. Please install one.')
         try:
             cmd = [
                 self._probe_executable,
@@ -153,7 +155,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
         filecodec = self.get_audio_codec(path)
         if filecodec is None:
-            raise PostProcessingError(u'WARNING: unable to obtain file audio codec with ffprobe')
+            raise PostProcessingError('WARNING: unable to obtain file audio codec with ffprobe')
 
         uses_avconv = self._uses_avconv()
         more_opts = []
@@ -202,7 +204,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 extension = 'wav'
                 more_opts += ['-f', 'wav']
 
-        prefix, sep, ext = path.rpartition(u'.')  # not os.path.splitext, since the latter does not work on unicode in all setups
+        prefix, sep, ext = path.rpartition('.')  # not os.path.splitext, since the latter does not work on unicode in all setups
         new_path = prefix + sep + extension
 
         # If we download foo.mp3 and convert it to... foo.mp3, then don't delete foo.mp3, silly.
@@ -211,16 +213,16 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
         try:
             if self._nopostoverwrites and os.path.exists(encodeFilename(new_path)):
-                self._downloader.to_screen(u'[youtube] Post-process file %s exists, skipping' % new_path)
+                self._downloader.to_screen('[youtube] Post-process file %s exists, skipping' % new_path)
             else:
-                self._downloader.to_screen(u'[' + self._executable + '] Destination: ' + new_path)
+                self._downloader.to_screen('[' + self._executable + '] Destination: ' + new_path)
                 self.run_ffmpeg(path, new_path, acodec, more_opts)
         except:
             etype, e, tb = sys.exc_info()
             if isinstance(e, AudioConversionError):
-                msg = u'audio conversion failed: ' + e.msg
+                msg = 'audio conversion failed: ' + e.msg
             else:
-                msg = u'error running ' + self._executable
+                msg = 'error running ' + self._executable
             raise PostProcessingError(msg)
 
         # Try to update the date time for extracted audio file.
@@ -228,7 +230,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
             try:
                 os.utime(encodeFilename(new_path), (time.time(), information['filetime']))
             except:
-                self._downloader.report_warning(u'Cannot update utime of audio file')
+                self._downloader.report_warning('Cannot update utime of audio file')
 
         information['filepath'] = new_path
         return self._nopostoverwrites, information
@@ -241,12 +243,12 @@ class FFmpegVideoConvertor(FFmpegPostProcessor):
 
     def run(self, information):
         path = information['filepath']
-        prefix, sep, ext = path.rpartition(u'.')
+        prefix, sep, ext = path.rpartition('.')
         outpath = prefix + sep + self._preferedformat
         if information['ext'] == self._preferedformat:
-            self._downloader.to_screen(u'[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
+            self._downloader.to_screen('[ffmpeg] Not converting video file %s - already is in target format %s' % (path, self._preferedformat))
             return True, information
-        self._downloader.to_screen(u'[' + 'ffmpeg' + '] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath)
+        self._downloader.to_screen('[' + 'ffmpeg' + '] Converting video from %s to %s, Destination: ' % (information['ext'], self._preferedformat) + outpath)
         self.run_ffmpeg(path, outpath, [])
         information['filepath'] = outpath
         information['format'] = self._preferedformat
@@ -453,11 +455,11 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
         return cls._lang_map.get(code[:2])
 
     def run(self, information):
-        if information['ext'] != u'mp4':
-            self._downloader.to_screen(u'[ffmpeg] Subtitles can only be embedded in mp4 files')
+        if information['ext'] != 'mp4':
+            self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4 files')
             return True, information
         if not information.get('subtitles'):
-            self._downloader.to_screen(u'[ffmpeg] There aren\'t any subtitles to embed')
+            self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to embed')
             return True, information
 
         sub_langs = [key for key in information['subtitles']]
@@ -472,8 +474,8 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
                 opts.extend(['-metadata:s:s:%d' % i, 'language=%s' % lang_code])
         opts.extend(['-f', 'mp4'])
 
-        temp_filename = filename + u'.temp'
-        self._downloader.to_screen(u'[ffmpeg] Embedding subtitles in \'%s\'' % filename)
+        temp_filename = filename + '.temp'
+        self._downloader.to_screen('[ffmpeg] Embedding subtitles in \'%s\'' % filename)
         self.run_ffmpeg_multiple_files(input_files, temp_filename, opts)
         os.remove(encodeFilename(filename))
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
@@ -494,13 +496,13 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
             metadata['artist'] = info['uploader_id']
 
         if not metadata:
-            self._downloader.to_screen(u'[ffmpeg] There isn\'t any metadata to add')
+            self._downloader.to_screen('[ffmpeg] There isn\'t any metadata to add')
             return True, info
 
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
 
-        if info['ext'] == u'm4a':
+        if info['ext'] == 'm4a':
             options = ['-vn', '-acodec', 'copy']
         else:
             options = ['-c', 'copy']
@@ -508,7 +510,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
         for (name, value) in metadata.items():
             options.extend(['-metadata', '%s=%s' % (name, value)])
 
-        self._downloader.to_screen(u'[ffmpeg] Adding metadata to \'%s\'' % filename)
+        self._downloader.to_screen('[ffmpeg] Adding metadata to \'%s\'' % filename)
         self.run_ffmpeg(filename, temp_filename, options)
         os.remove(encodeFilename(filename))
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
@@ -519,7 +521,7 @@ class FFmpegMergerPP(FFmpegPostProcessor):
     def run(self, info):
         filename = info['filepath']
         args = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0', '-shortest']
-        self._downloader.to_screen(u'[ffmpeg] Merging formats into "%s"' % filename)
+        self._downloader.to_screen('[ffmpeg] Merging formats into "%s"' % filename)
         self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args)
         return True, info
 
@@ -530,7 +532,7 @@ class FFmpegAudioFixPP(FFmpegPostProcessor):
         temp_filename = prepend_extension(filename, 'temp')
 
         options = ['-vn', '-acodec', 'copy']
-        self._downloader.to_screen(u'[ffmpeg] Fixing audio file "%s"' % filename)
+        self._downloader.to_screen('[ffmpeg] Fixing audio file "%s"' % filename)
         self.run_ffmpeg(filename, temp_filename, options)
 
         os.remove(encodeFilename(filename))

From 8865bdeb377bc653cea66305af9bf530440771c4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 13:06:02 +0100
Subject: [PATCH 0636/1937] Remove useless u prefixes

---
 youtube_dl/extractor/cbs.py         | 2 +-
 youtube_dl/extractor/clipfish.py    | 4 ++--
 youtube_dl/extractor/eighttracks.py | 2 +-
 youtube_dl/extractor/gamekings.py   | 2 +-
 youtube_dl/extractor/mooshare.py    | 2 +-
 youtube_dl/extractor/mtv.py         | 2 +-
 youtube_dl/extractor/nhl.py         | 6 ++++--
 youtube_dl/extractor/sohu.py        | 6 +++---
 youtube_dl/extractor/space.py       | 3 ++-
 youtube_dl/extractor/tudou.py       | 2 +-
 youtube_dl/extractor/youporn.py     | 4 ++--
 11 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index db48dc24f..e43756ec6 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -45,4 +45,4 @@ class CBSIE(InfoExtractor):
         real_id = self._search_regex(
             r"video\.settings\.pid\s*=\s*'([^']+)';",
             webpage, 'real video ID')
-        return self.url_result(u'theplatform:%s' % real_id)
+        return self.url_result('theplatform:%s' % real_id)
diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py
index 669919a2c..a5c3cb7c6 100644
--- a/youtube_dl/extractor/clipfish.py
+++ b/youtube_dl/extractor/clipfish.py
@@ -24,7 +24,7 @@ class ClipfishIE(InfoExtractor):
             'title': 'FIFA 14 - E3 2013 Trailer',
             'duration': 82,
         },
-        u'skip': 'Blocked in the US'
+        'skip': 'Blocked in the US'
     }
 
     def _real_extract(self, url):
@@ -34,7 +34,7 @@ class ClipfishIE(InfoExtractor):
         info_url = ('http://www.clipfish.de/devxml/videoinfo/%s?ts=%d' %
                     (video_id, int(time.time())))
         doc = self._download_xml(
-            info_url, video_id, note=u'Downloading info page')
+            info_url, video_id, note='Downloading info page')
         title = doc.find('title').text
         video_url = doc.find('filename').text
         if video_url is None:
diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index c1b4c729e..f4c1e2a72 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -125,7 +125,7 @@ class EightTracksIE(InfoExtractor):
             info = {
                 'id': compat_str(track_data['id']),
                 'url': track_data['track_file_stream_url'],
-                'title': track_data['performer'] + u' - ' + track_data['name'],
+                'title': track_data['performer'] + ' - ' + track_data['name'],
                 'raw_title': track_data['name'],
                 'uploader_id': data['user']['login'],
                 'ext': 'm4a',
diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 11fee3d31..cf8e90d7d 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -11,7 +11,7 @@ class GamekingsIE(InfoExtractor):
         'url': 'http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
         # MD5 is flaky, seems to change regularly
         # 'md5': '2f32b1f7b80fdc5cb616efb4f387f8a3',
-        u'info_dict': {
+        'info_dict': {
             'id': '20130811',
             'ext': 'mp4',
             'title': 'Phoenix Wright: Ace Attorney \u2013 Dual Destinies Review',
diff --git a/youtube_dl/extractor/mooshare.py b/youtube_dl/extractor/mooshare.py
index f50e432a3..34a4bec3a 100644
--- a/youtube_dl/extractor/mooshare.py
+++ b/youtube_dl/extractor/mooshare.py
@@ -49,7 +49,7 @@ class MooshareIE(InfoExtractor):
         page = self._download_webpage(url, video_id, 'Downloading page')
 
         if re.search(r'>Video Not Found or Deleted<', page) is not None:
-            raise ExtractorError(u'Video %s does not exist' % video_id, expected=True)
+            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
         hash_key = self._html_search_regex(r'<input type="hidden" name="hash" value="([^"]+)">', page, 'hash')
         title = self._html_search_regex(r'(?m)<div class="blockTitle">\s*<h2>Watch ([^<]+)</h2>', page, 'title')
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 506d2d5a0..b482d6d4d 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -164,7 +164,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         if mgid is None or ':' not in mgid:
             mgid = self._search_regex(
                 [r'data-mgid="(.*?)"', r'swfobject.embedSWF\(".*?(mgid:.*?)"'],
-                webpage, u'mgid')
+                webpage, 'mgid')
         return self._get_videos_info(mgid)
 
 
diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index 719eb51a4..bdcf7e239 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -4,9 +4,11 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urlparse,
     compat_urllib_parse,
+)
+from ..utils import (
     unified_strdate,
 )
 
@@ -122,7 +124,7 @@ class NHLVideocenterIE(NHLBaseInfoExtractor):
         response = self._download_webpage(request_url, playlist_title)
         response = self._fix_json(response)
         if not response.strip():
-            self._downloader.report_warning(u'Got an empty reponse, trying '
+            self._downloader.report_warning('Got an empty reponse, trying '
                                             'adding the "newvideos" parameter')
             response = self._download_webpage(request_url + '&newvideos=true',
                                               playlist_title)
diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 331b345dc..07f514a46 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -32,7 +32,7 @@ class SohuIE(InfoExtractor):
             data_url = base_data_url + str(vid_id)
             data_json = self._download_webpage(
                 data_url, video_id,
-                note=u'Downloading JSON data for ' + str(vid_id))
+                note='Downloading JSON data for ' + str(vid_id))
             return json.loads(data_json)
 
         mobj = re.match(self._VALID_URL, url)
@@ -53,7 +53,7 @@ class SohuIE(InfoExtractor):
                    for q in QUALITIES
                    if data['data'][q + 'Vid'] != 0]
         if not vid_ids:
-            raise ExtractorError(u'No formats available for this video')
+            raise ExtractorError('No formats available for this video')
 
         # For now, we just pick the highest available quality
         vid_id = vid_ids[-1]
@@ -71,7 +71,7 @@ class SohuIE(InfoExtractor):
                         (allot, prot, clipsURL[i], su[i]))
             part_str = self._download_webpage(
                 part_url, video_id,
-                note=u'Downloading part %d of %d' % (i + 1, part_count))
+                note='Downloading part %d of %d' % (i + 1, part_count))
 
             part_info = part_str.split('|')
             video_url = '%s%s?key=%s' % (part_info[0], su[i], part_info[3])
diff --git a/youtube_dl/extractor/space.py b/youtube_dl/extractor/space.py
index d34aefeaa..c2d0d36a6 100644
--- a/youtube_dl/extractor/space.py
+++ b/youtube_dl/extractor/space.py
@@ -33,5 +33,6 @@ class SpaceIE(InfoExtractor):
             # Other videos works fine with the info from the object
             brightcove_url = BrightcoveIE._extract_brightcove_url(webpage)
         if brightcove_url is None:
-            raise ExtractorError(u'The webpage does not contain a video', expected=True)
+            raise ExtractorError(
+                'The webpage does not contain a video', expected=True)
         return self.url_result(brightcove_url, BrightcoveIE.ie_key())
diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index be51f4f87..161e47624 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -73,7 +73,7 @@ class TudouIE(InfoExtractor):
         result = []
         len_parts = len(parts)
         if len_parts > 1:
-            self.to_screen(u'%s: found %s parts' % (video_id, len_parts))
+            self.to_screen('%s: found %s parts' % (video_id, len_parts))
         for part in parts:
             part_id = part['k']
             final_url = self._url_for_id(part_id, quality)
diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 97ef9c17e..8123928be 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -49,7 +49,7 @@ class YouPornIE(InfoExtractor):
         try:
             params = json.loads(json_params)
         except:
-            raise ExtractorError(u'Invalid JSON')
+            raise ExtractorError('Invalid JSON')
 
         self.report_extraction(video_id)
         try:
@@ -103,7 +103,7 @@ class YouPornIE(InfoExtractor):
         self._sort_formats(formats)
 
         if not formats:
-            raise ExtractorError(u'ERROR: no known formats available for video')
+            raise ExtractorError('ERROR: no known formats available for video')
 
         return {
             'id': video_id,

From e075a44afb320e4c4d42f80ea75057bbc20ba51a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 13:07:32 +0100
Subject: [PATCH 0637/1937] [tests] Remove useless u prefixes

---
 test/test_download.py | 4 ++--
 test/test_utils.py    | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/test_download.py b/test/test_download.py
index b2615f338..a009aa475 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -97,7 +97,7 @@ def generator(test_case):
             return
         for other_ie in other_ies:
             if not other_ie.working():
-                print_skipping(u'test depends on %sIE, marked as not WORKING' % other_ie.ie_key())
+                print_skipping('test depends on %sIE, marked as not WORKING' % other_ie.ie_key())
                 return
 
         params = get_params(test_case.get('params', {}))
@@ -143,7 +143,7 @@ def generator(test_case):
                         raise
 
                     if try_num == RETRIES:
-                        report_warning(u'Failed due to network errors, skipping...')
+                        report_warning('Failed due to network errors, skipping...')
                         return
 
                     print('Retrying: {0} failed tries\n\n##########\n\n'.format(try_num))
diff --git a/test/test_utils.py b/test/test_utils.py
index 9dc879e0d..baa3a2156 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -171,7 +171,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(find('media:song/url').text, 'http://server.com/download.mp3')
 
     def test_smuggle_url(self):
-        data = {u"ö": u"ö", u"abc": [3]}
+        data = {"ö": "ö", "abc": [3]}
         url = 'https://foo.bar/baz?x=y#a'
         smug_url = smuggle_url(url, data)
         unsmug_url, unsmug_data = unsmuggle_url(smug_url)

From a03aaaed2e0f4653a6b40e92bb91eb848e041de3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 13:08:42 +0100
Subject: [PATCH 0638/1937] Declare Python 3.2 compatibility

---
 README.md | 2 +-
 setup.py  | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 6669c1d7c..d6e7ff902 100644
--- a/README.md
+++ b/README.md
@@ -30,7 +30,7 @@ Alternatively, refer to the developer instructions below for how to check out an
 # DESCRIPTION
 **youtube-dl** is a small command-line program to download videos from
 YouTube.com and a few more sites. It requires the Python interpreter, version
-2.6, 2.7, or 3.3+, and it is not platform specific. It should work on
+2.6, 2.7, or 3.2+, and it is not platform specific. It should work on
 your Unix box, on Windows or on Mac OS X. It is released to the public domain,
 which means you can modify it, redistribute it or use it however you like.
 
diff --git a/setup.py b/setup.py
index 5a0d084cc..4686260e0 100644
--- a/setup.py
+++ b/setup.py
@@ -102,7 +102,9 @@ setup(
         "Programming Language :: Python :: 2.6",
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.3"
+        "Programming Language :: Python :: 3.2",
+        "Programming Language :: Python :: 3.3",
+        "Programming Language :: Python :: 3.4",
     ],
 
     **params

From ceb3367320e06be2307adc9ff134718eb96ad38d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Nov 2014 20:02:40 +0600
Subject: [PATCH 0639/1937] [gorillavid] Generalize extraction with countdown
 timeout and support faststream.in (Closes #4297)

---
 youtube_dl/extractor/gorillavid.py | 30 ++++++++++++++++++++++++++----
 1 file changed, 26 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index 65b153417..506521097 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import re
+import time
 
 from .common import InfoExtractor
 from ..utils import (
@@ -9,6 +10,7 @@ from ..utils import (
     determine_ext,
     compat_urllib_parse,
     compat_urllib_request,
+    int_or_none,
 )
 
 
@@ -16,7 +18,7 @@ class GorillaVidIE(InfoExtractor):
     IE_DESC = 'GorillaVid.in, daclips.in and movpod.in'
     _VALID_URL = r'''(?x)
         https?://(?P<host>(?:www\.)?
-            (?:daclips\.in|gorillavid\.in|movpod\.in))/
+            (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in))/
         (?:embed-)?(?P<id>[0-9a-zA-Z]+)(?:-[0-9]+x[0-9]+\.html)?
     '''
 
@@ -49,6 +51,16 @@ class GorillaVidIE(InfoExtractor):
             'title': 'Micro Pig piglets ready on 16th July 2009-bG0PdrCdxUc',
             'thumbnail': 're:http://.*\.jpg',
         }
+    }, {
+        # video with countdown timeout
+        'url': 'http://fastvideo.in/1qmdn1lmsmbw',
+        'md5': '8b87ec3f6564a3108a0e8e66594842ba',
+        'info_dict': {
+            'id': '1qmdn1lmsmbw',
+            'ext': 'mp4',
+            'title': 'Man of Steel - Trailer',
+            'thumbnail': 're:http://.*\.jpg',
+        },
     }, {
         'url': 'http://movpod.in/0wguyyxi1yca',
         'only_matching': True,
@@ -71,6 +83,12 @@ class GorillaVidIE(InfoExtractor):
             ''', webpage))
 
         if fields['op'] == 'download1':
+            countdown = int_or_none(self._search_regex(
+                r'<span id="countdown_str">(?:[Ww]ait)?\s*<span id="cxc">(\d+)</span>\s*(?:seconds?)?</span>',
+                webpage, 'countdown', default=None))
+            if countdown:
+                self._sleep(countdown, video_id)
+
             post = compat_urllib_parse.urlencode(fields)
 
             req = compat_urllib_request.Request(url, post)
@@ -78,9 +96,13 @@ class GorillaVidIE(InfoExtractor):
 
             webpage = self._download_webpage(req, video_id, 'Downloading video page')
 
-        title = self._search_regex(r'style="z-index: [0-9]+;">([^<]+)</span>', webpage, 'title')
-        video_url = self._search_regex(r'file\s*:\s*\'(http[^\']+)\',', webpage, 'file url')
-        thumbnail = self._search_regex(r'image\s*:\s*\'(http[^\']+)\',', webpage, 'thumbnail', fatal=False)
+        title = self._search_regex(
+            r'style="z-index: [0-9]+;">([^<]+)</span>',
+            webpage, 'title', default=None) or self._og_search_title(webpage)
+        video_url = self._search_regex(
+            r'file\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'file url')
+        thumbnail = self._search_regex(
+            r'image\s*:\s*["\'](http[^"\']+)["\'],', webpage, 'thumbnail', fatal=False)
 
         formats = [{
             'format_id': 'sd',

From e293711802c010e2720f8df00fde34d0af5f6ac3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Nov 2014 21:00:18 +0600
Subject: [PATCH 0640/1937] [udemy] Set session cookies to API requests (Closes
 #4124, closes #4219, closes #4308)

---
 youtube_dl/extractor/udemy.py | 67 +++++++++++++++++++++++++----------
 1 file changed, 48 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 1a7d01c67..cf7bd0ae5 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -45,6 +45,19 @@ class UdemyIE(InfoExtractor):
         self._handle_error(response)
         return response
 
+    def _download_json_cookies(self, url, video_id, note):
+        headers = {
+            'X-Udemy-Snail-Case': 'true',
+            'X-Requested-With': 'XMLHttpRequest',
+        }
+        for cookie in self._downloader.cookiejar:
+            if cookie.name == 'client_id':
+                headers['X-Udemy-Client-Id'] = cookie.value
+            elif cookie.name == 'access_token':
+                headers['X-Udemy-Bearer-Token'] = cookie.value
+        request = compat_urllib_request.Request(url, headers=headers)
+        return self._download_json(request, video_id, note)
+
     def _real_initialize(self):
         self._login()
 
@@ -62,7 +75,9 @@ class UdemyIE(InfoExtractor):
         if login_popup == '<div class="run-command close-popup redirect" data-url="https://www.udemy.com/"></div>':
             return
 
-        csrf = self._html_search_regex(r'<input type="hidden" name="csrf" value="(.+?)"', login_popup, 'csrf token')
+        csrf = self._html_search_regex(
+            r'<input type="hidden" name="csrf" value="(.+?)"',
+            login_popup, 'csrf token')
 
         login_form = {
             'email': username,
@@ -71,42 +86,52 @@ class UdemyIE(InfoExtractor):
             'displayType': 'json',
             'isSubmitted': '1',
         }
-        request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
-        response = self._download_json(request, None, 'Logging in as %s' % username)
+        request = compat_urllib_request.Request(
+            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
+        response = self._download_json(
+            request, None, 'Logging in as %s' % username)
 
         if 'returnUrl' not in response:
             raise ExtractorError('Unable to log in')
 
+
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         lecture_id = mobj.group('id')
 
-        lecture = self._download_json(
-            'https://www.udemy.com/api-1.1/lectures/%s' % lecture_id, lecture_id, 'Downloading lecture JSON')
+        lecture = self._download_json_cookies(
+            'https://www.udemy.com/api-1.1/lectures/%s' % lecture_id,
+            lecture_id, 'Downloading lecture JSON')
 
-        if lecture['assetType'] != 'Video':
-            raise ExtractorError('Lecture %s is not a video' % lecture_id, expected=True)
+        asset_type = lecture.get('assetType') or lecture.get('asset_type')
+        if asset_type != 'Video':
+            raise ExtractorError(
+                'Lecture %s is not a video' % lecture_id, expected=True)
 
         asset = lecture['asset']
 
-        stream_url = asset['streamUrl']
+        stream_url = asset.get('streamUrl') or asset.get('stream_url')
         mobj = re.search(r'(https?://www\.youtube\.com/watch\?v=.*)', stream_url)
         if mobj:
             return self.url_result(mobj.group(1), 'Youtube')
 
         video_id = asset['id']
-        thumbnail = asset['thumbnailUrl']
+        thumbnail = asset.get('thumbnailUrl') or asset.get('thumbnail_url')
         duration = asset['data']['duration']
 
-        download_url = asset['downloadUrl']
+        download_url = asset.get('downloadUrl') or asset.get('download_url')
+
+        video = download_url.get('Video') or download_url.get('video')
+        video_480p = download_url.get('Video480p') or download_url.get('video_480p')
 
         formats = [
             {
-                'url': download_url['Video480p'][0],
+                'url': video_480p[0],
                 'format_id': '360p',
             },
             {
-                'url': download_url['Video'][0],
+                'url': video[0],
                 'format_id': '720p',
             },
         ]
@@ -139,26 +164,30 @@ class UdemyCourseIE(UdemyIE):
         mobj = re.match(self._VALID_URL, url)
         course_path = mobj.group('coursepath')
 
-        response = self._download_json(
-            'https://www.udemy.com/api-1.1/courses/%s' % course_path, course_path, 'Downloading course JSON')
+        response = self._download_json_cookies(
+            'https://www.udemy.com/api-1.1/courses/%s' % course_path,
+            course_path, 'Downloading course JSON')
 
         course_id = int(response['id'])
         course_title = response['title']
 
         webpage = self._download_webpage(
-            'https://www.udemy.com/course/subscribe/?courseId=%s' % course_id, course_id, 'Enrolling in the course')
+            'https://www.udemy.com/course/subscribe/?courseId=%s' % course_id,
+            course_id, 'Enrolling in the course')
 
         if self._SUCCESSFULLY_ENROLLED in webpage:
             self.to_screen('%s: Successfully enrolled in' % course_id)
         elif self._ALREADY_ENROLLED in webpage:
             self.to_screen('%s: Already enrolled in' % course_id)
 
-        response = self._download_json('https://www.udemy.com/api-1.1/courses/%s/curriculum' % course_id,
-                                       course_id, 'Downloading course curriculum')
+        response = self._download_json_cookies(
+            'https://www.udemy.com/api-1.1/courses/%s/curriculum' % course_id,
+            course_id, 'Downloading course curriculum')
 
         entries = [
-            self.url_result('https://www.udemy.com/%s/#/lecture/%s' % (course_path, asset['id']), 'Udemy')
-            for asset in response if asset.get('assetType') == 'Video'
+            self.url_result(
+                'https://www.udemy.com/%s/#/lecture/%s' % (course_path, asset['id']), 'Udemy')
+            for asset in response if asset.get('assetType') or asset.get('asset_type') == 'Video'
         ]
 
         return self.playlist_result(entries, course_id, course_title)

From 81a7a521c5bdbd862b7c7d161425295fed63d212 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Nov 2014 21:02:46 +0600
Subject: [PATCH 0641/1937] [gorillavid] Remove unused import

---
 youtube_dl/extractor/gorillavid.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index 506521097..9a2e020e9 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import time
 
 from .common import InfoExtractor
 from ..utils import (

From 61c0663c1e40a6bfabb2a7ac933ce468f2989808 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 26 Nov 2014 21:25:43 +0600
Subject: [PATCH 0642/1937] [udemy] Generalize download json and fix login

---
 youtube_dl/extractor/udemy.py | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index cf7bd0ae5..0e4d386a8 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -40,12 +40,7 @@ class UdemyIE(InfoExtractor):
                 error_str += ' - %s' % error_data.get('formErrors')
             raise ExtractorError(error_str, expected=True)
 
-    def _download_json(self, url, video_id, note='Downloading JSON metadata'):
-        response = super(UdemyIE, self)._download_json(url, video_id, note)
-        self._handle_error(response)
-        return response
-
-    def _download_json_cookies(self, url, video_id, note):
+    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'):
         headers = {
             'X-Udemy-Snail-Case': 'true',
             'X-Requested-With': 'XMLHttpRequest',
@@ -55,8 +50,16 @@ class UdemyIE(InfoExtractor):
                 headers['X-Udemy-Client-Id'] = cookie.value
             elif cookie.name == 'access_token':
                 headers['X-Udemy-Bearer-Token'] = cookie.value
-        request = compat_urllib_request.Request(url, headers=headers)
-        return self._download_json(request, video_id, note)
+
+        if isinstance(url_or_request, compat_urllib_request.Request):
+            for header, value in headers.items():
+                url_or_request.add_header(header, value)
+        else:
+            url_or_request = compat_urllib_request.Request(url_or_request, headers=headers)
+
+        response = super(UdemyIE, self)._download_json(url_or_request, video_id, note)
+        self._handle_error(response)
+        return response
 
     def _real_initialize(self):
         self._login()
@@ -87,7 +90,7 @@ class UdemyIE(InfoExtractor):
             'isSubmitted': '1',
         }
         request = compat_urllib_request.Request(
-            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
+            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         response = self._download_json(
             request, None, 'Logging in as %s' % username)
 
@@ -100,7 +103,7 @@ class UdemyIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         lecture_id = mobj.group('id')
 
-        lecture = self._download_json_cookies(
+        lecture = self._download_json(
             'https://www.udemy.com/api-1.1/lectures/%s' % lecture_id,
             lecture_id, 'Downloading lecture JSON')
 
@@ -164,7 +167,7 @@ class UdemyCourseIE(UdemyIE):
         mobj = re.match(self._VALID_URL, url)
         course_path = mobj.group('coursepath')
 
-        response = self._download_json_cookies(
+        response = self._download_json(
             'https://www.udemy.com/api-1.1/courses/%s' % course_path,
             course_path, 'Downloading course JSON')
 
@@ -180,7 +183,7 @@ class UdemyCourseIE(UdemyIE):
         elif self._ALREADY_ENROLLED in webpage:
             self.to_screen('%s: Already enrolled in' % course_id)
 
-        response = self._download_json_cookies(
+        response = self._download_json(
             'https://www.udemy.com/api-1.1/courses/%s/curriculum' % course_id,
             course_id, 'Downloading course curriculum')
 

From a1008af412f51aac82c121eaed07f742055d010b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 27 Nov 2014 00:24:19 +0600
Subject: [PATCH 0643/1937] [gorillavid] Update IE_DESC

---
 youtube_dl/extractor/gorillavid.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index 9a2e020e9..1ac1da856 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class GorillaVidIE(InfoExtractor):
-    IE_DESC = 'GorillaVid.in, daclips.in and movpod.in'
+    IE_DESC = 'GorillaVid.in, daclips.in, movpod.in and fastvideo.in'
     _VALID_URL = r'''(?x)
         https?://(?P<host>(?:www\.)?
             (?:daclips\.in|gorillavid\.in|movpod\.in|fastvideo\.in))/

From dcddc10a5032996474e157946b00b8df039eebda Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 20:01:20 +0100
Subject: [PATCH 0644/1937] [test_unicode_literals] Arm unicode_literals check

From now on, the line

from __future__ import unicode_literals

should be contained in every single Python file lest we run into any more 2.x/3.x issues.
Going forward, we're likely to develop on 3.x only and would likely miss subtle bugs otherwise.
---
 devscripts/bash-completion.py            |  2 ++
 devscripts/check-porn.py                 |  1 +
 devscripts/gh-pages/add-version.py       |  1 +
 devscripts/gh-pages/generate-download.py |  2 ++
 devscripts/gh-pages/sign-versions.py     |  4 +++-
 devscripts/gh-pages/update-copyright.py  |  4 ++--
 devscripts/gh-pages/update-feed.py       |  1 +
 devscripts/gh-pages/update-sites.py      |  1 +
 devscripts/make_readme.py                |  2 ++
 devscripts/prepare_manpage.py            |  1 +
 devscripts/zsh-completion.py             |  2 ++
 test/test_unicode_literals.py            | 13 ++++++-------
 test/test_write_annotations.py           |  1 +
 test/test_write_info_json.py             |  7 ++++---
 test/test_youtube_lists.py               |  1 +
 youtube_dl/__main__.py                   |  1 +
 youtube_dl/aes.py                        |  2 ++
 youtube_dl/extractor/__init__.py         |  2 ++
 youtube_dl/extractor/bambuser.py         |  2 +-
 youtube_dl/postprocessor/__init__.py     |  1 +
 youtube_dl/postprocessor/common.py       |  2 ++
 youtube_dl/postprocessor/xattrpp.py      |  2 ++
 youtube_dl/version.py                    |  1 +
 23 files changed, 42 insertions(+), 14 deletions(-)

diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py
index 70a3f552c..cd26cc089 100755
--- a/devscripts/bash-completion.py
+++ b/devscripts/bash-completion.py
@@ -1,4 +1,6 @@
 #!/usr/bin/env python
+from __future__ import unicode_literals
+
 import os
 from os.path import dirname as dirn
 import sys
diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py
index 86aa37b5f..216282712 100644
--- a/devscripts/check-porn.py
+++ b/devscripts/check-porn.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import unicode_literals
 
 """
 This script employs a VERY basic heuristic ('porn' in webpage.lower()) to check
diff --git a/devscripts/gh-pages/add-version.py b/devscripts/gh-pages/add-version.py
index 35865b2f3..867ea0048 100755
--- a/devscripts/gh-pages/add-version.py
+++ b/devscripts/gh-pages/add-version.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+from __future__ import unicode_literals
 
 import json
 import sys
diff --git a/devscripts/gh-pages/generate-download.py b/devscripts/gh-pages/generate-download.py
index e90c787fd..392e3ba21 100755
--- a/devscripts/gh-pages/generate-download.py
+++ b/devscripts/gh-pages/generate-download.py
@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+from __future__ import unicode_literals
+
 import hashlib
 import urllib.request
 import json
diff --git a/devscripts/gh-pages/sign-versions.py b/devscripts/gh-pages/sign-versions.py
index 953a5162e..fa389c358 100755
--- a/devscripts/gh-pages/sign-versions.py
+++ b/devscripts/gh-pages/sign-versions.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+from __future__ import unicode_literals, with_statement
 
 import rsa
 import json
@@ -29,4 +30,5 @@ signature = hexlify(rsa.pkcs1.sign(json.dumps(versions_info, sort_keys=True).enc
 print('signature: ' + signature)
 
 versions_info['signature'] = signature
-json.dump(versions_info, open('update/versions.json', 'w'), indent=4, sort_keys=True)
+with open('update/versions.json', 'w') as versionsf:
+    json.dump(versions_info, versionsf, indent=4, sort_keys=True)
diff --git a/devscripts/gh-pages/update-copyright.py b/devscripts/gh-pages/update-copyright.py
index 5bc5c6514..3663c8afe 100755
--- a/devscripts/gh-pages/update-copyright.py
+++ b/devscripts/gh-pages/update-copyright.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-from __future__ import with_statement
+from __future__ import with_statement, unicode_literals
 
 import datetime
 import glob
@@ -13,7 +13,7 @@ year = str(datetime.datetime.now().year)
 for fn in glob.glob('*.html*'):
     with io.open(fn, encoding='utf-8') as f:
         content = f.read()
-    newc = re.sub(u'(?P<copyright>Copyright © 2006-)(?P<year>[0-9]{4})', u'Copyright © 2006-' + year, content)
+    newc = re.sub(r'(?P<copyright>Copyright © 2006-)(?P<year>[0-9]{4})', 'Copyright © 2006-' + year, content)
     if content != newc:
         tmpFn = fn + '.part'
         with io.open(tmpFn, 'wt', encoding='utf-8') as outf:
diff --git a/devscripts/gh-pages/update-feed.py b/devscripts/gh-pages/update-feed.py
index 46373a8db..e93eb60fb 100755
--- a/devscripts/gh-pages/update-feed.py
+++ b/devscripts/gh-pages/update-feed.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+from __future__ import unicode_literals
 
 import datetime
 import io
diff --git a/devscripts/gh-pages/update-sites.py b/devscripts/gh-pages/update-sites.py
index 4a6bb5e35..f0f0481c7 100755
--- a/devscripts/gh-pages/update-sites.py
+++ b/devscripts/gh-pages/update-sites.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+from __future__ import unicode_literals
 
 import sys
 import os
diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py
index 70fa942dd..8fbce0796 100755
--- a/devscripts/make_readme.py
+++ b/devscripts/make_readme.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import io
 import sys
 import re
diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py
index d9c857015..f66bebfea 100644
--- a/devscripts/prepare_manpage.py
+++ b/devscripts/prepare_manpage.py
@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 
 import io
 import os.path
diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py
index e8d71928a..f200f2c80 100755
--- a/devscripts/zsh-completion.py
+++ b/devscripts/zsh-completion.py
@@ -1,4 +1,6 @@
 #!/usr/bin/env python
+from __future__ import unicode_literals
+
 import os
 from os.path import dirname as dirn
 import sys
diff --git a/test/test_unicode_literals.py b/test/test_unicode_literals.py
index a4ba7bad0..2cc431b0b 100644
--- a/test/test_unicode_literals.py
+++ b/test/test_unicode_literals.py
@@ -9,14 +9,13 @@ rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 IGNORED_FILES = [
     'setup.py',  # http://bugs.python.org/issue13943
+    'conf.py',
+    'buildserver.py',
 ]
 
 
 class TestUnicodeLiterals(unittest.TestCase):
     def test_all_files(self):
-        print('Skipping this test (not yet fully implemented)')
-        return
-
         for dirpath, _, filenames in os.walk(rootDir):
             for basename in filenames:
                 if not basename.endswith('.py'):
@@ -30,10 +29,10 @@ class TestUnicodeLiterals(unittest.TestCase):
 
                 if "'" not in code and '"' not in code:
                     continue
-                imps = 'from __future__ import unicode_literals'
-                self.assertTrue(
-                    imps in code,
-                    ' %s  missing in %s' % (imps, fn))
+                self.assertRegexpMatches(
+                    code,
+                    r'(?:#.*\n*)?from __future__ import (?:[a-z_]+,\s*)*unicode_literals',
+                    'unicode_literals import  missing in %s' % fn)
 
                 m = re.search(r'(?<=\s)u[\'"](?!\)|,|$)', code)
                 if m is not None:
diff --git a/test/test_write_annotations.py b/test/test_write_annotations.py
index 852553ada..780636c77 100644
--- a/test/test_write_annotations.py
+++ b/test/test_write_annotations.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
+from __future__ import unicode_literals
 
 # Allow direct execution
 import os
diff --git a/test/test_write_info_json.py b/test/test_write_info_json.py
index 90426a559..0396ef262 100644
--- a/test/test_write_info_json.py
+++ b/test/test_write_info_json.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 # coding: utf-8
+from __future__ import unicode_literals
 
 # Allow direct execution
 import os
@@ -32,7 +33,7 @@ params = get_params({
 TEST_ID = 'BaW_jenozKc'
 INFO_JSON_FILE = TEST_ID + '.info.json'
 DESCRIPTION_FILE = TEST_ID + '.mp4.description'
-EXPECTED_DESCRIPTION = u'''test chars:  "'/\ä↭𝕐
+EXPECTED_DESCRIPTION = '''test chars:  "'/\ä↭𝕐
 test URL: https://github.com/rg3/youtube-dl/issues/1892
 
 This is a test video for youtube-dl.
@@ -53,11 +54,11 @@ class TestInfoJSON(unittest.TestCase):
         self.assertTrue(os.path.exists(INFO_JSON_FILE))
         with io.open(INFO_JSON_FILE, 'r', encoding='utf-8') as jsonf:
             jd = json.load(jsonf)
-        self.assertEqual(jd['upload_date'], u'20121002')
+        self.assertEqual(jd['upload_date'], '20121002')
         self.assertEqual(jd['description'], EXPECTED_DESCRIPTION)
         self.assertEqual(jd['id'], TEST_ID)
         self.assertEqual(jd['extractor'], 'youtube')
-        self.assertEqual(jd['title'], u'''youtube-dl test video "'/\ä↭𝕐''')
+        self.assertEqual(jd['title'], '''youtube-dl test video "'/\ä↭𝕐''')
         self.assertEqual(jd['uploader'], 'Philipp Hagemeister')
 
         self.assertTrue(os.path.exists(DESCRIPTION_FILE))
diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index 4ceba64ae..c889b6f15 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import unicode_literals
 
 # Allow direct execution
 import os
diff --git a/youtube_dl/__main__.py b/youtube_dl/__main__.py
index 3fe29c91f..65a0f891c 100755
--- a/youtube_dl/__main__.py
+++ b/youtube_dl/__main__.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import unicode_literals
 
 # Execute with
 # $ python youtube_dl/__main__.py (2.6+)
diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py
index 2eeb96a5a..5efd0f836 100644
--- a/youtube_dl/aes.py
+++ b/youtube_dl/aes.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 __all__ = ['aes_encrypt', 'key_expansion', 'aes_ctr_decrypt', 'aes_cbc_decrypt', 'aes_decrypt_text']
 
 import base64
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5c096542b..bbac03913 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 from .abc import ABCIE
 from .academicearth import AcademicEarthCourseIE
 from .addanime import AddAnimeIE
diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index ea2ba0fe2..1ca0b7cf2 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -18,7 +18,7 @@ class BambuserIE(InfoExtractor):
     _TEST = {
         'url': 'http://bambuser.com/v/4050584',
         # MD5 seems to be flaky, see https://travis-ci.org/rg3/youtube-dl/jobs/14051016#L388
-        # u'md5': 'fba8f7693e48fd4e8641b3fd5539a641',
+        # 'md5': 'fba8f7693e48fd4e8641b3fd5539a641',
         'info_dict': {
             'id': '4050584',
             'ext': 'flv',
diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index 6ac67cbae..fb367ebe4 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -1,3 +1,4 @@
+from __future__ import unicode_literals
 
 from .atomicparsley import AtomicParsleyPP
 from .ffmpeg import (
diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py
index 788f94d02..e54ae678d 100644
--- a/youtube_dl/postprocessor/common.py
+++ b/youtube_dl/postprocessor/common.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 from ..utils import PostProcessingError
 
 
diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py
index d1991c4f8..f6c63fe97 100644
--- a/youtube_dl/postprocessor/xattrpp.py
+++ b/youtube_dl/postprocessor/xattrpp.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 import os
 import subprocess
 import sys
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 82c204592..131d72a67 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,3 @@
+from __future__ import unicode_literals
 
 __version__ = '2014.11.26'

From 7c8ea53b964f25780fc8427d450e25de6ee75cf6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 22:01:06 +0100
Subject: [PATCH 0645/1937] release 2014.11.26.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 131d72a67..b18b061e5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.11.26'
+__version__ = '2014.11.26.1'

From 29e950f7c899fe6cc7ff42d4d6ccc32ac644a71f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 22:06:27 +0100
Subject: [PATCH 0646/1937] release 2014.11.26.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b18b061e5..128594d25 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.11.26.1'
+__version__ = '2014.11.26.2'

From 72476fcc42b2bb059c1d3b6f42e7b2bc1c33293f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 22:08:30 +0100
Subject: [PATCH 0647/1937] release 2014.11.26.3

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 128594d25..2f7a02a71 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.11.26.2'
+__version__ = '2014.11.26.3'

From 155f9550c0f7c0f671d7607f1be671e26a75f39d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 22:52:28 +0100
Subject: [PATCH 0648/1937] [test/helper] Fix newlines in output of missing
 test fields

---
 test/helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index 2e320e2cf..9a7f0746e 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -141,7 +141,7 @@ def expect_info_dict(self, expected_dict, got_dict):
     if missing_keys:
         def _repr(v):
             if isinstance(v, compat_str):
-                return "'%s'" % v.replace('\\', '\\\\').replace("'", "\\'")
+                return "'%s'" % v.replace('\\', '\\\\').replace("'", "\\'").replace('\n', '\\n')
             else:
                 return repr(v)
         info_dict_str = ''.join(

From e232f787f675e95c85a0b2471e496a8b1683fd95 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 22:52:52 +0100
Subject: [PATCH 0649/1937] [buzzfeed] Add new extractor (Fixes #4259)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/buzzfeed.py | 52 ++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 youtube_dl/extractor/buzzfeed.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bbac03913..9387feef1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -38,6 +38,7 @@ from .bpb import BpbIE
 from .br import BRIE
 from .breakcom import BreakIE
 from .brightcove import BrightcoveIE
+from .buzzfeed import BuzzFeedIE
 from .byutv import BYUtvIE
 from .c56 import C56IE
 from .canal13cl import Canal13clIE
diff --git a/youtube_dl/extractor/buzzfeed.py b/youtube_dl/extractor/buzzfeed.py
new file mode 100644
index 000000000..9c891af51
--- /dev/null
+++ b/youtube_dl/extractor/buzzfeed.py
@@ -0,0 +1,52 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+
+
+class BuzzFeedIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?buzzfeed\.com/[^?#]*?/(?P<id>[^?#]+)'
+    _TEST = {
+        'url': 'http://www.buzzfeed.com/abagg/this-angry-ram-destroys-a-punching-bag-like-a-boss?utm_term=4ldqpia',
+        'info_dict': {
+            'id': 'this-angry-ram-destroys-a-punching-bag-like-a-boss',
+            'title': 'This Angry Ram Destroys A Punching Bag Like A Boss',
+            'description': 'Rambro!',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': 'aVCR29aE_OQ',
+                'ext': 'mp4',
+                'upload_date': '20141024',
+                'uploader_id': 'Buddhanz1',
+                'description': 'He likes to stay in shape with his heavy bag, he wont stop until its on the ground\n\nFollow Angry Ram on Facebook for regular updates -\nhttps://www.facebook.com/pages/Angry-Ram/1436897249899558?ref=hl',
+                'uploader': 'Buddhanz',
+                'title': 'Angry Ram destroys a punching bag',
+            }
+        }]
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        webpage = self._download_webpage(url, playlist_id)
+
+        all_buckets = re.findall(
+            r'<div class="video-embed-videopost[^"]*".*?rel:bf_bucket_data=\'([^\']+)\'',
+            webpage)
+        entries = []
+        for bd_json in all_buckets:
+            bd = json.loads(bd_json)
+            if 'video' not in bd:
+                continue
+            entries.append(self.url_result(bd['video']['url']))
+
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+            'entries': entries,
+        }

From ad0d800fc3ab62f925076d851a2cb933ac3e8887 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 26 Nov 2014 22:53:02 +0100
Subject: [PATCH 0650/1937] release 2014.11.26.4

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2f7a02a71..0cad177c5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.11.26.3'
+__version__ = '2014.11.26.4'

From 4207558e8bd3f9586f50e80cde4725cec636fcff Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 27 Nov 2014 15:44:35 +0100
Subject: [PATCH 0651/1937] [buzzfeed] Add support for more video types (#4259)

---
 youtube_dl/extractor/buzzfeed.py | 32 +++++++++++++++++++++++++++-----
 1 file changed, 27 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/buzzfeed.py b/youtube_dl/extractor/buzzfeed.py
index 9c891af51..a40a1bbc4 100644
--- a/youtube_dl/extractor/buzzfeed.py
+++ b/youtube_dl/extractor/buzzfeed.py
@@ -9,7 +9,7 @@ from .common import InfoExtractor
 
 class BuzzFeedIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?buzzfeed\.com/[^?#]*?/(?P<id>[^?#]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.buzzfeed.com/abagg/this-angry-ram-destroys-a-punching-bag-like-a-boss?utm_term=4ldqpia',
         'info_dict': {
             'id': 'this-angry-ram-destroys-a-punching-bag-like-a-boss',
@@ -27,21 +27,43 @@ class BuzzFeedIE(InfoExtractor):
                 'title': 'Angry Ram destroys a punching bag',
             }
         }]
-    }
+    }, {
+        'url': 'http://www.buzzfeed.com/sheridanwatson/look-at-this-cute-dog-omg?utm_term=4ldqpia',
+        'params': {
+            'skip_download': True,  # Got enough YouTube download tests
+        },
+        'info_dict': {
+            'description': 'Munchkin the Teddy Bear is back !',
+            'title': 'You Need To Stop What You\'re Doing And Watching This Dog Walk On A Treadmill',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': 'mVmBL8B-In0',
+                'ext': 'mp4',
+                'upload_date': '20141124',
+                'uploader_id': 'CindysMunchkin',
+                'description': '© 2014 Munchkin the Shih Tzu\nAll rights reserved\nFacebook: http://facebook.com/MunchkintheShihTzu',
+                'uploader': 'Munchkin the Shih Tzu',
+                'title': 'Munchkin the Teddy Bear gets her exercise',
+            },
+        }]
+    }]
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
         webpage = self._download_webpage(url, playlist_id)
 
         all_buckets = re.findall(
-            r'<div class="video-embed-videopost[^"]*".*?rel:bf_bucket_data=\'([^\']+)\'',
+            r'(?s)<div class="video-embed[^"]*"..*?rel:bf_bucket_data=\'([^\']+)\'',
             webpage)
+
         entries = []
         for bd_json in all_buckets:
             bd = json.loads(bd_json)
-            if 'video' not in bd:
+            video = bd.get('video') or bd.get('progload_video')
+            if not video:
                 continue
-            entries.append(self.url_result(bd['video']['url']))
+            entries.append(self.url_result(video['url']))
 
         return {
             '_type': 'playlist',

From 93b8a10e3bb95871644b4931c8eff5d500107125 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 27 Nov 2014 15:44:49 +0100
Subject: [PATCH 0652/1937] release 2014.11.27

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0cad177c5..9476bedfa 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.11.26.4'
+__version__ = '2014.11.27'

From cb6444e1971f001d14683e5040beccbf1d253d94 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 28 Nov 2014 20:38:47 +0600
Subject: [PATCH 0653/1937] [noco] Add support for multi language videos
 (Closes #4326)

---
 youtube_dl/extractor/noco.py | 50 ++++++++++++++++++++----------------
 1 file changed, 28 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 18c869b39..7d2ff7b9a 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -20,6 +20,7 @@ class NocoIE(InfoExtractor):
     _VALID_URL = r'http://(?:(?:www\.)?noco\.tv/emission/|player\.noco\.tv/\?idvideo=)(?P<id>\d+)'
     _LOGIN_URL = 'http://noco.tv/do.php'
     _API_URL_TEMPLATE = 'https://api.noco.tv/1.1/%s?ts=%s&tk=%s'
+    _SUB_LANG_TEMPLATE = '&sub_lang=%s'
     _NETRC_MACHINE = 'noco'
 
     _TEST = {
@@ -60,10 +61,12 @@ class NocoIE(InfoExtractor):
         if 'erreur' in login:
             raise ExtractorError('Unable to login: %s' % clean_html(login['erreur']), expected=True)
 
-    def _call_api(self, path, video_id, note):
+    def _call_api(self, path, video_id, note, sub_lang=None):
         ts = compat_str(int(time.time() * 1000))
         tk = hashlib.md5((hashlib.md5(ts.encode('ascii')).hexdigest() + '#8S?uCraTedap6a').encode('ascii')).hexdigest()
         url = self._API_URL_TEMPLATE % (path, ts, tk)
+        if sub_lang:
+            url += self._SUB_LANG_TEMPLATE % sub_lang
 
         resp = self._download_json(url, video_id, note)
 
@@ -91,31 +94,34 @@ class NocoIE(InfoExtractor):
 
         formats = []
 
-        for format_id, fmt in medias['fr']['video_list']['none']['quality_list'].items():
+        for lang, lang_dict in medias['fr']['video_list'].items():
+            for format_id, fmt in lang_dict['quality_list'].items():
+                format_id_extended = '%s-%s' % (lang, format_id) if lang != 'none' else format_id
 
-            video = self._call_api(
-                'shows/%s/video/%s/fr' % (video_id, format_id.lower()),
-                video_id, 'Downloading %s video JSON' % format_id)
+                video = self._call_api(
+                    'shows/%s/video/%s/fr' % (video_id, format_id.lower()),
+                    video_id, 'Downloading %s video JSON' % format_id_extended,
+                    lang if lang != 'none' else None)
 
-            file_url = video['file']
-            if not file_url:
-                continue
+                file_url = video['file']
+                if not file_url:
+                    continue
 
-            if file_url in ['forbidden', 'not found']:
-                popmessage = video['popmessage']
-                self._raise_error(popmessage['title'], popmessage['message'])
+                if file_url in ['forbidden', 'not found']:
+                    popmessage = video['popmessage']
+                    self._raise_error(popmessage['title'], popmessage['message'])
 
-            formats.append({
-                'url': file_url,
-                'format_id': format_id,
-                'width': fmt['res_width'],
-                'height': fmt['res_lines'],
-                'abr': fmt['audiobitrate'],
-                'vbr': fmt['videobitrate'],
-                'filesize': fmt['filesize'],
-                'format_note': qualities[format_id]['quality_name'],
-                'preference': qualities[format_id]['priority'],
-            })
+                formats.append({
+                    'url': file_url,
+                    'format_id': format_id_extended,
+                    'width': fmt['res_width'],
+                    'height': fmt['res_lines'],
+                    'abr': fmt['audiobitrate'],
+                    'vbr': fmt['videobitrate'],
+                    'filesize': fmt['filesize'],
+                    'format_note': qualities[format_id]['quality_name'],
+                    'preference': qualities[format_id]['priority'],
+                })
 
         self._sort_formats(formats)
 

From e987e91fcc2437a219834025e12432879e4fd93b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 29 Nov 2014 22:16:35 +0600
Subject: [PATCH 0654/1937] [playvid] Capture and output error message

---
 youtube_dl/extractor/playvid.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/playvid.py b/youtube_dl/extractor/playvid.py
index b1322f13f..cd3905acb 100644
--- a/youtube_dl/extractor/playvid.py
+++ b/youtube_dl/extractor/playvid.py
@@ -4,6 +4,8 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
+    clean_html,
     compat_urllib_parse,
 )
 
@@ -28,6 +30,11 @@ class PlayvidIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
+        m_error = re.search(
+            r'<div class="block-error">\s*<div class="heading">\s*<div>(?P<msg>.+?)</div>\s*</div>', webpage)
+        if m_error:
+            raise ExtractorError(clean_html(m_error.group('msg')), expected=True)
+
         video_title = None
         duration = None
         video_thumbnail = None

From 96b7c7fe3fb099c1c6be1d80ee5b291045e8db77 Mon Sep 17 00:00:00 2001
From: Yuriy Melnyk <ymelnik@yandex.ru>
Date: Sat, 29 Nov 2014 17:58:34 +0200
Subject: [PATCH 0655/1937] [bliptv] Fix resolution of lookup id in some videos

In some videos (for example, http://blip.tv/play/gbk766dkj4Yn) resolving
lookup id would fail, because page at
http://blip.tv/play/gbk766dkj4Yn.x?p=1 would have no "config.id" in
it. Fixed by requesting different URL and inspecting the URL which the
client is redirected to.
---
 youtube_dl/extractor/bliptv.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index f2b02643d..a18c64dc0 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -64,6 +64,20 @@ class BlipTVIE(SubtitlesInfoExtractor):
                 'uploader': 'redvsblue',
                 'uploader_id': '792887',
             }
+        },
+        {
+            'url': 'http://blip.tv/play/gbk766dkj4Yn',
+            'md5': 'fe0a33f022d49399a241e84a8ea8b8e3',
+            'info_dict': {
+                'id': '1749452',
+                'ext': 'mp4',
+                'upload_date': '20090208',
+                'description': 'Witness the first appearance of the Nostalgia Critic character, as Doug reviews the movie Transformers.',
+                'title': 'Nostalgia Critic: Transformers',
+                'timestamp': 1234068723,
+                'uploader': 'NostalgiaCritic',
+                'uploader_id': '246467',
+            }
         }
     ]
 
@@ -74,11 +88,13 @@ class BlipTVIE(SubtitlesInfoExtractor):
         # See https://github.com/rg3/youtube-dl/issues/857 and
         # https://github.com/rg3/youtube-dl/issues/4197
         if lookup_id:
-            info_page = self._download_webpage(
-                'http://blip.tv/play/%s.x?p=1' % lookup_id, lookup_id, 'Resolving lookup id')
-            video_id = self._search_regex(r'config\.id\s*=\s*"([0-9]+)', info_page, 'video_id')
-        else:
-            video_id = mobj.group('id')
+            urlh = self._request_webpage(
+                'http://blip.tv/play/%s' % lookup_id, lookup_id, 'Resolving lookup id')
+            url = compat_urlparse.urlparse(urlh.geturl())
+            qs = compat_urlparse.parse_qs(url.query)
+            mobj = re.match(self._VALID_URL, qs['file'][0])
+
+        video_id = mobj.group('id')
 
         rss = self._download_xml('http://blip.tv/rss/flash/%s' % video_id, video_id, 'Downloading video RSS')
 

From 248a0b890f1e1a122079cbe855599e5fcec080e8 Mon Sep 17 00:00:00 2001
From: Yuriy Melnyk <ymelnik@yandex.ru>
Date: Sat, 29 Nov 2014 18:14:57 +0200
Subject: [PATCH 0656/1937] [bliptv] Fix \n\n at the end of real_url

See https://github.com/rg3/youtube-dl/issues/3544#issuecomment-53166516
---
 youtube_dl/extractor/bliptv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index a18c64dc0..da47f27bd 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -130,7 +130,7 @@ class BlipTVIE(SubtitlesInfoExtractor):
             msg = self._download_webpage(
                 url + '?showplayer=20140425131715&referrer=http://blip.tv&mask=7&skin=flashvars&view=url',
                 video_id, 'Resolving URL for %s' % role)
-            real_url = compat_urlparse.parse_qs(msg)['message'][0]
+            real_url = compat_urlparse.parse_qs(msg.strip())['message'][0]
 
             media_type = media_content.get('type')
             if media_type == 'text/srt' or url.endswith('.srt'):

From 37ea8164d38a03c56c454f369abd82e1a74312e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 29 Nov 2014 21:36:24 +0100
Subject: [PATCH 0657/1937] [youtube] Don't confirm age when initializing

It seems that all the videos with age restriction use now the age gate method, which doesn't require any confirmation.
---
 youtube_dl/extractor/youtube.py | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 115fc6840..93e9b7d6d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -39,7 +39,6 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
     _TWOFACTOR_URL = 'https://accounts.google.com/SecondFactor'
     _LANG_URL = r'https://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
-    _AGE_URL = 'https://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
     _NETRC_MACHINE = 'youtube'
     # If True it will raise an error if no login info is provided
     _LOGIN_REQUIRED = False
@@ -176,21 +175,6 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             return False
         return True
 
-    def _confirm_age(self):
-        age_form = {
-            'next_url': '/',
-            'action_confirm': 'Confirm',
-        }
-        req = compat_urllib_request.Request(
-            self._AGE_URL,
-            compat_urllib_parse.urlencode(age_form).encode('ascii')
-        )
-
-        self._download_webpage(
-            req, None,
-            note='Confirming age', errnote='Unable to confirm age',
-            fatal=False)
-
     def _real_initialize(self):
         if self._downloader is None:
             return
@@ -199,7 +183,6 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                 return
         if not self._login():
             return
-        self._confirm_age()
 
 
 class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):

From 42939b6129833e3fb9f4c22e4e9f4056df193af2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 30 Nov 2014 00:03:59 +0100
Subject: [PATCH 0658/1937] [youtube] Use a cookie for seeting the language

This way, we don't have to do an aditional request
---
 youtube_dl/extractor/common.py  |  6 ++++++
 youtube_dl/extractor/youtube.py | 23 +++++------------------
 2 files changed, 11 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 3e84d0e62..e80a2dad0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -13,6 +13,7 @@ import time
 import xml.etree.ElementTree
 
 from ..compat import (
+    compat_cookiejar,
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse_urlparse,
@@ -817,6 +818,11 @@ class InfoExtractor(object):
                 self._downloader.report_warning(msg)
         return res
 
+    def _set_cookie(self, domain, name, value, expire_time=None):
+        cookie = compat_cookiejar.Cookie(0, name, value, None, None, domain, None,
+            None, '/', True, False, expire_time, '', None, None, None)
+        self._downloader.cookiejar.set_cookie(cookie)
+
 
 class SearchInfoExtractor(InfoExtractor):
     """
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 93e9b7d6d..96b49fed1 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -7,6 +7,7 @@ import itertools
 import json
 import os.path
 import re
+import time
 import traceback
 
 from .common import InfoExtractor, SearchInfoExtractor
@@ -38,16 +39,14 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     """Provide base functions for Youtube extractors"""
     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
     _TWOFACTOR_URL = 'https://accounts.google.com/SecondFactor'
-    _LANG_URL = r'https://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
     _NETRC_MACHINE = 'youtube'
     # If True it will raise an error if no login info is provided
     _LOGIN_REQUIRED = False
 
     def _set_language(self):
-        return bool(self._download_webpage(
-            self._LANG_URL, None,
-            note='Setting language', errnote='unable to set language',
-            fatal=False))
+        self._set_cookie('.youtube.com', 'PREF', 'f1=50000000&hl=en',
+            # YouTube sets the expire time to about two months
+            expire_time=time.time() + 60*24*3600)
 
     def _login(self):
         """
@@ -178,9 +177,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     def _real_initialize(self):
         if self._downloader is None:
             return
-        if self._get_login_info()[0] is not None:
-            if not self._set_language():
-                return
+        self._set_language()
         if not self._login():
             return
 
@@ -667,16 +664,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
         # Get video webpage
         url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
-        pref_cookies = [
-            c for c in self._downloader.cookiejar
-            if c.domain == '.youtube.com' and c.name == 'PREF']
-        for pc in pref_cookies:
-            if 'hl=' in pc.value:
-                pc.value = re.sub(r'hl=[^&]+', 'hl=en', pc.value)
-            else:
-                if pc.value:
-                    pc.value += '&'
-                pc.value += 'hl=en'
         video_webpage = self._download_webpage(url, video_id)
 
         # Attempt to extract SWF player URL

From 313d4572ce782a639e42b454edadc0b24d7799a1 Mon Sep 17 00:00:00 2001
From: t0mm0 <dev@onairsoftware.co.uk>
Date: Sun, 30 Nov 2014 16:04:04 +0000
Subject: [PATCH 0659/1937] [xminus] update tkn extraction regex

---
 youtube_dl/extractor/xminus.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xminus.py b/youtube_dl/extractor/xminus.py
index a3fd8f48c..f7e2e8ac9 100644
--- a/youtube_dl/extractor/xminus.py
+++ b/youtube_dl/extractor/xminus.py
@@ -50,7 +50,7 @@ class XMinusIE(InfoExtractor):
             webpage, 'view count', fatal=False))
 
         enc_token = self._html_search_regex(
-            r'data-mt="(.*?)"', webpage, 'enc_token')
+            r'minus_track\.tkn="(.+?)"', webpage, 'enc_token')
         token = ''.join(
             c if pos == 3 else compat_chr(compat_ord(c) - 1)
             for pos, c in enumerate(reversed(enc_token)))

From c056efa2e3e450ef70f0b23559273e2ec0f4e218 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 Nov 2014 22:37:56 +0600
Subject: [PATCH 0660/1937] [bbccouk] Fix extraction (#4104, #4214)

---
 youtube_dl/extractor/bbccouk.py | 122 +++++++++++++++++++++-----------
 1 file changed, 81 insertions(+), 41 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 6a507e113..beb6cfc8a 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -1,9 +1,11 @@
 from __future__ import unicode_literals
 
 import re
+import xml.etree.ElementTree
 
 from .subtitles import SubtitlesInfoExtractor
 from ..utils import ExtractorError
+from ..compat import compat_HTTPError
 
 
 class BBCCoUkIE(SubtitlesInfoExtractor):
@@ -55,7 +57,22 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
                 'skip_download': True,
             },
             'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
-        }
+        },
+        {
+            'url': 'http://www.bbc.co.uk/iplayer/episode/p026c7jt/tomorrows-worlds-the-unearthly-history-of-science-fiction-2-invasion',
+            'info_dict': {
+                'id': 'b03k3pb7',
+                'ext': 'flv',
+                'title': "Tomorrow's Worlds: The Unearthly History of Science Fiction",
+                'description': '2. Invasion',
+                'duration': 3600,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+            'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
+        },
     ]
 
     def _extract_asx_playlist(self, connection, programme_id):
@@ -102,6 +119,10 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
         return playlist.findall('./{http://bbc.co.uk/2008/emp/playlist}item')
 
     def _extract_medias(self, media_selection):
+        error = media_selection.find('./{http://bbc.co.uk/2008/mp/mediaselection}error')
+        if error is not None:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error.get('id')), expected=True)
         return media_selection.findall('./{http://bbc.co.uk/2008/mp/mediaselection}media')
 
     def _extract_connections(self, media):
@@ -158,54 +179,73 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
             subtitles[lang] = srt
         return subtitles
 
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        group_id = mobj.group('id')
-
-        webpage = self._download_webpage(url, group_id, 'Downloading video page')
-        if re.search(r'id="emp-error" class="notinuk">', webpage):
-            raise ExtractorError('Currently BBC iPlayer TV programmes are available to play in the UK only',
-                                 expected=True)
-
-        playlist = self._download_xml('http://www.bbc.co.uk/iplayer/playlist/%s' % group_id, group_id,
-                                      'Downloading playlist XML')
-
-        no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
-        if no_items is not None:
-            reason = no_items.get('reason')
-            if reason == 'preAvailability':
-                msg = 'Episode %s is not yet available' % group_id
-            elif reason == 'postAvailability':
-                msg = 'Episode %s is no longer available' % group_id
+    def _download_media_selector(self, programme_id):
+        try:
+            media_selection = self._download_xml(
+                'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s' % programme_id,
+                programme_id, 'Downloading media selection XML')
+        except ExtractorError as ee:
+            if isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 403:
+                media_selection = xml.etree.ElementTree.fromstring(ee.cause.read().encode('utf-8'))
             else:
-                msg = 'Episode %s is not available: %s' % (group_id, reason)
-            raise ExtractorError(msg, expected=True)
+                raise
 
         formats = []
         subtitles = None
 
-        for item in self._extract_items(playlist):
-            kind = item.get('kind')
-            if kind != 'programme' and kind != 'radioProgramme':
-                continue
-            title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
-            description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
+        for media in self._extract_medias(media_selection):
+            kind = media.get('kind')
+            if kind == 'audio':
+                formats.extend(self._extract_audio(media, programme_id))
+            elif kind == 'video':
+                formats.extend(self._extract_video(media, programme_id))
+            elif kind == 'captions':
+                subtitles = self._extract_captions(media, programme_id)
 
-            programme_id = item.get('identifier')
-            duration = int(item.get('duration'))
+        return formats, subtitles
 
-            media_selection = self._download_xml(
-                'http://open.live.bbc.co.uk/mediaselector/5/select/version/2.0/mediaset/pc/vpid/%s' % programme_id,
-                programme_id, 'Downloading media selection XML')
+    def _real_extract(self, url):
+        group_id = self._match_id(url)
 
-            for media in self._extract_medias(media_selection):
-                kind = media.get('kind')
-                if kind == 'audio':
-                    formats.extend(self._extract_audio(media, programme_id))
-                elif kind == 'video':
-                    formats.extend(self._extract_video(media, programme_id))
-                elif kind == 'captions':
-                    subtitles = self._extract_captions(media, programme_id)
+        webpage = self._download_webpage(url, group_id, 'Downloading video page')
+
+        programme_id = self._search_regex(
+            r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False)
+        if programme_id:
+            player = self._download_json(
+                'http://www.bbc.co.uk/iplayer/episode/%s.json' % group_id,
+                group_id)['jsConf']['player']
+            title = player['title']
+            description = player['subtitle']
+            duration = player['duration']
+            formats, subtitles = self._download_media_selector(programme_id)
+        else:
+            playlist = self._download_xml(
+                'http://www.bbc.co.uk/iplayer/playlist/%s' % group_id,
+                group_id, 'Downloading playlist XML')
+
+            no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
+            if no_items is not None:
+                reason = no_items.get('reason')
+                if reason == 'preAvailability':
+                    msg = 'Episode %s is not yet available' % group_id
+                elif reason == 'postAvailability':
+                    msg = 'Episode %s is no longer available' % group_id
+                elif reason == 'noMedia':
+                    msg = 'Episode %s is not currently available' % group_id
+                else:
+                    msg = 'Episode %s is not available: %s' % (group_id, reason)
+                raise ExtractorError(msg, expected=True)
+
+            for item in self._extract_items(playlist):
+                kind = item.get('kind')
+                if kind != 'programme' and kind != 'radioProgramme':
+                    continue
+                title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
+                description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
+                programme_id = item.get('identifier')
+                duration = int(item.get('duration'))
+                formats, subtitles = self._download_media_selector(programme_id)
 
         if self._downloader.params.get('listsubtitles', False):
             self._list_available_subtitles(programme_id, subtitles)

From 10a404c335295d76678560eedaa5520c2aafae6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 30 Nov 2014 18:56:14 +0100
Subject: [PATCH 0661/1937] [youtube] Add format 313 (fixes #4339)

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 96b49fed1..eaa5651fe 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -285,6 +285,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
+        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'VP9'},
 
         # Dash webm audio
         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},

From b766eb2707c5fcd0ba55c1bc2a5d3e98aaa3b2c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 30 Nov 2014 19:18:39 +0100
Subject: [PATCH 0662/1937] [youtube] Update test

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index eaa5651fe..4b19061a9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -379,8 +379,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             'info_dict': {
                 'id': 'IB3lcPjvWLA',
                 'ext': 'm4a',
-                'title': 'Afrojack - The Spark ft. Spree Wilson',
-                'description': 'md5:9717375db5a9a3992be4668bbf3bc0a8',
+                'title': 'Afrojack, Spree Wilson - The Spark ft. Spree Wilson',
+                'description': 'md5:12e7067fa6735a77bdcbb58cb1187d2d',
                 'uploader': 'AfrojackVEVO',
                 'uploader_id': 'AfrojackVEVO',
                 'upload_date': '20131011',

From 4472f84f0c09eba9720e3447fc8c0b7b19a8c20e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 30 Nov 2014 19:42:54 +0100
Subject: [PATCH 0663/1937] [test/test_subtitles] Update checksum for vimeo
 subtitle file

---
 test/test_subtitles.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 2eeb31bc6..7c4cd8218 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -238,7 +238,7 @@ class TestVimeoSubtitles(BaseTestSubtitles):
     def test_subtitles(self):
         self.DL.params['writesubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '8062383cf4dec168fc40a088aa6d5888')
+        self.assertEqual(md5(subtitles['en']), '26399116d23ae3cf2c087cea94bc43b4')
 
     def test_subtitles_lang(self):
         self.DL.params['writesubtitles'] = True

From 4e62ebe25013b07dd43e27eea37710b2c374455f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 30 Nov 2014 20:56:32 +0100
Subject: [PATCH 0664/1937] [youtube] Try to extract the video_info from the
 webpage before requesting the 'get_video_info' pages

The YouTube player doesn't seem to use them except for embedded videos, so we can skip a network request.
But they still provide better error mesagges (for removed videos for example).
---
 youtube_dl/extractor/youtube.py | 60 +++++++++++++--------------------
 1 file changed, 24 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4b19061a9..6d48761c6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -675,7 +675,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             player_url = None
 
         # Get video info
-        self.report_video_info_webpage_download(video_id)
         if re.search(r'player-age-gate-content">', video_webpage) is not None:
             age_gate = True
             # We simulate the access to the video from www.youtube.com/v/{video_id}
@@ -694,15 +693,30 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             video_info = compat_parse_qs(video_info_webpage)
         else:
             age_gate = False
-            for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
-                video_info_url = (proto + '://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
-                                  % (video_id, el_type))
-                video_info_webpage = self._download_webpage(video_info_url, video_id,
-                                                            note=False,
-                                                            errnote='unable to download video info webpage')
-                video_info = compat_parse_qs(video_info_webpage)
-                if 'token' in video_info:
-                    break
+            try:
+                # Try looking directly into the video webpage
+                mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});', video_webpage)
+                if not mobj:
+                    raise ValueError('Could not find ytplayer.config')  # caught below
+                json_code = uppercase_escape(mobj.group(1))
+                ytplayer_config = json.loads(json_code)
+                args = ytplayer_config['args']
+                # Convert to the same format returned by compat_parse_qs
+                video_info = dict((k, [v]) for k, v in args.items())
+                if 'url_encoded_fmt_stream_map' not in args:
+                    raise ValueError('No stream_map present')  # caught below
+            except ValueError:
+                # We fallback to the get_video_info pages (used by the embed page)
+                self.report_video_info_webpage_download(video_id)
+                for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
+                    video_info_url = (proto + '://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
+                        % (video_id, el_type))
+                    video_info_webpage = self._download_webpage(video_info_url,
+                        video_id, note=False,
+                        errnote='unable to download video info webpage')
+                    video_info = compat_parse_qs(video_info_webpage)
+                    if 'token' in video_info:
+                        break
         if 'token' not in video_info:
             if 'reason' in video_info:
                 raise ExtractorError(
@@ -827,32 +841,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         if self._downloader.params.get('writeannotations', False):
             video_annotations = self._extract_annotations(video_id)
 
-        # Decide which formats to download
-        try:
-            mobj = re.search(r';ytplayer\.config\s*=\s*({.*?});', video_webpage)
-            if not mobj:
-                raise ValueError('Could not find vevo ID')
-            json_code = uppercase_escape(mobj.group(1))
-            ytplayer_config = json.loads(json_code)
-            args = ytplayer_config['args']
-            # Easy way to know if the 's' value is in url_encoded_fmt_stream_map
-            # this signatures are encrypted
-            if 'url_encoded_fmt_stream_map' not in args:
-                raise ValueError('No stream_map present')  # caught below
-            re_signature = re.compile(r'[&,]s=')
-            m_s = re_signature.search(args['url_encoded_fmt_stream_map'])
-            if m_s is not None:
-                self.to_screen('%s: Encrypted signatures detected.' % video_id)
-                video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']]
-            m_s = re_signature.search(args.get('adaptive_fmts', ''))
-            if m_s is not None:
-                if 'adaptive_fmts' in video_info:
-                    video_info['adaptive_fmts'][0] += ',' + args['adaptive_fmts']
-                else:
-                    video_info['adaptive_fmts'] = [args['adaptive_fmts']]
-        except ValueError:
-            pass
-
         def _map_to_format_list(urlmap):
             formats = []
             for itag, video_real_url in urlmap.items():

From 7160532d41af171e6ffa41986b04bc51b934a5ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 30 Nov 2014 21:07:50 +0100
Subject: [PATCH 0665/1937] [youtube] Simplify code for getting the dash
 manifest url

video_info contains now the 'ytplayer.config.args' dictionary
---
 youtube_dl/extractor/youtube.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6d48761c6..6268b012f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -933,10 +933,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 # However, in the case of an age restriction there won't be any embedded dashmpd in the video_webpage.
                 # Luckily, it seems, this case uses some kind of default signature (len == 86), so the
                 # combination of get_video_info and the _static_decrypt_signature() decryption fallback will work here.
-                if age_gate:
-                    dash_manifest_url = video_info.get('dashmpd')[0]
-                else:
-                    dash_manifest_url = ytplayer_config['args']['dashmpd']
+                dash_manifest_url = video_info.get('dashmpd')[0]
 
                 def decrypt_sig(mobj):
                     s = mobj.group(1)

From c522adb1f0747c5b7f4c8c60e6edcc6f1809f718 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 30 Nov 2014 21:45:49 +0100
Subject: [PATCH 0666/1937] [youtube] Add a normal age-gate test video

---
 youtube_dl/extractor/youtube.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6268b012f..2642ecfff 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -402,7 +402,20 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'title': 'Burning Everyone\'s Koran',
                 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
             }
-        }
+        },
+        # Normal age-gate video (No vevo, embed allowed)
+        {
+            'url': 'http://youtube.com/watch?v=HtVdAasjOgU',
+            'info_dict': {
+                'id': 'HtVdAasjOgU',
+                'ext': 'mp4',
+                'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
+                'description': 'md5:eca57043abae25130f58f655ad9a7771',
+                'uploader': 'The Witcher',
+                'uploader_id': 'WitcherGame',
+                'upload_date': '20140605',
+            },
+        },
     ]
 
     def __init__(self, *args, **kwargs):

From f2b44a2513d788e0e6a7f54497a7ac0e4090c1ff Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Sun, 30 Nov 2014 19:06:23 +0100
Subject: [PATCH 0667/1937] [myspace] Use player_url for faster download It
 keeps reconnecting without it. Download time decreased from 7+ minutes to 25
 seconds for me.

---
 youtube_dl/extractor/myspace.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index c3c1a27c3..24da36212 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -48,6 +48,8 @@ class MySpaceIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         webpage = self._download_webpage(url, video_id)
+        player_url = self._search_regex(
+            r'playerSwf":"([^"?]*)', webpage, 'player URL')
 
         if mobj.group('mediatype').startswith('music/song'):
             # songs don't store any useful info in the 'context' variable
@@ -79,6 +81,7 @@ class MySpaceIE(InfoExtractor):
         info.update({
             'url': rtmp_url,
             'play_path': play_path,
+            'player_url': player_url,
             'ext': 'flv',
         })
         return info

From 03fd72d9967805aff7f28213385cf04e420d136e Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Sun, 30 Nov 2014 19:07:36 +0100
Subject: [PATCH 0668/1937] [myspace] Add more data to info dict `uploader` is
 an artist `playlist` is an album

---
 youtube_dl/extractor/myspace.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index 24da36212..1a118a37f 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -60,7 +60,9 @@ class MySpaceIE(InfoExtractor):
             info = {
                 'id': video_id,
                 'title': self._og_search_title(webpage),
+                'uploader': search_data('artist-name'),
                 'uploader_id': search_data('artist-username'),
+                'playlist': search_data('album-title'),
                 'thumbnail': self._og_search_thumbnail(webpage),
             }
         else:

From 1940fadd5307a11fd64ab26c43fbc31764c213a1 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Sun, 30 Nov 2014 19:36:24 +0100
Subject: [PATCH 0669/1937] [myspace] Handle non-playable songs I'm adding this
 because sometimes there is a song page, but you cannot play it. Example:
 https://myspace.com/starset2/music/song/let-it-die-maniac-agenda-remix-bonus-track-95799916-106964439
 It will be useful for downloading whole album with songs like this.

---
 youtube_dl/extractor/myspace.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index 1a118a37f..843e1f24a 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -53,9 +53,17 @@ class MySpaceIE(InfoExtractor):
 
         if mobj.group('mediatype').startswith('music/song'):
             # songs don't store any useful info in the 'context' variable
+            song_data = self._search_regex(
+                r'''<button.*data-song-id=(["\'])%s\1.*''' % video_id,
+                webpage, 'song_data', default=None, group=0)
+            if song_data is None:
+                self.to_screen(
+                    '%s: No downloadable song on this page' % video_id)
+                return
             def search_data(name):
                 return self._search_regex(
-                    r'data-%s="(.*?)"' % name, webpage, name)
+                    r'''data-%s=([\'"])(.*?)\1''' % name,
+                    song_data, name, default='', group=2)
             streamUrl = search_data('stream-url')
             info = {
                 'id': video_id,

From 3266f0c68e9512f226043d868630a7179f45d390 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Sun, 30 Nov 2014 20:00:16 +0100
Subject: [PATCH 0670/1937] [myspace] Redirect to other extractors There are
 many songs just linked from Vevo/YouTube to MySpace. Vevo example:
 https://myspace.com/threedaysgrace/music/song/animal-i-have-become-28400208-28218041
 YouTube example:
 https://myspace.com/starset2/music/song/first-light-95799905-106964426

---
 youtube_dl/extractor/myspace.py | 13 +++++++++++++
 youtube_dl/extractor/vevo.py    |  2 +-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index 843e1f24a..dd0a0415e 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_str,
 )
+from ..utils import ExtractorError
 
 
 class MySpaceIE(InfoExtractor):
@@ -65,6 +66,18 @@ class MySpaceIE(InfoExtractor):
                     r'''data-%s=([\'"])(.*?)\1''' % name,
                     song_data, name, default='', group=2)
             streamUrl = search_data('stream-url')
+            if not streamUrl:
+                vevo_id = search_data('vevo-id')
+                youtube_id = search_data('youtube-id')
+                if vevo_id:
+                    self.to_screen('Vevo video detected: %s' % vevo_id)
+                    return self.url_result('vevo:%s' % vevo_id, ie='Vevo')
+                elif youtube_id:
+                    self.to_screen('Youtube video detected: %s' % youtube_id)
+                    return self.url_result(youtube_id, ie='Youtube')
+                else:
+                    raise ExtractorError(
+                        'Found song but don\'t know how to download it')
             info = {
                 'id': video_id,
                 'title': self._og_search_title(webpage),
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 5b1a3ec78..c912c3cbe 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -13,7 +13,7 @@ from ..utils import (
 class VevoIE(InfoExtractor):
     """
     Accepts urls from vevo.com or in the format 'vevo:{id}'
-    (currently used by MTVIE)
+    (currently used by MTVIE and MySpaceIE)
     """
     _VALID_URL = r'''(?x)
         (?:https?://www\.vevo\.com/watch/(?:[^/]+/(?:[^/]+/)?)?|

From a196a53265a1df05270fe2a5ae83257b3629d30c Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Sun, 30 Nov 2014 19:57:35 +0100
Subject: [PATCH 0671/1937] [myspace] Update tests

---
 youtube_dl/extractor/myspace.py | 33 ++++++++++++++++++++++++++++-----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index dd0a0415e..b4b1fb51d 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -29,19 +29,42 @@ class MySpaceIE(InfoExtractor):
                 'skip_download': True,
             },
         },
-        # song
+        # songs
         {
-            'url': 'https://myspace.com/spiderbags/music/song/darkness-in-my-heart-39008454-27041242',
+            'url': 'https://myspace.com/killsorrow/music/song/of-weakened-soul...-93388656-103880681',
+            'md5': 'f1d7323321f6b7775bf1e3754c1707dc',
             'info_dict': {
-                'id': '39008454',
+                'id': '93388656',
                 'ext': 'flv',
-                'title': 'Darkness In My Heart',
-                'uploader_id': 'spiderbags',
+                'playlist': 'The Demo',
+                'title': 'Of weakened soul...',
+                'uploader': 'Killsorrow',
+                'uploader_id': 'killsorrow',
             },
             'params': {
                 # rtmp download
                 'skip_download': True,
             },
+        }, {
+            'add_ie': ['Vevo'],
+            'url': 'https://myspace.com/threedaysgrace/music/song/animal-i-have-become-28400208-28218041',
+            'info_dict': {
+                'id': u'USZM20600099',
+                'title': u'Animal I Have Become',
+                'uploader': u'Three Days Grace',
+                'timestamp': int,
+            },
+            'skip': 'VEVO is only available in some countries',
+        }, {
+            'add_ie': ['Youtube'],
+            'url': 'https://myspace.com/starset2/music/song/first-light-95799905-106964426',
+            'info_dict': {
+                'id': 'ypWvQgnJrSU',
+                'title': 'Starset - First Light',
+                'uploader': 'Jacob Soren',
+                'uploader_id': 'SorenPromotions',
+                'upload_date': '20140725',
+            }
         },
     ]
 

From 95c673a148f1a6cd8571b4dfa931d98092500fc2 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Sun, 30 Nov 2014 21:36:00 +0100
Subject: [PATCH 0672/1937] [myspace] Add extractor for albums

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/myspace.py  | 43 ++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9387feef1..0339d1386 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -242,7 +242,7 @@ from .muenchentv import MuenchenTVIE
 from .musicplayon import MusicPlayOnIE
 from .musicvault import MusicVaultIE
 from .muzu import MuzuTVIE
-from .myspace import MySpaceIE
+from .myspace import MySpaceIE, MySpaceAlbumIE
 from .myspass import MySpassIE
 from .myvideo import MyVideoIE
 from .naver import NaverIE
diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index b4b1fb51d..759bc7fed 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -131,3 +131,46 @@ class MySpaceIE(InfoExtractor):
             'ext': 'flv',
         })
         return info
+
+
+class MySpaceAlbumIE(InfoExtractor):
+    IE_NAME = 'MySpace:album'
+    _VALID_URL = r'https?://myspace\.com/([^/]+)/music/album/(?P<title>.*-)(?P<id>\d+)'
+
+    _TESTS = [{
+        'url': 'https://myspace.com/starset2/music/album/transmissions-19455773',
+        'info_dict': {
+            'title': 'Transmissions',
+            'id': '19455773',
+        },
+        'playlist_count': 14,
+        'skip': 'this album is only available in some countries',
+    }, {
+        'url': 'https://myspace.com/killsorrow/music/album/the-demo-18596029',
+        'info_dict': {
+            'title': 'The Demo',
+            'id': '18596029',
+        },
+        'playlist_count': 5,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        playlist_id = mobj.group('id')
+        display_id = mobj.group('title') + playlist_id
+        webpage = self._download_webpage(url, display_id)
+        tracks_paths = re.findall(r'"music:song" content="(.*?)"', webpage)
+        if not tracks_paths:
+            self.to_screen('%s: No songs found, try using proxy' % display_id)
+            return
+        entries = [
+            self.url_result(t_path, ie=MySpaceIE.ie_key())
+            for t_path in tracks_paths]
+        title = self._og_search_title(webpage)
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'display_id': display_id,
+            'title': title,
+            'entries': entries,
+        }

From 19e92770c9fe5c08297f8100952d1bdb0e90db71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 30 Nov 2014 23:44:37 +0100
Subject: [PATCH 0673/1937] [myspace] Replace removed test video and fix the
 others

---
 youtube_dl/extractor/myspace.py | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index 759bc7fed..0ead81095 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -1,3 +1,4 @@
+# encoding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -15,14 +16,14 @@ class MySpaceIE(InfoExtractor):
 
     _TESTS = [
         {
-            'url': 'https://myspace.com/coldplay/video/viva-la-vida/100008689',
+            'url': 'https://myspace.com/fiveminutestothestage/video/little-big-town/109594919',
             'info_dict': {
-                'id': '100008689',
+                'id': '109594919',
                 'ext': 'flv',
-                'title': 'Viva La Vida',
-                'description': 'The official Viva La Vida video, directed by Hype Williams',
-                'uploader': 'Coldplay',
-                'uploader_id': 'coldplay',
+                'title': 'Little Big Town',
+                'description': 'This country quartet was all smiles while playing a sold out show at the Pacific Amphitheatre in Orange County, California.',
+                'uploader': 'Five Minutes to the Stage',
+                'uploader_id': 'fiveminutestothestage',
             },
             'params': {
                 # rtmp download
@@ -32,7 +33,6 @@ class MySpaceIE(InfoExtractor):
         # songs
         {
             'url': 'https://myspace.com/killsorrow/music/song/of-weakened-soul...-93388656-103880681',
-            'md5': 'f1d7323321f6b7775bf1e3754c1707dc',
             'info_dict': {
                 'id': '93388656',
                 'ext': 'flv',
@@ -49,10 +49,12 @@ class MySpaceIE(InfoExtractor):
             'add_ie': ['Vevo'],
             'url': 'https://myspace.com/threedaysgrace/music/song/animal-i-have-become-28400208-28218041',
             'info_dict': {
-                'id': u'USZM20600099',
-                'title': u'Animal I Have Become',
-                'uploader': u'Three Days Grace',
+                'id': 'USZM20600099',
+                'ext': 'mp4',
+                'title': 'Animal I Have Become',
+                'uploader': 'Three Days Grace',
                 'timestamp': int,
+                'upload_date': '20060502',
             },
             'skip': 'VEVO is only available in some countries',
         }, {
@@ -60,7 +62,9 @@ class MySpaceIE(InfoExtractor):
             'url': 'https://myspace.com/starset2/music/song/first-light-95799905-106964426',
             'info_dict': {
                 'id': 'ypWvQgnJrSU',
+                'ext': 'mp4',
                 'title': 'Starset - First Light',
+                'description': 'md5:2d5db6c9d11d527683bcda818d332414',
                 'uploader': 'Jacob Soren',
                 'uploader_id': 'SorenPromotions',
                 'upload_date': '20140725',

From 954f36f890713f39bba7e5a08559856c367c89e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 1 Dec 2014 00:03:24 +0100
Subject: [PATCH 0674/1937] [myspace] Cleanup

---
 youtube_dl/extractor/myspace.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index 0ead81095..e62614670 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -36,7 +36,6 @@ class MySpaceIE(InfoExtractor):
             'info_dict': {
                 'id': '93388656',
                 'ext': 'flv',
-                'playlist': 'The Demo',
                 'title': 'Of weakened soul...',
                 'uploader': 'Killsorrow',
                 'uploader_id': 'killsorrow',
@@ -85,13 +84,14 @@ class MySpaceIE(InfoExtractor):
                 r'''<button.*data-song-id=(["\'])%s\1.*''' % video_id,
                 webpage, 'song_data', default=None, group=0)
             if song_data is None:
-                self.to_screen(
+                # some songs in an album are not playable
+                self.report_warning(
                     '%s: No downloadable song on this page' % video_id)
                 return
             def search_data(name):
                 return self._search_regex(
-                    r'''data-%s=([\'"])(.*?)\1''' % name,
-                    song_data, name, default='', group=2)
+                    r'''data-%s=([\'"])(?P<data>.*?)\1''' % name,
+                    song_data, name, default='', group='data')
             streamUrl = search_data('stream-url')
             if not streamUrl:
                 vevo_id = search_data('vevo-id')
@@ -110,7 +110,6 @@ class MySpaceIE(InfoExtractor):
                 'title': self._og_search_title(webpage),
                 'uploader': search_data('artist-name'),
                 'uploader_id': search_data('artist-username'),
-                'playlist': search_data('album-title'),
                 'thumbnail': self._og_search_thumbnail(webpage),
             }
         else:
@@ -165,16 +164,16 @@ class MySpaceAlbumIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
         tracks_paths = re.findall(r'"music:song" content="(.*?)"', webpage)
         if not tracks_paths:
-            self.to_screen('%s: No songs found, try using proxy' % display_id)
-            return
+            raise ExtractorError(
+                '%s: No songs found, try using proxy' % display_id,
+                expected=True)
         entries = [
             self.url_result(t_path, ie=MySpaceIE.ie_key())
             for t_path in tracks_paths]
-        title = self._og_search_title(webpage)
         return {
             '_type': 'playlist',
             'id': playlist_id,
             'display_id': display_id,
-            'title': title,
+            'title': self._og_search_title(webpage),
             'entries': entries,
         }

From d128cfe3931eeb779a13678e43e09e012a64aca7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Dec 2014 20:18:42 +0600
Subject: [PATCH 0675/1937] [slideshare] Fix description extraction

---
 youtube_dl/extractor/slideshare.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/slideshare.py b/youtube_dl/extractor/slideshare.py
index 53c3c9220..5864b9936 100644
--- a/youtube_dl/extractor/slideshare.py
+++ b/youtube_dl/extractor/slideshare.py
@@ -39,7 +39,7 @@ class SlideshareIE(InfoExtractor):
         ext = info['jsplayer']['video_extension']
         video_url = compat_urlparse.urljoin(bucket, doc + '-SD.' + ext)
         description = self._html_search_regex(
-            r'<p\s+(?:style="[^"]*"\s+)?class="description.*?"[^>]*>(.*?)</p>', webpage,
+            r'<p\s+(?:style="[^"]*"\s+)?class=".*?description.*?"[^>]*>(.*?)</p>', webpage,
             'description', fatal=False)
 
         return {

From 94aa25b99541be80a975fd48c07d096fc8c3203c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 1 Dec 2014 16:15:09 +0100
Subject: [PATCH 0676/1937] Credit @Tithen-Firion for the myspace changes
 (#4341)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index d4b0b75fa..bd2e967e3 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -88,3 +88,4 @@ Dao Hoang Son
 Oskar Jauch
 Matthew Rayfield
 t0mm0
+Tithen-Firion

From 1434bffa1ff2fb1e4d80efc900a13a3541923119 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 1 Dec 2014 18:10:04 +0200
Subject: [PATCH 0677/1937] [tunein] Use station API

---
 youtube_dl/extractor/tunein.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/tunein.py b/youtube_dl/extractor/tunein.py
index 8c29f1634..4ce5aeeba 100644
--- a/youtube_dl/extractor/tunein.py
+++ b/youtube_dl/extractor/tunein.py
@@ -19,6 +19,7 @@ class TuneInIE(InfoExtractor):
         |tun\.in/(?P<redirect_id>[A-Za-z0-9]+)
     )
     '''
+    _API_URL_TEMPLATE = 'http://tunein.com/tuner/tune/?stationId={0:}&tuneType=Station'
 
     _INFO_DICT = {
         'id': '34682',
@@ -56,13 +57,10 @@ class TuneInIE(InfoExtractor):
             mobj = re.match(self._VALID_URL, url)
         station_id = mobj.group('id')
 
-        webpage = self._download_webpage(
-            url, station_id, note='Downloading station webpage')
+        station_info = self._download_json(
+            self._API_URL_TEMPLATE.format(station_id),
+            station_id, note='Downloading station JSON')
 
-        payload = self._html_search_regex(
-            r'(?m)TuneIn\.payload\s*=\s*(\{[^$]+?)$', webpage, 'JSON data')
-        json_data = json.loads(payload)
-        station_info = json_data['Station']['broadcast']
         title = station_info['Title']
         thumbnail = station_info.get('Logo')
         location = station_info.get('Location')

From df015c69eab2e7617664ed2d2982832e08c8b843 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 1 Dec 2014 17:28:34 +0100
Subject: [PATCH 0678/1937] release 2014.12.01

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9476bedfa..847a47c45 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.11.27'
+__version__ = '2014.12.01'

From c41a9650c3b8c5430cd21661d668ac9fb346e397 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 1 Dec 2014 17:35:03 +0100
Subject: [PATCH 0679/1937] [youtube] Extract framerate from the dash manifest

Not all videos have 60 fps, for example they can have 48 fps.
---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2642ecfff..1cba40387 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -971,6 +971,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                         'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
                         'asr': int_or_none(r.attrib.get('audioSamplingRate')),
                         'filesize': filesize,
+                        'fps': int_or_none(r.attrib.get('frameRate')),
                     }
                     try:
                         existing_format = next(

From 05177b34a62e3a45417ec1937650d43c20ebdfab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 1 Dec 2014 18:20:36 +0100
Subject: [PATCH 0680/1937] [rutube] Extract m3u8 formats (fixes #3984)

---
 youtube_dl/extractor/rutube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 0c8790da2..6941d96fb 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -53,6 +53,7 @@ class RutubeIE(InfoExtractor):
         m3u8_url = options['video_balancer'].get('m3u8')
         if m3u8_url is None:
             raise ExtractorError('Couldn\'t find m3u8 manifest url')
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
 
         return {
             'id': video['id'],
@@ -60,8 +61,7 @@ class RutubeIE(InfoExtractor):
             'description': video['description'],
             'duration': video['duration'],
             'view_count': video['hits'],
-            'url': m3u8_url,
-            'ext': 'mp4',
+            'formats': formats,
             'thumbnail': video['thumbnail_url'],
             'uploader': author.get('name'),
             'uploader_id': compat_str(author['id']) if author else None,

From b1399a144deb810a6d599e500391befe93142cf9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Dec 2014 20:45:43 +0600
Subject: [PATCH 0681/1937] [rts] Add support for the new URL format and
 extract display id (Closes #4349)

---
 youtube_dl/extractor/rts.py | 44 ++++++++++++++++++++++++++++++-------
 1 file changed, 36 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index e8199b114..82d783078 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 class RTSIE(InfoExtractor):
     IE_DESC = 'RTS.ch'
-    _VALID_URL = r'^https?://(?:www\.)?rts\.ch/(?:[^/]+/){2,}(?P<id>[0-9]+)-.*?\.html'
+    _VALID_URL = r'^https?://(?:www\.)?rts\.ch/(?:(?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|play/tv/-/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+))'
 
     _TESTS = [
         {
@@ -23,6 +23,7 @@ class RTSIE(InfoExtractor):
             'md5': '753b877968ad8afaeddccc374d4256a5',
             'info_dict': {
                 'id': '3449373',
+                'display_id': 'les-enfants-terribles',
                 'ext': 'mp4',
                 'duration': 1488,
                 'title': 'Les Enfants Terribles',
@@ -30,7 +31,8 @@ class RTSIE(InfoExtractor):
                 'uploader': 'Divers',
                 'upload_date': '19680921',
                 'timestamp': -40280400,
-                'thumbnail': 're:^https?://.*\.image'
+                'thumbnail': 're:^https?://.*\.image',
+                'view_count': int,
             },
         },
         {
@@ -38,6 +40,7 @@ class RTSIE(InfoExtractor):
             'md5': 'c148457a27bdc9e5b1ffe081a7a8337b',
             'info_dict': {
                 'id': '5624067',
+                'display_id': 'entre-ciel-et-mer',
                 'ext': 'mp4',
                 'duration': 3720,
                 'title': 'Les yeux dans les cieux - Mon homard au Canada',
@@ -45,7 +48,8 @@ class RTSIE(InfoExtractor):
                 'uploader': 'Passe-moi les jumelles',
                 'upload_date': '20140404',
                 'timestamp': 1396635300,
-                'thumbnail': 're:^https?://.*\.image'
+                'thumbnail': 're:^https?://.*\.image',
+                'view_count': int,
             },
         },
         {
@@ -53,6 +57,7 @@ class RTSIE(InfoExtractor):
             'md5': 'b4326fecd3eb64a458ba73c73e91299d',
             'info_dict': {
                 'id': '5745975',
+                'display_id': '1-2-kloten-fribourg-5-2-second-but-pour-gotteron-par-kwiatowski',
                 'ext': 'mp4',
                 'duration': 48,
                 'title': '1/2, Kloten - Fribourg (5-2): second but pour Gottéron par Kwiatowski',
@@ -60,7 +65,8 @@ class RTSIE(InfoExtractor):
                 'uploader': 'Hockey',
                 'upload_date': '20140403',
                 'timestamp': 1396556882,
-                'thumbnail': 're:^https?://.*\.image'
+                'thumbnail': 're:^https?://.*\.image',
+                'view_count': int,
             },
             'skip': 'Blocked outside Switzerland',
         },
@@ -69,6 +75,7 @@ class RTSIE(InfoExtractor):
             'md5': '9bb06503773c07ce83d3cbd793cebb91',
             'info_dict': {
                 'id': '5745356',
+                'display_id': 'londres-cachee-par-un-epais-smog',
                 'ext': 'mp4',
                 'duration': 33,
                 'title': 'Londres cachée par un épais smog',
@@ -76,7 +83,8 @@ class RTSIE(InfoExtractor):
                 'uploader': 'Le Journal en continu',
                 'upload_date': '20140403',
                 'timestamp': 1396537322,
-                'thumbnail': 're:^https?://.*\.image'
+                'thumbnail': 're:^https?://.*\.image',
+                'view_count': int,
             },
         },
         {
@@ -84,6 +92,7 @@ class RTSIE(InfoExtractor):
             'md5': 'dd8ef6a22dff163d063e2a52bc8adcae',
             'info_dict': {
                 'id': '5706148',
+                'display_id': 'urban-hippie-de-damien-krisl-03-04-2014',
                 'ext': 'mp3',
                 'duration': 123,
                 'title': '"Urban Hippie", de Damien Krisl',
@@ -92,22 +101,40 @@ class RTSIE(InfoExtractor):
                 'timestamp': 1396551600,
             },
         },
+        {
+            'url': 'http://www.rts.ch/play/tv/-/video/le-19h30?id=6348260',
+            'md5': '968777c8779e5aa2434be96c54e19743',
+            'info_dict': {
+                'id': '6348260',
+                'display_id': 'le-19h30',
+                'ext': 'mp4',
+                'duration': 1796,
+                'title': 'Le 19h30',
+                'description': '',
+                'uploader': 'Le 19h30',
+                'upload_date': '20141201',
+                'timestamp': 1417458600,
+                'thumbnail': 're:^https?://.*\.image',
+                'view_count': int,
+            },
+        }
     ]
 
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url)
-        video_id = m.group('id')
+        video_id = m.group('id') or m.group('id_new')
+        display_id = m.group('display_id') or m.group('display_id_new')
 
         def download_json(internal_id):
             return self._download_json(
                 'http://www.rts.ch/a/%s.html?f=json/article' % internal_id,
-                video_id)
+                display_id)
 
         all_info = download_json(video_id)
 
         # video_id extracted out of URL is not always a real id
         if 'video' not in all_info and 'audio' not in all_info:
-            page = self._download_webpage(url, video_id)
+            page = self._download_webpage(url, display_id)
             internal_id = self._html_search_regex(
                 r'<(?:video|audio) data-id="([0-9]+)"', page,
                 'internal video id')
@@ -143,6 +170,7 @@ class RTSIE(InfoExtractor):
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'formats': formats,
             'title': info['title'],
             'description': info.get('intro'),

From 752c8c9b7635137c519ac9b0872c7155322e51f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Dec 2014 20:53:19 +0600
Subject: [PATCH 0682/1937] [rts] Improve _VALID_URL

---
 youtube_dl/extractor/rts.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index 82d783078..c31049c0e 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 class RTSIE(InfoExtractor):
     IE_DESC = 'RTS.ch'
-    _VALID_URL = r'^https?://(?:www\.)?rts\.ch/(?:(?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|play/tv/-/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+))'
+    _VALID_URL = r'^https?://(?:www\.)?rts\.ch/(?:(?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|play/tv/[^/]+/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+))'
 
     _TESTS = [
         {
@@ -117,6 +117,10 @@ class RTSIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*\.image',
                 'view_count': int,
             },
+        },
+        {
+            'url': 'http://www.rts.ch/play/tv/le-19h30/video/le-chantier-du-nouveau-parlement-vaudois-a-permis-une-trouvaille-historique?id=6348280',
+            'only_matching': True,
         }
     ]
 

From 706d7d4ee737f0e382df0bae3c8146dd5839995d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Dec 2014 21:18:07 +0600
Subject: [PATCH 0683/1937] [YoutubeDL] Avoid negative timestamps on Windows

---
 youtube_dl/YoutubeDL.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 21c7c298a..f89ac4e1d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -787,6 +787,10 @@ class YoutubeDL(object):
             info_dict['display_id'] = info_dict['id']
 
         if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
+            # Working around negative timestamps in Windows
+            # (see http://bugs.python.org/issue1646728)
+            if info_dict['timestamp'] < 0 and os.name == 'nt':
+                info_dict['timestamp'] = 0
             upload_date = datetime.datetime.utcfromtimestamp(
                 info_dict['timestamp'])
             info_dict['upload_date'] = upload_date.strftime('%Y%m%d')

From ac5633592afa9d70c05170333a0fb9013ba868da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Dec 2014 22:23:23 +0600
Subject: [PATCH 0684/1937] [24video] Add extractor (Closes #4350)

---
 youtube_dl/extractor/__init__.py        |   1 +
 youtube_dl/extractor/twentyfourvideo.py | 109 ++++++++++++++++++++++++
 2 files changed, 110 insertions(+)
 create mode 100644 youtube_dl/extractor/twentyfourvideo.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0339d1386..8b513ffd1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -417,6 +417,7 @@ from .tutv import TutvIE
 from .tvigle import TvigleIE
 from .tvp import TvpIE
 from .tvplay import TVPlayIE
+from .twentyfourvideo import TwentyFourVideoIE
 from .twitch import TwitchIE
 from .ubu import UbuIE
 from .udemy import (
diff --git a/youtube_dl/extractor/twentyfourvideo.py b/youtube_dl/extractor/twentyfourvideo.py
new file mode 100644
index 000000000..67e8bfea0
--- /dev/null
+++ b/youtube_dl/extractor/twentyfourvideo.py
@@ -0,0 +1,109 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    int_or_none,
+)
+
+
+class TwentyFourVideoIE(InfoExtractor):
+    IE_NAME = '24video'
+    _VALID_URL = r'https?://(?:www\.)?24video\.net/(?:video/(?:view|xml)/|player/new24_play\.swf\?id=)(?P<id>\d+)'
+
+    _TESTS = [
+        {
+            'url': 'http://www.24video.net/video/view/1044982',
+            'md5': '48dd7646775690a80447a8dca6a2df76',
+            'info_dict': {
+                'id': '1044982',
+                'ext': 'mp4',
+                'title': 'Эротика каменного века',
+                'description': 'Как смотрели порно в каменном веке.',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'uploader': 'SUPERTELO',
+                'duration': 31,
+                'timestamp': 1275937857,
+                'upload_date': '20100607',
+                'age_limit': 18,
+                'like_count': int,
+                'dislike_count': int,
+            },
+        },
+        {
+            'url': 'http://www.24video.net/player/new24_play.swf?id=1044982',
+            'only_matching': True,
+        }
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'http://www.24video.net/video/view/%s' % video_id, video_id)
+
+        title = self._og_search_title(webpage)
+        description = self._html_search_regex(
+            r'<span itemprop="description">([^<]+)</span>', webpage, 'description', fatal=False)
+        thumbnail = self._og_search_thumbnail(webpage)
+        duration = int_or_none(self._og_search_property(
+            'duration', webpage, 'duration', fatal=False))
+        timestamp = parse_iso8601(self._search_regex(
+            r'<time id="video-timeago" datetime="([^"]+)" itemprop="uploadDate">',
+            webpage, 'upload date'))
+
+        uploader = self._html_search_regex(
+            r'Загрузил\s*<a href="/jsecUser/movies/[^"]+" class="link">([^<]+)</a>',
+            webpage, 'uploader', fatal=False)
+
+        view_count = int_or_none(self._html_search_regex(
+            r'<span class="video-views">(\d+) просмотр',
+            webpage, 'view count', fatal=False))
+        comment_count = int_or_none(self._html_search_regex(
+            r'<div class="comments-title" id="comments-count">(\d+) комментари',
+            webpage, 'comment count', fatal=False))
+
+        formats = []
+
+        pc_video = self._download_xml(
+            'http://www.24video.net/video/xml/%s?mode=play' % video_id,
+            video_id, 'Downloading PC video URL').find('.//video')
+
+        formats.append({
+            'url': pc_video.attrib['url'],
+            'format_id': 'pc',
+            'quality': 1,
+        })
+
+        like_count = int_or_none(pc_video.get('ratingPlus'))
+        dislike_count = int_or_none(pc_video.get('ratingMinus'))
+        age_limit = 18 if pc_video.get('adult') == 'true' else 0
+
+        mobile_video = self._download_xml(
+            'http://www.24video.net/video/xml/%s' % video_id,
+            video_id, 'Downloading mobile video URL').find('.//video')
+
+        formats.append({
+            'url': mobile_video.attrib['url'],
+            'format_id': 'mobile',
+            'quality': 0,
+        })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'duration': duration,
+            'timestamp': timestamp,
+            'view_count': view_count,
+            'comment_count': comment_count,
+            'like_count': like_count,
+            'dislike_count': dislike_count,
+            'age_limit': age_limit,
+            'formats': formats,
+        }

From c492970b4b2d9b3afa5fe817e4da748816126ebd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Dec 2014 22:24:47 +0600
Subject: [PATCH 0685/1937] [rts] Improve _VALID_URL

---
 youtube_dl/extractor/rts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index c31049c0e..dc59a5e5c 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 class RTSIE(InfoExtractor):
     IE_DESC = 'RTS.ch'
-    _VALID_URL = r'^https?://(?:www\.)?rts\.ch/(?:(?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|play/tv/[^/]+/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+))'
+    _VALID_URL = r'https?://(?:www\.)?rts\.ch/(?:(?:[^/]+/){2,}(?P<id>[0-9]+)-(?P<display_id>.+?)\.html|play/tv/[^/]+/video/(?P<display_id_new>.+?)\?id=(?P<id_new>[0-9]+))'
 
     _TESTS = [
         {

From 62d8b56655eceb763fe0150866092ef99ec6fb84 Mon Sep 17 00:00:00 2001
From: Adrian Kretz <adriankretz@gmail.com>
Date: Tue, 2 Dec 2014 23:23:18 +0100
Subject: [PATCH 0686/1937] [nhl] Support videos which don't have mp4-extension
 (fixes #4348)

---
 youtube_dl/extractor/nhl.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index bdcf7e239..72dcbf424 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urlparse,
     compat_urllib_parse,
+    compat_urllib_parse_urlparse
 )
 from ..utils import (
     unified_strdate,
@@ -24,9 +25,13 @@ class NHLBaseInfoExtractor(InfoExtractor):
 
         initial_video_url = info['publishPoint']
         if info['formats'] == '1':
+            parsed_url = compat_urllib_parse_urlparse(initial_video_url)
+            path = parsed_url.path
+            extension_index = path.rfind('.')
+            path = path[:extension_index] + '_sd' + path[extension_index:]
             data = compat_urllib_parse.urlencode({
                 'type': 'fvod',
-                'path': initial_video_url.replace('.mp4', '_sd.mp4'),
+                'path': compat_urlparse.urlunparse(parsed_url[:2] + (path,) + parsed_url[3:])
             })
             path_url = 'http://video.nhl.com/videocenter/servlets/encryptvideopath?' + data
             path_doc = self._download_xml(
@@ -73,6 +78,17 @@ class NHLIE(NHLBaseInfoExtractor):
             'duration': 0,
             'upload_date': '20141011',
         },
+    }, {
+        'url': 'http://video.mapleleafs.nhl.com/videocenter/console?id=58665&catid=802',
+        'md5': 'c78fc64ea01777e426cfc202b746c825',
+        'info_dict': {
+            'id': '58665',
+            'ext': 'flv',
+            'title': 'Classic Game In Six - April 22, 1979',
+            'description': 'It was the last playoff game for the Leafs in the decade, and the last time the Leafs and Habs played in the playoffs. Great game, not a great ending.',
+            'duration': 400,
+            'upload_date': '20100129'
+        },
     }, {
         'url': 'http://video.flames.nhl.com/videocenter/console?id=630616',
         'only_matching': True,

From ea2ee40357eb2c43f8968a4eeba47e1cf97667ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 2 Dec 2014 23:56:30 +0100
Subject: [PATCH 0687/1937] [nhl.com:videocenter] Don't match url with 'id=*'
 before 'catid' in the query

Since the order extractors are added is not defined, it would match instead of NHLIE.
---
 youtube_dl/extractor/nhl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index bdcf7e239..2a84f6761 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -90,7 +90,7 @@ class NHLIE(NHLBaseInfoExtractor):
 class NHLVideocenterIE(NHLBaseInfoExtractor):
     IE_NAME = 'nhl.com:videocenter'
     IE_DESC = 'NHL videocenter category'
-    _VALID_URL = r'https?://video\.(?P<team>[^.]*)\.nhl\.com/videocenter/(console\?.*?catid=(?P<catid>[0-9]+)(?![&?]id=).*?)?$'
+    _VALID_URL = r'https?://video\.(?P<team>[^.]*)\.nhl\.com/videocenter/(console\?[^(id=)]*catid=(?P<catid>[0-9]+)(?![&?]id=).*?)?$'
     _TEST = {
         'url': 'http://video.canucks.nhl.com/videocenter/console?catid=999',
         'info_dict': {

From 163c8babaab947d243f4b8cc4e1fa0bc4b4e1603 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 3 Dec 2014 00:08:26 +0100
Subject: [PATCH 0688/1937] [nhl] Simplify

---
 youtube_dl/extractor/nhl.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index 7d74b110b..0244368e9 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -26,9 +26,7 @@ class NHLBaseInfoExtractor(InfoExtractor):
         initial_video_url = info['publishPoint']
         if info['formats'] == '1':
             parsed_url = compat_urllib_parse_urlparse(initial_video_url)
-            path = parsed_url.path
-            extension_index = path.rfind('.')
-            path = path[:extension_index] + '_sd' + path[extension_index:]
+            path = parsed_url.path.replace('.', '_sd.', 1)
             data = compat_urllib_parse.urlencode({
                 'type': 'fvod',
                 'path': compat_urlparse.urlunparse(parsed_url[:2] + (path,) + parsed_url[3:])

From c627f7d48c8e778958aaf03eb091364fbe12f2ca Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 3 Dec 2014 12:15:34 +0100
Subject: [PATCH 0689/1937] release 2014.12.03

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 847a47c45..61902a8cc 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.01'
+__version__ = '2014.12.03'

From 233d37fb6b420b01315c48488c84dc5b6e7aaea0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 3 Dec 2014 13:25:49 +0100
Subject: [PATCH 0690/1937] [brightcove] Make sure that the 'ext' variable is
 set (fixes #4360)

---
 youtube_dl/extractor/brightcove.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 2db7f9fef..bf18a97e0 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -265,6 +265,7 @@ class BrightcoveIE(InfoExtractor):
                 url = rend['defaultURL']
                 if not url:
                     continue
+                ext = None
                 if rend['remote']:
                     url_comp = compat_urllib_parse_urlparse(url)
                     if url_comp.path.endswith('.m3u8'):
@@ -276,7 +277,7 @@ class BrightcoveIE(InfoExtractor):
                         # akamaihd.net, but they don't use f4m manifests
                         url = url.replace('control/', '') + '?&v=3.3.0&fp=13&r=FEEFJ&g=RTSJIMBMPFPB'
                         ext = 'flv'
-                else:
+                if ext is None:
                     ext = determine_ext(url)
                 size = rend.get('size')
                 formats.append({

From de9bd74bc2e06a6d5396151050fd1eeafdd54a33 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 3 Dec 2014 21:17:11 +0600
Subject: [PATCH 0691/1937] [ted] Fix type_watch links extraction

---
 youtube_dl/extractor/ted.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index f8a87afda..72160503c 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -199,8 +199,9 @@ class TEDIE(SubtitlesInfoExtractor):
         webpage = self._download_webpage(url, name)
 
         config_json = self._html_search_regex(
-            r"data-config='([^']+)", webpage, 'config')
-        config = json.loads(config_json)
+            r'"pages\.jwplayer"\s*,\s*({.+?})\s*\)\s*</script>',
+            webpage, 'config')
+        config = json.loads(config_json)['config']
         video_url = config['video']['url']
         thumbnail = config.get('image', {}).get('url')
 

From 0403b06985bcb004efb3683bca6a56a47e3227b1 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Wed, 3 Dec 2014 11:49:53 +0100
Subject: [PATCH 0692/1937] [soundcloud] Improve_VALID_URL Add support for
 links from Audiomack

---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 3c1d058db..ab9483d2d 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -32,7 +32,7 @@ class SoundcloudIE(InfoExtractor):
                             (?P<title>[\w\d-]+)/?
                             (?P<token>[^?]+?)?(?:[?].*)?$)
                        |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)
-                          (?:/?\?secret_token=(?P<secret_token>[^&]+?))?$)
+                          (?:/?\?secret_token=(?P<secret_token>[^&]+))?)
                        |(?P<player>(?:w|player|p.)\.soundcloud\.com/player/?.*?url=.*)
                     )
                     '''

From 9095aa38ac7361b55aa0e9fd3fe4fccf23718e6c Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Wed, 3 Dec 2014 14:53:12 +0100
Subject: [PATCH 0693/1937] [audiomack] Update test

---
 youtube_dl/extractor/audiomack.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index eeeec768f..04386f7f7 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -24,17 +24,17 @@ class AudiomackIE(InfoExtractor):
         },
         # hosted on soundcloud via audiomack
         {
+            'add_ie': ['Soundcloud'],
             'url': 'http://www.audiomack.com/song/xclusiveszone/take-kare',
             'file': '172419696.mp3',
             'info_dict':
             {
-                'ext': 'mp3',
+                'description': 'md5:1fc3272ed7a635cce5be1568c2822997',
                 'title': 'Young Thug ft Lil Wayne - Take Kare',
-                "upload_date": "20141016",
-                "description": "New track produced by London On Da Track called “Take Kare\"\n\nhttp://instagram.com/theyoungthugworld\nhttps://www.facebook.com/ThuggerThuggerCashMoney\n",
-                "uploader": "Young Thug World"
+                'uploader':'Young Thug World',
+                'upload_date':'20141016',
             }
-        }
+        },
     ]
 
     def _real_extract(self, url):

From dce2a3cf9e4640ec9ae54f1997596d00591a97c3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 01:33:30 +0100
Subject: [PATCH 0694/1937] [break] Remove md5sum from test

---
 youtube_dl/extractor/breakcom.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/breakcom.py b/youtube_dl/extractor/breakcom.py
index 2c0e5eea2..4bcc897c9 100644
--- a/youtube_dl/extractor/breakcom.py
+++ b/youtube_dl/extractor/breakcom.py
@@ -14,7 +14,6 @@ class BreakIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?break\.com/video/(?:[^/]+/)*.+-(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.break.com/video/when-girls-act-like-guys-2468056',
-        'md5': '33aa4ff477ecd124d18d7b5d23b87ce5',
         'info_dict': {
             'id': '2468056',
             'ext': 'mp4',

From eecd6a467dd6b28cd0cdd63a9cebd5a8bfafd4c4 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Wed, 3 Dec 2014 19:07:09 +0100
Subject: [PATCH 0695/1937] [vgtv] Update tests

---
 youtube_dl/extractor/vgtv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index 70578a4cc..2f111bf7e 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -17,7 +17,7 @@ class VGTVIE(InfoExtractor):
             'info_dict': {
                 'id': '84196',
                 'ext': 'mp4',
-                'title': 'Hevnen er søt episode 1:10 - Abu',
+                'title': 'Hevnen er søt: Episode 10 - Abu',
                 'description': 'md5:e25e4badb5f544b04341e14abdc72234',
                 'thumbnail': 're:^https?://.*\.jpg',
                 'duration': 648.000,
@@ -35,7 +35,7 @@ class VGTVIE(InfoExtractor):
                 'title': 'OPPTAK: VGTV følger EM-kvalifiseringen',
                 'description': 'md5:3772d9c0dc2dff92a886b60039a7d4d3',
                 'thumbnail': 're:^https?://.*\.jpg',
-                'duration': 9056.000,
+                'duration': 9103.0,
                 'timestamp': 1410113864,
                 'upload_date': '20140907',
                 'view_count': int,

From 29f400b97d00cc7bd8b1a7549417584a45e38df8 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 4 Dec 2014 02:54:25 +0100
Subject: [PATCH 0696/1937] [tvp] Update extractor

---
 youtube_dl/extractor/tvp.py | 121 +++++++++++++++++++++++++++++-------
 1 file changed, 98 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index a64580005..6b95e2ed1 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -1,37 +1,112 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 
 
 class TvpIE(InfoExtractor):
     IE_NAME = 'tvp.pl'
-    _VALID_URL = r'https?://www\.tvp\.pl/.*?wideo/(?P<date>\d+)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?P<type>vod|www)\.tvp\.pl/.*/(?P<id>\d+)$'
 
-    _TEST = {
-        'url': 'http://www.tvp.pl/warszawa/magazyny/campusnews/wideo/31102013/12878238',
-        'md5': '148408967a6a468953c0a75cbdaf0d7a',
-        'info_dict': {
-            'id': '12878238',
-            'ext': 'wmv',
-            'title': '31.10.2013 - Odcinek 2',
-            'description': '31.10.2013 - Odcinek 2',
+    _TESTS = [
+        {
+            'url': 'http://www.tvp.pl/warszawa/magazyny/campusnews/wideo/31102013/12878238',
+            'info_dict': {
+                'id': '12878238',
+                'ext': 'wmv',
+                'title': 'CAMPUSnews, 31.10.2013 - Odcinek 2',
+                'description': '',
+            },
+            'skip': 'Download has to use same server IP as extraction. Therefore, a good (load-balancing) DNS resolver will make the download fail.',
+        }, {
+            'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem/wideo/odc-2/4278035',
+            'info_dict': {
+                'id': '4278035',
+                'ext': 'wmv',
+                'title': 'Ogniem i mieczem, odc. 2',
+                'description': 'Bohun dowiaduje się o złamaniu przez kniahinię danego mu słowa i wyrusza do Rozłogów. Helenie w ostatniej chwili udaje się uciec dzięki pomocy Zagłoby.',
+            },
+            'skip': 'As above',
+        }, {
+            'url': 'http://vod.tvp.pl/seriale/obyczajowe/czas-honoru/sezon-1-1-13/i-seria-odc-13/194536',
+            'info_dict': {
+                'id': '194536',
+                'ext': 'mp4',
+                'title': 'Czas honoru, I seria – odc. 13',
+                'description': 'WŁADEK\nCzesław prosi Marię o dostarczenie Władkowi zarazki tyfusu. Jeśli zachoruje zostanie przewieziony do szpitala skąd łatwiej będzie go odbić. Czy matka zdecyduje się zarazić syna? Karol odwiedza Wandę przyznaje się, że ją oszukiwał, ale ostrzega też, że grozi jej aresztowanie i nalega, żeby wyjechała z Warszawy. Czy dziewczyna zdecyduje się znów oddalić od ukochanego? Rozpoczyna się akcja odbicia Władka.',
+            },
+        }, {
+            'url': 'http://www.tvp.pl/there-can-be-anything-so-i-shortened-it/17916176',
+            'info_dict': {
+                'id': '17916176',
+                'ext': 'mp4',
+                'title': 'rozmaitosci, TVP Gorzów pokaże filmy studentów z podroży dookoła świata',
+                'description': '',
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': 'true',
+            },
+        }, {
+            'url': 'http://vod.tvp.pl/seriale/obyczajowe/na-sygnale/sezon-2-27-/odc-39/17834272',
+            'info_dict': {
+                'id': '17834272',
+                'ext': 'mp4',
+                'title': 'Na sygnale, odc. 39',
+                'description': 'Ekipa Wiktora ratuje młodą matkę, która spadła ze schodów trzymając na rękach noworodka. Okazuje się, że dziewczyna jest surogatką, a biologiczni rodzice dziecka próbują zmusić ją do oddania synka…',
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': 'true',
+            },
         },
-        'skip': 'Download has to use same server IP as extraction. Therefore, a good (load-balancing) DNS resolver will make the download fail.'
-    }
+    ]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        json_url = 'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s' % video_id
-        params = self._download_json(
-            json_url, video_id, "Downloading video metadata")
-        video_url = params['video_url']
-
-        return {
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        webpage = self._download_webpage(
+            'http://www.tvp.pl/sess/tvplayer.php?object_id=%s' % video_id, video_id)
+        title = self._og_search_title(webpage)
+        series = self._search_regex(
+            r'{name:\s*([\'"])SeriesTitle\1,\s*value:\s*\1(?P<series>.*?)\1},',
+            webpage, 'series', group='series', default=None)
+        if series is not None and series not in title:
+            title = '%s, %s' % (series, title)
+        info_dict = {
             'id': video_id,
-            'title': self._og_search_title(webpage),
-            'ext': 'wmv',
-            'url': video_url,
-            'description': self._og_search_description(webpage),
+            'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
+            'description': self._og_search_description(webpage, default=''),
         }
+        if mobj.group('type') == 'vod' and info_dict['description'] == '':
+            info_dict.update({
+                'description': self._html_search_regex(
+                    r'(?s)<div\s+class=[\'"]opis.*?</div>',
+                    self._download_webpage(url, video_id), 'description', group=0),
+            })
+
+        video_url = self._search_regex(
+            r'0:{src:([\'"])(?P<url>.*?)\1', webpage, 'formats', group='url', default=None)
+        if video_url is None:
+            video_url = self._download_json(
+                'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s' % video_id,
+                video_id)['video_url']
+
+        ext = video_url.rsplit('.', 1)[-1]
+        if ext != 'ism/manifest':
+            if '/' in ext:
+                ext = 'mp4'
+            info_dict.update({
+                'ext': ext,
+                'url': video_url,
+            })
+        else:
+            m3u8_url = re.sub('([^/]*)\.ism/manifest', r'\1.ism/\1.m3u8', video_url)
+            formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
+            info_dict.update({
+                'formats': formats,
+            })
+        return info_dict

From 6ce2c6783b2c1516fe9b1b5cb88d28231db45f8c Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 4 Dec 2014 05:14:09 +0100
Subject: [PATCH 0697/1937] [tvp] Add extractor

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/tvp.py      | 56 ++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8b513ffd1..b09ee303d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -415,7 +415,7 @@ from .tunein import TuneInIE
 from .turbo import TurboIE
 from .tutv import TutvIE
 from .tvigle import TvigleIE
-from .tvp import TvpIE
+from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
 from .twentyfourvideo import TwentyFourVideoIE
 from .twitch import TwitchIE
diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index 6b95e2ed1..2248a9fdf 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -110,3 +110,59 @@ class TvpIE(InfoExtractor):
                 'formats': formats,
             })
         return info_dict
+
+
+class TvpSeriesIE(InfoExtractor):
+    IE_NAME = 'tvp.pl:Series'
+    _VALID_URL = r'https?://vod\.tvp\.pl/(?:[^/]+/){2}(?P<id>[^/]+)/?$'
+
+    _TESTS = [
+        {
+            'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem',
+            'info_dict': {
+                'title': 'Ogniem i mieczem',
+                'id': '4278026',
+            },
+            'playlist_count': 4,
+        }, {
+            'url': 'http://vod.tvp.pl/audycje/podroze/boso-przez-swiat',
+            'info_dict': {
+                'title': 'Boso przez świat',
+                'id': '9329207',
+            },
+            'playlist_count': 86,
+        }
+    ]
+
+    def _force_download_webpage(self, url, v_id, tries=0):
+        if tries >= 5:
+            raise ExtractorError(
+                '%s: Cannot download webpage, try again later' % v_id)
+        # Sometimes happen, but in my tests second try always succeeded
+        try:
+            return self._download_webpage(url, v_id)
+        except IncompleteRead as e:
+            return self._force_download_webpage(url, v_id, tries+1)
+    
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._force_download_webpage(url, display_id)
+        title = self._html_search_regex(
+            r'(?s) id=[\'"]path[\'"]>(.*?)</span>', webpage, 'series')
+        title = title.split(' / ', 2)[-1]
+        playlist_id = self._search_regex(r'nodeId:\s*(\d+)', webpage, 'playlist id')
+        playlist = self._force_download_webpage(
+            'http://vod.tvp.pl/vod/seriesAjax?type=series&nodeId=%s&recommend'
+            'edId=0&sort=&page=0&pageSize=1000000' % playlist_id, display_id)
+        videos_paths = re.findall(
+            '(?s)class="shortTitle">.*?href="(/[^"]+)', playlist)
+        entries = [
+            self.url_result('http://vod.tvp.pl%s' % v_path, ie=TvpIE.ie_key())
+            for v_path in videos_paths]
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'display_id': display_id,
+            'title': title,
+            'entries': entries,
+        }

From b92c54869345b60a29882297ac1832429f964d03 Mon Sep 17 00:00:00 2001
From: Zack Fernandes <zack@ohno.io>
Date: Wed, 3 Dec 2014 20:22:48 -0800
Subject: [PATCH 0698/1937] [foxgay] Initial support

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/foxgay.py   | 48 ++++++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+)
 create mode 100644 youtube_dl/extractor/foxgay.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8b513ffd1..b2c7fd586 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -121,6 +121,7 @@ from .fktv import (
 from .flickr import FlickrIE
 from .folketinget import FolketingetIE
 from .fourtube import FourTubeIE
+from .foxgay import FoxgayIE
 from .franceculture import FranceCultureIE
 from .franceinter import FranceInterIE
 from .francetv import (
diff --git a/youtube_dl/extractor/foxgay.py b/youtube_dl/extractor/foxgay.py
new file mode 100644
index 000000000..a146a9180
--- /dev/null
+++ b/youtube_dl/extractor/foxgay.py
@@ -0,0 +1,48 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class FoxgayIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?foxgay\.com/videos/(?:\S+-)?(?P<id>\d+)\.shtml'
+    _TEST = {
+        'url': 'http://foxgay.com/videos/fuck-turkish-style-2582.shtml',
+        'md5': '80d72beab5d04e1655a56ad37afe6841',
+        'info_dict': {
+            'id': '2582',
+            'ext': 'mp4',
+            'title': 'md5:6122f7ae0fc6b21ebdf59c5e083ce25a',
+            'description': 'md5:5e51dc4405f1fd315f7927daed2ce5cf',
+            'age_limit': 18,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        # Download the initial webpage and extract all available data.
+        webpage = self._download_webpage(url, video_id)
+        title = self._html_search_regex(
+            r'<title>(?P<title>.*?)</title>',
+            webpage, 'title', fatal=False)
+        description = self._html_search_regex(
+            r'<div class="ico_desc"><h2>(?P<description>.*?)</h2>',
+            webpage, 'description', fatal=False)
+
+        # Find the URL for the iFrame which contains the actual video.
+        iframe = self._download_webpage(
+            self._html_search_regex(r'iframe src="(?P<frame>.*?)"', webpage, 'video frame'),
+            video_id)
+        video_url = self._html_search_regex(
+            r"v_path = '(?P<vid>http://.*?)'", iframe, 'url')
+        thumb_url = self._html_search_regex(
+            r"t_path = '(?P<thumb>http://.*?)'", iframe, 'thumbnail', fatal=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': video_url,
+            'description': description,
+            'thumbnail': thumb_url,
+            'age_limit': 18,
+        }

From ecd7ea1e6bb5e91dc3a4391fbd49c85dab337bd9 Mon Sep 17 00:00:00 2001
From: Zack Fernandes <zack@ohno.io>
Date: Wed, 3 Dec 2014 22:22:36 -0800
Subject: [PATCH 0699/1937] [myvidster] Added support for Myvidster

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/myvidster.py | 28 ++++++++++++++++++++++++++++
 2 files changed, 29 insertions(+)
 create mode 100644 youtube_dl/extractor/myvidster.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8b513ffd1..f4f66fb0b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -245,6 +245,7 @@ from .muzu import MuzuTVIE
 from .myspace import MySpaceIE, MySpaceAlbumIE
 from .myspass import MySpassIE
 from .myvideo import MyVideoIE
+from .myvidster import MyVidsterIE
 from .naver import NaverIE
 from .nba import NBAIE
 from .nbc import (
diff --git a/youtube_dl/extractor/myvidster.py b/youtube_dl/extractor/myvidster.py
new file mode 100644
index 000000000..7dd36ee46
--- /dev/null
+++ b/youtube_dl/extractor/myvidster.py
@@ -0,0 +1,28 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class MyVidsterIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?myvidster\.com/video/(?P<id>\d+)/.*$'
+
+    _TEST = {
+        'url': 'http://www.myvidster.com/video/32059805/Hot_chemistry_with_raw_love_making',
+        'md5': '95296d0231c1363222c3441af62dc4ca',
+        'info_dict': {
+            'id': '3685814',
+            'title': 'md5:7d8427d6d02c4fbcef50fe269980c749',
+            'upload_date': '20141027',
+            'uploader_id': 'utkualp',
+            'ext': 'mp4',
+        },
+        'add_ie': ['XHamster'],
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        return self.url_result(self._html_search_regex(
+            r'rel="videolink" href="(?P<real_url>.*)">',
+            webpage, 'real video url'))

From d4e06d4a83b223e82814cd160bdf62e3a82f33e4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 07:57:18 +0100
Subject: [PATCH 0700/1937] [options] Standardize mentoined configuration file
 location (Fixes #4367)

---
 youtube_dl/options.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 2e8c71508..041ca83d8 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -163,7 +163,10 @@ def parseOpts(overrideArguments=None):
     general.add_option(
         '--ignore-config',
         action='store_true',
-        help='Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: do not read the user configuration in ~/.config/youtube-dl.conf (%APPDATA%/youtube-dl/config.txt on Windows)')
+        help='Do not read configuration files. '
+        'When given in the global configuration file /etc/youtube-dl.conf: '
+        'Do not read the user configuration in ~/.config/youtube-dl/config '
+        '(%APPDATA%/youtube-dl/config.txt on Windows)')
     general.add_option(
         '--flat-playlist',
         action='store_const', dest='extract_flat', const='in_playlist',

From abe70fa04440398f2262d34d7229cbb40db3e70f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 4 Dec 2014 08:21:29 +0100
Subject: [PATCH 0701/1937] [audiomack] Modernize test definition

---
 youtube_dl/extractor/audiomack.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 04386f7f7..9ce4982d9 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -26,9 +26,10 @@ class AudiomackIE(InfoExtractor):
         {
             'add_ie': ['Soundcloud'],
             'url': 'http://www.audiomack.com/song/xclusiveszone/take-kare',
-            'file': '172419696.mp3',
             'info_dict':
             {
+                'id': '172419696',
+                'ext': 'mp3',
                 'description': 'md5:1fc3272ed7a635cce5be1568c2822997',
                 'title': 'Young Thug ft Lil Wayne - Take Kare',
                 'uploader':'Young Thug World',

From 191cc41ba49a5d19e798131060c6976dbb2d357e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 08:22:20 +0100
Subject: [PATCH 0702/1937] [foxgay] Add thumbnail to test definition

---
 youtube_dl/extractor/foxgay.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/foxgay.py b/youtube_dl/extractor/foxgay.py
index a146a9180..08b8ea362 100644
--- a/youtube_dl/extractor/foxgay.py
+++ b/youtube_dl/extractor/foxgay.py
@@ -14,14 +14,14 @@ class FoxgayIE(InfoExtractor):
             'title': 'md5:6122f7ae0fc6b21ebdf59c5e083ce25a',
             'description': 'md5:5e51dc4405f1fd315f7927daed2ce5cf',
             'age_limit': 18,
+            'thumbnail': 're:https?://.*\.jpg$',
         },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
-        # Download the initial webpage and extract all available data.
         webpage = self._download_webpage(url, video_id)
+
         title = self._html_search_regex(
             r'<title>(?P<title>.*?)</title>',
             webpage, 'title', fatal=False)

From 5f5e993dc61e68402d60f35df66a08b8bcca808c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 08:22:53 +0100
Subject: [PATCH 0703/1937] [bbccouk] Remove unused import

---
 youtube_dl/extractor/bbccouk.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index beb6cfc8a..01c02d360 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import re
 import xml.etree.ElementTree
 
 from .subtitles import SubtitlesInfoExtractor

From 810fb84d5edaccfdebbf1c4e441d7747f2688b7b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 08:27:40 +0100
Subject: [PATCH 0704/1937] pep8 and minor beautification all around

---
 youtube_dl/extractor/audiomack.py | 10 +++++-----
 youtube_dl/extractor/common.py    |  3 ++-
 youtube_dl/extractor/myspace.py   |  1 +
 youtube_dl/extractor/udemy.py     |  5 +----
 youtube_dl/extractor/youtube.py   | 13 ++++++++-----
 5 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 04386f7f7..622b20989 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -26,13 +26,13 @@ class AudiomackIE(InfoExtractor):
         {
             'add_ie': ['Soundcloud'],
             'url': 'http://www.audiomack.com/song/xclusiveszone/take-kare',
-            'file': '172419696.mp3',
-            'info_dict':
-            {
+            'info_dict': {
+                'id': '172419696',
+                'ext': 'mp3',
                 'description': 'md5:1fc3272ed7a635cce5be1568c2822997',
                 'title': 'Young Thug ft Lil Wayne - Take Kare',
-                'uploader':'Young Thug World',
-                'upload_date':'20141016',
+                'uploader': 'Young Thug World',
+                'upload_date': '20141016',
             }
         },
     ]
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e80a2dad0..5b5e33cea 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -819,7 +819,8 @@ class InfoExtractor(object):
         return res
 
     def _set_cookie(self, domain, name, value, expire_time=None):
-        cookie = compat_cookiejar.Cookie(0, name, value, None, None, domain, None,
+        cookie = compat_cookiejar.Cookie(
+            0, name, value, None, None, domain, None,
             None, '/', True, False, expire_time, '', None, None, None)
         self._downloader.cookiejar.set_cookie(cookie)
 
diff --git a/youtube_dl/extractor/myspace.py b/youtube_dl/extractor/myspace.py
index e62614670..83414a232 100644
--- a/youtube_dl/extractor/myspace.py
+++ b/youtube_dl/extractor/myspace.py
@@ -88,6 +88,7 @@ class MySpaceIE(InfoExtractor):
                 self.report_warning(
                     '%s: No downloadable song on this page' % video_id)
                 return
+
             def search_data(name):
                 return self._search_regex(
                     r'''data-%s=([\'"])(?P<data>.*?)\1''' % name,
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 0e4d386a8..5271611ac 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -97,11 +97,8 @@ class UdemyIE(InfoExtractor):
         if 'returnUrl' not in response:
             raise ExtractorError('Unable to log in')
 
-
-
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        lecture_id = mobj.group('id')
+        lecture_id = self._match_id(url)
 
         lecture = self._download_json(
             'https://www.udemy.com/api-1.1/lectures/%s' % lecture_id,
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1cba40387..f452a90d8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -44,9 +44,10 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     _LOGIN_REQUIRED = False
 
     def _set_language(self):
-        self._set_cookie('.youtube.com', 'PREF', 'f1=50000000&hl=en',
+        self._set_cookie(
+            '.youtube.com', 'PREF', 'f1=50000000&hl=en',
             # YouTube sets the expire time to about two months
-            expire_time=time.time() + 60*24*3600)
+            expire_time=time.time() + 2 * 30 * 24 * 3600)
 
     def _login(self):
         """
@@ -722,9 +723,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 # We fallback to the get_video_info pages (used by the embed page)
                 self.report_video_info_webpage_download(video_id)
                 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
-                    video_info_url = (proto + '://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
-                        % (video_id, el_type))
-                    video_info_webpage = self._download_webpage(video_info_url,
+                    video_info_url = (
+                        '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
+                        % (proto, video_id, el_type))
+                    video_info_webpage = self._download_webpage(
+                        video_info_url,
                         video_id, note=False,
                         errnote='unable to download video info webpage')
                     video_info = compat_parse_qs(video_info_webpage)

From 8b97115358c89063cef27edc26f39cf64cbe4a87 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 08:28:41 +0100
Subject: [PATCH 0705/1937] Credit @zackfern for foxgay (#4371)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index bd2e967e3..4d3b9ddc1 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -89,3 +89,4 @@ Oskar Jauch
 Matthew Rayfield
 t0mm0
 Tithen-Firion
+Zack Fernandes

From 39c52bbd32efa5367eae31af59066f045bc7fe7e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 08:31:55 +0100
Subject: [PATCH 0706/1937] [myvidster] Enforce age limit in test

---
 youtube_dl/extractor/myvidster.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/myvidster.py b/youtube_dl/extractor/myvidster.py
index 7dd36ee46..a94ab8358 100644
--- a/youtube_dl/extractor/myvidster.py
+++ b/youtube_dl/extractor/myvidster.py
@@ -4,7 +4,7 @@ from .common import InfoExtractor
 
 
 class MyVidsterIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?myvidster\.com/video/(?P<id>\d+)/.*$'
+    _VALID_URL = r'http://(?:www\.)?myvidster\.com/video/(?P<id>\d+)/'
 
     _TEST = {
         'url': 'http://www.myvidster.com/video/32059805/Hot_chemistry_with_raw_love_making',
@@ -15,6 +15,7 @@ class MyVidsterIE(InfoExtractor):
             'upload_date': '20141027',
             'uploader_id': 'utkualp',
             'ext': 'mp4',
+            'age_limit': 18,
         },
         'add_ie': ['XHamster'],
     }

From 9776bc7f57f061d133b204c056b1cebee775ddad Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 08:34:12 +0100
Subject: [PATCH 0707/1937] release 2014.12.04

---
 README.md             | 8 ++++----
 youtube_dl/version.py | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index d6e7ff902..18be078d7 100644
--- a/README.md
+++ b/README.md
@@ -65,10 +65,10 @@ which means you can modify it, redistribute it or use it however you like.
                                      this is not possible instead of searching.
     --ignore-config                  Do not read configuration files. When given
                                      in the global configuration file /etc
-                                     /youtube-dl.conf: do not read the user
-                                     configuration in ~/.config/youtube-dl.conf
-                                     (%APPDATA%/youtube-dl/config.txt on
-                                     Windows)
+                                     /youtube-dl.conf: Do not read the user
+                                     configuration in ~/.config/youtube-
+                                     dl/config (%APPDATA%/youtube-dl/config.txt
+                                     on Windows)
     --flat-playlist                  Do not extract the videos of a playlist,
                                      only list them.
 
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 61902a8cc..7ff8633fe 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.03'
+__version__ = '2014.12.04'

From 225e4b9633285b66adc914a61d8f55ca125eb91d Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 4 Dec 2014 14:08:08 +0100
Subject: [PATCH 0708/1937] [tvp] Remove unnecessary code

---
 youtube_dl/extractor/tvp.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index 2248a9fdf..bd7ee25a0 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -79,15 +79,8 @@ class TvpIE(InfoExtractor):
             'id': video_id,
             'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
-            'description': self._og_search_description(webpage, default=''),
+            'description': self._og_search_description(webpage),
         }
-        if mobj.group('type') == 'vod' and info_dict['description'] == '':
-            info_dict.update({
-                'description': self._html_search_regex(
-                    r'(?s)<div\s+class=[\'"]opis.*?</div>',
-                    self._download_webpage(url, video_id), 'description', group=0),
-            })
-
         video_url = self._search_regex(
             r'0:{src:([\'"])(?P<url>.*?)\1', webpage, 'formats', group='url', default=None)
         if video_url is None:

From 995ad69c54899a0cfc84fd89083f07919acdbb83 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 4 Dec 2014 14:11:27 +0100
Subject: [PATCH 0709/1937] [common] Add new parameters for _download_webpage

---
 youtube_dl/extractor/common.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e80a2dad0..b633ea9b9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -360,9 +360,19 @@ class InfoExtractor(object):
 
         return content
 
-    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
+    def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, tries=1, timeout=5):
         """ Returns the data of the page as a string """
-        res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal)
+        success = False
+        try_count = 0
+        while success is False:
+            try:
+                res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal)
+                success = True
+            except compat_http_client.IncompleteRead as e:
+                try_count += 1
+                if try_count >= tries:
+                    raise e
+                self._sleep(timeout, video_id)
         if res is False:
             return res
         else:

From 2415951ead0c4996d7892ef859d06970c4595701 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Thu, 4 Dec 2014 14:12:09 +0100
Subject: [PATCH 0710/1937] [tvp] Modernize

---
 youtube_dl/extractor/tvp.py | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index bd7ee25a0..cf21d1202 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -127,26 +127,15 @@ class TvpSeriesIE(InfoExtractor):
         }
     ]
 
-    def _force_download_webpage(self, url, v_id, tries=0):
-        if tries >= 5:
-            raise ExtractorError(
-                '%s: Cannot download webpage, try again later' % v_id)
-        # Sometimes happen, but in my tests second try always succeeded
-        try:
-            return self._download_webpage(url, v_id)
-        except IncompleteRead as e:
-            return self._force_download_webpage(url, v_id, tries+1)
-    
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        webpage = self._force_download_webpage(url, display_id)
+        webpage = self._download_webpage(url, display_id, tries=5)
         title = self._html_search_regex(
-            r'(?s) id=[\'"]path[\'"]>(.*?)</span>', webpage, 'series')
-        title = title.split(' / ', 2)[-1]
+            r'(?s) id=[\'"]path[\'"]>(?:.*? / ){2}(.*?)</span>', webpage, 'series')
         playlist_id = self._search_regex(r'nodeId:\s*(\d+)', webpage, 'playlist id')
-        playlist = self._force_download_webpage(
+        playlist = self._download_webpage(
             'http://vod.tvp.pl/vod/seriesAjax?type=series&nodeId=%s&recommend'
-            'edId=0&sort=&page=0&pageSize=1000000' % playlist_id, display_id)
+            'edId=0&sort=&page=0&pageSize=10000' % playlist_id, display_id, tries=5)
         videos_paths = re.findall(
             '(?s)class="shortTitle">.*?href="(/[^"]+)', playlist)
         entries = [

From b128c9ed68394c5c6dd1c46f81d21314ed03ac18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Dec 2014 20:12:06 +0600
Subject: [PATCH 0711/1937] [vine:user] Add support for another URL format
 (Closes #4365)

---
 youtube_dl/extractor/vine.py | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 42995226e..4970b2f23 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -63,29 +63,36 @@ class VineIE(InfoExtractor):
 
 class VineUserIE(InfoExtractor):
     IE_NAME = 'vine:user'
-    _VALID_URL = r'(?:https?://)?vine\.co/(?P<user>[^/]+)/?(\?.*)?$'
+    _VALID_URL = r'(?:https?://)?vine\.co/(?P<u>u/)?(?P<user>[^/]+)/?(\?.*)?$'
     _VINE_BASE_URL = "https://vine.co/"
-    _TEST = {
-        'url': 'https://vine.co/Visa',
-        'info_dict': {
-            'id': 'Visa',
+    _TESTS = [
+        {
+            'url': 'https://vine.co/Visa',
+            'info_dict': {
+                'id': 'Visa',
+            },
+            'playlist_mincount': 46,
         },
-        'playlist_mincount': 46,
-    }
+        {
+            'url': 'https://vine.co/u/941705360593584128',
+            'only_matching': True,
+        },
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         user = mobj.group('user')
+        u = mobj.group('u')
 
-        profile_url = "%sapi/users/profiles/vanity/%s" % (
-            self._VINE_BASE_URL, user)
+        profile_url = "%sapi/users/profiles/%s%s" % (
+            self._VINE_BASE_URL, 'vanity/' if not u else '', user)
         profile_data = self._download_json(
             profile_url, user, note='Downloading user profile data')
 
         user_id = profile_data['data']['userId']
         timeline_data = []
         for pagenum in itertools.count(1):
-            timeline_url = "%sapi/timelines/users/%s?page=%s" % (
+            timeline_url = "%sapi/timelines/users/%s?page=%s&size=100" % (
                 self._VINE_BASE_URL, user_id, pagenum)
             timeline_page = self._download_json(
                 timeline_url, user, note='Downloading page %d' % pagenum)

From 1139a54d9bb9d214a8451301a44af6d03dd450b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Dec 2014 21:19:08 +0600
Subject: [PATCH 0712/1937] [foxnews] Add extractor (Closes #4352)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/foxnews.py  | 94 ++++++++++++++++++++++++++++++++
 2 files changed, 95 insertions(+)
 create mode 100644 youtube_dl/extractor/foxnews.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 647352b59..0a5151d5a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -122,6 +122,7 @@ from .flickr import FlickrIE
 from .folketinget import FolketingetIE
 from .fourtube import FourTubeIE
 from .foxgay import FoxgayIE
+from .foxnews import FoxNewsIE
 from .franceculture import FranceCultureIE
 from .franceinter import FranceInterIE
 from .francetv import (
diff --git a/youtube_dl/extractor/foxnews.py b/youtube_dl/extractor/foxnews.py
new file mode 100644
index 000000000..917f76b1e
--- /dev/null
+++ b/youtube_dl/extractor/foxnews.py
@@ -0,0 +1,94 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    int_or_none,
+)
+
+
+class FoxNewsIE(InfoExtractor):
+    _VALID_URL = r'https?://video\.foxnews\.com/v/(?:video-embed\.html\?video_id=)?(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'http://video.foxnews.com/v/3937480/frozen-in-time/#sp=show-clips',
+            'md5': '32aaded6ba3ef0d1c04e238d01031e5e',
+            'info_dict': {
+                'id': '3937480',
+                'ext': 'flv',
+                'title': 'Frozen in Time',
+                'description': 'Doctors baffled by 16-year-old girl that is the size of a toddler',
+                'duration': 265,
+                'timestamp': 1304411491,
+                'upload_date': '20110503',
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+        },
+        {
+            'url': 'http://video.foxnews.com/v/3922535568001/rep-luis-gutierrez-on-if-obamas-immigration-plan-is-legal/#sp=show-clips',
+            'md5': '5846c64a1ea05ec78175421b8323e2df',
+            'info_dict': {
+                'id': '3922535568001',
+                'ext': 'mp4',
+                'title': "Rep. Luis Gutierrez on if Obama's immigration plan is legal",
+                'description': "Congressman discusses the president's executive action",
+                'duration': 292,
+                'timestamp': 1417662047,
+                'upload_date': '20141204',
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+        },
+        {
+            'url': 'http://video.foxnews.com/v/video-embed.html?video_id=3937480&d=video.foxnews.com',
+            'only_matching': True,
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        video = self._download_json(
+            'http://video.foxnews.com/v/feed/video/%s.js?template=fox' % video_id, video_id)
+
+        item = video['channel']['item']
+        title = item['title']
+        description = item['description']
+        timestamp = parse_iso8601(item['dc-date'])
+
+        media_group = item['media-group']
+        duration = None
+        formats = []
+        for media in media_group['media-content']:
+            attributes = media['@attributes']
+            video_url = attributes['url']
+            if video_url.endswith('.f4m'):
+                formats.extend(self._extract_f4m_formats(video_url + '?hdcore=3.4.0&plugin=aasp-3.4.0.132.124', video_id))
+            elif video_url.endswith('.m3u8'):
+                formats.extend(self._extract_m3u8_formats(video_url, video_id, 'flv'))
+            elif not video_url.endswith('.smil'):
+                duration = int_or_none(attributes.get('duration'))
+                formats.append({
+                    'url': video_url,
+                    'format_id': media['media-category']['@attributes']['label'],
+                    'preference': 1,
+                    'vbr': int_or_none(attributes.get('bitrate')),
+                    'filesize': int_or_none(attributes.get('fileSize'))
+                })
+        self._sort_formats(formats)
+
+        media_thumbnail = media_group['media-thumbnail']['@attributes']
+        thumbnails = [{
+            'url': media_thumbnail['url'],
+            'width': int_or_none(media_thumbnail.get('width')),
+            'height': int_or_none(media_thumbnail.get('height')),
+        }] if media_thumbnail else []
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'timestamp': timestamp,
+            'formats': formats,
+            'thumbnails': thumbnails,
+        }

From 4349c07dd7cd07620365b36093f0a148c41ce434 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 17:02:05 +0100
Subject: [PATCH 0713/1937] [minhateca] Add extractor (Fixes #4094)

---
 test/test_utils.py                |  1 +
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/minhateca.py | 71 +++++++++++++++++++++++++++++++
 youtube_dl/utils.py               |  7 ++-
 4 files changed, 78 insertions(+), 2 deletions(-)
 create mode 100644 youtube_dl/extractor/minhateca.py

diff --git a/test/test_utils.py b/test/test_utils.py
index baa3a2156..04f1bf283 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -376,6 +376,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_filesize('2 MiB'), 2097152)
         self.assertEqual(parse_filesize('5 GB'), 5000000000)
         self.assertEqual(parse_filesize('1.2Tb'), 1200000000000)
+        self.assertEqual(parse_filesize('1,24 KB'), 1240)
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 647352b59..a56ec4fb5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -217,6 +217,7 @@ from .mdr import MDRIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
+from .minhateca import MinhatecaIE
 from .ministrygrid import MinistryGridIE
 from .mit import TechTVMITIE, MITIE, OCWMITIE
 from .mitele import MiTeleIE
diff --git a/youtube_dl/extractor/minhateca.py b/youtube_dl/extractor/minhateca.py
new file mode 100644
index 000000000..077c9b19d
--- /dev/null
+++ b/youtube_dl/extractor/minhateca.py
@@ -0,0 +1,71 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_request,
+)
+from ..utils import (
+    int_or_none,
+    parse_filesize,
+)
+
+
+class MinhatecaIE(InfoExtractor):
+    _VALID_URL = r'https?://minhateca\.com\.br/[^?#]+,(?P<id>[0-9]+)\.'
+    _TEST = {
+        'url': 'http://minhateca.com.br/pereba/misc/youtube-dl+test+video,125848331.mp4(video)',
+        'info_dict': {
+            'id': '125848331',
+            'ext': 'mp4',
+            'title': 'youtube-dl test video',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'filesize_approx': 1530000,
+            'duration': 9,
+            'view_count': int,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        token = self._html_search_regex(
+            r'<input name="__RequestVerificationToken".*?value="([^"]+)"',
+            webpage, 'request token')
+        token_data = [
+            ('fileId', video_id),
+            ('__RequestVerificationToken', token),
+        ]
+        req = compat_urllib_request.Request(
+            'http://minhateca.com.br/action/License/Download',
+            data=compat_urllib_parse.urlencode(token_data))
+        req.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        data = self._download_json(
+            req, video_id, note='Downloading metadata')
+
+        video_url = data['redirectUrl']
+        title_str = self._html_search_regex(
+            r'<h1.*?>(.*?)</h1>', webpage, 'title')
+        title, _, ext = title_str.rpartition('.')
+        filesize_approx = parse_filesize(self._html_search_regex(
+            r'<p class="fileSize">(.*?)</p>',
+            webpage, 'file size approximation', fatal=False))
+        duration = int_or_none(self._html_search_regex(
+            r'(?s)<p class="fileLeng[ht][th]">.*?([0-9]+)\s*s',
+            webpage, 'duration', fatal=False))
+        view_count = int_or_none(self._html_search_regex(
+            r'<p class="downloadsCounter">([0-9]+)</p>',
+            webpage, 'view count', fatal=False))
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'ext': ext,
+            'filesize_approx': filesize_approx,
+            'duration': duration,
+            'view_count': view_count,
+            'thumbnail': self._og_search_thumbnail(webpage),
+        }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4d3cbac74..5e9ae7a42 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1090,11 +1090,14 @@ def parse_filesize(s):
     }
 
     units_re = '|'.join(re.escape(u) for u in _UNIT_TABLE)
-    m = re.match(r'(?P<num>[0-9]+(?:\.[0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
+    m = re.match(
+        r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
     if not m:
         return None
 
-    return int(float(m.group('num')) * _UNIT_TABLE[m.group('unit')])
+    num_str = m.group('num').replace(',', '.')
+    mult = _UNIT_TABLE[m.group('unit')]
+    return int(float(num_str) * mult)
 
 
 def get_term_width():

From ab07963b5cc79812c6fb7e4f9e363533d8123830 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 17:02:23 +0100
Subject: [PATCH 0714/1937] release 2014.12.04.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7ff8633fe..cce669059 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.04'
+__version__ = '2014.12.04.1'

From e8df5cee12378acd708b6686130a73c5edc06f0e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 17:35:40 +0100
Subject: [PATCH 0715/1937] [minhateca] Fix duration parsing

---
 test/test_utils.py                |  3 +++
 youtube_dl/extractor/minhateca.py |  5 +++--
 youtube_dl/utils.py               | 19 +++++++++++++++----
 3 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 04f1bf283..badab1370 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -220,6 +220,9 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_duration('0s'), 0)
         self.assertEqual(parse_duration('01:02:03.05'), 3723.05)
         self.assertEqual(parse_duration('T30M38S'), 1838)
+        self.assertEqual(parse_duration('5 s'), 5)
+        self.assertEqual(parse_duration('3 min'), 180)
+        self.assertEqual(parse_duration('2.5 hours'), 9000)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
diff --git a/youtube_dl/extractor/minhateca.py b/youtube_dl/extractor/minhateca.py
index 077c9b19d..14934b7ec 100644
--- a/youtube_dl/extractor/minhateca.py
+++ b/youtube_dl/extractor/minhateca.py
@@ -8,6 +8,7 @@ from ..compat import (
 )
 from ..utils import (
     int_or_none,
+    parse_duration,
     parse_filesize,
 )
 
@@ -52,8 +53,8 @@ class MinhatecaIE(InfoExtractor):
         filesize_approx = parse_filesize(self._html_search_regex(
             r'<p class="fileSize">(.*?)</p>',
             webpage, 'file size approximation', fatal=False))
-        duration = int_or_none(self._html_search_regex(
-            r'(?s)<p class="fileLeng[ht][th]">.*?([0-9]+)\s*s',
+        duration = parse_duration(self._html_search_regex(
+            r'(?s)<p class="fileLeng[ht][th]">.*?class="bold">(.*?)<',
             webpage, 'duration', fatal=False))
         view_count = int_or_none(self._html_search_regex(
             r'<p class="downloadsCounter">([0-9]+)</p>',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5e9ae7a42..5efb4c59a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1206,18 +1206,29 @@ def parse_duration(s):
 
     m = re.match(
         r'''(?ix)T?
+        (?:
+            (?P<only_mins>[0-9.]+)\s*(?:mins?|minutes?)\s*|
+            (?P<only_hours>[0-9.]+)\s*(?:hours?)|
+
             (?:
                 (?:(?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*)?
                 (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
             )?
-            (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?$''', s)
+            (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
+        )$''', s)
     if not m:
         return None
-    res = int(m.group('secs'))
+    res = 0
+    if m.group('only_mins'):
+        return float_or_none(m.group('only_mins'), invscale=60)
+    if m.group('only_hours'):
+        return float_or_none(m.group('only_hours'), invscale=60 * 60)
+    if m.group('secs'):
+        res += int(m.group('secs'))
     if m.group('mins'):
         res += int(m.group('mins')) * 60
-        if m.group('hours'):
-            res += int(m.group('hours')) * 60 * 60
+    if m.group('hours'):
+        res += int(m.group('hours')) * 60 * 60
     if m.group('ms'):
         res += float(m.group('ms'))
     return res

From 81028ff9ebdfdccc680d404ce49a3fa32ac965c0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 17:43:34 +0100
Subject: [PATCH 0716/1937] [xminus] Capture description (#4300)

---
 youtube_dl/extractor/xminus.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/xminus.py b/youtube_dl/extractor/xminus.py
index f7e2e8ac9..8c6241aed 100644
--- a/youtube_dl/extractor/xminus.py
+++ b/youtube_dl/extractor/xminus.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import (
     compat_chr,
@@ -25,6 +27,7 @@ class XMinusIE(InfoExtractor):
             'tbr': 320,
             'filesize_approx': 5900000,
             'view_count': int,
+            'description': 'md5:03238c5b663810bc79cf42ef3c03e371',
         }
     }
 
@@ -48,6 +51,11 @@ class XMinusIE(InfoExtractor):
         view_count = int_or_none(self._html_search_regex(
             r'<div class="quality.*?► ([0-9]+)',
             webpage, 'view count', fatal=False))
+        description = self._html_search_regex(
+            r'(?s)<div id="song_texts">(.*?)</div><br',
+            webpage, 'song lyrics', fatal=False)
+        if description:
+            description = re.sub(' *\r *', '\n', description)
 
         enc_token = self._html_search_regex(
             r'minus_track\.tkn="(.+?)"', webpage, 'enc_token')
@@ -64,4 +72,5 @@ class XMinusIE(InfoExtractor):
             'filesize_approx': filesize_approx,
             'tbr': tbr,
             'view_count': view_count,
+            'description': description,
         }

From a154eb3d159baa2b7f8f4dcd9c1215d7a3486651 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 4 Dec 2014 17:43:39 +0100
Subject: [PATCH 0717/1937] release 2014.12.04.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index cce669059..0f902b1e9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.04.1'
+__version__ = '2014.12.04.2'

From 122c2f87c17984e4ae04cbb3c3631b547dc61105 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 5 Dec 2014 10:59:55 +0100
Subject: [PATCH 0718/1937] [tagesschau] Modernize

---
 youtube_dl/extractor/tagesschau.py | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index b87047451..17e123112 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -28,14 +28,8 @@ class TagesschauIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        if video_id.startswith('-'):
-            display_id = video_id.strip('-')
-        else:
-            display_id = video_id
-
+        video_id = self._match_id(url)
+        display_id = video_id.lstrip('-')
         webpage = self._download_webpage(url, display_id)
 
         playerpage = self._download_webpage(

From ebb6419960eafbdde055d016b4f95620c724effb Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Fri, 5 Dec 2014 12:07:06 +0100
Subject: [PATCH 0719/1937] [common] Split _download_json Add ability for
 extractor to use _parse_json

---
 youtube_dl/extractor/common.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e80a2dad0..7cbd846f6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -390,6 +390,10 @@ class InfoExtractor(object):
             url_or_request, video_id, note, errnote, fatal=fatal)
         if (not fatal) and json_string is False:
             return None
+        return self._parse_json(
+            json_string, video_id, transform_source=transform_source, fatal=fatal)
+
+    def _parse_json(self, json_string, video_id, transform_source=None, fatal=True):
         if transform_source:
             json_string = transform_source(json_string)
         try:

From d958fa9ff961fb17b55aba0d70e5175e2c63f631 Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Fri, 5 Dec 2014 00:09:17 +0100
Subject: [PATCH 0720/1937] [howstuffworks] Rewrite extractor

---
 youtube_dl/extractor/howstuffworks.py | 139 ++++++++------------------
 1 file changed, 40 insertions(+), 99 deletions(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index fccc23884..c46e8b911 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -1,134 +1,75 @@
 from __future__ import unicode_literals
 
-import re
-import json
-import random
-import string
-
 from .common import InfoExtractor
-from ..utils import find_xpath_attr
 
 
 class HowStuffWorksIE(InfoExtractor):
     _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*\d+-(?P<id>.+?)-video\.htm'
     _TESTS = [
         {
-            'url': 'http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm',
+            'url': 'http://adventure.howstuffworks.com/39521-deadliest-catch-nautical-collision-video.htm',
             'info_dict': {
-                'id': '450221',
-                'display_id': 'cool-jobs-iditarod-musher',
-                'ext': 'flv',
-                'title': 'Cool Jobs - Iditarod Musher',
-                'description': 'md5:82bb58438a88027b8186a1fccb365f90',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'id': '553475',
+                'ext': 'mp4',
+                'title': 'Deadliest Catch: Nautical Collision',
+                'description': 'Check out this clip to get an exclusive look at the Deadliest Catch crew back in action.',
+                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/39521.jpg',
             },
-            'params': {
-                # md5 is not consistent
-                'skip_download': True
-            }
         },
         {
             'url': 'http://adventure.howstuffworks.com/7199-survival-zone-food-and-water-in-the-savanna-video.htm',
             'info_dict': {
                 'id': '453464',
-                'display_id': 'survival-zone-food-and-water-in-the-savanna',
                 'ext': 'mp4',
                 'title': 'Survival Zone: Food and Water In the Savanna',
-                'description': 'md5:7e1c89f6411434970c15fa094170c371',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'description': 'Learn how to find both food and water while trekking in the African savannah. In this video from the Discovery Channel.',
+                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/7199.jpg',
             },
-            'params': {
-                # md5 is not consistent
-                'skip_download': True
-            }
         },
         {
-            'url': 'http://entertainment.howstuffworks.com/arts/2706-sword-swallowing-1-by-dan-meyer-video.htm',
+            'url': 'http://entertainment.howstuffworks.com/arts/34476-time-warp-fire-breathing-disaster-video.htm',
             'info_dict': {
-                'id': '440011',
-                'display_id': 'sword-swallowing-1-by-dan-meyer',
-                'ext': 'flv',
-                'title': 'Sword Swallowing #1 by Dan Meyer',
-                'description': 'md5:b2409e88172913e2e7d3d1159b0ef735',
-                'thumbnail': 're:^https?://.*\.jpg$',
+                'id': '487036',
+                'ext': 'mp4',
+                'title': 'Time Warp: Fire-Breathing Disaster',
+                'description': 'Fire-breathers are safe from flames when they angle their heads up as they blow -- see what would happen if they looked down instead. Check out this clip from Discovery\'s "Time Warp" series to learn more.',
+                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/34476.jpg',
             },
-            'params': {
-                # md5 is not consistent
-                'skip_download': True
-            }
         },
     ]
 
+    def _extract_clip_info(self, key, clip_info, name=None, **kargs):
+        if name is None:
+            name = key
+        return self._html_search_regex(
+            r"\s*%s\s*: '?([^'\n]*[^'\n,])" % key, clip_info, name, **kargs)
+
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('id')
+        display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-
-        content_id = self._search_regex(r'var siteSectionId="(\d+)";', webpage, 'content id')
-
-        mp4 = self._search_regex(
-            r'''(?xs)var\s+clip\s*=\s*{\s*
-                .+?\s*
-                content_id\s*:\s*%s\s*,\s*
-                .+?\s*
-                mp4\s*:\s*\[(.*?),?\]\s*
-                };\s*
-                videoData\.push\(clip\);''' % content_id,
-            webpage, 'mp4', fatal=False, default=None)
-
-        smil = self._download_xml(
-            'http://services.media.howstuffworks.com/videos/%s/smil-service.smil' % content_id,
-            content_id, 'Downloading video SMIL')
-
-        http_base = find_xpath_attr(
-            smil,
-            './{0}head/{0}meta'.format('{http://www.w3.org/2001/SMIL20/Language}'),
-            'name',
-            'httpBase').get('content')
-
-        def random_string(str_len=0):
-            return ''.join([random.choice(string.ascii_uppercase) for _ in range(str_len)])
-
-        URL_SUFFIX = '?v=2.11.3&fp=LNX 11,2,202,356&r=%s&g=%s' % (random_string(5), random_string(12))
-
+        clip_info = self._search_regex('(?s)var clip = {(.*?)};', webpage, 'clip info')
+        video_id = self._extract_clip_info('content_id', clip_info, 'video id')
         formats = []
-
-        if mp4:
-            for video in json.loads('[%s]' % mp4):
-                bitrate = video['bitrate']
-                fmt = {
-                    'url': video['src'].replace('http://pmd.video.howstuffworks.com', http_base) + URL_SUFFIX,
-                    'format_id': bitrate,
-                }
-                m = re.search(r'(?P<vbr>\d+)[Kk]', bitrate)
-                if m:
-                    fmt['vbr'] = int(m.group('vbr'))
-                formats.append(fmt)
-        else:
-            for video in smil.findall(
-                    './/{0}body/{0}switch/{0}video'.format('{http://www.w3.org/2001/SMIL20/Language}')):
-                vbr = int(video.attrib['system-bitrate']) / 1000
-                formats.append({
-                    'url': '%s/%s%s' % (http_base, video.attrib['src'], URL_SUFFIX),
-                    'format_id': '%dk' % vbr,
-                    'vbr': vbr,
-                })
-
+        m3u8_url = self._extract_clip_info('m3u8', clip_info, 'm3u8 url', default=None)
+        if m3u8_url is not None:
+            formats += self._extract_m3u8_formats(m3u8_url , video_id, 'mp4')
+        mp4 = self._parse_json(
+            self._extract_clip_info(
+                'mp4', clip_info, 'formats').replace('},]','}]'), video_id)
+        for video in mp4:
+            formats.append({
+                'url': video['src'],
+                'format_id': video['bitrate'],
+                'tbr': int(video['bitrate'].rstrip('k')),
+            })
         self._sort_formats(formats)
 
-        title = self._og_search_title(webpage)
-        TITLE_SUFFIX = ' : HowStuffWorks'
-        if title.endswith(TITLE_SUFFIX):
-            title = title[:-len(TITLE_SUFFIX)]
-
-        description = self._og_search_description(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
-
         return {
-            'id': content_id,
+            'id': video_id,
             'display_id': display_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
+            'title': self._extract_clip_info('clip_title', clip_info, 'title'),
+            'description': self._extract_clip_info('caption', clip_info, 'description', fatal=False),
+            'thumbnail': self._extract_clip_info('video_still_url', clip_info, 'thumbnail'),
+            'duration': self._extract_clip_info('duration', clip_info),
             'formats': formats,
         }

From 90644a68436e53e3ee74af53841c750549816cd7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 5 Dec 2014 22:08:30 +0600
Subject: [PATCH 0721/1937] [azubu] Add extractor (Closes #4379)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/azubu.py    | 93 ++++++++++++++++++++++++++++++++
 2 files changed, 94 insertions(+)
 create mode 100644 youtube_dl/extractor/azubu.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e0c4a3486..4c9c44b31 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -24,6 +24,7 @@ from .arte import (
 )
 from .audiomack import AudiomackIE
 from .auengine import AUEngineIE
+from .azubu import AzubuIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
 from .bbccouk import BBCCoUkIE
diff --git a/youtube_dl/extractor/azubu.py b/youtube_dl/extractor/azubu.py
new file mode 100644
index 000000000..0961d339f
--- /dev/null
+++ b/youtube_dl/extractor/azubu.py
@@ -0,0 +1,93 @@
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..utils import float_or_none
+
+
+class AzubuIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?azubu\.tv/[^/]+#!/play/(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'http://www.azubu.tv/GSL#!/play/15575/2014-hot6-cup-last-big-match-ro8-day-1',
+            'md5': 'a88b42fcf844f29ad6035054bd9ecaf4',
+            'info_dict': {
+                'id': '15575',
+                'ext': 'mp4',
+                'title': '2014 HOT6 CUP LAST BIG MATCH Ro8 Day 1',
+                'description': 'md5:d06bdea27b8cc4388a90ad35b5c66c01',
+                'thumbnail': 're:^https?://.*\.jpe?g',
+                'timestamp': 1417523507.334,
+                'upload_date': '20141202',
+                'duration': 9988.7,
+                'uploader': 'GSL',
+                'uploader_id': 414310,
+                'view_count': int,
+            },
+        },
+        {
+            'url': 'http://www.azubu.tv/FnaticTV#!/play/9344/-fnatic-at-worlds-2014:-toyz---%22i-love-rekkles,-he-has-amazing-mechanics%22-',
+            'md5': 'b72a871fe1d9f70bd7673769cdb3b925',
+            'info_dict': {
+                'id': '9344',
+                'ext': 'mp4',
+                'title': 'Fnatic at Worlds 2014: Toyz - "I love Rekkles, he has amazing mechanics"',
+                'description': 'md5:4a649737b5f6c8b5c5be543e88dc62af',
+                'thumbnail': 're:^https?://.*\.jpe?g',
+                'timestamp': 1410530893.320,
+                'upload_date': '20140912',
+                'duration': 172.385,
+                'uploader': 'FnaticTV',
+                'uploader_id': 272749,
+                'view_count': int,
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        data = self._download_json(
+            'http://www.azubu.tv/api/video/%s' % video_id, video_id)['data']
+
+        title = data['title'].strip()
+        description = data['description']
+        thumbnail = data['thumbnail']
+        view_count = data['view_count']
+        uploader = data['user']['username']
+        uploader_id = data['user']['id']
+
+        stream_params = json.loads(data['stream_params'])
+
+        timestamp = float_or_none(stream_params['creationDate'], 1000)
+        duration = float_or_none(stream_params['length'], 1000)
+
+        renditions = stream_params.get('renditions') or []
+        video = stream_params.get('FLVFullLength') or stream_params.get('videoFullLength')
+        if video:
+            renditions.append(video)
+
+        formats = [{
+            'url': fmt['url'],
+            'width': fmt['frameWidth'],
+            'height': fmt['frameHeight'],
+            'vbr': float_or_none(fmt['encodingRate'], 1000),
+            'filesize': fmt['size'],
+            'vcodec': fmt['videoCodec'],
+            'container': fmt['videoContainer'],
+        } for fmt in renditions if fmt['url']]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'view_count': view_count,
+            'formats': formats,
+        }

From e638e83662e5baa414b8d80145e46c1912f7998d Mon Sep 17 00:00:00 2001
From: Tithen-Firion <Tithen-Firion@users.noreply.github.com>
Date: Fri, 5 Dec 2014 19:46:49 +0100
Subject: [PATCH 0722/1937] [howstuffworks] Update extractor

---
 youtube_dl/extractor/howstuffworks.py | 77 ++++++++++++++++++---------
 1 file changed, 51 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index c46e8b911..9478e9e67 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -1,19 +1,20 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import find_xpath_attr
 
 
 class HowStuffWorksIE(InfoExtractor):
     _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*\d+-(?P<id>.+?)-video\.htm'
     _TESTS = [
         {
-            'url': 'http://adventure.howstuffworks.com/39521-deadliest-catch-nautical-collision-video.htm',
+            'url': 'http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm',
             'info_dict': {
-                'id': '553475',
-                'ext': 'mp4',
-                'title': 'Deadliest Catch: Nautical Collision',
-                'description': 'Check out this clip to get an exclusive look at the Deadliest Catch crew back in action.',
-                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/39521.jpg',
+                'id': '450221',
+                'ext': 'flv',
+                'title': 'Cool Jobs - Iditarod Musher',
+                'description': 'Cold sleds, freezing temps and warm dog breath... an Iditarod musher\'s dream. Kasey-Dee Gardner jumps on a sled to find out what the big deal is.',
+                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/5266.jpg',
             },
         },
         {
@@ -27,49 +28,73 @@ class HowStuffWorksIE(InfoExtractor):
             },
         },
         {
-            'url': 'http://entertainment.howstuffworks.com/arts/34476-time-warp-fire-breathing-disaster-video.htm',
+            'url': 'http://entertainment.howstuffworks.com/arts/2706-sword-swallowing-1-by-dan-meyer-video.htm',
             'info_dict': {
-                'id': '487036',
-                'ext': 'mp4',
-                'title': 'Time Warp: Fire-Breathing Disaster',
-                'description': 'Fire-breathers are safe from flames when they angle their heads up as they blow -- see what would happen if they looked down instead. Check out this clip from Discovery\'s "Time Warp" series to learn more.',
-                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/34476.jpg',
+                'id': '440011',
+                'ext': 'flv',
+                'title': 'Sword Swallowing #1 by Dan Meyer',
+                'description': 'Video footage (1 of 3) used by permission of the owner Dan Meyer through Sword Swallowers Association International <www.swordswallow.org>',
+                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/118306353233.jpg',
             },
         },
     ]
 
-    def _extract_clip_info(self, key, clip_info, name=None, **kargs):
-        if name is None:
-            name = key
-        return self._html_search_regex(
-            r"\s*%s\s*: '?([^'\n]*[^'\n,])" % key, clip_info, name, **kargs)
-
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
         clip_info = self._search_regex('(?s)var clip = {(.*?)};', webpage, 'clip info')
-        video_id = self._extract_clip_info('content_id', clip_info, 'video id')
+
+        def extract_clip_info(key, clip_info, name=None, **kargs):
+            if name is None:
+                name = key
+            return self._html_search_regex(
+                r"\s*%s\s*: '?([^'\n]*[^'\n,])" % key, clip_info, name, **kargs)
+
+        video_id = extract_clip_info('content_id', clip_info, 'video id')
         formats = []
-        m3u8_url = self._extract_clip_info('m3u8', clip_info, 'm3u8 url', default=None)
+        m3u8_url = extract_clip_info('m3u8', clip_info, 'm3u8 url', default=None)
         if m3u8_url is not None:
             formats += self._extract_m3u8_formats(m3u8_url , video_id, 'mp4')
         mp4 = self._parse_json(
-            self._extract_clip_info(
+            extract_clip_info(
                 'mp4', clip_info, 'formats').replace('},]','}]'), video_id)
         for video in mp4:
             formats.append({
                 'url': video['src'],
                 'format_id': video['bitrate'],
-                'tbr': int(video['bitrate'].rstrip('k')),
+                'vbr': int(video['bitrate'].rstrip('k')),
             })
+
+        if not formats:
+            smil = self._download_xml(
+                'http://services.media.howstuffworks.com/videos/%s/smil-service.smil' % video_id,
+                video_id, 'Downloading video SMIL')
+
+            http_base = find_xpath_attr(
+                smil,
+                './{0}head/{0}meta'.format('{http://www.w3.org/2001/SMIL20/Language}'),
+                'name',
+                'httpBase').get('content')
+
+            URL_SUFFIX = '?v=2.11.3&fp=LNX 11,2,202,356&r=A&g=A'
+
+            for video in smil.findall(
+                    './{0}body/{0}switch/{0}video'.format('{http://www.w3.org/2001/SMIL20/Language}')):
+                vbr = int(video.attrib['system-bitrate']) / 1000
+                formats.append({
+                    'url': '%s/%s%s' % (http_base, video.attrib['src'], URL_SUFFIX),
+                    'format_id': '%dk' % vbr,
+                    'vbr': vbr,
+                })
+
         self._sort_formats(formats)
 
         return {
             'id': video_id,
             'display_id': display_id,
-            'title': self._extract_clip_info('clip_title', clip_info, 'title'),
-            'description': self._extract_clip_info('caption', clip_info, 'description', fatal=False),
-            'thumbnail': self._extract_clip_info('video_still_url', clip_info, 'thumbnail'),
-            'duration': self._extract_clip_info('duration', clip_info),
+            'title': extract_clip_info('clip_title', clip_info, 'title'),
+            'description': extract_clip_info('caption', clip_info, 'description', fatal=False),
+            'thumbnail': extract_clip_info('video_still_url', clip_info, 'thumbnail'),
+            'duration': extract_clip_info('duration', clip_info),
             'formats': formats,
         }

From 045c48847ad066fc273337c96b25eacc35d146cd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 00:42:39 +0100
Subject: [PATCH 0723/1937] [tagesschau] Add suppot for sendung (Fixes #4378)

---
 youtube_dl/extractor/tagesschau.py | 101 ++++++++++++++++++++++-------
 1 file changed, 79 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index 17e123112..aed670941 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -4,10 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import parse_filesize
 
 
 class TagesschauIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/video/video(?P<id>-?[0-9]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/ts|video/video)(?P<id>-?[0-9]+)\.html'
 
     _TESTS = [{
         'url': 'http://www.tagesschau.de/multimedia/video/video1399128.html',
@@ -19,6 +20,16 @@ class TagesschauIE(InfoExtractor):
             'description': 'md5:69da3c61275b426426d711bde96463ab',
             'thumbnail': 're:^http:.*\.jpg$',
         },
+    }, {
+        'url': 'http://www.tagesschau.de/multimedia/sendung/ts-5727.html',
+        'md5': '3c54c1f6243d279b706bde660ceec633',
+        'info_dict': {
+            'id': '5727',
+            'ext': 'mp4',
+            'description': 'md5:695c01bfd98b7e313c501386327aea59',
+            'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr',
+            'thumbnail': 're:^http:.*\.jpg$',
+        }
     }]
 
     _FORMATS = {
@@ -32,32 +43,78 @@ class TagesschauIE(InfoExtractor):
         display_id = video_id.lstrip('-')
         webpage = self._download_webpage(url, display_id)
 
-        playerpage = self._download_webpage(
-            'http://www.tagesschau.de/multimedia/video/video%s~player_autoplay-true.html' % video_id,
-            display_id, 'Downloading player page')
+        player_url = self._html_search_meta(
+            'twitter:player', webpage, 'player URL', default=None)
+        if player_url:
+            playerpage = self._download_webpage(
+                player_url, display_id, 'Downloading player page')
 
-        medias = re.findall(
-            r'"(http://media.+?)", type:"video/(.+?)", quality:"(.+?)"',
-            playerpage)
-
-        formats = []
-        for url, ext, res in medias:
-            f = {
-                'format_id': res + '_' + ext,
-                'url': url,
-                'ext': ext,
-            }
-            f.update(self._FORMATS.get(res, {}))
-            formats.append(f)
+            medias = re.findall(
+                r'"(http://media.+?)", type:"video/(.+?)", quality:"(.+?)"',
+                playerpage)
+            formats = []
+            for url, ext, res in medias:
+                f = {
+                    'format_id': res + '_' + ext,
+                    'url': url,
+                    'ext': ext,
+                }
+                f.update(self._FORMATS.get(res, {}))
+                formats.append(f)
+            thumbnail_fn = re.findall(r'"(/multimedia/.+?\.jpg)"', playerpage)[-1]
+            title = self._og_search_title(webpage).strip()
+            description = self._og_search_description(webpage).strip()
+        else:
+            download_text = self._search_regex(
+                r'(?s)<p>Wir bieten dieses Video in folgenden Formaten zum Download an:</p>\s*<div class="controls">(.*?)</div>\s*<p>',
+                webpage, 'download links')
+            links = re.finditer(
+                r'<div class="button" title="(?P<title>[^"]*)"><a href="(?P<url>[^"]+)">(?P<name>.+?)</a></div>',
+                webpage)
+            formats = []
+            for l in links:
+                format_id = self._search_regex(
+                    r'.*/[^/.]+\.([^/]+)\.[^/.]+', l.group('url'), 'format ID')
+                format = {
+                    'format_id': format_id,
+                    'url': l.group('url'),
+                    'format_name': l.group('name'),
+                }
+                m = re.match(
+                    r'''(?x)
+                        Video:\s*(?P<vcodec>[a-zA-Z0-9/._-]+)\s*&\#10;
+                        (?P<width>[0-9]+)x(?P<height>[0-9]+)px&\#10;
+                        (?P<vbr>[0-9]+)kbps&\#10;
+                        Audio:\s*(?P<abr>[0-9]+)kbps,\s*(?P<audio_desc>[A-Za-z\.0-9]+)&\#10;
+                        Gr&ouml;&szlig;e:\s*(?P<filesize_approx>[0-9.,]+\s+[a-zA-Z]*B)''',
+                    l.group('title'))
+                if m:
+                    format.update({
+                        'format_note': m.group('audio_desc'),
+                        'vcodec': m.group('vcodec'),
+                        'width': int(m.group('width')),
+                        'height': int(m.group('height')),
+                        'abr': int(m.group('abr')),
+                        'vbr': int(m.group('vbr')),
+                        'filesize_approx': parse_filesize(m.group('filesize_approx')),
+                    })
+                formats.append(format)
+            thumbnail_fn = self._search_regex(
+                r'(?s)<img alt="Sendungsbild".*?src="([^"]+)"',
+                webpage, 'thumbnail', fatal=False)
+            description = self._html_search_regex(
+                r'(?s)<p class="teasertext">(.*?)</p>',
+                webpage, 'description', fatal=False)
+            title = self._html_search_regex(
+                r'<span class="headline".*?>(.*?)</span>', webpage, 'title')
 
         self._sort_formats(formats)
-
-        thumbnail = re.findall(r'"(/multimedia/.+?\.jpg)"', playerpage)[-1]
+        thumbnail = 'http://www.tagesschau.de' + thumbnail_fn
 
         return {
             'id': display_id,
-            'title': self._og_search_title(webpage).strip(),
-            'thumbnail': 'http://www.tagesschau.de' + thumbnail,
+            'title': title,
+            'thumbnail': thumbnail,
             'formats': formats,
-            'description': self._og_search_description(webpage).strip(),
+            'description': description,
         }

From 1a7c6c69d3a1f16507b9fff87257cd7f2fef7f25 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 00:43:04 +0100
Subject: [PATCH 0724/1937] release 2014.12.06

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0f902b1e9..c568920d9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.04.2'
+__version__ = '2014.12.06'

From 58daf5ebed87fb92d0b4866777d448e1142e346b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 00:48:29 +0100
Subject: [PATCH 0725/1937] [youporn] Fix JSON parameter regexp (Fixes #4384)

---
 youtube_dl/extractor/youporn.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 8123928be..d9c06a2ee 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -45,7 +45,9 @@ class YouPornIE(InfoExtractor):
         age_limit = self._rta_search(webpage)
 
         # Get JSON parameters
-        json_params = self._search_regex(r'var currentVideo = new Video\((.*)\);', webpage, 'JSON parameters')
+        json_params = self._search_regex(
+            r'var currentVideo = new Video\((.*)\)[,;]',
+            webpage, 'JSON parameters')
         try:
             params = json.loads(json_params)
         except:

From 0df23ba9f9ddef005ec5f592ccb43b41564b5767 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 00:48:34 +0100
Subject: [PATCH 0726/1937] release 2014.12.06.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c568920d9..3ac525557 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.06'
+__version__ = '2014.12.06.1'

From 71155991218dc8c392e8e3431ab71a0f43687349 Mon Sep 17 00:00:00 2001
From: cryptonaut <cryptonaut64@gmail.com>
Date: Fri, 5 Dec 2014 21:49:41 -0800
Subject: [PATCH 0727/1937] [adultswim] Updated to work with new site format
 (fixes #4317)

---
 youtube_dl/extractor/adultswim.py | 206 +++++++++++++++++-------------
 1 file changed, 114 insertions(+), 92 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 0d05cbb4b..e4c78a81c 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -2,123 +2,147 @@
 from __future__ import unicode_literals
 
 import re
+import json
 
 from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+)
 
 
 class AdultSwimIE(InfoExtractor):
-    _VALID_URL = r'https?://video\.adultswim\.com/(?P<path>.+?)(?:\.html)?(?:\?.*)?(?:#.*)?$'
-    _TEST = {
-        'url': 'http://video.adultswim.com/rick-and-morty/close-rick-counters-of-the-rick-kind.html?x=y#title',
+    _VALID_URL = r'https?://(?:www\.)?adultswim\.com/videos/(?P<is_playlist>playlists/)?(?P<show_path>[^/]+)/(?P<episode_path>[^/?#]+)/?'
+
+    _TESTS = [{
+        'url': 'http://adultswim.com/videos/rick-and-morty/pilot',
+        'playlist': [
+           {
+                'md5': '247572debc75c7652f253c8daa51a14d',
+                'info_dict': {
+                    'id': 'rQxZvXQ4ROaSOqq-or2Mow-0',
+                    'ext': 'flv',
+                    'title': 'Rick and Morty - Pilot Part 1',
+                    'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
+                },
+            },
+            {
+                'md5': '77b0e037a4b20ec6b98671c4c379f48d',
+                'info_dict': {
+                    'id': 'rQxZvXQ4ROaSOqq-or2Mow-3',
+                    'ext': 'flv',
+                    'title': 'Rick and Morty - Pilot Part 4',
+                    'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
+                },
+            },
+        ],
+        'info_dict': {
+            'title': 'Rick and Morty - Pilot',
+            'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
+        }
+    }, {
+        'url': 'http://www.adultswim.com/videos/playlists/american-parenting/putting-francine-out-of-business/',
         'playlist': [
             {
-                'md5': '4da359ec73b58df4575cd01a610ba5dc',
+                'md5': '2eb5c06d0f9a1539da3718d897f13ec5',
                 'info_dict': {
-                    'id': '8a250ba1450996e901453d7f02ca02f5',
+                    'id': '-t8CamQlQ2aYZ49ItZCFog-0',
                     'ext': 'flv',
-                    'title': 'Rick and Morty Close Rick-Counters of the Rick Kind part 1',
-                    'description': 'Rick has a run in with some old associates, resulting in a fallout with Morty. You got any chips, broh?',
-                    'uploader': 'Rick and Morty',
-                    'thumbnail': 'http://i.cdn.turner.com/asfix/repository/8a250ba13f865824013fc9db8b6b0400/thumbnail_267549017116827057.jpg'
-                }
-            },
-            {
-                'md5': 'ffbdf55af9331c509d95350bd0cc1819',
-                'info_dict': {
-                    'id': '8a250ba1450996e901453d7f4bd102f6',
-                    'ext': 'flv',
-                    'title': 'Rick and Morty Close Rick-Counters of the Rick Kind part 2',
-                    'description': 'Rick has a run in with some old associates, resulting in a fallout with Morty. You got any chips, broh?',
-                    'uploader': 'Rick and Morty',
-                    'thumbnail': 'http://i.cdn.turner.com/asfix/repository/8a250ba13f865824013fc9db8b6b0400/thumbnail_267549017116827057.jpg'
-                }
-            },
-            {
-                'md5': 'b92409635540304280b4b6c36bd14a0a',
-                'info_dict': {
-                    'id': '8a250ba1450996e901453d7fa73c02f7',
-                    'ext': 'flv',
-                    'title': 'Rick and Morty Close Rick-Counters of the Rick Kind part 3',
-                    'description': 'Rick has a run in with some old associates, resulting in a fallout with Morty. You got any chips, broh?',
-                    'uploader': 'Rick and Morty',
-                    'thumbnail': 'http://i.cdn.turner.com/asfix/repository/8a250ba13f865824013fc9db8b6b0400/thumbnail_267549017116827057.jpg'
-                }
-            },
-            {
-                'md5': 'e8818891d60e47b29cd89d7b0278156d',
-                'info_dict': {
-                    'id': '8a250ba1450996e901453d7fc8ba02f8',
-                    'ext': 'flv',
-                    'title': 'Rick and Morty Close Rick-Counters of the Rick Kind part 4',
-                    'description': 'Rick has a run in with some old associates, resulting in a fallout with Morty. You got any chips, broh?',
-                    'uploader': 'Rick and Morty',
-                    'thumbnail': 'http://i.cdn.turner.com/asfix/repository/8a250ba13f865824013fc9db8b6b0400/thumbnail_267549017116827057.jpg'
-                }
+                    'title': 'American Dad - Putting Francine Out of Business',
+                    'description': 'Stan hatches a plan to get Francine out of the real estate business.Watch more American Dad on [adult swim].'
+                },
             }
-        ]
-    }
+        ],
+        'info_dict': {
+            'title': 'American Dad - Putting Francine Out of Business',
+            'description': 'Stan hatches a plan to get Francine out of the real estate business.Watch more American Dad on [adult swim].'
+        },
+    }]
 
-    _video_extensions = {
-        '3500': 'flv',
-        '640': 'mp4',
-        '150': 'mp4',
-        'ipad': 'm3u8',
-        'iphone': 'm3u8'
-    }
-    _video_dimensions = {
-        '3500': (1280, 720),
-        '640': (480, 270),
-        '150': (320, 180)
-    }
+    @staticmethod
+    def find_video_info(collection, slug):
+        for video in collection.get('videos'):
+            if video.get('slug') == slug:
+                return video
+
+    @staticmethod
+    def find_collection_by_linkURL(collections, linkURL):
+        for collection in collections:
+            if collection.get('linkURL') == linkURL:
+                return collection
+
+    @staticmethod
+    def find_collection_containing_video(collections, slug):
+        for collection in collections:
+            for video in collection.get('videos'):
+                if video.get('slug') == slug:
+                    return collection, video
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_path = mobj.group('path')
+        show_path = mobj.group('show_path')
+        episode_path = mobj.group('episode_path')
+        is_playlist = True if mobj.group('is_playlist') else False
 
-        webpage = self._download_webpage(url, video_path)
-        episode_id = self._html_search_regex(
-            r'<link rel="video_src" href="http://i\.adultswim\.com/adultswim/adultswimtv/tools/swf/viralplayer.swf\?id=([0-9a-f]+?)"\s*/?\s*>',
-            webpage, 'episode_id')
-        title = self._og_search_title(webpage)
+        webpage = self._download_webpage(url, episode_path)
 
-        index_url = 'http://asfix.adultswim.com/asfix-svc/episodeSearch/getEpisodesByIDs?networkName=AS&ids=%s' % episode_id
-        idoc = self._download_xml(index_url, title, 'Downloading episode index', 'Unable to download episode index')
+        # Extract the value of `bootstrappedData` from the Javascript in the page.
+        bootstrappedDataJS = self._search_regex(r'var bootstrappedData = ({.*});', webpage, episode_path)
 
-        episode_el = idoc.find('.//episode')
-        show_title = episode_el.attrib.get('collectionTitle')
-        episode_title = episode_el.attrib.get('title')
-        thumbnail = episode_el.attrib.get('thumbnailUrl')
-        description = episode_el.find('./description').text.strip()
+        try:
+            bootstrappedData = json.loads(bootstrappedDataJS)
+        except ValueError as ve:
+            errmsg = '%s: Failed to parse JSON ' % episode_path
+            raise ExtractorError(errmsg, cause=ve)
+
+        # Downloading videos from a /videos/playlist/ URL needs to be handled differently.
+        # NOTE: We are only downloading one video (the current one) not the playlist
+        if is_playlist:
+            collections = bootstrappedData['playlists']['collections']
+            collection = self.find_collection_by_linkURL(collections, show_path)
+            video_info = self.find_video_info(collection, episode_path)
+
+            show_title = video_info['showTitle']
+            segment_ids = [video_info['videoPlaybackID']]
+        else:
+            collections = bootstrappedData['show']['collections']
+            collection, video_info = self.find_collection_containing_video(collections, episode_path)
+
+            show = bootstrappedData['show']
+            show_title = show['title']
+            segment_ids = [clip['videoPlaybackID'] for clip in video_info['clips']]
+
+        episode_id = video_info['id']
+        episode_title = video_info['title']
+        episode_description = video_info['description']
+        episode_duration = video_info.get('duration')
 
         entries = []
-        segment_els = episode_el.findall('./segments/segment')
+        for part_num, segment_id in enumerate(segment_ids):
+            segment_url = 'http://www.adultswim.com/videos/api/v0/assets?id=%s&platform=mobile' % segment_id
 
-        for part_num, segment_el in enumerate(segment_els):
-            segment_id = segment_el.attrib.get('id')
-            segment_title = '%s %s part %d' % (show_title, episode_title, part_num + 1)
-            thumbnail = segment_el.attrib.get('thumbnailUrl')
-            duration = segment_el.attrib.get('duration')
+            segment_title = '%s - %s' % (show_title, episode_title)
+            if len(segment_ids) > 1:
+                segment_title += ' Part %d' % (part_num + 1)
 
-            segment_url = 'http://asfix.adultswim.com/asfix-svc/episodeservices/getCvpPlaylist?networkName=AS&id=%s' % segment_id
             idoc = self._download_xml(
                 segment_url, segment_title,
                 'Downloading segment information', 'Unable to download segment information')
 
+            segment_duration = idoc.find('.//trt').text.strip()
+
             formats = []
             file_els = idoc.findall('.//files/file')
 
             for file_el in file_els:
                 bitrate = file_el.attrib.get('bitrate')
-                type = file_el.attrib.get('type')
-                width, height = self._video_dimensions.get(bitrate, (None, None))
+                ftype = file_el.attrib.get('type')
+
                 formats.append({
-                    'format_id': '%s-%s' % (bitrate, type),
-                    'url': file_el.text,
-                    'ext': self._video_extensions.get(bitrate, 'mp4'),
+                    'format_id': '%s_%s' % (bitrate, ftype),
+                    'url': file_el.text.strip(),
                     # The bitrate may not be a number (for example: 'iphone')
                     'tbr': int(bitrate) if bitrate.isdigit() else None,
-                    'height': height,
-                    'width': width
+                    'quality': 1 if ftype == 'hd' else -1
                 })
 
             self._sort_formats(formats)
@@ -127,18 +151,16 @@ class AdultSwimIE(InfoExtractor):
                 'id': segment_id,
                 'title': segment_title,
                 'formats': formats,
-                'uploader': show_title,
-                'thumbnail': thumbnail,
-                'duration': duration,
-                'description': description
+                'duration': segment_duration,
+                'description': episode_description
             })
 
         return {
             '_type': 'playlist',
             'id': episode_id,
-            'display_id': video_path,
+            'display_id': episode_path,
             'entries': entries,
-            'title': '%s %s' % (show_title, episode_title),
-            'description': description,
-            'thumbnail': thumbnail
+            'title': '%s - %s' % (show_title, episode_title),
+            'description': episode_description,
+            'duration': episode_duration
         }

From 754f0008ec6b87316ce3e4807f150726ff2af3c5 Mon Sep 17 00:00:00 2001
From: netanel <netanelmaman0@gmail.com>
Date: Sat, 6 Dec 2014 09:20:35 +0200
Subject: [PATCH 0728/1937] fix increment operator

---
 youtube_dl/extractor/eighttracks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index 101a09699..f093592a8 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -135,7 +135,7 @@ class EightTracksIE(InfoExtractor):
                     if download_tries > 3:
                         raise
                     else:
-                        ++download_tries
+                        download_tries += 1
                         time.sleep(avg_song_duration)
 
             api_data = json.loads(api_json)

From 1a9f8b1ad4f56c42e28f45bb727d0fc20f71c7a7 Mon Sep 17 00:00:00 2001
From: cryptonaut <cryptonaut64@gmail.com>
Date: Sat, 6 Dec 2014 01:42:58 -0800
Subject: [PATCH 0729/1937] [nba] Improve _VALID_URL regex (fixes #4387)

Allows for optional trailing / or /index.html
---
 youtube_dl/extractor/nba.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index f69fe0925..842ea66a1 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class NBAIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?:nba/)?video(?P<id>/[^?]*?)(?:/index\.html)?(?:\?.*)?$'
+    _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?:nba/)?video(?P<id>/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
     _TEST = {
         'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html',
         'md5': 'c0edcfc37607344e2ff8f13c378c88a4',

From 0b5cc1983ecc7f407092d5431807341d82610aa8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 11:15:25 +0100
Subject: [PATCH 0730/1937] [nba] Modernize

---
 youtube_dl/extractor/nba.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 842ea66a1..cd021332b 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -24,9 +24,7 @@ class NBAIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         video_url = 'http://ht-mobile.cdn.turner.com/nba/big' + video_id + '_nba_1280x720.mp4'
@@ -37,7 +35,7 @@ class NBAIE(InfoExtractor):
 
         description = self._og_search_description(webpage)
         duration = parse_duration(
-            self._html_search_meta('duration', webpage, 'duration', fatal=False))
+            self._html_search_meta('duration', webpage, 'duration'))
 
         return {
             'id': shortened_video_id,

From 6a3e0103bb84ca65e11e8473ff41bf7796d998a2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 11:26:17 +0100
Subject: [PATCH 0731/1937] [nba] Add test for #4387

---
 youtube_dl/extractor/nba.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index cd021332b..2b3a3daa7 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 class NBAIE(InfoExtractor):
     _VALID_URL = r'https?://(?:watch\.|www\.)?nba\.com/(?:nba/)?video(?P<id>/[^?]*?)/?(?:/index\.html)?(?:\?.*)?$'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.nba.com/video/games/nets/2012/12/04/0021200253-okc-bkn-recap.nba/index.html',
         'md5': 'c0edcfc37607344e2ff8f13c378c88a4',
         'info_dict': {
@@ -21,7 +21,10 @@ class NBAIE(InfoExtractor):
             'description': 'Kevin Durant scores 32 points and dishes out six assists as the Thunder beat the Nets in Brooklyn.',
             'duration': 181,
         },
-    }
+    }, {
+        'url': 'http://www.nba.com/video/games/hornets/2014/12/05/0021400276-nyk-cha-play5.nba/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 5f9b83944da7c531f82ace0ce9275c2661e7187e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 12:14:26 +0100
Subject: [PATCH 0732/1937] [ffmpeg] Improve version check and call it from hls
 (Fixes #4377)

---
 test/test_utils.py                 | 6 ++++++
 youtube_dl/downloader/hls.py       | 4 ++++
 youtube_dl/postprocessor/ffmpeg.py | 6 +++---
 youtube_dl/utils.py                | 2 +-
 4 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index badab1370..5c7bf5175 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -48,6 +48,7 @@ from youtube_dl.utils import (
     intlist_to_bytes,
     args_to_str,
     parse_filesize,
+    version_tuple,
 )
 
 
@@ -381,5 +382,10 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_filesize('1.2Tb'), 1200000000000)
         self.assertEqual(parse_filesize('1,24 KB'), 1240)
 
+    def test_version_tuple(self):
+        self.assertEqual(version_tuple('1'), (1,))
+        self.assertEqual(version_tuple('10.23.344'), (10, 23, 344))
+        self.assertEqual(version_tuple('10-6'), (10, 6))  # avconv style
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 954beffd5..ad26cfa40 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -4,6 +4,7 @@ import os
 import re
 import subprocess
 
+from ..postprocessor.ffmpeg import FFmpegPostProcessor
 from .common import FileDownloader
 from ..utils import (
     compat_urlparse,
@@ -32,6 +33,9 @@ class HlsFD(FileDownloader):
             return False
         cmd = [program] + args
 
+        ffpp = FFmpegPostProcessor(downloader=self)
+        ffpp.check_version()
+
         retval = subprocess.call(cmd)
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 9303b8378..965ded4c1 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -37,11 +37,11 @@ class FFmpegPostProcessor(PostProcessor):
         if not self._executable:
             raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')
 
-        REQUIRED_VERSION = '1.0'
+        required_version = '10-0' if self._uses_avconv() else '1.0'
         if is_outdated_version(
-                self._versions[self._executable], REQUIRED_VERSION):
+                self._versions[self._executable], required_version):
             warning = 'Your copy of %s is outdated, update %s to version %s or newer if you encounter any errors.' % (
-                self._executable, self._executable, REQUIRED_VERSION)
+                self._executable, self._executable, required_version)
             if self._downloader:
                 self._downloader.report_warning(warning)
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5efb4c59a..2e70cc791 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1502,7 +1502,7 @@ def limit_length(s, length):
 
 
 def version_tuple(v):
-    return [int(e) for e in v.split('.')]
+    return tuple(int(e) for e in re.split(r'[-.]', v))
 
 
 def is_outdated_version(version, limit, assume_new=True):

From 9ff6772790975ecf0d06c5511bfa7f1dee0955c9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 12:20:54 +0100
Subject: [PATCH 0733/1937] [youtube] Modernize

---
 youtube_dl/extractor/youtube.py | 18 ++++--------------
 1 file changed, 4 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f452a90d8..addef9594 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1229,7 +1229,7 @@ class YoutubeTopListIE(YoutubePlaylistIE):
 
 class YoutubeChannelIE(InfoExtractor):
     IE_DESC = 'YouTube.com channels'
-    _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)"
+    _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/(?P<id>[0-9A-Za-z_-]+)'
     _MORE_PAGES_INDICATOR = 'yt-uix-load-more'
     _MORE_PAGES_URL = 'https://www.youtube.com/c4_browse_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
     IE_NAME = 'youtube:channel'
@@ -1247,13 +1247,8 @@ class YoutubeChannelIE(InfoExtractor):
         return ids_in_page
 
     def _real_extract(self, url):
-        # Extract channel id
-        mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError('Invalid URL: %s' % url)
+        channel_id = self._match_id(url)
 
-        # Download channel page
-        channel_id = mobj.group(1)
         video_ids = []
         url = 'https://www.youtube.com/channel/%s/videos' % channel_id
         channel_page = self._download_webpage(url, channel_id)
@@ -1290,7 +1285,7 @@ class YoutubeChannelIE(InfoExtractor):
 
 class YoutubeUserIE(InfoExtractor):
     IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
-    _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)([A-Za-z0-9_-]+)'
+    _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
     _TEMPLATE_URL = 'https://gdata.youtube.com/feeds/api/users/%s'
     _GDATA_PAGE_SIZE = 50
     _GDATA_URL = 'https://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d&alt=json'
@@ -1318,12 +1313,7 @@ class YoutubeUserIE(InfoExtractor):
             return super(YoutubeUserIE, cls).suitable(url)
 
     def _real_extract(self, url):
-        # Extract username
-        mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError('Invalid URL: %s' % url)
-
-        username = mobj.group(1)
+        username = self._match_id(url)
 
         # Download video ids using YouTube Data API. Result size per
         # query is limited (currently to 50 videos) so we need to query

From db175341c7d80a9f78ddd78d8e93b1286f725a0f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 12:32:01 +0100
Subject: [PATCH 0734/1937] Credit @cryptonaut for adultswim (#4388)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 4d3b9ddc1..0b2ef8540 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -90,3 +90,4 @@ Matthew Rayfield
 t0mm0
 Tithen-Firion
 Zack Fernandes
+cryptonaut

From 47d7c64274f9cde33daab5f54b3707ffb19d2f20 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 12:36:23 +0100
Subject: [PATCH 0735/1937] [test_utils] Make test more realistically (#4377)

---
 test/test_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 5c7bf5175..aaa293ff8 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -385,7 +385,7 @@ class TestUtil(unittest.TestCase):
     def test_version_tuple(self):
         self.assertEqual(version_tuple('1'), (1,))
         self.assertEqual(version_tuple('10.23.344'), (10, 23, 344))
-        self.assertEqual(version_tuple('10-6'), (10, 6))  # avconv style
+        self.assertEqual(version_tuple('10.1-6'), (10, 1, 6))  # avconv style
 
 if __name__ == '__main__':
     unittest.main()

From 6591fdf51f194d4d140fb3da582361766a8ec01c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 13:59:10 +0100
Subject: [PATCH 0736/1937] [tagesschau] Look at the right place for download
 links

---
 youtube_dl/extractor/tagesschau.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py
index aed670941..bfe07b024 100644
--- a/youtube_dl/extractor/tagesschau.py
+++ b/youtube_dl/extractor/tagesschau.py
@@ -70,7 +70,7 @@ class TagesschauIE(InfoExtractor):
                 webpage, 'download links')
             links = re.finditer(
                 r'<div class="button" title="(?P<title>[^"]*)"><a href="(?P<url>[^"]+)">(?P<name>.+?)</a></div>',
-                webpage)
+                download_text)
             formats = []
             for l in links:
                 format_id = self._search_regex(

From 7d70cf4157306e4b14c4f3967fad25911299acdf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 13:59:37 +0100
Subject: [PATCH 0737/1937] [nba] Remove unused import

---
 youtube_dl/extractor/nba.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/nba.py b/youtube_dl/extractor/nba.py
index 2b3a3daa7..862b706bf 100644
--- a/youtube_dl/extractor/nba.py
+++ b/youtube_dl/extractor/nba.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     remove_end,

From 158f8cadc0c29e9d43e1440a672a3f72f0e454c7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 14:01:59 +0100
Subject: [PATCH 0738/1937] [adultswim] PEP8

---
 youtube_dl/extractor/adultswim.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index e4c78a81c..39e4ca296 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -16,7 +16,7 @@ class AdultSwimIE(InfoExtractor):
     _TESTS = [{
         'url': 'http://adultswim.com/videos/rick-and-morty/pilot',
         'playlist': [
-           {
+            {
                 'md5': '247572debc75c7652f253c8daa51a14d',
                 'info_dict': {
                     'id': 'rQxZvXQ4ROaSOqq-or2Mow-0',

From b82f815f373818ba99ee43660e9255e8f4ecac62 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 6 Dec 2014 14:02:19 +0100
Subject: [PATCH 0739/1937] Allow iterators for playlist result entries

---
 youtube_dl/YoutubeDL.py         | 20 ++++++++++++++------
 youtube_dl/extractor/common.py  |  4 ++--
 youtube_dl/extractor/youtube.py | 18 ++++++++++--------
 3 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index f89ac4e1d..56dc3d461 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -7,6 +7,7 @@ import collections
 import datetime
 import errno
 import io
+import itertools
 import json
 import locale
 import os
@@ -654,21 +655,28 @@ class YoutubeDL(object):
             if playlistend == -1:
                 playlistend = None
 
-            if isinstance(ie_result['entries'], list):
-                n_all_entries = len(ie_result['entries'])
-                entries = ie_result['entries'][playliststart:playlistend]
+            ie_entries = ie_result['entries']
+            if isinstance(ie_entries, list):
+                n_all_entries = len(ie_entries)
+                entries = ie_entries[playliststart:playlistend]
                 n_entries = len(entries)
                 self.to_screen(
                     "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
-            else:
-                assert isinstance(ie_result['entries'], PagedList)
-                entries = ie_result['entries'].getslice(
+            elif isinstance(ie_entries, PagedList):
+                entries = ie_entries.getslice(
                     playliststart, playlistend)
                 n_entries = len(entries)
                 self.to_screen(
                     "[%s] playlist %s: Downloading %d videos" %
                     (ie_result['extractor'], playlist, n_entries))
+            else:  # iterable
+                entries = list(itertools.islice(
+                    ie_entries, playliststart, playlistend))
+                n_entries = len(entries)
+                self.to_screen(
+                    "[%s] playlist %s: Downloading %d videos" %
+                    (ie_result['extractor'], playlist, n_entries))
 
             for i, entry in enumerate(entries, 1):
                 self.to_screen('[download] Downloading video #%s of %s' % (i, n_entries))
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 5b5e33cea..cb6081dd0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -158,8 +158,8 @@ class InfoExtractor(object):
 
 
     _type "playlist" indicates multiple videos.
-    There must be a key "entries", which is a list or a PagedList object, each
-    element of which is a valid dictionary under this specfication.
+    There must be a key "entries", which is a list, an iterable, or a PagedList
+    object, each element of which is a valid dictionary by this specification.
 
     Additionally, playlists can have "title" and "id" attributes with the same
     semantics as videos (see above).
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index addef9594..8b6e591a4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1262,8 +1262,12 @@ class YoutubeChannelIE(InfoExtractor):
             # The videos are contained in a single page
             # the ajax pages can't be used, they are empty
             video_ids = self.extract_videos_from_page(channel_page)
-        else:
-            # Download all channel pages using the json-based channel_ajax query
+            entries = [
+                self.url_result(video_id, 'Youtube', video_id=video_id)
+                for video_id in video_ids]
+            return self.playlist_result(entries, channel_id)
+
+        def _entries():
             for pagenum in itertools.count(1):
                 url = self._MORE_PAGES_URL % (pagenum, channel_id)
                 page = self._download_json(
@@ -1271,16 +1275,14 @@ class YoutubeChannelIE(InfoExtractor):
                     transform_source=uppercase_escape)
 
                 ids_in_page = self.extract_videos_from_page(page['content_html'])
-                video_ids.extend(ids_in_page)
+                for video_id in ids_in_page:
+                    yield self.url_result(
+                        video_id, 'Youtube', video_id=video_id)
 
                 if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
                     break
 
-        self._downloader.to_screen('[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
-
-        url_entries = [self.url_result(video_id, 'Youtube', video_id=video_id)
-                       for video_id in video_ids]
-        return self.playlist_result(url_entries, channel_id)
+        return self.playlist_result(_entries(), channel_id)
 
 
 class YoutubeUserIE(InfoExtractor):

From 6ac4e8065a506f92db919ff063cf4d2f00bc81f0 Mon Sep 17 00:00:00 2001
From: Grom PE <i@grompe.org.ru>
Date: Sat, 6 Dec 2014 20:15:41 +0700
Subject: [PATCH 0740/1937] Fix utils.py for PyPy on Windows

The line
```python
from __future__ import unicode_literals
```
introduced in commit [ecc0c5ee01f0e5bdd6af0c32cb5b4adcb2a2f78c](https://github.com/rg3/youtube-dl/commit/ecc0c5ee01f0e5bdd6af0c32cb5b4adcb2a2f78c) broke youtube-dl for PyPy on Windows, making it unable to locate WinAPI functions.
Error: "TypeError: function name must be a string or integer"

Adding "b" prefix to strings with WinAPI function names fixes it.
---
 youtube_dl/utils.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2e70cc791..2b0f4e589 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -808,22 +808,22 @@ def _windows_write_string(s, out):
 
     GetStdHandle = ctypes.WINFUNCTYPE(
         ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
-        ("GetStdHandle", ctypes.windll.kernel32))
+        (b"GetStdHandle", ctypes.windll.kernel32))
     h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
 
     WriteConsoleW = ctypes.WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
         ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
-        ctypes.wintypes.LPVOID)(("WriteConsoleW", ctypes.windll.kernel32))
+        ctypes.wintypes.LPVOID)((b"WriteConsoleW", ctypes.windll.kernel32))
     written = ctypes.wintypes.DWORD(0)
 
-    GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)(("GetFileType", ctypes.windll.kernel32))
+    GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b"GetFileType", ctypes.windll.kernel32))
     FILE_TYPE_CHAR = 0x0002
     FILE_TYPE_REMOTE = 0x8000
     GetConsoleMode = ctypes.WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
         ctypes.POINTER(ctypes.wintypes.DWORD))(
-        ("GetConsoleMode", ctypes.windll.kernel32))
+        (b"GetConsoleMode", ctypes.windll.kernel32))
     INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
 
     def not_a_console(handle):

From 04d02a9d57c834d3044871ce7dcc90932e686e46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Dec 2014 21:24:20 +0600
Subject: [PATCH 0741/1937] [twitch] Add login support (#3986)

---
 youtube_dl/extractor/twitch.py | 42 ++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 36aa1ad6e..397d167e8 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import itertools
@@ -5,6 +6,8 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    compat_urllib_parse,
+    compat_urllib_request,
     ExtractorError,
     parse_iso8601,
 )
@@ -24,6 +27,7 @@ class TwitchIE(InfoExtractor):
         """
     _PAGE_LIMIT = 100
     _API_BASE = 'https://api.twitch.tv'
+    _LOGIN_URL = 'https://secure.twitch.tv/user/login'
     _TESTS = [{
         'url': 'http://www.twitch.tv/riotgames/b/577357806',
         'info_dict': {
@@ -109,6 +113,44 @@ class TwitchIE(InfoExtractor):
             'view_count': info['views'],
         }
 
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_page = self._download_webpage(
+            self._LOGIN_URL, None, 'Downloading login page')
+
+        authenticity_token = self._search_regex(
+            r'<input name="authenticity_token" type="hidden" value="([^"]+)"',
+            login_page, 'authenticity token')
+
+        login_form = {
+            'utf8': '✓'.encode('utf-8'),
+            'authenticity_token': authenticity_token,
+            'redirect_on_login': '',
+            'embed_form': 'false',
+            'mp_source_action': '',
+            'follow': '',
+            'user[login]': username,
+            'user[password]': password,
+        }
+
+        request = compat_urllib_request.Request(
+            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+        request.add_header('Referer', self._LOGIN_URL)
+        response = self._download_webpage(
+            request, None, 'Logging in as %s' % username)
+
+        m = re.search(
+            r"id=([\"'])login_error_message\1[^>]*>(?P<msg>[^<]+)", response)
+        if m:
+            raise ExtractorError(
+                'Unable to login: %s' % m.group('msg').strip(), expected=True)
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         if mobj.group('chapterid'):

From 6d0886204a920e64606688b1217835d10e47d281 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 6 Dec 2014 23:01:52 +0600
Subject: [PATCH 0742/1937] [radio.de] Add support for radio.de websites
 (Closes #4393)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/radiode.py  | 55 ++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/radiode.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4c9c44b31..33a22d76e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -307,6 +307,7 @@ from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
 from .pyvideo import PyvideoIE
 from .quickvid import QuickVidIE
+from .radiode import RadioDeIE
 from .radiofrance import RadioFranceIE
 from .rai import RaiIE
 from .rbmaradio import RBMARadioIE
diff --git a/youtube_dl/extractor/radiode.py b/youtube_dl/extractor/radiode.py
new file mode 100644
index 000000000..f95bc9454
--- /dev/null
+++ b/youtube_dl/extractor/radiode.py
@@ -0,0 +1,55 @@
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+
+
+class RadioDeIE(InfoExtractor):
+    IE_NAME = 'radio.de'
+    _VALID_URL = r'https?://(?P<id>.+?)\.(?:radio\.(?:de|at|fr|pt|es|pl|it)|rad\.io)'
+    _TEST = {
+        'url': 'http://ndr2.radio.de/',
+        'md5': '3b4cdd011bc59174596b6145cda474a4',
+        'info_dict': {
+            'id': 'ndr2',
+            'ext': 'mp3',
+            'title': 're:^NDR 2 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'md5:591c49c702db1a33751625ebfb67f273',
+            'thumbnail': 're:^https?://.*\.png',
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        radio_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, radio_id)
+
+        broadcast = json.loads(self._search_regex(
+            r'_getBroadcast\s*=\s*function\(\s*\)\s*{\s*return\s+({.+?})\s*;\s*}',
+            webpage, 'broadcast'))
+
+        title = self._live_title(broadcast['name'])
+        description = broadcast.get('description') or broadcast.get('shortDescription')
+        thumbnail = broadcast.get('picture4Url') or broadcast.get('picture4TransUrl')
+
+        formats = [{
+            'url': stream['streamUrl'],
+            'ext': stream['streamContentFormat'].lower(),
+            'acodec': stream['streamContentFormat'],
+            'abr': stream['bitRate'],
+            'asr': stream['sampleRate']
+        } for stream in broadcast['streamUrls']]
+        self._sort_formats(formats)
+
+        return {
+            'id': radio_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'is_live': True,
+            'formats': formats,
+        }

From c84890f7082237b02a5fc61657b2ea513836a85e Mon Sep 17 00:00:00 2001
From: Adrian Kretz <adriankretz@gmail.com>
Date: Sat, 6 Dec 2014 18:28:21 +0100
Subject: [PATCH 0743/1937] [prosiebensat1] Add support for playlists (fixes
 #4357)

---
 youtube_dl/extractor/prosiebensat1.py | 47 +++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 32d747ede..c6e539e5a 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -8,6 +8,7 @@ from .common import InfoExtractor
 from ..utils import (
     compat_urllib_parse,
     unified_strdate,
+    ExtractorError,
 )
 
 
@@ -152,6 +153,15 @@ class ProSiebenSat1IE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'http://www.prosieben.de/tv/joko-gegen-klaas/videos/playlists/episode-8-ganze-folge-playlist',
+            'info_dict': {
+                'id': '439664',
+                'title': 'Episode 8 - Ganze Folge - Playlist',
+                'description': 'Das finale und härteste Duell aller Zeiten ist vorbei! Der Weltmeister für dieses Jahr steht! Alle packenden Duelle der achten Episode von "Joko gegen Klaas - das Duell um die Welt" seht ihr hier noch einmal in voller Länge!',
+            },
+            'playlist_count': 2,
+        },
     ]
 
     _CLIPID_REGEXES = [
@@ -178,11 +188,48 @@ class ProSiebenSat1IE(InfoExtractor):
         r'<span style="padding-left: 4px;line-height:20px; color:#404040">(\d{2}\.\d{2}\.\d{4})</span>',
         r'(\d{2}\.\d{2}\.\d{4}) \| \d{2}:\d{2} Min<br/>',
     ]
+    _ITEM_TYPE_REGEXES = [
+        r"'itemType'\s*:\s*'([^']*)'",
+    ]
+    _ITEM_ID_REGEXES = [
+        r"'itemId'\s*:\s*'([^']*)'",
+    ]
+    _PLAYLIST_CLIPS_REGEXES = [
+        r'data-qvt=.+?<a href="([^"]+)"',
+    ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
+        item_type = self._html_search_regex(self._ITEM_TYPE_REGEXES, webpage, 'item type', default='CLIP')
+        if item_type == 'CLIP':
+            return self._clip_extract(url, webpage)
+        elif item_type == 'PLAYLIST':
+            playlist_id = self._html_search_regex(self._ITEM_ID_REGEXES, webpage, 'playlist id')
+
+            for regex in self._PLAYLIST_CLIPS_REGEXES:
+                playlist_clips = re.findall(regex, webpage, re.DOTALL)
+                if playlist_clips:
+                    title = self._html_search_regex(self._TITLE_REGEXES, webpage, 'title')
+                    description = self._html_search_regex(self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False)
+                    root_url = re.match('(.+?//.+?)/', url).group(1)
+
+                    return {
+                        '_type': 'playlist',
+                        'id': playlist_id,
+                        'title': title,
+                        'description': description,
+                        'entries': [self._clip_extract(root_url + clip_path) for clip_path in playlist_clips]
+                    }
+        else:
+            raise ExtractorError('Unknown item type "%s"' % item_type)
+
+    def _clip_extract(self, url, webpage=None):
+        if webpage is None:
+            video_id = self._match_id(url)
+            webpage = self._download_webpage(url, video_id)
+
         clip_id = self._html_search_regex(self._CLIPID_REGEXES, webpage, 'clip id')
 
         access_token = 'testclient'

From 8d1c8cae9ce9027058d90b246b9fa30ea88602ef Mon Sep 17 00:00:00 2001
From: Adrian Kretz <adriankretz@gmail.com>
Date: Sat, 6 Dec 2014 19:21:05 +0100
Subject: [PATCH 0744/1937] [prosiebensat1] Fix broken tests

---
 youtube_dl/extractor/prosiebensat1.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index c6e539e5a..8c9451f40 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -86,7 +86,7 @@ class ProSiebenSat1IE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Im Interview: Kai Wiesinger',
                 'description': 'md5:e4e5370652ec63b95023e914190b4eb9',
-                'upload_date': '20140225',
+                'upload_date': '20140203',
                 'duration': 522.56,
             },
             'params': {
@@ -101,7 +101,7 @@ class ProSiebenSat1IE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Jagd auf Fertigkost im Elsthal - Teil 2',
                 'description': 'md5:2669cde3febe9bce13904f701e774eb6',
-                'upload_date': '20140225',
+                'upload_date': '20141014',
                 'duration': 2410.44,
             },
             'params': {
@@ -168,6 +168,7 @@ class ProSiebenSat1IE(InfoExtractor):
         r'"clip_id"\s*:\s+"(\d+)"',
         r'clipid: "(\d+)"',
         r'clip[iI]d=(\d+)',
+        r"'itemImageUrl'\s*:\s*'/dynamic/thumbnails/full/\d+/(\d+)",
     ]
     _TITLE_REGEXES = [
         r'<h2 class="subtitle" itemprop="name">\s*(.+?)</h2>',

From acf5cbfe93ec4f828fef75486fa8b4e587a64baa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Dec 2014 01:46:30 +0600
Subject: [PATCH 0745/1937] [extractor/common] Add description to
 playlist_result

---
 youtube_dl/extractor/common.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index cb6081dd0..2faaf6226 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -439,7 +439,7 @@ class InfoExtractor(object):
         return video_info
 
     @staticmethod
-    def playlist_result(entries, playlist_id=None, playlist_title=None):
+    def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None):
         """Returns a playlist"""
         video_info = {'_type': 'playlist',
                       'entries': entries}
@@ -447,6 +447,8 @@ class InfoExtractor(object):
             video_info['id'] = playlist_id
         if playlist_title:
             video_info['title'] = playlist_title
+        if playlist_description:
+            video_info['description'] = playlist_description
         return video_info
 
     def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):

From 6a52eed80edbb0f1e65a74def32d84236ac6e640 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Dec 2014 01:46:44 +0600
Subject: [PATCH 0746/1937] [prosiebensat1] Improve and simplify

---
 youtube_dl/extractor/prosiebensat1.py | 74 +++++++++++++--------------
 1 file changed, 36 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 8c9451f40..1262793c8 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -8,7 +8,6 @@ from .common import InfoExtractor
 from ..utils import (
     compat_urllib_parse,
     unified_strdate,
-    ExtractorError,
 )
 
 
@@ -158,7 +157,7 @@ class ProSiebenSat1IE(InfoExtractor):
             'info_dict': {
                 'id': '439664',
                 'title': 'Episode 8 - Ganze Folge - Playlist',
-                'description': 'Das finale und härteste Duell aller Zeiten ist vorbei! Der Weltmeister für dieses Jahr steht! Alle packenden Duelle der achten Episode von "Joko gegen Klaas - das Duell um die Welt" seht ihr hier noch einmal in voller Länge!',
+                'description': 'md5:63b8963e71f481782aeea877658dec84',
             },
             'playlist_count': 2,
         },
@@ -189,48 +188,19 @@ class ProSiebenSat1IE(InfoExtractor):
         r'<span style="padding-left: 4px;line-height:20px; color:#404040">(\d{2}\.\d{2}\.\d{4})</span>',
         r'(\d{2}\.\d{2}\.\d{4}) \| \d{2}:\d{2} Min<br/>',
     ]
-    _ITEM_TYPE_REGEXES = [
+    _PAGE_TYPE_REGEXES = [
+        r'<meta name="page_type" content="([^"]+)">',
         r"'itemType'\s*:\s*'([^']*)'",
     ]
-    _ITEM_ID_REGEXES = [
+    _PLAYLIST_ID_REGEXES = [
+        r'content[iI]d=(\d+)',
         r"'itemId'\s*:\s*'([^']*)'",
     ]
-    _PLAYLIST_CLIPS_REGEXES = [
-        r'data-qvt=.+?<a href="([^"]+)"',
+    _PLAYLIST_CLIP_REGEXES = [
+        r'(?s)data-qvt=.+?<a href="([^"]+)"',
     ]
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        item_type = self._html_search_regex(self._ITEM_TYPE_REGEXES, webpage, 'item type', default='CLIP')
-        if item_type == 'CLIP':
-            return self._clip_extract(url, webpage)
-        elif item_type == 'PLAYLIST':
-            playlist_id = self._html_search_regex(self._ITEM_ID_REGEXES, webpage, 'playlist id')
-
-            for regex in self._PLAYLIST_CLIPS_REGEXES:
-                playlist_clips = re.findall(regex, webpage, re.DOTALL)
-                if playlist_clips:
-                    title = self._html_search_regex(self._TITLE_REGEXES, webpage, 'title')
-                    description = self._html_search_regex(self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False)
-                    root_url = re.match('(.+?//.+?)/', url).group(1)
-
-                    return {
-                        '_type': 'playlist',
-                        'id': playlist_id,
-                        'title': title,
-                        'description': description,
-                        'entries': [self._clip_extract(root_url + clip_path) for clip_path in playlist_clips]
-                    }
-        else:
-            raise ExtractorError('Unknown item type "%s"' % item_type)
-
-    def _clip_extract(self, url, webpage=None):
-        if webpage is None:
-            video_id = self._match_id(url)
-            webpage = self._download_webpage(url, video_id)
-
+    def _extract_clip(self, url, webpage):
         clip_id = self._html_search_regex(self._CLIPID_REGEXES, webpage, 'clip id')
 
         access_token = 'testclient'
@@ -329,3 +299,31 @@ class ProSiebenSat1IE(InfoExtractor):
             'duration': duration,
             'formats': formats,
         }
+
+    def _extract_playlist(self, url, webpage):
+        playlist_id = self._html_search_regex(
+            self._PLAYLIST_ID_REGEXES, webpage, 'playlist id')
+        for regex in self._PLAYLIST_CLIP_REGEXES:
+            playlist_clips = re.findall(regex, webpage)
+            if playlist_clips:
+                title = self._html_search_regex(
+                    self._TITLE_REGEXES, webpage, 'title')
+                description = self._html_search_regex(
+                    self._DESCRIPTION_REGEXES, webpage, 'description', fatal=False)
+                entries = [
+                    self.url_result(
+                        re.match('(.+?//.+?)/', url).group(1) + clip_path,
+                        'ProSiebenSat1')
+                    for clip_path in playlist_clips]
+                return self.playlist_result(entries, playlist_id, title, description)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        page_type = self._search_regex(
+            self._PAGE_TYPE_REGEXES, webpage,
+            'page type', default='clip').lower()
+        if page_type == 'clip':
+            return self._extract_clip(url, webpage)
+        elif page_type == 'playlist':
+            return self._extract_playlist(url, webpage)

From 0ef4d4ab7e15833031cd43211a38464a9ab9aa17 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Dec 2014 01:48:45 +0600
Subject: [PATCH 0747/1937] Credit @akretz for prosiebensat1 playlist support
 (#4394)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 0b2ef8540..bfa00f91b 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -91,3 +91,4 @@ t0mm0
 Tithen-Firion
 Zack Fernandes
 cryptonaut
+Adrian Kretz

From 16ae61f65515229b5cb4f5a60936bed3fa1b2b23 Mon Sep 17 00:00:00 2001
From: cryptonaut <cryptonaut64@gmail.com>
Date: Sat, 6 Dec 2014 12:47:29 -0800
Subject: [PATCH 0748/1937] Handle --get-url with merged formats (fixes #2883)

Outputs one URL per line
---
 youtube_dl/YoutubeDL.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 56dc3d461..248367039 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -942,8 +942,12 @@ class YoutubeDL(object):
         if self.params.get('forceid', False):
             self.to_stdout(info_dict['id'])
         if self.params.get('forceurl', False):
-            # For RTMP URLs, also include the playpath
-            self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
+            if info_dict.get('requested_formats') is not None:
+                for f in info_dict['requested_formats']:
+                    self.to_stdout(f['url'] + f.get('play_path', ''))
+            else:
+                # For RTMP URLs, also include the playpath
+                self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
         if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
             self.to_stdout(info_dict['thumbnail'])
         if self.params.get('forcedescription', False) and info_dict.get('description') is not None:

From 603c92080fd9923f4a17f9ee67fac99494a92bc7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 7 Dec 2014 11:26:07 +0100
Subject: [PATCH 0749/1937] [nhl] Make sure we add '_sd' before the extension
 (fixes #4397)

'.replace' would find the first dot in the path.
---
 youtube_dl/extractor/nhl.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index 0244368e9..b2f40344f 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import re
 import json
+import os
 
 from .common import InfoExtractor
 from ..compat import (
@@ -26,7 +27,8 @@ class NHLBaseInfoExtractor(InfoExtractor):
         initial_video_url = info['publishPoint']
         if info['formats'] == '1':
             parsed_url = compat_urllib_parse_urlparse(initial_video_url)
-            path = parsed_url.path.replace('.', '_sd.', 1)
+            filename, ext = os.path.splitext(parsed_url.path)
+            path = '%s_sd%s' % (filename, ext)
             data = compat_urllib_parse.urlencode({
                 'type': 'fvod',
                 'path': compat_urlparse.urlunparse(parsed_url[:2] + (path,) + parsed_url[3:])

From 684712076f1ade9dbe432ad95d5796ec278b73ce Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Sun, 7 Dec 2014 11:51:02 +0100
Subject: [PATCH 0750/1937] add direct screenwavemedia.com URL support

---
 youtube_dl/extractor/__init__.py              |  2 +-
 .../{cinemassacre.py => screenwavemedia.py}   | 76 +++++++++++++++----
 2 files changed, 61 insertions(+), 17 deletions(-)
 rename youtube_dl/extractor/{cinemassacre.py => screenwavemedia.py} (64%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 33a22d76e..c0dcdaf02 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -50,7 +50,6 @@ from .cbsnews import CBSNewsIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chilloutzone import ChilloutzoneIE
-from .cinemassacre import CinemassacreIE
 from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
 from .clipsyndicate import ClipsyndicateIE
@@ -335,6 +334,7 @@ from .savefrom import SaveFromIE
 from .sbs import SBSIE
 from .scivee import SciVeeIE
 from .screencast import ScreencastIE
+from .screenwavemedia import ScreenwaveMediaIE
 from .servingsys import ServingSysIE
 from .sexu import SexuIE
 from .sexykarma import SexyKarmaIE
diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/screenwavemedia.py
similarity index 64%
rename from youtube_dl/extractor/cinemassacre.py
rename to youtube_dl/extractor/screenwavemedia.py
index b7fa73c3b..3fb3bfa32 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -9,15 +9,18 @@ from ..utils import (
     int_or_none,
 )
 
+class ScreenwaveMediaIE(InfoExtractor):
+    _VALID_URL = r'(?:http://)?(?' \
+        r':(?P<generic>player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?P<video_id>.+))' \
+        r'|(?P<cinemassacre>(?:www\.)?cinemassacre\.com/(?P<cm_date_Y>[0-9]{4})/(?P<cm_date_m>[0-9]{2})/(?P<cm_date_d>[0-9]{2})/(?P<cm_display_id>[^?#/]+))' \
+        r')'
 
-class CinemassacreIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?cinemassacre\.com/(?P<date_Y>[0-9]{4})/(?P<date_m>[0-9]{2})/(?P<date_d>[0-9]{2})/(?P<display_id>[^?#/]+)'
     _TESTS = [
         {
             'url': 'http://cinemassacre.com/2012/11/10/avgn-the-movie-trailer/',
             'md5': 'fde81fbafaee331785f58cd6c0d46190',
             'info_dict': {
-                'id': '19911',
+                'id': 'Cinemasssacre-19911',
                 'ext': 'mp4',
                 'upload_date': '20121110',
                 'title': '“Angry Video Game Nerd: The Movie” – Trailer',
@@ -28,7 +31,7 @@ class CinemassacreIE(InfoExtractor):
             'url': 'http://cinemassacre.com/2013/10/02/the-mummys-hand-1940',
             'md5': 'd72f10cd39eac4215048f62ab477a511',
             'info_dict': {
-                'id': '521be8ef82b16',
+                'id': 'Cinemasssacre-521be8ef82b16',
                 'ext': 'mp4',
                 'upload_date': '20131002',
                 'title': 'The Mummy’s Hand (1940)',
@@ -36,18 +39,16 @@ class CinemassacreIE(InfoExtractor):
         }
     ]
 
-    def _real_extract(self, url):
+    def _cinemassacre_get_info(self, url):
         mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
+        display_id = mobj.group('cm_display_id')
 
         webpage = self._download_webpage(url, display_id)
-        video_date = mobj.group('date_Y') + mobj.group('date_m') + mobj.group('date_d')
-        mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?P<full_video_id>(?:Cinemassacre-)?(?P<video_id>.+?)))"', webpage)
+        video_date = mobj.group('cm_date_Y') + mobj.group('cm_date_m') + mobj.group('cm_date_d')
+        mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"', webpage)
         if not mobj:
             raise ExtractorError('Can\'t extract embed url and video id')
         playerdata_url = mobj.group('embed_url')
-        video_id = mobj.group('video_id')
-        full_video_id = mobj.group('full_video_id')
 
         video_title = self._html_search_regex(
             r'<title>(?P<title>.+?)\|', webpage, 'title')
@@ -56,10 +57,28 @@ class CinemassacreIE(InfoExtractor):
             webpage, 'description', flags=re.DOTALL, fatal=False)
         video_thumbnail = self._og_search_thumbnail(webpage)
 
-        playerdata = self._download_webpage(playerdata_url, video_id, 'Downloading player webpage')
+        return {
+            'title': video_title,
+            'description': video_description,
+            'upload_date': video_date,
+            'thumbnail': video_thumbnail,
+            '_embed_url': playerdata_url,
+        }
 
+    def _screenwavemedia_get_info(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        if not mobj:
+            raise ExtractorError('Can\'t extract embed url and video id')
+        video_id = mobj.group('video_id')
+
+        playerdata = self._download_webpage(url, video_id, 'Downloading player webpage')
+
+        vidtitle = self._search_regex(
+            r'\'vidtitle\'\s*:\s*"([^\']+)"', playerdata, 'vidtitle').replace('\\/', '/')
         vidurl = self._search_regex(
             r'\'vidurl\'\s*:\s*"([^\']+)"', playerdata, 'vidurl').replace('\\/', '/')
+        pageurl = self._search_regex(
+            r'\'pageurl\'\s*:\s*"([^\']+)"', playerdata, 'pageurl', fatal=False).replace('\\/', '/')
 
         videolist_url = None
 
@@ -67,7 +86,7 @@ class CinemassacreIE(InfoExtractor):
         if mobj:
             videoserver = mobj.group('videoserver')
             mobj = re.search(r'\'vidid\'\s*:\s*"(?P<vidid>[^\']+)"', playerdata)
-            vidid = mobj.group('vidid') if mobj else full_video_id
+            vidid = mobj.group('vidid') if mobj else video_id
             videolist_url = 'http://%s/vod/smil:%s.smil/jwplayer.smil' % (videoserver, vidid)
         else:
             mobj = re.search(r"file\s*:\s*'(?P<smil>http.+?/jwplayer\.smil)'", playerdata)
@@ -110,9 +129,34 @@ class CinemassacreIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'title': video_title,
+            'title': vidtitle,
             'formats': formats,
-            'description': video_description,
-            'upload_date': video_date,
-            'thumbnail': video_thumbnail,
+            '_episode_page': pageurl,
         }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        swm_info = None
+        site_info = None
+
+        if mobj.group('generic'):
+            swm_info = self._screenwavemedia_get_info(url)
+            url = swm_info['_episode_page']
+            mobj = re.match(self._VALID_URL, url)
+
+        if mobj:
+            if mobj.group('cinemassacre'):
+                site_info = self._cinemassacre_get_info(url)
+
+        if not swm_info:
+            if site_info:
+                swm_info = self._screenwavemedia_get_info(site_info['_embed_url'])
+
+        if not swm_info:
+            raise ExtractorError("Failed to extract metadata for this URL")
+
+        if site_info:
+            swm_info.update(site_info)
+
+        return swm_info

From ce36339575da408c5d795f691d07d2e310292b28 Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Sun, 7 Dec 2014 11:52:03 +0100
Subject: [PATCH 0751/1937] add teamfourstar.com support

---
 youtube_dl/extractor/screenwavemedia.py | 32 +++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 3fb3bfa32..cf9b8e0de 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    month_by_name,
     int_or_none,
 )
 
@@ -13,6 +14,7 @@ class ScreenwaveMediaIE(InfoExtractor):
     _VALID_URL = r'(?:http://)?(?' \
         r':(?P<generic>player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?P<video_id>.+))' \
         r'|(?P<cinemassacre>(?:www\.)?cinemassacre\.com/(?P<cm_date_Y>[0-9]{4})/(?P<cm_date_m>[0-9]{2})/(?P<cm_date_d>[0-9]{2})/(?P<cm_display_id>[^?#/]+))' \
+        r'|(?P<teamfourstar>(?:www\.)?teamfourstar\.com/video/(?P<tfs_display_id>[a-z0-9\-]+)/?)' \
         r')'
 
     _TESTS = [
@@ -65,6 +67,34 @@ class ScreenwaveMediaIE(InfoExtractor):
             '_embed_url': playerdata_url,
         }
 
+    def _teamfourstar_get_info(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('tfs_display_id')
+        webpage = self._download_webpage(url, display_id)
+
+        mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"', webpage)
+        if not mobj:
+            raise ExtractorError('Can\'t extract embed url and video id')
+        playerdata_url = mobj.group('embed_url')
+
+        video_title = self._html_search_regex(
+            r'<div class="heroheadingtitle">(?P<title>.+?)</div>', webpage, 'title')
+        video_date = self._html_search_regex(
+            r'<div class="heroheadingdate">(?P<date>.+?)</div>', webpage, 'date')
+        mobj = re.match('(?P<month>[A-Z][a-z]+) (?P<day>\d+), (?P<year>\d+)', video_date)
+        video_date = '%04u%02u%02u' % (int(mobj.group('year')), month_by_name(mobj.group('month')), int(mobj.group('day')))
+        video_description = self._html_search_regex(
+            r'<div class="postcontent">(?P<description>.+?)</div>', webpage, 'description', flags=re.DOTALL)
+        video_thumbnail = self._og_search_thumbnail(webpage)
+
+        return {
+            'title': video_title,
+            'description': video_description,
+            'upload_date': video_date,
+            'thumbnail': video_thumbnail,
+            '_embed_url': playerdata_url,
+        }
+
     def _screenwavemedia_get_info(self, url):
         mobj = re.match(self._VALID_URL, url)
         if not mobj:
@@ -148,6 +178,8 @@ class ScreenwaveMediaIE(InfoExtractor):
         if mobj:
             if mobj.group('cinemassacre'):
                 site_info = self._cinemassacre_get_info(url)
+            elif mobj.group('teamfourstar'):
+                site_info = self._teamfourstar_get_info(url)
 
         if not swm_info:
             if site_info:

From 819039ee63fc74f15d8c2138791d025a74743590 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Dec 2014 22:03:02 +0600
Subject: [PATCH 0752/1937] [tvigle] Update test and modernize

---
 youtube_dl/extractor/tvigle.py | 31 ++++++++++++++-----------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index d81d1d1a6..ba65996dc 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -1,32 +1,30 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     float_or_none,
-    str_to_int,
+    parse_age_limit,
 )
 
 
 class TvigleIE(InfoExtractor):
     IE_NAME = 'tvigle'
     IE_DESC = 'Интернет-телевидение Tvigle.ru'
-    _VALID_URL = r'http://(?:www\.)?tvigle\.ru/(?:[^/]+/)+(?P<display_id>[^/]+)/$'
+    _VALID_URL = r'http://(?:www\.)?tvigle\.ru/(?:[^/]+/)+(?P<id>[^/]+)/$'
 
     _TESTS = [
         {
-            'url': 'http://www.tvigle.ru/video/brat/',
-            'md5': 'ff4344a4894b0524441fb6f8218dc716',
+            'url': 'http://www.tvigle.ru/video/sokrat/',
+            'md5': '36514aed3657d4f70b4b2cef8eb520cd',
             'info_dict': {
-                'id': '5118490',
-                'display_id': 'brat',
-                'ext': 'mp4',
-                'title': 'Брат',
-                'description': 'md5:d16ac7c0b47052ea51fddb92c4e413eb',
-                'duration': 5722.6,
-                'age_limit': 16,
+                'id': '1848932',
+                'display_id': 'sokrat',
+                'ext': 'flv',
+                'title': 'Сократ',
+                'description': 'md5:a05bd01be310074d5833efc6743be95e',
+                'duration': 6586,
+                'age_limit': 0,
             },
         },
         {
@@ -44,8 +42,7 @@ class TvigleIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
+        display_id = self._match_id(url)
 
         webpage = self._download_webpage(url, display_id)
 
@@ -60,8 +57,8 @@ class TvigleIE(InfoExtractor):
         title = item['title']
         description = item['description']
         thumbnail = item['thumbnail']
-        duration = float_or_none(item['durationMilliseconds'], 1000)
-        age_limit = str_to_int(item['ageRestrictions'])
+        duration = float_or_none(item.get('durationMilliseconds'), 1000)
+        age_limit = parse_age_limit(item.get('ageRestrictions'))
 
         formats = []
         for vcodec, fmts in item['videos'].items():

From 09a5dd2d3be02a5fca7639a77c5dcf970bfa0e8f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 8 Dec 2014 17:17:15 +0100
Subject: [PATCH 0753/1937] [bliptv] Add support for audio-only files (Fixes
 #4404)

---
 youtube_dl/extractor/bliptv.py | 42 ++++++++++++++++++++++++++--------
 1 file changed, 32 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index da47f27bd..14b814120 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -4,13 +4,17 @@ import re
 
 from .common import InfoExtractor
 from .subtitles import SubtitlesInfoExtractor
-from ..utils import (
-    compat_urllib_request,
-    unescapeHTML,
-    parse_iso8601,
-    compat_urlparse,
-    clean_html,
+
+from ..compat import (
     compat_str,
+    compat_urllib_request,
+    compat_urlparse,
+)
+from ..utils import (
+    clean_html,
+    int_or_none,
+    parse_iso8601,
+    unescapeHTML,
 )
 
 
@@ -78,7 +82,25 @@ class BlipTVIE(SubtitlesInfoExtractor):
                 'uploader': 'NostalgiaCritic',
                 'uploader_id': '246467',
             }
-        }
+        },
+        {
+            # https://github.com/rg3/youtube-dl/pull/4404
+            'note': 'Audio only',
+            'url': 'http://blip.tv/hilarios-productions/weekly-manga-recap-kingdom-7119982',
+            'md5': '76c0a56f24e769ceaab21fbb6416a351',
+            'info_dict': {
+                'id': '7103299',
+                'ext': 'flv',
+                'title': 'Weekly Manga Recap: Kingdom',
+                'description': 'And then Shin breaks the enemy line, and he&apos;s all like HWAH! And then he slices a guy and it&apos;s all like FWASHING! And... it&apos;s really hard to describe the best parts of this series without breaking down into sound effects, okay?',
+                'timestamp': 1417660321,
+                'upload_date': '20141204',
+                'uploader': 'The Rollo T',
+                'uploader_id': '407429',
+                'duration': 7251,
+                'vcodec': 'none',
+            }
+        },
     ]
 
     def _real_extract(self, url):
@@ -145,11 +167,11 @@ class BlipTVIE(SubtitlesInfoExtractor):
                     'url': real_url,
                     'format_id': role,
                     'format_note': media_type,
-                    'vcodec': media_content.get(blip('vcodec')),
+                    'vcodec': media_content.get(blip('vcodec')) or 'none',
                     'acodec': media_content.get(blip('acodec')),
                     'filesize': media_content.get('filesize'),
-                    'width': int(media_content.get('width')),
-                    'height': int(media_content.get('height')),
+                    'width': int_or_none(media_content.get('width')),
+                    'height': int_or_none(media_content.get('height')),
                 })
         self._sort_formats(formats)
 

From f58766ce5cc5a1c6d651560591bad119787e94b2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 9 Dec 2014 10:58:06 +0100
Subject: [PATCH 0754/1937] [extractor/common] Document ie_key in url results

---
 youtube_dl/extractor/common.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2faaf6226..cfd25cd21 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -174,9 +174,10 @@ class InfoExtractor(object):
     _type "url" indicates that the video must be extracted from another
     location, possibly by a different extractor. Its only required key is:
     "url" - the next URL to extract.
-
-    Additionally, it may have properties believed to be identical to the
-    resolved entity, for example "title" if the title of the referred video is
+    The key "ie_key" can be set to the class name (minus the trailing "IE",
+    e.g. "Youtube") if the extractor class is known in advance.
+    Additionally, the dictionary may have any properties of the resolved entity
+    known in advance, for example "title" if the title of the referred video is
     known ahead of time.
 
 

From e89a2aabedf24c145698a96a192a28a55a16fe1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Dec 2014 22:28:28 +0600
Subject: [PATCH 0755/1937] [extractor/common] Add generic SMIL formats
 extraction routine

---
 youtube_dl/extractor/common.py | 43 ++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index cfd25cd21..2277ec6ab 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -793,6 +793,49 @@ class InfoExtractor(object):
         self._sort_formats(formats)
         return formats
 
+    # TODO: improve extraction
+    def _extract_smil_formats(self, smil_url, video_id):
+        smil = self._download_xml(
+            smil_url, video_id, 'Downloading SMIL file',
+            'Unable to download SMIL file')
+
+        base = smil.find('./head/meta').get('base')
+
+        formats = []
+        rtmp_count = 0
+        for video in smil.findall('./body/switch/video'):
+            src = video.get('src')
+            if not src:
+                continue
+            bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
+            width = int_or_none(video.get('width'))
+            height = int_or_none(video.get('height'))
+            proto = video.get('proto')
+            if not proto:
+                if base:
+                    if base.startswith('rtmp'):
+                        proto = 'rtmp'
+                    elif base.startswith('http'):
+                        proto = 'http'
+            ext = video.get('ext')
+            if proto == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(src, video_id, ext))
+            elif proto == 'rtmp':
+                rtmp_count += 1
+                streamer = video.get('streamer') or base
+                formats.append({
+                    'url': streamer,
+                    'play_path': src,
+                    'ext': 'flv',
+                    'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
+                    'tbr': bitrate,
+                    'width': width,
+                    'height': height,
+                })
+        self._sort_formats(formats)
+
+        return formats
+
     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()

From b89a9386874630c6bfa5cecc59af200ac3206813 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 9 Dec 2014 22:29:01 +0600
Subject: [PATCH 0756/1937] [bet] Add extractor (Closes #4416)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/bet.py      | 108 +++++++++++++++++++++++++++++++
 2 files changed, 109 insertions(+)
 create mode 100644 youtube_dl/extractor/bet.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 33a22d76e..9f2dc803b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -30,6 +30,7 @@ from .bandcamp import BandcampIE, BandcampAlbumIE
 from .bbccouk import BBCCoUkIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
+from .bet import BetIE
 from .bild import BildIE
 from .bilibili import BiliBiliIE
 from .blinkx import BlinkxIE
diff --git a/youtube_dl/extractor/bet.py b/youtube_dl/extractor/bet.py
new file mode 100644
index 000000000..c1fc433f7
--- /dev/null
+++ b/youtube_dl/extractor/bet.py
@@ -0,0 +1,108 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    compat_urllib_parse,
+    xpath_text,
+    xpath_with_ns,
+    int_or_none,
+    parse_iso8601,
+)
+
+
+class BetIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?bet\.com/(?:[^/]+/)+(?P<id>.+?)\.html'
+    _TESTS = [
+        {
+            'url': 'http://www.bet.com/news/politics/2014/12/08/in-bet-exclusive-obama-talks-race-and-racism.html',
+            'info_dict': {
+                'id': '417cd61c-c793-4e8e-b006-e445ecc45add',
+                'display_id': 'in-bet-exclusive-obama-talks-race-and-racism',
+                'ext': 'flv',
+                'title': 'BET News Presents: A Conversation With President Obama',
+                'description': 'md5:5a88d8ae912c1b33e090290af7ec33c6',
+                'duration': 1534,
+                'timestamp': 1418075340,
+                'upload_date': '20141208',
+                'uploader': 'admin',
+                'thumbnail': 're:(?i)^https?://.*\.jpg$',
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        },
+        {
+            'url': 'http://www.bet.com/video/news/national/2014/justice-for-ferguson-a-community-reacts.html',
+            'info_dict': {
+                'id': '4160e53b-ad41-43b1-980f-8d85f63121f4',
+                'display_id': 'justice-for-ferguson-a-community-reacts',
+                'ext': 'flv',
+                'title': 'Justice for Ferguson: A Community Reacts',
+                'description': 'A BET News special.',
+                'duration': 1696,
+                'timestamp': 1416942360,
+                'upload_date': '20141125',
+                'uploader': 'admin',
+                'thumbnail': 're:(?i)^https?://.*\.jpg$',
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            },
+        }
+    ]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        media_url = compat_urllib_parse.unquote(self._search_regex(
+            [r'mediaURL\s*:\s*"([^"]+)"', r"var\s+mrssMediaUrl\s*=\s*'([^']+)'"],
+            webpage, 'media URL'))
+
+        mrss = self._download_xml(media_url, display_id)
+
+        item = mrss.find('./channel/item')
+
+        NS_MAP = {
+            'dc': 'http://purl.org/dc/elements/1.1/',
+            'media': 'http://search.yahoo.com/mrss/',
+            'ka': 'http://kickapps.com/karss',
+        }
+
+        title = xpath_text(item, './title', 'title')
+        description = xpath_text(
+            item, './description', 'description', fatal=False)
+
+        video_id = xpath_text(item, './guid', 'video id', fatal=False)
+
+        timestamp = parse_iso8601(xpath_text(
+            item, xpath_with_ns('./dc:date', NS_MAP),
+            'upload date', fatal=False))
+        uploader = xpath_text(
+            item, xpath_with_ns('./dc:creator', NS_MAP),
+            'uploader', fatal=False)
+
+        media_content = item.find(
+            xpath_with_ns('./media:content', NS_MAP))
+        duration = int_or_none(media_content.get('duration'))
+        smil_url = media_content.get('url')
+
+        thumbnail = media_content.find(
+            xpath_with_ns('./media:thumbnail', NS_MAP)).get('url')
+
+        formats = self._extract_smil_formats(smil_url, display_id)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'duration': duration,
+            'formats': formats,
+        }

From 83e865a37051ceeb0024490dce6849b8fd6a6943 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 9 Dec 2014 23:11:26 +0100
Subject: [PATCH 0757/1937] Fix PEP8 issue E713

---
 youtube_dl/compat.py                          | 2 +-
 youtube_dl/postprocessor/execafterdownload.py | 2 +-
 youtube_dl/update.py                          | 2 +-
 youtube_dl/utils.py                           | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 27596687d..f4a85443e 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -247,7 +247,7 @@ else:
                 userhome = compat_getenv('HOME')
             elif 'USERPROFILE' in os.environ:
                 userhome = compat_getenv('USERPROFILE')
-            elif not 'HOMEPATH' in os.environ:
+            elif 'HOMEPATH' not in os.environ:
                 return path
             else:
                 try:
diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py
index 09db43611..75c0f7bbe 100644
--- a/youtube_dl/postprocessor/execafterdownload.py
+++ b/youtube_dl/postprocessor/execafterdownload.py
@@ -14,7 +14,7 @@ class ExecAfterDownloadPP(PostProcessor):
 
     def run(self, information):
         cmd = self.exec_cmd
-        if not '{}' in cmd:
+        if '{}' not in cmd:
             cmd += ' {}'
 
         cmd = cmd.replace('{}', shlex_quote(information['filepath']))
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 4c07a558e..2d2703368 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -79,7 +79,7 @@ def update_self(to_screen, verbose):
             to_screen(compat_str(traceback.format_exc()))
         to_screen('ERROR: can\'t obtain versions info. Please try again later.')
         return
-    if not 'signature' in versions_info:
+    if 'signature' not in versions_info:
         to_screen('ERROR: the versions file is not signed or corrupted. Aborting.')
         return
     signature = versions_info['signature']
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2e70cc791..a95d2c942 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1024,7 +1024,7 @@ def smuggle_url(url, data):
 
 
 def unsmuggle_url(smug_url, default=None):
-    if not '#__youtubedl_smuggle' in smug_url:
+    if '#__youtubedl_smuggle' not in smug_url:
         return smug_url, default
     url, _, sdata = smug_url.rpartition('#')
     jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]

From f20b52778b32023b5b56245c36f5e638d7879515 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 10 Dec 2014 12:21:40 +0100
Subject: [PATCH 0758/1937] release 2014.12.10

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3ac525557..21b7758bf 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.06.1'
+__version__ = '2014.12.10'

From 774e208f94574ef9c62e82ac5ad0c57f245a8752 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 10 Dec 2014 13:21:24 +0100
Subject: [PATCH 0759/1937] [youtube] Handle missing DASH manifest (Fixes
 #4421, fixes #4420)

---
 youtube_dl/extractor/youtube.py | 113 +++++++++++++++++++-------------
 1 file changed, 68 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8b6e591a4..329690742 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -417,6 +417,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'upload_date': '20140605',
             },
         },
+        # video_info is None (https://github.com/rg3/youtube-dl/issues/4421)
+        {
+            'url': '__2ABJjxzNo',
+            'info_dict': {
+                'id': '__2ABJjxzNo',
+                'ext': 'mp4',
+                'upload_date': '20100430',
+                'uploader_id': 'deadmau5',
+                'description': 'md5:12c56784b8032162bb936a5f76d55360',
+                'uploader': 'deadmau5',
+                'title': 'Deadmau5 - Some Chords (HD)',
+            },
+            'expected_warnings': [
+                'DASH manifest missing',
+            ]
+        }
     ]
 
     def __init__(self, *args, **kwargs):
@@ -666,6 +682,45 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
         return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
 
+    def _parse_dash_manifest(self, video_id, dash_manifest_url):
+        def decrypt_sig(mobj):
+            s = mobj.group(1)
+            dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
+            return '/signature/%s' % dec_s
+        dash_manifest_url = re.sub(r'/s/([\w\.]+)', decrypt_sig, dash_manifest_url)
+        dash_doc = self._download_xml(
+            dash_manifest_url, video_id,
+            note='Downloading DASH manifest',
+            errnote='Could not download DASH manifest')
+
+        formats = []
+        for r in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}Representation'):
+            url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
+            if url_el is None:
+                continue
+            format_id = r.attrib['id']
+            video_url = url_el.text
+            filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
+            f = {
+                'format_id': format_id,
+                'url': video_url,
+                'width': int_or_none(r.attrib.get('width')),
+                'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
+                'asr': int_or_none(r.attrib.get('audioSamplingRate')),
+                'filesize': filesize,
+                'fps': int_or_none(r.attrib.get('frameRate')),
+            }
+            try:
+                existing_format = next(
+                    fo for fo in formats
+                    if fo['format_id'] == format_id)
+            except StopIteration:
+                f.update(self._formats.get(format_id, {}))
+                formats.append(f)
+            else:
+                existing_format.update(f)
+        return formats
+
     def _real_extract(self, url):
         proto = (
             'http' if self._downloader.params.get('prefer_insecure', False)
@@ -943,51 +998,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
         # Look for the DASH manifest
         if self._downloader.params.get('youtube_include_dash_manifest', True):
-            try:
-                # The DASH manifest used needs to be the one from the original video_webpage.
-                # The one found in get_video_info seems to be using different signatures.
-                # However, in the case of an age restriction there won't be any embedded dashmpd in the video_webpage.
-                # Luckily, it seems, this case uses some kind of default signature (len == 86), so the
-                # combination of get_video_info and the _static_decrypt_signature() decryption fallback will work here.
-                dash_manifest_url = video_info.get('dashmpd')[0]
-
-                def decrypt_sig(mobj):
-                    s = mobj.group(1)
-                    dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
-                    return '/signature/%s' % dec_s
-                dash_manifest_url = re.sub(r'/s/([\w\.]+)', decrypt_sig, dash_manifest_url)
-                dash_doc = self._download_xml(
-                    dash_manifest_url, video_id,
-                    note='Downloading DASH manifest',
-                    errnote='Could not download DASH manifest')
-                for r in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}Representation'):
-                    url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
-                    if url_el is None:
-                        continue
-                    format_id = r.attrib['id']
-                    video_url = url_el.text
-                    filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
-                    f = {
-                        'format_id': format_id,
-                        'url': video_url,
-                        'width': int_or_none(r.attrib.get('width')),
-                        'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
-                        'asr': int_or_none(r.attrib.get('audioSamplingRate')),
-                        'filesize': filesize,
-                        'fps': int_or_none(r.attrib.get('frameRate')),
-                    }
-                    try:
-                        existing_format = next(
-                            fo for fo in formats
-                            if fo['format_id'] == format_id)
-                    except StopIteration:
-                        f.update(self._formats.get(format_id, {}))
-                        formats.append(f)
-                    else:
-                        existing_format.update(f)
-
-            except (ExtractorError, KeyError) as e:
-                self.report_warning('Skipping DASH manifest: %r' % e, video_id)
+            dash_mpd = video_info.get('dashmpd')
+            if not dash_mpd:
+                self.report_warning('%s: DASH manifest missing' % video_id)
+            else:
+                dash_manifest_url = dash_mpd[0]
+                try:
+                    dash_formats = self._parse_dash_manifest(
+                        video_id, dash_manifest_url)
+                except (ExtractorError, KeyError) as e:
+                    self.report_warning(
+                        'Skipping DASH manifest: %r' % e, video_id)
+                else:
+                    formats.extend(dash_formats)
 
         self._sort_formats(formats)
 

From b8a6114309971ff4fcfd7b617fcf0321a3ecfc1b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 10 Dec 2014 13:21:49 +0100
Subject: [PATCH 0760/1937] release 2014.12.10.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 21b7758bf..bbd779764 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.10'
+__version__ = '2014.12.10.1'

From da276600148ce17714a93ea9f1de8fd38736f468 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 10 Dec 2014 14:39:00 +0100
Subject: [PATCH 0761/1937] [youtube] Pass in all variables to DASH manifest
 (Fixes #4424)

---
 youtube_dl/extractor/youtube.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 329690742..42441d9fa 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -682,7 +682,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
         return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
 
-    def _parse_dash_manifest(self, video_id, dash_manifest_url):
+    def _parse_dash_manifest(
+            self, video_id, dash_manifest_url, player_url, age_gate):
         def decrypt_sig(mobj):
             s = mobj.group(1)
             dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
@@ -1005,7 +1006,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 dash_manifest_url = dash_mpd[0]
                 try:
                     dash_formats = self._parse_dash_manifest(
-                        video_id, dash_manifest_url)
+                        video_id, dash_manifest_url, player_url, age_gate)
                 except (ExtractorError, KeyError) as e:
                     self.report_warning(
                         'Skipping DASH manifest: %r' % e, video_id)

From 5f3544baa36b76b31b030e07ef6c120b1dd432c4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 10 Dec 2014 14:39:06 +0100
Subject: [PATCH 0762/1937] release 2014.12.10.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index bbd779764..4ca1f095a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.10.1'
+__version__ = '2014.12.10.2'

From 2fc9f2b41d362f15c7d61fd2b16c541e53d03b87 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 10 Dec 2014 15:18:34 +0100
Subject: [PATCH 0763/1937] [facebook] Make thumbnail and duration optional

Fixes #4425.
Looks like both properties aren't given to us anymore. For now, just fall back to not returning them.
---
 youtube_dl/extractor/facebook.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 2139f68aa..1ad4e77a8 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -13,9 +13,10 @@ from ..compat import (
     compat_urllib_request,
 )
 from ..utils import (
-    urlencode_postdata,
     ExtractorError,
+    int_or_none,
     limit_length,
+    urlencode_postdata,
 )
 
 
@@ -36,7 +37,6 @@ class FacebookIE(InfoExtractor):
         'info_dict': {
             'id': '637842556329505',
             'ext': 'mp4',
-            'duration': 38,
             'title': 're:Did you know Kei Nishikori is the first Asian man to ever reach a Grand Slam',
         }
     }, {
@@ -107,9 +107,7 @@ class FacebookIE(InfoExtractor):
         self._login()
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         url = 'https://www.facebook.com/video/video.php?v=%s' % video_id
         webpage = self._download_webpage(url, video_id)
 
@@ -149,6 +147,6 @@ class FacebookIE(InfoExtractor):
             'id': video_id,
             'title': video_title,
             'url': video_url,
-            'duration': int(video_data['video_duration']),
-            'thumbnail': video_data['thumbnail_src'],
+            'duration': int_or_none(video_data.get('video_duration')),
+            'thumbnail': video_data.get('thumbnail_src'),
         }

From 72e3ffeb743712c51f476196c1b22f4de23253f5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 10 Dec 2014 15:19:08 +0100
Subject: [PATCH 0764/1937] release 2014.12.10.3

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4ca1f095a..0345b7eb6 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.10.2'
+__version__ = '2014.12.10.3'

From a81bbebf44b6f405fcb3959b3fc2d2dc03b3bb59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Dec 2014 20:22:49 +0600
Subject: [PATCH 0765/1937] [smotri:broadcast] Fix extraction

---
 youtube_dl/extractor/smotri.py | 35 +++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 0751efc61..646af3cc9 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -274,15 +274,18 @@ class SmotriBroadcastIE(InfoExtractor):
         broadcast_page = self._download_webpage(broadcast_url, broadcast_id, 'Downloading broadcast page')
 
         if re.search('>Режиссер с логином <br/>"%s"<br/> <span>не существует<' % broadcast_id, broadcast_page) is not None:
-            raise ExtractorError('Broadcast %s does not exist' % broadcast_id, expected=True)
+            raise ExtractorError(
+                'Broadcast %s does not exist' % broadcast_id, expected=True)
 
         # Adult content
         if re.search('EroConfirmText">', broadcast_page) is not None:
 
             (username, password) = self._get_login_info()
             if username is None:
-                raise ExtractorError('Erotic broadcasts allowed only for registered users, '
-                                     'use --username and --password options to provide account credentials.', expected=True)
+                raise ExtractorError(
+                    'Erotic broadcasts allowed only for registered users, '
+                    'use --username and --password options to provide account credentials.',
+                    expected=True)
 
             login_form = {
                 'login-hint53': '1',
@@ -291,9 +294,11 @@ class SmotriBroadcastIE(InfoExtractor):
                 'password': password,
             }
 
-            request = compat_urllib_request.Request(broadcast_url + '/?no_redirect=1', compat_urllib_parse.urlencode(login_form))
+            request = compat_urllib_request.Request(
+                broadcast_url + '/?no_redirect=1', compat_urllib_parse.urlencode(login_form))
             request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-            broadcast_page = self._download_webpage(request, broadcast_id, 'Logging in and confirming age')
+            broadcast_page = self._download_webpage(
+                request, broadcast_id, 'Logging in and confirming age')
 
             if re.search('>Неверный логин или пароль<', broadcast_page) is not None:
                 raise ExtractorError('Unable to log in: bad username or password', expected=True)
@@ -303,7 +308,7 @@ class SmotriBroadcastIE(InfoExtractor):
             adult_content = False
 
         ticket = self._html_search_regex(
-            'window\.broadcast_control\.addFlashVar\\(\'file\', \'([^\']+)\'\\);',
+            r"window\.broadcast_control\.addFlashVar\('file'\s*,\s*'([^']+)'\)",
             broadcast_page, 'broadcast ticket')
 
         url = 'http://smotri.com/broadcast/view/url/?ticket=%s' % ticket
@@ -312,26 +317,31 @@ class SmotriBroadcastIE(InfoExtractor):
         if broadcast_password:
             url += '&pass=%s' % hashlib.md5(broadcast_password.encode('utf-8')).hexdigest()
 
-        broadcast_json_page = self._download_webpage(url, broadcast_id, 'Downloading broadcast JSON')
+        broadcast_json_page = self._download_webpage(
+            url, broadcast_id, 'Downloading broadcast JSON')
 
         try:
             broadcast_json = json.loads(broadcast_json_page)
 
             protected_broadcast = broadcast_json['_pass_protected'] == 1
             if protected_broadcast and not broadcast_password:
-                raise ExtractorError('This broadcast is protected by a password, use the --video-password option', expected=True)
+                raise ExtractorError(
+                    'This broadcast is protected by a password, use the --video-password option',
+                    expected=True)
 
             broadcast_offline = broadcast_json['is_play'] == 0
             if broadcast_offline:
                 raise ExtractorError('Broadcast %s is offline' % broadcast_id, expected=True)
 
             rtmp_url = broadcast_json['_server']
-            if not rtmp_url.startswith('rtmp://'):
+            mobj = re.search(r'^rtmp://[^/]+/(?P<app>.+)/?$', rtmp_url)
+            if not mobj:
                 raise ExtractorError('Unexpected broadcast rtmp URL')
 
             broadcast_playpath = broadcast_json['_streamName']
+            broadcast_app = '%s/%s' % (mobj.group('app'), broadcast_json['_vidURL'])
             broadcast_thumbnail = broadcast_json['_imgURL']
-            broadcast_title = broadcast_json['title']
+            broadcast_title = self._live_title(broadcast_json['title'])
             broadcast_description = broadcast_json['description']
             broadcaster_nick = broadcast_json['nick']
             broadcaster_login = broadcast_json['login']
@@ -352,6 +362,9 @@ class SmotriBroadcastIE(InfoExtractor):
             'age_limit': 18 if adult_content else 0,
             'ext': 'flv',
             'play_path': broadcast_playpath,
+            'player_url': 'http://pics.smotri.com/broadcast_play.swf',
+            'app': broadcast_app,
             'rtmp_live': True,
-            'rtmp_conn': rtmp_conn
+            'rtmp_conn': rtmp_conn,
+            'is_live': True,
         }

From 8560c618422214745caa4766292afa9c0d3f35c2 Mon Sep 17 00:00:00 2001
From: Adrian Kretz <adriankretz@gmail.com>
Date: Wed, 10 Dec 2014 17:29:03 +0100
Subject: [PATCH 0766/1937] [zdf] Add support for channels

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/zdf.py      | 85 ++++++++++++++++++++++++++++++--
 2 files changed, 83 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9f2dc803b..027fb06f9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -526,7 +526,7 @@ from .youtube import (
     YoutubeUserIE,
     YoutubeWatchLaterIE,
 )
-from .zdf import ZDFIE
+from .zdf import ZDFIE, ZDFChannelIE
 from .zingmp3 import (
     ZingMp3SongIE,
     ZingMp3AlbumIE,
diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 9ff00e26c..221f16686 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -86,6 +86,26 @@ def extract_from_xml_url(ie, video_id, xml_url):
     }
 
 
+def extract_channel_from_xml_url(ie, channel_id, xml_url):
+    doc = ie._download_xml(
+        xml_url, channel_id,
+        note='Downloading channel info',
+        errnote='Failed to download channel info')
+
+    title = doc.find('.//information/title').text
+    description = doc.find('.//information/detail').text
+    assets = [{'id': asset.find('./details/assetId').text,
+               'type': asset.find('./type').text,
+               } for asset in doc.findall('.//teasers/teaser')]
+
+    return {
+        'id': channel_id,
+        'title': title,
+        'description': description,
+        'assets': assets,
+    }
+
+
 class ZDFIE(InfoExtractor):
     _VALID_URL = r'^https?://www\.zdf\.de/ZDFmediathek(?P<hash>#)?/(.*beitrag/(?:video/)?)(?P<id>[0-9]+)(?:/[^/?]+)?(?:\?.*)?'
 
@@ -104,8 +124,67 @@ class ZDFIE(InfoExtractor):
         'skip': 'Videos on ZDF.de are depublicised in short order',
     }
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
+    def _extract_video(self, video_id):
         xml_url = 'http://www.zdf.de/ZDFmediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id
         return extract_from_xml_url(self, video_id, xml_url)
+
+    def _real_extract(self, url):
+        return self._extract_video(self._match_id(url))
+
+
+class ZDFChannelIE(ZDFIE):
+    _VALID_URL = r'^https?://www\.zdf\.de/ZDFmediathek(?P<hash>#)?/(.*kanaluebersicht/)(?P<id>[0-9]+)'
+
+    _TEST = {
+        'url': 'http://www.zdf.de/ZDFmediathek#/kanaluebersicht/1586442/sendung/Titanic',
+        'info_dict': {
+            'id': '1586442',
+            'title': 'Titanic',
+            'description': 'md5:444c048cfe3fdc2561be7de4bcbf1d04',
+        },
+        'playlist_count': 3,
+    }
+
+    def _extract_channel(self, channel_id):
+        def load_chunks(channel_id, chunk_length):
+            offset = 0
+            while True:
+                url = ('http://www.zdf.de/ZDFmediathek/xmlservice/web/aktuellste?ak=web&offset=%d&maxLength=%d&id=%s'
+                       % (offset, chunk_length, channel_id))
+                result = extract_channel_from_xml_url(self, channel_id, url)
+                yield result
+                if len(result['assets']) < chunk_length:
+                    return
+                offset += chunk_length
+
+        def load_channel(channel_id):
+            chunks = list(load_chunks(channel_id, 50))  # The server rejects higher values
+            assets = [asset for chunk in chunks for asset in chunk['assets']]
+            video_ids = [asset['id'] for asset in
+                         filter(lambda asset: asset['type'] == 'video',
+                                assets)]
+            topic_ids = [asset['id'] for asset in
+                         filter(lambda asset: asset['type'] == 'thema',
+                                assets)]
+            if topic_ids:
+                video_ids = reduce(list.__add__,
+                                   [load_channel(topic_id)['video_ids']
+                                    for topic_id in topic_ids],
+                                   video_ids)
+
+            result = chunks[0]
+            result['video_ids'] = video_ids
+            return result
+
+        channel = load_channel(channel_id)
+        return {
+            '_type': 'playlist',
+            'id': channel['id'],
+            'title': channel['title'],
+            'description': channel['description'],
+            'entries': [self._extract_video(video_id)
+                        for video_id in channel['video_ids']],
+        }
+
+    def _real_extract(self, url):
+        return self._extract_channel(self._match_id(url))

From 4bb4a18876f5489db77365528638da8d46890a38 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 11 Dec 2014 10:08:17 +0100
Subject: [PATCH 0767/1937] [youtube] Fix imports

---
 youtube_dl/extractor/youtube.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 42441d9fa..6216028cf 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -14,23 +14,24 @@ from .common import InfoExtractor, SearchInfoExtractor
 from .subtitles import SubtitlesInfoExtractor
 from ..jsinterp import JSInterpreter
 from ..swfinterp import SWFInterpreter
-from ..utils import (
+from ..compat import (
     compat_chr,
     compat_parse_qs,
     compat_urllib_parse,
     compat_urllib_request,
     compat_urlparse,
     compat_str,
-
+)
+from ..utils import (
     clean_html,
-    get_element_by_id,
-    get_element_by_attribute,
     ExtractorError,
+    get_element_by_attribute,
+    get_element_by_id,
     int_or_none,
     OnDemandPagedList,
+    orderedSet,
     unescapeHTML,
     unified_strdate,
-    orderedSet,
     uppercase_escape,
 )
 

From f8795e102b35198db550432c2eee43a1c2571093 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 11 Dec 2014 10:29:30 +0100
Subject: [PATCH 0768/1937] [utils] Add "yesterday" as a date keyword

---
 youtube_dl/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a95d2c942..75f9594e6 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -712,8 +712,10 @@ def date_from_str(date_str):
     Return a datetime object from a string in the format YYYYMMDD or
     (now|today)[+-][0-9](day|week|month|year)(s)?"""
     today = datetime.date.today()
-    if date_str == 'now'or date_str == 'today':
+    if date_str in ('now', 'today'):
         return today
+    if date_str == 'yesterday':
+        return today - datetime.timedelta(days=1)
     match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
     if match is not None:
         sign = match.group('sign')

From e0b9d47387327fb8d25d5bb2f2f4f80a3b77a4a8 Mon Sep 17 00:00:00 2001
From: 5moufl <malouito@gmail.com>
Date: Thu, 11 Dec 2014 14:25:26 +0100
Subject: [PATCH 0769/1937] [BehindKink] Update  URL extraction

---
 youtube_dl/extractor/behindkink.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/behindkink.py b/youtube_dl/extractor/behindkink.py
index 31fdc0dcc..84d8ef849 100644
--- a/youtube_dl/extractor/behindkink.py
+++ b/youtube_dl/extractor/behindkink.py
@@ -34,8 +34,7 @@ class BehindKinkIE(InfoExtractor):
         webpage = self._download_webpage(url, display_id)
 
         video_url = self._search_regex(
-            r"'file':\s*'([^']+)'",
-            webpage, 'URL base')
+            r'<source src="(.*?)" type="video/mp4" />', webpage, 'video URL')
 
         video_id = url_basename(video_url)
         video_id = video_id.split('_')[0]

From d6f78a354d77d32c9929ca99cb82d49aa7d7f8ab Mon Sep 17 00:00:00 2001
From: 5moufl <malouito@gmail.com>
Date: Thu, 11 Dec 2014 14:26:59 +0100
Subject: [PATCH 0770/1937] [BehindKink] Replace test

Old one is not accessible anymore
---
 youtube_dl/extractor/behindkink.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/behindkink.py b/youtube_dl/extractor/behindkink.py
index 84d8ef849..9e4f1c90f 100644
--- a/youtube_dl/extractor/behindkink.py
+++ b/youtube_dl/extractor/behindkink.py
@@ -10,15 +10,15 @@ from ..utils import url_basename
 class BehindKinkIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?behindkink\.com/(?P<year>[0-9]{4})/(?P<month>[0-9]{2})/(?P<day>[0-9]{2})/(?P<id>[^/#?_]+)'
     _TEST = {
-        'url': 'http://www.behindkink.com/2014/08/14/ab1576-performers-voice-finally-heard-the-bill-is-killed/',
-        'md5': '41ad01222b8442089a55528fec43ec01',
+        'url': 'http://www.behindkink.com/2014/12/05/what-are-you-passionate-about-marley-blaze/',
+        'md5': '507b57d8fdcd75a41a9a7bdb7989c762',
         'info_dict': {
-            'id': '36370',
+            'id': '37127',
             'ext': 'mp4',
-            'title': 'AB1576 - PERFORMERS VOICE FINALLY HEARD - THE BILL IS KILLED!',
-            'description': 'The adult industry voice was finally heard as Assembly Bill 1576 remained\xa0 in suspense today at the Senate Appropriations Hearing. AB1576 was, among other industry damaging issues, a condom mandate...',
-            'upload_date': '20140814',
-            'thumbnail': 'http://www.behindkink.com/wp-content/uploads/2014/08/36370_AB1576_Win.jpg',
+            'title': 'What are you passionate about – Marley Blaze',
+            'description': 'Getting a better understanding of the talent that comes through the doors of the Armory is one of our missions at Behind Kink. Asking the question what are you passionate about helps us get a littl...',
+            'upload_date': '20141205',
+            'thumbnail': 'http://www.behindkink.com/wp-content/uploads/2014/12/blaze-1.jpg',
             'age_limit': 18,
         }
     }

From 09c82fbc9a6fc4c5dbb3b84031f31b39e1f9cee2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Dec 2014 20:06:19 +0600
Subject: [PATCH 0771/1937] [behindkink] Simplify

---
 youtube_dl/extractor/behindkink.py | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/behindkink.py b/youtube_dl/extractor/behindkink.py
index 9e4f1c90f..8c947526f 100644
--- a/youtube_dl/extractor/behindkink.py
+++ b/youtube_dl/extractor/behindkink.py
@@ -16,7 +16,7 @@ class BehindKinkIE(InfoExtractor):
             'id': '37127',
             'ext': 'mp4',
             'title': 'What are you passionate about – Marley Blaze',
-            'description': 'Getting a better understanding of the talent that comes through the doors of the Armory is one of our missions at Behind Kink. Asking the question what are you passionate about helps us get a littl...',
+            'description': 'md5:aee8e9611b4ff70186f752975d9b94b4',
             'upload_date': '20141205',
             'thumbnail': 'http://www.behindkink.com/wp-content/uploads/2014/12/blaze-1.jpg',
             'age_limit': 18,
@@ -26,25 +26,19 @@ class BehindKinkIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         display_id = mobj.group('id')
-        year = mobj.group('year')
-        month = mobj.group('month')
-        day = mobj.group('day')
-        upload_date = year + month + day
 
         webpage = self._download_webpage(url, display_id)
 
         video_url = self._search_regex(
-            r'<source src="(.*?)" type="video/mp4" />', webpage, 'video URL')
-
-        video_id = url_basename(video_url)
-        video_id = video_id.split('_')[0]
+            r'<source src="([^"]+)"', webpage, 'video URL')
+        video_id =  url_basename(video_url).split('_')[0]
+        upload_date = mobj.group('year') + mobj.group('month') + mobj.group('day')
 
         return {
             'id': video_id,
-            'url': video_url,
-            'ext': 'mp4',
-            'title': self._og_search_title(webpage),
             'display_id': display_id,
+            'url': video_url,
+            'title': self._og_search_title(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
             'description': self._og_search_description(webpage),
             'upload_date': upload_date,

From bf671b605e013db8ecfa539bfe03689db830448b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 11 Dec 2014 20:09:52 +0600
Subject: [PATCH 0772/1937] [behindkink] Remove superfluous whitespace

---
 youtube_dl/extractor/behindkink.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/behindkink.py b/youtube_dl/extractor/behindkink.py
index 8c947526f..1bdc25812 100644
--- a/youtube_dl/extractor/behindkink.py
+++ b/youtube_dl/extractor/behindkink.py
@@ -31,7 +31,7 @@ class BehindKinkIE(InfoExtractor):
 
         video_url = self._search_regex(
             r'<source src="([^"]+)"', webpage, 'video URL')
-        video_id =  url_basename(video_url).split('_')[0]
+        video_id = url_basename(video_url).split('_')[0]
         upload_date = mobj.group('year') + mobj.group('month') + mobj.group('day')
 
         return {

From 3c1b81b95735bf8fb065ba7a32f220c37b8a10dc Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 11 Dec 2014 16:58:45 +0200
Subject: [PATCH 0773/1937] [ntv] Rename `flash_ver` to `flash_version` in the
 format dict

RTMP downloader uses `flash_version`
---
 youtube_dl/extractor/ntv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ntv.py b/youtube_dl/extractor/ntv.py
index 13c8d79cd..ee740cd9c 100644
--- a/youtube_dl/extractor/ntv.py
+++ b/youtube_dl/extractor/ntv.py
@@ -130,7 +130,7 @@ class NTVIE(InfoExtractor):
                 'rtmp_conn': 'B:1',
                 'player_url': 'http://www.ntv.ru/swf/vps1.swf?update=20131128',
                 'page_url': 'http://www.ntv.ru',
-                'flash_ver': 'LNX 11,2,202,341',
+                'flash_version': 'LNX 11,2,202,341',
                 'rtmp_live': True,
                 'ext': 'flv',
                 'filesize': int(size.text),

From 24270b0301aacc307688d1da71ee0de6e24f3197 Mon Sep 17 00:00:00 2001
From: Adrian Kretz <adriankretz@gmail.com>
Date: Wed, 10 Dec 2014 21:51:05 +0100
Subject: [PATCH 0774/1937] [youtube] The case that
 'url_encoded_fmt_stream_map' or 'adaptive_fmts' is the empty string is
 handled accordingly (fixes #4431)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 42441d9fa..faac98ce0 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -934,7 +934,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'url': video_info['conn'][0],
                 'player_url': player_url,
             }]
-        elif len(video_info.get('url_encoded_fmt_stream_map', [])) >= 1 or len(video_info.get('adaptive_fmts', [])) >= 1:
+        elif len(video_info.get('url_encoded_fmt_stream_map', [''])[0]) >= 1 or len(video_info.get('adaptive_fmts', [''])[0]) >= 1:
             encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
             if 'rtmpe%3Dyes' in encoded_url_map:
                 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)

From 158731f83e2af39b3ecd8af3eaf4ce5b26ff6908 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 11 Dec 2014 17:07:50 +0200
Subject: [PATCH 0775/1937] [tvplay] Don't raise an exception if
 `is_geo_blocked` is True

Videos which return `is_geo_blocked' to be True can actually be downloaded from
the country to which the video is restricted
---
 youtube_dl/extractor/tvplay.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index eb9473754..0157392cc 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -182,8 +182,8 @@ class TVPlayIE(InfoExtractor):
             'http://playapi.mtgx.tv/v1/videos/%s' % video_id, video_id, 'Downloading video JSON')
 
         if video['is_geo_blocked']:
-            raise ExtractorError(
-                'This content is not available in your country due to copyright reasons', expected=True)
+            self.report_warning(
+                'This content might not be available in your country due to copyright reasons')
 
         streams = self._download_json(
             'http://playapi.mtgx.tv/v1/videos/stream/%s' % video_id, video_id, 'Downloading streams JSON')

From e52a40abf7519ba38a0607ea2069e39d684a12e2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 11 Dec 2014 16:28:07 +0100
Subject: [PATCH 0776/1937] [youtube] Add test case for #4431

---
 youtube_dl/extractor/youtube.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6216028cf..4f6898c1a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -433,6 +433,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             'expected_warnings': [
                 'DASH manifest missing',
             ]
+        },
+        # Olympics (https://github.com/rg3/youtube-dl/issues/4431)
+        {
+            'url': 'lqQg6PlCWgI',
+            'info_dict': {
+                'id': 'lqQg6PlCWgI',
+                'ext': 'mp4',
+                
+            }
         }
     ]
 

From 624dcebff617d3e995d1f0ddb27d604a881b2de2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 11 Dec 2014 16:32:33 +0100
Subject: [PATCH 0777/1937] [youtube] Make category optional (#4442)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f8d5cdd5d..003aeaee6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -866,7 +866,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
         m_cat_container = self._search_regex(
             r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
-            video_webpage, 'categories', fatal=False)
+            video_webpage, 'categories', default=None)
         if m_cat_container:
             category = self._html_search_regex(
                 r'(?s)<a[^<]+>(.*?)</a>', m_cat_container, 'category',

From 75111274edeec5f7088730b7f9c5c623dae77f28 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 11 Dec 2014 16:33:28 +0100
Subject: [PATCH 0778/1937] [youtube] Do not warn if DASH manifest is missing
 (#4442)

---
 youtube_dl/extractor/youtube.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 003aeaee6..f41086762 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1010,9 +1010,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         # Look for the DASH manifest
         if self._downloader.params.get('youtube_include_dash_manifest', True):
             dash_mpd = video_info.get('dashmpd')
-            if not dash_mpd:
-                self.report_warning('%s: DASH manifest missing' % video_id)
-            else:
+            if dash_mpd:
                 dash_manifest_url = dash_mpd[0]
                 try:
                     dash_formats = self._parse_dash_manifest(

From cbe2bd914d65fc717d31034234102cd25c7f7313 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 11 Dec 2014 16:34:37 +0100
Subject: [PATCH 0779/1937] [youtube] Amend test

---
 youtube_dl/extractor/youtube.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f41086762..7b6179a2a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -440,9 +440,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             'info_dict': {
                 'id': 'lqQg6PlCWgI',
                 'ext': 'mp4',
-                
+                'upload_date': '20120731',
+                'uploader_id': 'olympic',
+                'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
+                'uploader': 'Olympics',
+                'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
+            },
+            'params': {
+                'skip_download': 'requires avconv',
             }
-        }
+        },
     ]
 
     def __init__(self, *args, **kwargs):

From 9abd500a7402fe1f66e4177b61a7327b416b6b9d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 11 Dec 2014 17:07:59 +0100
Subject: [PATCH 0780/1937] [zdf:channel] Simplify (#4427)

---
 youtube_dl/extractor/zdf.py | 111 +++++++++++++-----------------------
 1 file changed, 40 insertions(+), 71 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 221f16686..74c76a9a0 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -1,12 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import functools
 import re
 
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
     unified_strdate,
+    OnDemandPagedList,
 )
 
 
@@ -86,28 +88,8 @@ def extract_from_xml_url(ie, video_id, xml_url):
     }
 
 
-def extract_channel_from_xml_url(ie, channel_id, xml_url):
-    doc = ie._download_xml(
-        xml_url, channel_id,
-        note='Downloading channel info',
-        errnote='Failed to download channel info')
-
-    title = doc.find('.//information/title').text
-    description = doc.find('.//information/detail').text
-    assets = [{'id': asset.find('./details/assetId').text,
-               'type': asset.find('./type').text,
-               } for asset in doc.findall('.//teasers/teaser')]
-
-    return {
-        'id': channel_id,
-        'title': title,
-        'description': description,
-        'assets': assets,
-    }
-
-
 class ZDFIE(InfoExtractor):
-    _VALID_URL = r'^https?://www\.zdf\.de/ZDFmediathek(?P<hash>#)?/(.*beitrag/(?:video/)?)(?P<id>[0-9]+)(?:/[^/?]+)?(?:\?.*)?'
+    _VALID_URL = r'(?:zdf:|zdf:video:|https?://www\.zdf\.de/ZDFmediathek(?:#)?/(.*beitrag/(?:video/)?))(?P<id>[0-9]+)(?:/[^/?]+)?(?:\?.*)?'
 
     _TEST = {
         'url': 'http://www.zdf.de/ZDFmediathek/beitrag/video/2037704/ZDFspezial---Ende-des-Machtpokers--?bc=sts;stt',
@@ -124,67 +106,54 @@ class ZDFIE(InfoExtractor):
         'skip': 'Videos on ZDF.de are depublicised in short order',
     }
 
-    def _extract_video(self, video_id):
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
         xml_url = 'http://www.zdf.de/ZDFmediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id
         return extract_from_xml_url(self, video_id, xml_url)
 
-    def _real_extract(self, url):
-        return self._extract_video(self._match_id(url))
-
-
-class ZDFChannelIE(ZDFIE):
-    _VALID_URL = r'^https?://www\.zdf\.de/ZDFmediathek(?P<hash>#)?/(.*kanaluebersicht/)(?P<id>[0-9]+)'
 
+class ZDFChannelIE(InfoExtractor):
+    _VALID_URL = r'(?:zdf:topic:|https?://www\.zdf\.de/ZDFmediathek(?:#)?/.*kanaluebersicht/)(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.zdf.de/ZDFmediathek#/kanaluebersicht/1586442/sendung/Titanic',
         'info_dict': {
             'id': '1586442',
-            'title': 'Titanic',
-            'description': 'md5:444c048cfe3fdc2561be7de4bcbf1d04',
         },
-        'playlist_count': 3,
+        'playlist_count': 4,
     }
+    _PAGE_SIZE = 50
 
-    def _extract_channel(self, channel_id):
-        def load_chunks(channel_id, chunk_length):
-            offset = 0
-            while True:
-                url = ('http://www.zdf.de/ZDFmediathek/xmlservice/web/aktuellste?ak=web&offset=%d&maxLength=%d&id=%s'
-                       % (offset, chunk_length, channel_id))
-                result = extract_channel_from_xml_url(self, channel_id, url)
-                yield result
-                if len(result['assets']) < chunk_length:
-                    return
-                offset += chunk_length
+    def _fetch_page(self, channel_id, page):
+        offset = page * self._PAGE_SIZE
+        xml_url = (
+            'http://www.zdf.de/ZDFmediathek/xmlservice/web/aktuellste?ak=web&offset=%d&maxLength=%d&id=%s'
+            % (offset, self._PAGE_SIZE, channel_id))
+        doc = self._download_xml(
+            xml_url, channel_id,
+            note='Downloading channel info',
+            errnote='Failed to download channel info')
 
-        def load_channel(channel_id):
-            chunks = list(load_chunks(channel_id, 50))  # The server rejects higher values
-            assets = [asset for chunk in chunks for asset in chunk['assets']]
-            video_ids = [asset['id'] for asset in
-                         filter(lambda asset: asset['type'] == 'video',
-                                assets)]
-            topic_ids = [asset['id'] for asset in
-                         filter(lambda asset: asset['type'] == 'thema',
-                                assets)]
-            if topic_ids:
-                video_ids = reduce(list.__add__,
-                                   [load_channel(topic_id)['video_ids']
-                                    for topic_id in topic_ids],
-                                   video_ids)
-
-            result = chunks[0]
-            result['video_ids'] = video_ids
-            return result
-
-        channel = load_channel(channel_id)
-        return {
-            '_type': 'playlist',
-            'id': channel['id'],
-            'title': channel['title'],
-            'description': channel['description'],
-            'entries': [self._extract_video(video_id)
-                        for video_id in channel['video_ids']],
-        }
+        title = doc.find('.//information/title').text
+        description = doc.find('.//information/detail').text
+        for asset in doc.findall('.//teasers/teaser'):
+            a_type = asset.find('./type').text
+            a_id = asset.find('./details/assetId').text
+            if a_type not in ('video', 'topic'):
+                continue
+            yield {
+                '_type': 'url',
+                'playlist_title': title,
+                'playlist_description': description,
+                'url': 'zdf:%s:%s' % (a_type, a_id),
+            }
 
     def _real_extract(self, url):
-        return self._extract_channel(self._match_id(url))
+        channel_id = self._match_id(url)
+        entries = OnDemandPagedList(
+            functools.partial(self._fetch_page, channel_id), self._PAGE_SIZE)
+
+        return {
+            '_type': 'playlist',
+            'id': channel_id,
+            'entries': entries,
+        }

From cb007f47c1685e1d17a8fc7be7920293610290a6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 11 Dec 2014 17:08:31 +0100
Subject: [PATCH 0781/1937] release 2014.12.11

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0345b7eb6..b2a285bb2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.10.3'
+__version__ = '2014.12.11'

From 9c1aa1d66819ebce6e6cce42da3065ec91720620 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 11 Dec 2014 23:16:40 +0100
Subject: [PATCH 0782/1937] [mixcloud] Fix metadata extraction (fixes #4443)

---
 youtube_dl/extractor/mixcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index bb8937c4d..55cc33a3e 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -70,7 +70,7 @@ class MixcloudIE(InfoExtractor):
             raise ExtractorError('Unable to extract track url')
 
         PREFIX = (
-            r'<div class="cloudcast-play-button-container[^"]*?"'
+            r'<span class="play-button[^"]*?"'
             r'(?:\s+[a-zA-Z0-9-]+(?:="[^"]+")?)*?\s+')
         title = self._html_search_regex(
             PREFIX + r'm-title="([^"]+)"', webpage, 'title')

From 807962f4a14c08d397dd7a015aaa3f8948f6af82 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 11 Dec 2014 23:50:25 +0100
Subject: [PATCH 0783/1937] [pornhd] Adapt to new sources scheme (Fixes #4446)

---
 youtube_dl/extractor/pornhd.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index bac484c67..63472b33e 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -45,13 +45,18 @@ class PornHdIE(InfoExtractor):
         thumbnail = self._search_regex(
             r"'poster'\s*:\s*'([^']+)'", webpage, 'thumbnail', fatal=False)
 
-        quality = qualities(['SD', 'HD'])
-        formats = [{
-            'url': source['file'],
-            'format_id': '%s-%s' % (source['label'], determine_ext(source['file'])),
-            'quality': quality(source['label']),
-        } for source in json.loads(js_to_json(self._search_regex(
-            r"(?s)'sources'\s*:\s*(\[.+?\])", webpage, 'sources')))]
+        quality = qualities(['sd', 'hd'])
+        sources = json.loads(js_to_json(self._search_regex(
+            r"(?s)'sources'\s*:\s*(\{.+?\})\s*\}\);", webpage, 'sources')))
+        formats = []
+        for container, s in sources.items():
+            for qname, video_url in s.items():
+                formats.append({
+                    'url': video_url,
+                    'container': container,
+                    'format_id': '%s-%s' % (container, qname),
+                    'quality': quality(qname),
+                })
         self._sort_formats(formats)
 
         return {

From f17e4c9c285470af3404532520599f914e73014c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 02:11:58 +0100
Subject: [PATCH 0784/1937] [screenwavemedia] Simplify (#3766)

---
 youtube_dl/YoutubeDL.py                 |  24 +--
 youtube_dl/extractor/__init__.py        |   2 +-
 youtube_dl/extractor/screenwavemedia.py | 220 +++++++++++-------------
 3 files changed, 113 insertions(+), 133 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 248367039..31531855e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -622,23 +622,17 @@ class YoutubeDL(object):
                 ie_result['url'], ie_key=ie_result.get('ie_key'),
                 extra_info=extra_info, download=False, process=False)
 
-            def make_result(embedded_info):
-                new_result = ie_result.copy()
-                for f in ('_type', 'url', 'ext', 'player_url', 'formats',
-                          'entries', 'ie_key', 'duration',
-                          'subtitles', 'annotations', 'format',
-                          'thumbnail', 'thumbnails'):
-                    if f in new_result:
-                        del new_result[f]
-                    if f in embedded_info:
-                        new_result[f] = embedded_info[f]
-                return new_result
-            new_result = make_result(info)
+            new_result = ie_result.copy()
+            for f in ('_type', 'id', 'url', 'ext', 'player_url', 'formats',
+                      'entries', 'ie_key', 'duration',
+                      'subtitles', 'annotations', 'format',
+                      'thumbnail', 'thumbnails'):
+                if f in new_result:
+                    del new_result[f]
+                if f in info:
+                    new_result[f] = info[f]
 
             assert new_result.get('_type') != 'url_transparent'
-            if new_result.get('_type') == 'compat_list':
-                new_result['entries'] = [
-                    make_result(e) for e in new_result['entries']]
 
             return self.process_ie_result(
                 new_result, download=download, extra_info=extra_info)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6b7660ab1..982a134bf 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -335,7 +335,7 @@ from .savefrom import SaveFromIE
 from .sbs import SBSIE
 from .scivee import SciVeeIE
 from .screencast import ScreencastIE
-from .screenwavemedia import ScreenwaveMediaIE
+from .screenwavemedia import CinemassacreIE, ScreenwaveMediaIE, TeamFourIE
 from .servingsys import ServingSysIE
 from .sexu import SexuIE
 from .sexykarma import SexyKarmaIE
diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index cf9b8e0de..108190471 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -6,109 +6,28 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    month_by_name,
     int_or_none,
+    month_by_name,
+    unified_strdate,
 )
 
+
 class ScreenwaveMediaIE(InfoExtractor):
-    _VALID_URL = r'(?:http://)?(?' \
-        r':(?P<generic>player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?P<video_id>.+))' \
-        r'|(?P<cinemassacre>(?:www\.)?cinemassacre\.com/(?P<cm_date_Y>[0-9]{4})/(?P<cm_date_m>[0-9]{2})/(?P<cm_date_d>[0-9]{2})/(?P<cm_display_id>[^?#/]+))' \
-        r'|(?P<teamfourstar>(?:www\.)?teamfourstar\.com/video/(?P<tfs_display_id>[a-z0-9\-]+)/?)' \
-        r')'
+    _VALID_URL = r'http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=(?P<id>.+)'
 
-    _TESTS = [
-        {
-            'url': 'http://cinemassacre.com/2012/11/10/avgn-the-movie-trailer/',
-            'md5': 'fde81fbafaee331785f58cd6c0d46190',
-            'info_dict': {
-                'id': 'Cinemasssacre-19911',
-                'ext': 'mp4',
-                'upload_date': '20121110',
-                'title': '“Angry Video Game Nerd: The Movie” – Trailer',
-                'description': 'md5:fb87405fcb42a331742a0dce2708560b',
-            },
-        },
-        {
-            'url': 'http://cinemassacre.com/2013/10/02/the-mummys-hand-1940',
-            'md5': 'd72f10cd39eac4215048f62ab477a511',
-            'info_dict': {
-                'id': 'Cinemasssacre-521be8ef82b16',
-                'ext': 'mp4',
-                'upload_date': '20131002',
-                'title': 'The Mummy’s Hand (1940)',
-            },
-        }
-    ]
-
-    def _cinemassacre_get_info(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('cm_display_id')
-
-        webpage = self._download_webpage(url, display_id)
-        video_date = mobj.group('cm_date_Y') + mobj.group('cm_date_m') + mobj.group('cm_date_d')
-        mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"', webpage)
-        if not mobj:
-            raise ExtractorError('Can\'t extract embed url and video id')
-        playerdata_url = mobj.group('embed_url')
-
-        video_title = self._html_search_regex(
-            r'<title>(?P<title>.+?)\|', webpage, 'title')
-        video_description = self._html_search_regex(
-            r'<div class="entry-content">(?P<description>.+?)</div>',
-            webpage, 'description', flags=re.DOTALL, fatal=False)
-        video_thumbnail = self._og_search_thumbnail(webpage)
-
-        return {
-            'title': video_title,
-            'description': video_description,
-            'upload_date': video_date,
-            'thumbnail': video_thumbnail,
-            '_embed_url': playerdata_url,
-        }
-
-    def _teamfourstar_get_info(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('tfs_display_id')
-        webpage = self._download_webpage(url, display_id)
-
-        mobj = re.search(r'src="(?P<embed_url>http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"', webpage)
-        if not mobj:
-            raise ExtractorError('Can\'t extract embed url and video id')
-        playerdata_url = mobj.group('embed_url')
-
-        video_title = self._html_search_regex(
-            r'<div class="heroheadingtitle">(?P<title>.+?)</div>', webpage, 'title')
-        video_date = self._html_search_regex(
-            r'<div class="heroheadingdate">(?P<date>.+?)</div>', webpage, 'date')
-        mobj = re.match('(?P<month>[A-Z][a-z]+) (?P<day>\d+), (?P<year>\d+)', video_date)
-        video_date = '%04u%02u%02u' % (int(mobj.group('year')), month_by_name(mobj.group('month')), int(mobj.group('day')))
-        video_description = self._html_search_regex(
-            r'<div class="postcontent">(?P<description>.+?)</div>', webpage, 'description', flags=re.DOTALL)
-        video_thumbnail = self._og_search_thumbnail(webpage)
-
-        return {
-            'title': video_title,
-            'description': video_description,
-            'upload_date': video_date,
-            'thumbnail': video_thumbnail,
-            '_embed_url': playerdata_url,
-        }
-
-    def _screenwavemedia_get_info(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        if not mobj:
-            raise ExtractorError('Can\'t extract embed url and video id')
-        video_id = mobj.group('video_id')
+    _TESTS = [{
+        'url': 'http://player.screenwavemedia.com/play/play.php?playerdiv=videoarea&companiondiv=squareAd&id=Cinemassacre-19911',
+        'only_matching': True,
+    }]
 
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
         playerdata = self._download_webpage(url, video_id, 'Downloading player webpage')
 
         vidtitle = self._search_regex(
-            r'\'vidtitle\'\s*:\s*"([^\']+)"', playerdata, 'vidtitle').replace('\\/', '/')
+            r'\'vidtitle\'\s*:\s*"([^"]+)"', playerdata, 'vidtitle').replace('\\/', '/')
         vidurl = self._search_regex(
-            r'\'vidurl\'\s*:\s*"([^\']+)"', playerdata, 'vidurl').replace('\\/', '/')
-        pageurl = self._search_regex(
-            r'\'pageurl\'\s*:\s*"([^\']+)"', playerdata, 'pageurl', fatal=False).replace('\\/', '/')
+            r'\'vidurl\'\s*:\s*"([^"]+)"', playerdata, 'vidurl').replace('\\/', '/')
 
         videolist_url = None
 
@@ -134,61 +53,128 @@ class ScreenwaveMediaIE(InfoExtractor):
                 file_ = src.partition(':')[-1]
                 width = int_or_none(video.get('width'))
                 height = int_or_none(video.get('height'))
-                bitrate = int_or_none(video.get('system-bitrate'))
+                bitrate = int_or_none(video.get('system-bitrate'), scale=1000)
                 format = {
                     'url': baseurl + file_,
                     'format_id': src.rpartition('.')[0].rpartition('_')[-1],
                 }
                 if width or height:
                     format.update({
-                        'tbr': bitrate // 1000 if bitrate else None,
+                        'tbr': bitrate,
                         'width': width,
                         'height': height,
                     })
                 else:
                     format.update({
-                        'abr': bitrate // 1000 if bitrate else None,
+                        'abr': bitrate,
                         'vcodec': 'none',
                     })
                 formats.append(format)
-            self._sort_formats(formats)
         else:
             formats = [{
                 'url': vidurl,
             }]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': vidtitle,
             'formats': formats,
-            '_episode_page': pageurl,
         }
 
+
+class CinemassacreIE(InfoExtractor):
+    _VALID_URL = 'https?://(?:www\.)?cinemassacre\.com/(?P<date_y>[0-9]{4})/(?P<date_m>[0-9]{2})/(?P<date_d>[0-9]{2})/(?P<display_id>[^?#/]+)'
+    _TESTS = [
+        {
+            'url': 'http://cinemassacre.com/2012/11/10/avgn-the-movie-trailer/',
+            'md5': 'fde81fbafaee331785f58cd6c0d46190',
+            'info_dict': {
+                'id': 'Cinemassacre-19911',
+                'ext': 'mp4',
+                'upload_date': '20121110',
+                'title': '“Angry Video Game Nerd: The Movie” – Trailer',
+                'description': 'md5:fb87405fcb42a331742a0dce2708560b',
+            },
+        },
+        {
+            'url': 'http://cinemassacre.com/2013/10/02/the-mummys-hand-1940',
+            'md5': 'd72f10cd39eac4215048f62ab477a511',
+            'info_dict': {
+                'id': 'Cinemassacre-521be8ef82b16',
+                'ext': 'mp4',
+                'upload_date': '20131002',
+                'title': 'The Mummy’s Hand (1940)',
+            },
+        }
+    ]
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('display_id')
+        video_date = mobj.group('date_y') + mobj.group('date_m') + mobj.group('date_d')
 
-        swm_info = None
-        site_info = None
+        webpage = self._download_webpage(url, display_id)
 
-        if mobj.group('generic'):
-            swm_info = self._screenwavemedia_get_info(url)
-            url = swm_info['_episode_page']
-            mobj = re.match(self._VALID_URL, url)
+        playerdata_url = self._search_regex(
+            r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
+            webpage, 'player data URL')
+        video_title = self._html_search_regex(
+            r'<title>(?P<title>.+?)\|', webpage, 'title')
+        video_description = self._html_search_regex(
+            r'<div class="entry-content">(?P<description>.+?)</div>',
+            webpage, 'description', flags=re.DOTALL, fatal=False)
+        video_thumbnail = self._og_search_thumbnail(webpage)
 
-        if mobj:
-            if mobj.group('cinemassacre'):
-                site_info = self._cinemassacre_get_info(url)
-            elif mobj.group('teamfourstar'):
-                site_info = self._teamfourstar_get_info(url)
+        return {
+            '_type': 'url_transparent',
+            'display_id': display_id,
+            'title': video_title,
+            'description': video_description,
+            'upload_date': video_date,
+            'thumbnail': video_thumbnail,
+            'url': playerdata_url,
+        }
 
-        if not swm_info:
-            if site_info:
-                swm_info = self._screenwavemedia_get_info(site_info['_embed_url'])
 
-        if not swm_info:
-            raise ExtractorError("Failed to extract metadata for this URL")
+class TeamFourIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?teamfourstar\.com/video/(?P<id>[a-z0-9\-]+)/?'
+    _TEST = {
+        'url': 'http://teamfourstar.com/video/a-moment-with-tfs-episode-4/',
+        'info_dict': {
+            'id': 'TeamFourStar-5292a02f20bfa',
+            'ext': 'mp4',
+            'upload_date': '20130401',
+            'description': 'Check out this and more on our website: http://teamfourstar.com\nTFS Store: http://sharkrobot.com/team-four-star\nFollow on Twitter: http://twitter.com/teamfourstar\nLike on FB: http://facebook.com/teamfourstar',
+            'title': 'A Moment With TFS Episode 4',
+        }
+    }
 
-        if site_info:
-            swm_info.update(site_info)
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
 
-        return swm_info
+        playerdata_url = self._search_regex(
+            r'src="(http://player\.screenwavemedia\.com/play/[a-zA-Z]+\.php\?[^"]*\bid=.+?)"',
+            webpage, 'player data URL')
+
+        video_title = self._html_search_regex(
+            r'<div class="heroheadingtitle">(?P<title>.+?)</div>',
+            webpage, 'title')
+        video_date = unified_strdate(self._html_search_regex(
+            r'<div class="heroheadingdate">(?P<date>.+?)</div>',
+            webpage, 'date', fatal=False))
+        video_description = self._html_search_regex(
+            r'(?s)<div class="postcontent">(?P<description>.+?)</div>',
+            webpage, 'description', fatal=False)
+        video_thumbnail = self._og_search_thumbnail(webpage)
+
+        return {
+            '_type': 'url_transparent',
+            'display_id': display_id,
+            'title': video_title,
+            'description': video_description,
+            'upload_date': video_date,
+            'thumbnail': video_thumbnail,
+            'url': playerdata_url,
+        }

From 0fa2b899d1c5affc55bfa2e65e3b0fda0de56b10 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 02:14:04 +0100
Subject: [PATCH 0785/1937] [Makefile] remove *.info.json in clean target

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 3e1debc7e..b846331b9 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json
 
 cleanall: clean
 	rm -f youtube-dl youtube-dl.exe

From 42bdd9d0516be5b71c89c9cccde16a880a14b0b1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 02:57:36 +0100
Subject: [PATCH 0786/1937] [cinchcast] Add new extractor (Fixes #4428)

---
 test/test_utils.py                |  3 ++
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/cinchcast.py | 53 +++++++++++++++++++++++++++++++
 youtube_dl/extractor/generic.py   | 20 ++++++++++--
 youtube_dl/utils.py               | 17 +++++++---
 5 files changed, 88 insertions(+), 6 deletions(-)
 create mode 100644 youtube_dl/extractor/cinchcast.py

diff --git a/test/test_utils.py b/test/test_utils.py
index aaa293ff8..d42df6d96 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -144,6 +144,9 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_strdate('2012/10/11 01:56:38 +0000'), '20121011')
         self.assertEqual(unified_strdate('1968-12-10'), '19681210')
         self.assertEqual(unified_strdate('28/01/2014 21:00:00 +0100'), '20140128')
+        self.assertEqual(
+            unified_strdate('11/26/2014 11:30:00 AM PST', day_first=False),
+            '20141126')
 
     def test_find_xpath_attr(self):
         testxml = '''<root>
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 982a134bf..746ee69e4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -51,6 +51,7 @@ from .cbsnews import CBSNewsIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chilloutzone import ChilloutzoneIE
+from .cinchcast import CinchcastIE
 from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
 from .clipsyndicate import ClipsyndicateIE
diff --git a/youtube_dl/extractor/cinchcast.py b/youtube_dl/extractor/cinchcast.py
new file mode 100644
index 000000000..8ce8b3128
--- /dev/null
+++ b/youtube_dl/extractor/cinchcast.py
@@ -0,0 +1,53 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    unified_strdate,
+    xpath_text,
+)
+
+
+class CinchcastIE(InfoExtractor):
+    _VALID_URL = r'https?://player\.cinchcast\.com/.*?assetId=(?P<id>[0-9]+)'
+    _TEST = {
+        # Actual test is run in generic, look for undergroundwellness
+        'url': 'http://player.cinchcast.com/?platformId=1&#038;assetType=single&#038;assetId=7141703',
+        'only_matching': True,
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        doc = self._download_xml(
+            'http://www.blogtalkradio.com/playerasset/mrss?assetType=single&assetId=%s' % video_id,
+            video_id)
+
+        item = doc.find('.//item')
+        title = xpath_text(item, './title', fatal=True)
+        date_str = xpath_text(
+            item, './{http://developer.longtailvideo.com/trac/}date')
+        upload_date = unified_strdate(date_str, day_first=False)
+        # duration is present but wrong
+        formats = []
+        formats.append({
+            'format_id': 'main',
+            'url': item.find(
+                './{http://search.yahoo.com/mrss/}content').attrib['url'],
+        })
+        backup_url = xpath_text(
+            item, './{http://developer.longtailvideo.com/trac/}backupContent')
+        if backup_url:
+            formats.append({
+                'preference': 2,  # seems to be more reliable
+                'format_id': 'backup',
+                'url': backup_url,
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'upload_date': upload_date,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 328301de3..2b4d8c62f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -467,8 +467,17 @@ class GenericIE(InfoExtractor):
             'expected_warnings': [
                 'URL could be a direct video link, returning it as such.'
             ]
-        }
-
+        },
+        # Cinchcast embed
+        {
+            'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
+            'info_dict': {
+                'id': '7141703',
+                'ext': 'mp3',
+                'upload_date': '20141126',
+                'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
+            }
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -962,6 +971,13 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'SBS')
 
+        # Look for embedded Cinchcast player
+        mobj = re.search(
+            r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1',
+            webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'), 'Cinchcast')
+
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
             webpage)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 75f9594e6..4b0567c93 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -166,7 +166,7 @@ def xpath_text(node, xpath, name=None, fatal=False):
         xpath = xpath.encode('ascii')
 
     n = node.find(xpath)
-    if n is None:
+    if n is None or n.text is None:
         if fatal:
             name = xpath if name is None else name
             raise ExtractorError('Could not find XML element %s' % name)
@@ -644,17 +644,19 @@ def parse_iso8601(date_str, delimiter='T'):
     return calendar.timegm(dt.timetuple())
 
 
-def unified_strdate(date_str):
+def unified_strdate(date_str, day_first=True):
     """Return a string with the date in the format YYYYMMDD"""
 
     if date_str is None:
         return None
-
     upload_date = None
     # Replace commas
     date_str = date_str.replace(',', ' ')
     # %z (UTC offset) is only supported in python>=3.2
     date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
+    # Remove AM/PM + timezone
+    date_str = re.sub(r'(?i)\s*(?:AM|PM)\s+[A-Z]+', '', date_str)
+
     format_expressions = [
         '%d %B %Y',
         '%d %b %Y',
@@ -669,7 +671,6 @@ def unified_strdate(date_str):
         '%d/%m/%Y',
         '%d/%m/%y',
         '%Y/%m/%d %H:%M:%S',
-        '%d/%m/%Y %H:%M:%S',
         '%Y-%m-%d %H:%M:%S',
         '%Y-%m-%d %H:%M:%S.%f',
         '%d.%m.%Y %H:%M',
@@ -681,6 +682,14 @@ def unified_strdate(date_str):
         '%Y-%m-%dT%H:%M:%S.%f',
         '%Y-%m-%dT%H:%M',
     ]
+    if day_first:
+        format_expressions.extend([
+            '%d/%m/%Y %H:%M:%S',
+        ])
+    else:
+        format_expressions.extend([
+            '%m/%d/%Y %H:%M:%S',
+        ])
     for expression in format_expressions:
         try:
             upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')

From 63e0f295645c768420bdf19b598393799c108652 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 02:59:52 +0100
Subject: [PATCH 0787/1937] [vine] Modernize

---
 youtube_dl/extractor/vine.py | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 4970b2f23..ef3dadeb5 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -25,26 +25,21 @@ class VineIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage('https://vine.co/v/' + video_id, video_id)
 
         data = json.loads(self._html_search_regex(
             r'window\.POST_DATA = { %s: ({.+?}) }' % video_id, webpage, 'vine data'))
 
-        formats = [
-            {
-                'url': data['videoLowURL'],
-                'ext': 'mp4',
-                'format_id': 'low',
-            },
-            {
-                'url': data['videoUrl'],
-                'ext': 'mp4',
-                'format_id': 'standard',
-            }
-        ]
+        formats = [{
+            'url': data['videoLowURL'],
+            'ext': 'mp4',
+            'format_id': 'low',
+        }, {
+            'url': data['videoUrl'],
+            'ext': 'mp4',
+            'format_id': 'standard',
+        }]
 
         return {
             'id': video_id,

From 6a5308ab499ef656fca65b6f22fd02c47486d94e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 03:02:56 +0100
Subject: [PATCH 0788/1937] release 2014.12.12

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b2a285bb2..eda34d8d1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.11'
+__version__ = '2014.12.12'

From f5e43bc695947989d5d563f53135fb8fcfcd4081 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 03:34:28 +0100
Subject: [PATCH 0789/1937] [vine] Provide alt_title (Fixes #4448)

---
 youtube_dl/extractor/common.py | 3 ++-
 youtube_dl/extractor/vine.py   | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2277ec6ab..c2e24def1 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -118,6 +118,7 @@ class InfoExtractor(object):
 
     The following fields are optional:
 
+    alt_title:      A secondary title of the video.
     display_id      An alternative identifier for the video, not necessarily
                     unique, but available before title. Typically, id is
                     something like "4234987", title "Dancing naked mole rats",
@@ -129,7 +130,7 @@ class InfoExtractor(object):
                         * "resolution" (optional, string "{width}x{height"},
                                         deprecated)
     thumbnail:      Full URL to a video thumbnail image.
-    description:    One-line video description.
+    description:    Full video description.
     uploader:       Full name of the video uploader.
     timestamp:      UNIX timestamp of the moment the video became available.
     upload_date:    Video upload date (YYYYMMDD).
diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index ef3dadeb5..0b58fe0fe 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -17,6 +17,7 @@ class VineIE(InfoExtractor):
             'id': 'b9KOOWX7HUx',
             'ext': 'mp4',
             'title': 'Chicken.',
+            'alt_title': 'Vine by Jack Dorsey',
             'description': 'Chicken.',
             'upload_date': '20130519',
             'uploader': 'Jack Dorsey',
@@ -44,6 +45,7 @@ class VineIE(InfoExtractor):
         return {
             'id': video_id,
             'title': self._og_search_title(webpage),
+            'alt_title': self._og_search_description(webpage),
             'description': data['description'],
             'thumbnail': data['thumbnailUrl'],
             'upload_date': unified_strdate(data['created']),

From d068ba24f3fa247b262a0aed6d94ac7f4f43de97 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 03:34:33 +0100
Subject: [PATCH 0790/1937] release 2014.12.12.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index eda34d8d1..7289ea4d2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.12'
+__version__ = '2014.12.12.1'

From 16040f46d64bad8dcc5f948288ef469dd787d3d3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 04:01:08 +0100
Subject: [PATCH 0791/1937] [utils] Work around PyPy stupidity with Windows
 DLLs (Fixes #4392)

---
 youtube_dl/compat.py | 19 +++++++++++++++++++
 youtube_dl/utils.py  |  9 +++++----
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index f4a85443e..cd46693b3 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -1,8 +1,10 @@
 from __future__ import unicode_literals
 
+import ctypes
 import getpass
 import optparse
 import os
+import platform
 import re
 import subprocess
 import sys
@@ -326,6 +328,22 @@ def workaround_optparse_bug9161():
         optparse.OptionGroup.add_option = _compat_add_option
 
 
+if platform.python_implementation() == 'PyPy':
+    # PyPy expects byte strings as Windows function names
+    # https://github.com/rg3/youtube-dl/pull/4392
+    def compat_WINFUNCTYPE(*args, **kwargs):
+        real = ctypes.WINFUNCTYPE(*args, **kwargs)
+
+        def resf(tpl, *args, **kwargs):
+            funcname, dll = tpl
+            return real((str(funcname), dll), *args, **kwargs)
+
+        return resf
+else:
+    def compat_WINFUNCTYPE(*args, **kwargs):
+        return ctypes.WINFUNCTYPE(*args, **kwargs)
+
+
 __all__ = [
     'compat_HTTPError',
     'compat_chr',
@@ -349,6 +367,7 @@ __all__ = [
     'compat_urllib_request',
     'compat_urlparse',
     'compat_urlretrieve',
+    'compat_WINFUNCTYPE',
     'compat_xml_parse_error',
     'shlex_quote',
     'subprocess_check_output',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4b0567c93..bbe554a65 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -41,6 +41,7 @@ from .compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urlparse,
+    compat_WINFUNCTYPE,
     shlex_quote,
 )
 
@@ -817,21 +818,21 @@ def _windows_write_string(s, out):
     if fileno not in WIN_OUTPUT_IDS:
         return False
 
-    GetStdHandle = ctypes.WINFUNCTYPE(
+    GetStdHandle = compat_WINFUNCTYPE(
         ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
         ("GetStdHandle", ctypes.windll.kernel32))
     h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
 
-    WriteConsoleW = ctypes.WINFUNCTYPE(
+    WriteConsoleW = compat_WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
         ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
         ctypes.wintypes.LPVOID)(("WriteConsoleW", ctypes.windll.kernel32))
     written = ctypes.wintypes.DWORD(0)
 
-    GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)(("GetFileType", ctypes.windll.kernel32))
+    GetFileType = compat_WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)(("GetFileType", ctypes.windll.kernel32))
     FILE_TYPE_CHAR = 0x0002
     FILE_TYPE_REMOTE = 0x8000
-    GetConsoleMode = ctypes.WINFUNCTYPE(
+    GetConsoleMode = compat_WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
         ctypes.POINTER(ctypes.wintypes.DWORD))(
         ("GetConsoleMode", ctypes.windll.kernel32))

From 4f978523161abb0af241a91da13158d6d91ff442 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 04:09:32 +0100
Subject: [PATCH 0792/1937] Remove unused imports

---
 youtube_dl/extractor/cinchcast.py       | 1 -
 youtube_dl/extractor/pornhd.py          | 1 -
 youtube_dl/extractor/screenwavemedia.py | 2 --
 youtube_dl/extractor/tvplay.py          | 1 -
 4 files changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/cinchcast.py b/youtube_dl/extractor/cinchcast.py
index 8ce8b3128..0c9a24bef 100644
--- a/youtube_dl/extractor/cinchcast.py
+++ b/youtube_dl/extractor/cinchcast.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    int_or_none,
     unified_strdate,
     xpath_text,
 )
diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index 63472b33e..954dfccb7 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -8,7 +8,6 @@ from ..utils import (
     int_or_none,
     js_to_json,
     qualities,
-    determine_ext,
 )
 
 
diff --git a/youtube_dl/extractor/screenwavemedia.py b/youtube_dl/extractor/screenwavemedia.py
index 108190471..6c9fdb7c1 100644
--- a/youtube_dl/extractor/screenwavemedia.py
+++ b/youtube_dl/extractor/screenwavemedia.py
@@ -5,9 +5,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
     int_or_none,
-    month_by_name,
     unified_strdate,
 )
 
diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 0157392cc..9a53a3c74 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -6,7 +6,6 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
-    ExtractorError,
     parse_iso8601,
     qualities,
 )

From 62651c556a1c61b5e73a958e2bc5c1c0624213bc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 04:23:34 +0100
Subject: [PATCH 0793/1937] [howstuffworks] Parse only once, but right (#4383)

---
 youtube_dl/extractor/howstuffworks.py | 53 ++++++++++++++-------------
 1 file changed, 28 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py
index 9478e9e67..e97339121 100644
--- a/youtube_dl/extractor/howstuffworks.py
+++ b/youtube_dl/extractor/howstuffworks.py
@@ -1,7 +1,12 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import find_xpath_attr
+from ..utils import (
+    find_xpath_attr,
+    int_or_none,
+    js_to_json,
+    unescapeHTML,
+)
 
 
 class HowStuffWorksIE(InfoExtractor):
@@ -14,7 +19,9 @@ class HowStuffWorksIE(InfoExtractor):
                 'ext': 'flv',
                 'title': 'Cool Jobs - Iditarod Musher',
                 'description': 'Cold sleds, freezing temps and warm dog breath... an Iditarod musher\'s dream. Kasey-Dee Gardner jumps on a sled to find out what the big deal is.',
-                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/5266.jpg',
+                'display_id': 'cool-jobs-iditarod-musher',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 161,
             },
         },
         {
@@ -24,7 +31,8 @@ class HowStuffWorksIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Survival Zone: Food and Water In the Savanna',
                 'description': 'Learn how to find both food and water while trekking in the African savannah. In this video from the Discovery Channel.',
-                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/7199.jpg',
+                'display_id': 'survival-zone-food-and-water-in-the-savanna',
+                'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
         {
@@ -34,7 +42,8 @@ class HowStuffWorksIE(InfoExtractor):
                 'ext': 'flv',
                 'title': 'Sword Swallowing #1 by Dan Meyer',
                 'description': 'Video footage (1 of 3) used by permission of the owner Dan Meyer through Sword Swallowers Association International <www.swordswallow.org>',
-                'thumbnail': 'http://s.hswstatic.com/gif/videos/480x360/118306353233.jpg',
+                'display_id': 'sword-swallowing-1-by-dan-meyer',
+                'thumbnail': 're:^https?://.*\.jpg$',
             },
         },
     ]
@@ -42,23 +51,17 @@ class HowStuffWorksIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        clip_info = self._search_regex('(?s)var clip = {(.*?)};', webpage, 'clip info')
+        clip_js = self._search_regex(
+            r'(?s)var clip = ({.*?});', webpage, 'clip info')
+        clip_info = self._parse_json(
+            clip_js, display_id, transform_source=js_to_json)
 
-        def extract_clip_info(key, clip_info, name=None, **kargs):
-            if name is None:
-                name = key
-            return self._html_search_regex(
-                r"\s*%s\s*: '?([^'\n]*[^'\n,])" % key, clip_info, name, **kargs)
-
-        video_id = extract_clip_info('content_id', clip_info, 'video id')
+        video_id = clip_info['content_id']
         formats = []
-        m3u8_url = extract_clip_info('m3u8', clip_info, 'm3u8 url', default=None)
-        if m3u8_url is not None:
-            formats += self._extract_m3u8_formats(m3u8_url , video_id, 'mp4')
-        mp4 = self._parse_json(
-            extract_clip_info(
-                'mp4', clip_info, 'formats').replace('},]','}]'), video_id)
-        for video in mp4:
+        m3u8_url = clip_info.get('m3u8')
+        if m3u8_url:
+            formats += self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
+        for video in clip_info.get('mp4', []):
             formats.append({
                 'url': video['src'],
                 'format_id': video['bitrate'],
@@ -80,7 +83,7 @@ class HowStuffWorksIE(InfoExtractor):
 
             for video in smil.findall(
                     './{0}body/{0}switch/{0}video'.format('{http://www.w3.org/2001/SMIL20/Language}')):
-                vbr = int(video.attrib['system-bitrate']) / 1000
+                vbr = int_or_none(video.attrib['system-bitrate'], scale=1000)
                 formats.append({
                     'url': '%s/%s%s' % (http_base, video.attrib['src'], URL_SUFFIX),
                     'format_id': '%dk' % vbr,
@@ -90,11 +93,11 @@ class HowStuffWorksIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': video_id,
+            'id': '%s' % video_id,
             'display_id': display_id,
-            'title': extract_clip_info('clip_title', clip_info, 'title'),
-            'description': extract_clip_info('caption', clip_info, 'description', fatal=False),
-            'thumbnail': extract_clip_info('video_still_url', clip_info, 'thumbnail'),
-            'duration': extract_clip_info('duration', clip_info),
+            'title': unescapeHTML(clip_info['clip_title']),
+            'description': unescapeHTML(clip_info.get('caption')),
+            'thumbnail': clip_info.get('video_still_url'),
+            'duration': clip_info.get('duration'),
             'formats': formats,
         }

From e2f89ec7aada2a58ca658a77401762b764adc112 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 11:33:55 +0100
Subject: [PATCH 0794/1937] Revert "[utils] Work around PyPy stupidity with
 Windows DLLs (Fixes #4392)"

This reverts commit 16040f46d64bad8dcc5f948288ef469dd787d3d3.
---
 youtube_dl/compat.py | 19 -------------------
 youtube_dl/utils.py  |  9 ++++-----
 2 files changed, 4 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index cd46693b3..f4a85443e 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -1,10 +1,8 @@
 from __future__ import unicode_literals
 
-import ctypes
 import getpass
 import optparse
 import os
-import platform
 import re
 import subprocess
 import sys
@@ -328,22 +326,6 @@ def workaround_optparse_bug9161():
         optparse.OptionGroup.add_option = _compat_add_option
 
 
-if platform.python_implementation() == 'PyPy':
-    # PyPy expects byte strings as Windows function names
-    # https://github.com/rg3/youtube-dl/pull/4392
-    def compat_WINFUNCTYPE(*args, **kwargs):
-        real = ctypes.WINFUNCTYPE(*args, **kwargs)
-
-        def resf(tpl, *args, **kwargs):
-            funcname, dll = tpl
-            return real((str(funcname), dll), *args, **kwargs)
-
-        return resf
-else:
-    def compat_WINFUNCTYPE(*args, **kwargs):
-        return ctypes.WINFUNCTYPE(*args, **kwargs)
-
-
 __all__ = [
     'compat_HTTPError',
     'compat_chr',
@@ -367,7 +349,6 @@ __all__ = [
     'compat_urllib_request',
     'compat_urlparse',
     'compat_urlretrieve',
-    'compat_WINFUNCTYPE',
     'compat_xml_parse_error',
     'shlex_quote',
     'subprocess_check_output',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index bbe554a65..4b0567c93 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -41,7 +41,6 @@ from .compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urlparse,
-    compat_WINFUNCTYPE,
     shlex_quote,
 )
 
@@ -818,21 +817,21 @@ def _windows_write_string(s, out):
     if fileno not in WIN_OUTPUT_IDS:
         return False
 
-    GetStdHandle = compat_WINFUNCTYPE(
+    GetStdHandle = ctypes.WINFUNCTYPE(
         ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
         ("GetStdHandle", ctypes.windll.kernel32))
     h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
 
-    WriteConsoleW = compat_WINFUNCTYPE(
+    WriteConsoleW = ctypes.WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
         ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
         ctypes.wintypes.LPVOID)(("WriteConsoleW", ctypes.windll.kernel32))
     written = ctypes.wintypes.DWORD(0)
 
-    GetFileType = compat_WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)(("GetFileType", ctypes.windll.kernel32))
+    GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)(("GetFileType", ctypes.windll.kernel32))
     FILE_TYPE_CHAR = 0x0002
     FILE_TYPE_REMOTE = 0x8000
-    GetConsoleMode = compat_WINFUNCTYPE(
+    GetConsoleMode = ctypes.WINFUNCTYPE(
         ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
         ctypes.POINTER(ctypes.wintypes.DWORD))(
         ("GetConsoleMode", ctypes.windll.kernel32))

From c6973bd4124bb7e15b937f2d337bd7af20562ca6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 12:42:33 +0100
Subject: [PATCH 0795/1937] [compat] Simplify kwarg detection code

This enables nuitka to compile youtube-dl.
---
 youtube_dl/compat.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index f4a85443e..46d438846 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -297,7 +297,9 @@ else:
 
 # Old 2.6 and 2.7 releases require kwargs to be bytes
 try:
-    (lambda x: x)(**{'x': 0})
+    def _testfunc(x):
+        pass
+    _testfunc(**{'x': 0})
 except TypeError:
     def compat_kwargs(kwargs):
         return dict((bytes(k), v) for k, v in kwargs.items())

From 73aeb2dc56068f50e996f2cc08c58db9258b21d4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 12:44:32 +0100
Subject: [PATCH 0796/1937] [goshgay] Modernize

---
 youtube_dl/extractor/goshgay.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/goshgay.py b/youtube_dl/extractor/goshgay.py
index 18474cbb7..ccf65c2c3 100644
--- a/youtube_dl/extractor/goshgay.py
+++ b/youtube_dl/extractor/goshgay.py
@@ -2,14 +2,14 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
-    compat_urlparse,
     ExtractorError,
 )
 
 
 class GoshgayIE(InfoExtractor):
-    _VALID_URL = r'^(?:https?://)www.goshgay.com/video(?P<id>\d+?)($|/)'
+    _VALID_URL = r'https?://www\.goshgay\.com/video(?P<id>\d+?)($|/)'
     _TEST = {
         'url': 'http://www.goshgay.com/video4116282',
         'md5': '268b9f3c3229105c57859e166dd72b03',
@@ -24,8 +24,8 @@ class GoshgayIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         webpage = self._download_webpage(url, video_id)
+
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
         family_friendly = self._html_search_meta(

From 82b34105d301f0aa6897b6695e390324e3722add Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 12:55:13 +0100
Subject: [PATCH 0797/1937] [goshgay] Fix extraction

---
 youtube_dl/extractor/goshgay.py | 47 +++++++++++++++------------------
 1 file changed, 21 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/goshgay.py b/youtube_dl/extractor/goshgay.py
index ccf65c2c3..b116d251d 100644
--- a/youtube_dl/extractor/goshgay.py
+++ b/youtube_dl/extractor/goshgay.py
@@ -2,22 +2,25 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_parse_qs,
+)
 from ..utils import (
-    ExtractorError,
+    parse_duration,
 )
 
 
 class GoshgayIE(InfoExtractor):
     _VALID_URL = r'https?://www\.goshgay\.com/video(?P<id>\d+?)($|/)'
     _TEST = {
-        'url': 'http://www.goshgay.com/video4116282',
-        'md5': '268b9f3c3229105c57859e166dd72b03',
+        'url': 'http://www.goshgay.com/video299069/diesel_sfw_xxx_video',
+        'md5': '027fcc54459dff0feb0bc06a7aeda680',
         'info_dict': {
-            'id': '4116282',
+            'id': '299069',
             'ext': 'flv',
-            'title': 'md5:089833a4790b5e103285a07337f245bf',
-            'thumbnail': 're:http://.*\.jpg',
+            'title': 'DIESEL SFW XXX Video',
+            'thumbnail': 're:^http://.*\.jpg$',
+            'duration': 79,
             'age_limit': 18,
         }
     }
@@ -26,33 +29,25 @@ class GoshgayIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        title = self._og_search_title(webpage)
-        thumbnail = self._og_search_thumbnail(webpage)
+        title = self._html_search_regex(
+            r'<h2>(.*?)<', webpage, 'title')
+        duration = parse_duration(self._html_search_regex(
+            r'<span class="duration">\s*-?\s*(.*?)</span>',
+            webpage, 'duration', fatal=False))
         family_friendly = self._html_search_meta(
             'isFamilyFriendly', webpage, default='false')
-        config_url = self._search_regex(
-            r"'config'\s*:\s*'([^']+)'", webpage, 'config URL')
 
-        config = self._download_xml(
-            config_url, video_id, 'Downloading player config XML')
-
-        if config is None:
-            raise ExtractorError('Missing config XML')
-        if config.tag != 'config':
-            raise ExtractorError('Missing config attribute')
-        fns = config.findall('file')
-        if len(fns) < 1:
-            raise ExtractorError('Missing media URI')
-        video_url = fns[0].text
-
-        url_comp = compat_urlparse.urlparse(url)
-        ref = "%s://%s%s" % (url_comp[0], url_comp[1], url_comp[2])
+        flashvars = compat_parse_qs(self._html_search_regex(
+            r'<embed.+?id="flash-player-embed".+?flashvars="([^"]+)"',
+            webpage, 'flashvars'))
+        thumbnail = flashvars.get('url_bigthumb', [None])[0]
+        video_url = flashvars['flv_url'][0]
 
         return {
             'id': video_id,
             'url': video_url,
             'title': title,
             'thumbnail': thumbnail,
-            'http_referer': ref,
+            'duration': duration,
             'age_limit': 0 if family_friendly == 'true' else 18,
         }

From 0d2fb1d19342aa1f9ae2f9509ca3cdb819c6b8d1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 13:03:16 +0100
Subject: [PATCH 0798/1937] [helsinki] Fix extraction

---
 youtube_dl/extractor/helsinki.py | 37 ++++++++------------------------
 1 file changed, 9 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/helsinki.py b/youtube_dl/extractor/helsinki.py
index 5268efa49..93107b306 100644
--- a/youtube_dl/extractor/helsinki.py
+++ b/youtube_dl/extractor/helsinki.py
@@ -2,9 +2,8 @@
 
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import js_to_json
 
 
 class HelsinkiIE(InfoExtractor):
@@ -24,39 +23,21 @@ class HelsinkiIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        formats = []
-
-        mobj = re.search(r'file=((\w+):[^&]+)', webpage)
-        if mobj:
-            formats.append({
-                'ext': mobj.group(2),
-                'play_path': mobj.group(1),
-                'url': 'rtmp://flashvideo.it.helsinki.fi/vod/',
-                'player_url': 'http://video.helsinki.fi/player.swf',
-                'format_note': 'sd',
-                'quality': 0,
-            })
-
-        mobj = re.search(r'hd\.file=((\w+):[^&]+)', webpage)
-        if mobj:
-            formats.append({
-                'ext': mobj.group(2),
-                'play_path': mobj.group(1),
-                'url': 'rtmp://flashvideo.it.helsinki.fi/vod/',
-                'player_url': 'http://video.helsinki.fi/player.swf',
-                'format_note': 'hd',
-                'quality': 1,
-            })
 
+        params = self._parse_json(self._html_search_regex(
+            r'(?s)jwplayer\("player"\).setup\((\{.*?\})\);',
+            webpage, 'player code'), video_id, transform_source=js_to_json)
+        formats = [{
+            'url': s['file'],
+            'ext': 'mp4',
+        } for s in params['sources']]
         self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': self._og_search_title(webpage).replace('Video: ', ''),
             'description': self._og_search_description(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
             'formats': formats,
         }

From 025f30ba38311cfffef7bafa31ae87b8365b47bd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 13:07:43 +0100
Subject: [PATCH 0799/1937] [channel9] Do not return compat_list results
 anymore

---
 youtube_dl/extractor/channel9.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py
index 2a05813f8..3dfc24f5b 100644
--- a/youtube_dl/extractor/channel9.py
+++ b/youtube_dl/extractor/channel9.py
@@ -236,16 +236,17 @@ class Channel9IE(InfoExtractor):
         if contents is None:
             return contents
 
-        session_meta = {'session_code': self._extract_session_code(html),
-                        'session_day': self._extract_session_day(html),
-                        'session_room': self._extract_session_room(html),
-                        'session_speakers': self._extract_session_speakers(html),
-                        }
+        session_meta = {
+            'session_code': self._extract_session_code(html),
+            'session_day': self._extract_session_day(html),
+            'session_room': self._extract_session_room(html),
+            'session_speakers': self._extract_session_speakers(html),
+        }
 
         for content in contents:
             content.update(session_meta)
 
-        return contents
+        return self.playlist_result(contents)
 
     def _extract_list(self, content_path):
         rss = self._download_xml(self._RSS_URL % content_path, content_path, 'Downloading RSS')

From 751536f5c80cdfdbf6ddfd9f9a673ffd3d8f5d55 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 13:09:54 +0100
Subject: [PATCH 0800/1937] [goldenmoustache] Remove view count

view count is not present anymore, so we can't extract it.
---
 youtube_dl/extractor/goldenmoustache.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/extractor/goldenmoustache.py b/youtube_dl/extractor/goldenmoustache.py
index 10001d4d9..4a07b5558 100644
--- a/youtube_dl/extractor/goldenmoustache.py
+++ b/youtube_dl/extractor/goldenmoustache.py
@@ -17,7 +17,6 @@ class GoldenMoustacheIE(InfoExtractor):
             'title': 'Suricate - Le Poker',
             'description': 'md5:3d1f242f44f8c8cb0a106f1fd08e5dc9',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'view_count': int,
         }
     }, {
         'url': 'http://www.goldenmoustache.com/le-lab-tout-effacer-mc-fly-et-carlito-55249/',
@@ -28,7 +27,6 @@ class GoldenMoustacheIE(InfoExtractor):
             'title': 'Le LAB - Tout Effacer (Mc Fly et Carlito)',
             'description': 'md5:9b7fbf11023fb2250bd4b185e3de3b2a',
             'thumbnail': 're:^https?://.*\.(?:png|jpg)$',
-            'view_count': int,
         }
     }]
 
@@ -42,9 +40,6 @@ class GoldenMoustacheIE(InfoExtractor):
             r'<title>(.*?)(?: - Golden Moustache)?</title>', webpage, 'title')
         thumbnail = self._og_search_thumbnail(webpage)
         description = self._og_search_description(webpage)
-        view_count = int_or_none(self._html_search_regex(
-            r'<strong>([0-9]+)</strong>\s*VUES</span>',
-            webpage, 'view count', fatal=False))
 
         return {
             'id': video_id,
@@ -53,5 +48,4 @@ class GoldenMoustacheIE(InfoExtractor):
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
-            'view_count': view_count,
         }

From 412c617d0f7d22e9b2f7c3b0252a76bcf8ffef2f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 15:55:55 +0100
Subject: [PATCH 0801/1937] [cnet] Update to new theplatform infrastructure
 (Fixes #2736)

---
 youtube_dl/YoutubeDL.py      | 16 +++++++---------
 youtube_dl/extractor/cnet.py | 33 ++++++++++++++-------------------
 2 files changed, 21 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 31531855e..e7abb7e0b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -622,15 +622,13 @@ class YoutubeDL(object):
                 ie_result['url'], ie_key=ie_result.get('ie_key'),
                 extra_info=extra_info, download=False, process=False)
 
-            new_result = ie_result.copy()
-            for f in ('_type', 'id', 'url', 'ext', 'player_url', 'formats',
-                      'entries', 'ie_key', 'duration',
-                      'subtitles', 'annotations', 'format',
-                      'thumbnail', 'thumbnails'):
-                if f in new_result:
-                    del new_result[f]
-                if f in info:
-                    new_result[f] = info[f]
+            force_properties = dict(
+                (k, v) for k, v in ie_result.items() if v is not None)
+            for f in ('_type', 'url'):
+                if f in force_properties:
+                    del force_properties[f]
+            new_result = info.copy()
+            new_result.update(force_properties)
 
             assert new_result.get('_type') != 'url_transparent'
 
diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py
index 710d5009b..a6f1f3a91 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cnet.py
@@ -15,23 +15,24 @@ class CNETIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?cnet\.com/videos/(?P<id>[^/]+)/'
     _TEST = {
         'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/',
-        'md5': '041233212a0d06b179c87cbcca1577b8',
         'info_dict': {
             'id': '56f4ea68-bd21-4852-b08c-4de5b8354c60',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Hands-on with Microsoft Windows 8.1 Update',
             'description': 'The new update to the Windows 8 OS brings improved performance for mouse and keyboard users.',
             'thumbnail': 're:^http://.*/flmswindows8.jpg$',
-            'uploader_id': 'sarah.mitroff@cbsinteractive.com',
+            'uploader_id': '6085384d-619e-11e3-b231-14feb5ca9861',
             'uploader': 'Sarah Mitroff',
+        },
+        'params': {
+            'skip_download': 'requires rtmpdump',
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('id')
-
+        display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
+
         data_json = self._html_search_regex(
             r"<div class=\"cnetVideoPlayer\"\s+.*?data-cnet-video-options='([^']+)'",
             webpage, 'data json')
@@ -42,37 +43,31 @@ class CNETIE(InfoExtractor):
         if not vdata:
             raise ExtractorError('Cannot find video data')
 
+        mpx_account = data['config']['players']['default']['mpx_account']
+        vid = vdata['files']['rtmp']
+        tp_link = 'http://link.theplatform.com/s/%s/%s' % (mpx_account, vid)
+
         video_id = vdata['id']
         title = vdata.get('headline')
         if title is None:
             title = vdata.get('title')
         if title is None:
             raise ExtractorError('Cannot find title!')
-        description = vdata.get('dek')
         thumbnail = vdata.get('image', {}).get('path')
         author = vdata.get('author')
         if author:
             uploader = '%s %s' % (author['firstName'], author['lastName'])
-            uploader_id = author.get('email')
+            uploader_id = author.get('id')
         else:
             uploader = None
             uploader_id = None
 
-        formats = [{
-            'format_id': '%s-%s-%s' % (
-                f['type'], f['format'],
-                int_or_none(f.get('bitrate'), 1000, default='')),
-            'url': f['uri'],
-            'tbr': int_or_none(f.get('bitrate'), 1000),
-        } for f in vdata['files']['data']]
-        self._sort_formats(formats)
-
         return {
+            '_type': 'url_transparent',
+            'url': tp_link,
             'id': video_id,
             'display_id': display_id,
             'title': title,
-            'formats': formats,
-            'description': description,
             'uploader': uploader,
             'uploader_id': uploader_id,
             'thumbnail': thumbnail,

From 3b0bec8d11cdd4ef17a4d53295ad2717193d5308 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 15:56:45 +0100
Subject: [PATCH 0802/1937] release 2014.12.12.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7289ea4d2..61c106e1a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.12.1'
+__version__ = '2014.12.12.2'

From 68d301ffd4dba86b06fb7cc54634b5762df14318 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Dec 2014 21:23:42 +0600
Subject: [PATCH 0803/1937] [giantbomb] Add extractor (Closes #4432)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/giantbomb.py | 81 +++++++++++++++++++++++++++++++
 2 files changed, 82 insertions(+)
 create mode 100644 youtube_dl/extractor/giantbomb.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 746ee69e4..6866f06d4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -148,6 +148,7 @@ from .gamestar import GameStarIE
 from .gametrailers import GametrailersIE
 from .gdcvault import GDCVaultIE
 from .generic import GenericIE
+from .giantbomb import GiantBombIE
 from .glide import GlideIE
 from .globo import GloboIE
 from .godtube import GodTubeIE
diff --git a/youtube_dl/extractor/giantbomb.py b/youtube_dl/extractor/giantbomb.py
new file mode 100644
index 000000000..87cd19147
--- /dev/null
+++ b/youtube_dl/extractor/giantbomb.py
@@ -0,0 +1,81 @@
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    unescapeHTML,
+    qualities,
+    int_or_none,
+)
+
+
+class GiantBombIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?giantbomb\.com/videos/(?P<display_id>[^/]+)/(?P<id>\d+-\d+)'
+    _TEST = {
+        'url': 'http://www.giantbomb.com/videos/quick-look-destiny-the-dark-below/2300-9782/',
+        'md5': '57badeface303ecf6b98b812de1b9018',
+        'info_dict': {
+            'id': '2300-9782',
+            'display_id': 'quick-look-destiny-the-dark-below',
+            'ext': 'mp4',
+            'title': 'Quick Look: Destiny: The Dark Below',
+            'description': 'md5:0aa3aaf2772a41b91d44c63f30dfad24',
+            'duration': 2399,
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
+
+        webpage = self._download_webpage(url, display_id)
+
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        video = json.loads(unescapeHTML(self._search_regex(
+            r'data-video="([^"]+)"', webpage, 'data-video')))
+
+        duration = int_or_none(video.get('lengthSeconds'))
+
+        quality = qualities([
+            'f4m_low', 'progressive_low', 'f4m_high',
+            'progressive_high', 'f4m_hd', 'progressive_hd'])
+
+        formats = []
+        for format_id, video_url in video['videoStreams'].items():
+            if format_id == 'f4m_stream':
+                continue
+            if video_url.endswith('.f4m'):
+                f4m_formats = self._extract_f4m_formats(video_url + '?hdcore=3.3.1', display_id)
+                if f4m_formats:
+                    f4m_formats[0]['quality'] = quality(format_id)
+                    formats.extend(f4m_formats)
+            else:
+                formats.append({
+                    'url': video_url,
+                    'format_id': format_id,
+                    'quality': quality(format_id),
+                })
+
+        if not formats:
+            youtube_id = video.get('youtubeID')
+            if youtube_id:
+                return self.url_result(youtube_id, 'Youtube')
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
+        }

From b3efb3ebae6bcedfb972e45c7266a31afa5591c6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 16:40:37 +0100
Subject: [PATCH 0804/1937] [README] More concise and nicer bug reporting
 instructions

---
 README.md | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 18be078d7..b3af18912 100644
--- a/README.md
+++ b/README.md
@@ -536,13 +536,11 @@ Most likely, you'll want to use various options. For a list of what can be done,
 
 # BUGS
 
-Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues> . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email.
+Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues> . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the irc channel #youtube-dl on freenode.
 
 Please include the full output of the command when run with `--verbose`. The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
 
-For discussions, join us in the irc channel #youtube-dl on freenode.
-
-When you submit a request, please re-read it once to avoid a couple of mistakes (you can and should use this as a checklist):
+Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist):
 
 ### Is the description of the issue itself sufficient?
 

From e56190b378367291abe4677f2357fcfc213b1a6a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 16:42:40 +0100
Subject: [PATCH 0805/1937] [Makefile] Add CONTRIBUTING.md (Fixes #2984)

---
 CONTRIBUTING.md                 | 150 ++++++++++++++++++++++++++++++++
 Makefile                        |   7 +-
 devscripts/make_contributing.py |  31 +++++++
 3 files changed, 186 insertions(+), 2 deletions(-)
 create mode 100644 CONTRIBUTING.md
 create mode 100755 devscripts/make_contributing.py

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 000000000..48840f8f6
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,150 @@
+Please include the full output of the command when run with `--verbose`. The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
+
+Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist):
+
+### Is the description of the issue itself sufficient?
+
+We often get issue reports that we cannot really decipher. While in most cases we eventually get the required information after asking back multiple times, this poses an unnecessary drain on our resources. Many contributors, including myself, are also not native speakers, so we may misread some parts.
+
+So please elaborate on what feature you are requesting, or what bug you want to be fixed. Make sure that it's obvious
+
+- What the problem is
+- How it could be fixed
+- How your proposed solution would look like
+
+If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a commiter myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
+
+For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the -v flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
+
+Site support requests **must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
+
+###  Are you using the latest version?
+
+Before reporting any issue, type youtube-dl -U. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well.
+
+###  Is the issue already documented?
+
+Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or at https://github.com/rg3/youtube-dl/search?type=Issues . If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity.
+
+###  Why are existing options not enough?
+
+Before requesting a new feature, please have a quick peek at [the list of supported options](https://github.com/rg3/youtube-dl/blob/master/README.md#synopsis). Many feature requests are for features that actually exist already! Please, absolutely do show off your work in the issue report and detail how the existing similar options do *not* solve your problem.
+
+###  Is there enough context in your bug report?
+
+People want to solve problems, and often think they do us a favor by breaking down their larger problems (e.g. wanting to skip already downloaded files) to a specific request (e.g. requesting us to look whether the file exists before downloading the info page). However, what often happens is that they break down the problem into two steps: One simple, and one impossible (or extremely complicated one).
+
+We are then presented with a very complicated request when the original problem could be solved far easier, e.g. by recording the downloaded video IDs in a separate file. To avoid this, you must include the greater context where it is non-obvious. In particular, every feature request that does not consist of adding support for a new site should contain a use case scenario that explains in what situation the missing feature would be useful.
+
+###  Does the issue involve one problem, and one problem only?
+
+Some of our users seem to think there is a limit of issues they can or should open. There is no limit of issues they can or should open. While it may seem appealing to be able to dump all your issues into one ticket, that means that someone who solves one of your issues cannot mark the issue as closed. Typically, reporting a bunch of issues leads to the ticket lingering since nobody wants to attack that behemoth, until someone mercifully splits the issue into multiple ones.
+
+In particular, every site support request issue should only pertain to services at one site (generally under a common domain, but always using the same backend technology). Do not request support for vimeo user videos, Whitehouse podcasts, and Google Plus pages in the same issue. Also, make sure that you don't post bug reports alongside feature requests. As a rule of thumb, a feature request does not include outputs of youtube-dl that are not immediately related to the feature at hand. Do not post reports of a network error alongside the request for a new video service.
+
+###  Is anyone going to need the feature?
+
+Only post features that you (or an incapicated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them.
+
+###  Is your question about youtube-dl?
+
+It may sound strange, but some bug reports we receive are completely unrelated to youtube-dl and relate to a different or even the reporter's own application. Please make sure that you are actually using youtube-dl. If you are using a UI for youtube-dl, report the bug to the maintainer of the actual application providing the UI. On the other hand, if your UI for youtube-dl fails in some way you believe is related to youtube-dl, by all means, go ahead and report the bug.
+
+# DEVELOPER INSTRUCTIONS
+
+Most users do not need to build youtube-dl and can [download the builds](http://rg3.github.io/youtube-dl/download.html) or get them from their distribution.
+
+To run youtube-dl as a developer, you don't need to build anything either. Simply execute
+
+    python -m youtube_dl
+
+To run the test, simply invoke your favorite test runner, or execute a test file directly; any of the following work:
+
+    python -m unittest discover
+    python test/test_download.py
+    nosetests
+
+If you want to create a build of youtube-dl yourself, you'll need
+
+* python
+* make
+* pandoc
+* zip
+* nosetests
+
+### Adding support for a new site
+
+If you want to add support for a new site, you can follow this quick list (assuming your service is called `yourextractor`):
+
+1. [Fork this repository](https://github.com/rg3/youtube-dl/fork)
+2. Check out the source code with `git clone git@github.com:YOUR_GITHUB_USERNAME/youtube-dl.git`
+3. Start a new git branch with `cd youtube-dl; git checkout -b yourextractor`
+4. Start with this simple template and save it to `youtube_dl/extractor/yourextractor.py`:
+    ```python
+    # coding: utf-8
+    from __future__ import unicode_literals
+
+    from .common import InfoExtractor
+
+
+    class YourExtractorIE(InfoExtractor):
+        _VALID_URL = r'https?://(?:www\.)?yourextractor\.com/watch/(?P<id>[0-9]+)'
+        _TEST = {
+            'url': 'http://yourextractor.com/watch/42',
+            'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
+            'info_dict': {
+                'id': '42',
+                'ext': 'mp4',
+                'title': 'Video title goes here',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                # TODO more properties, either as:
+                # * A value
+                # * MD5 checksum; start the string with md5:
+                # * A regular expression; start the string with re:
+                # * Any Python type (for example int or float)
+            }
+        }
+
+        def _real_extract(self, url):
+            video_id = self._match_id(url)
+            webpage = self._download_webpage(url, video_id)
+
+            # TODO more code goes here, for example ...
+            title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, 'title')
+
+            return {
+                'id': video_id,
+                'title': title,
+                'description': self._og_search_description(webpage),
+                # TODO more properties (see youtube_dl/extractor/common.py)
+            }
+    ```
+5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
+6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
+7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want.
+8. If you can, check the code with [pyflakes](https://pypi.python.org/pypi/pyflakes) (a good idea) and [pep8](https://pypi.python.org/pypi/pep8) (optional, ignore E501).
+9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
+
+        $ git add youtube_dl/extractor/__init__.py
+        $ git add youtube_dl/extractor/yourextractor.py
+        $ git commit -m '[yourextractor] Add new extractor'
+        $ git push origin yourextractor
+
+10. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.
+
+In any case, thank you very much for your contributions!
+
+# EMBEDDING YOUTUBE-DL
+
+youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new).
+
+From a Python program, you can embed youtube-dl in a more powerful fashion, like this:
+
+    import youtube_dl
+
+    ydl_opts = {}
+    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
+        ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
+
+Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
+
diff --git a/Makefile b/Makefile
index b846331b9..d5b6e4307 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
-all: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
+all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json CONTRIBUTING.md.tmp
 
 cleanall: clean
 	rm -f youtube-dl youtube-dl.exe
@@ -56,6 +56,9 @@ youtube-dl: youtube_dl/*.py youtube_dl/*/*.py
 README.md: youtube_dl/*.py youtube_dl/*/*.py
 	COLUMNS=80 python -m youtube_dl --help | python devscripts/make_readme.py
 
+CONTRIBUTING.md: README.md
+	python devscripts/make_contributing.py README.md CONTRIBUTING.md
+
 README.txt: README.md
 	pandoc -f markdown -t plain README.md -o README.txt
 
diff --git a/devscripts/make_contributing.py b/devscripts/make_contributing.py
new file mode 100755
index 000000000..99451c568
--- /dev/null
+++ b/devscripts/make_contributing.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+import argparse
+import io
+import re
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        'INFILE', help='README.md file name to read from')
+    parser.add_argument(
+        'OUTFILE', help='CONTRIBUTING.md file name to write to')
+    args = parser.parse_args()
+
+    with io.open(args.INFILE, encoding='utf-8') as inf:
+        readme = inf.read()
+
+    bug_text = re.search(
+        r'(?s)#\s*BUGS\s*[^\n]*\s*(.*?)#\s*COPYRIGHT', readme).group(1)
+    dev_text = re.search(
+        r'(?s)(#\s*DEVELOPER INSTRUCTIONS.*?)#\s*BUGS', readme).group(1)
+
+    out = bug_text + dev_text
+
+    with io.open(args.OUTFILE, 'w', encoding='utf-8') as outf:
+        outf.write(out)
+
+if __name__ == '__main__':
+    main()

From 13dcfd41bda5eaf40637d4054bec33cd804940b9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 16:47:22 +0100
Subject: [PATCH 0806/1937] [CONTRIBUTING.md] Remove the section about
 embedding; that is not applicable for youtube-dl contributors

---
 CONTRIBUTING.md                 | 14 --------------
 devscripts/make_contributing.py |  3 ++-
 2 files changed, 2 insertions(+), 15 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 48840f8f6..0ff7b395a 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -134,17 +134,3 @@ If you want to add support for a new site, you can follow this quick list (assum
 
 In any case, thank you very much for your contributions!
 
-# EMBEDDING YOUTUBE-DL
-
-youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/rg3/youtube-dl/issues/new).
-
-From a Python program, you can embed youtube-dl in a more powerful fashion, like this:
-
-    import youtube_dl
-
-    ydl_opts = {}
-    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
-        ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
-
-Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
-
diff --git a/devscripts/make_contributing.py b/devscripts/make_contributing.py
index 99451c568..5fa8cf851 100755
--- a/devscripts/make_contributing.py
+++ b/devscripts/make_contributing.py
@@ -20,7 +20,8 @@ def main():
     bug_text = re.search(
         r'(?s)#\s*BUGS\s*[^\n]*\s*(.*?)#\s*COPYRIGHT', readme).group(1)
     dev_text = re.search(
-        r'(?s)(#\s*DEVELOPER INSTRUCTIONS.*?)#\s*BUGS', readme).group(1)
+        r'(?s)(#\s*DEVELOPER INSTRUCTIONS.*?)#\s*EMBEDDING YOUTUBE-DL',
+        readme).group(1)
 
     out = bug_text + dev_text
 

From da3a2d8137fa89051b011d90875827504a9a9373 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 16:47:38 +0100
Subject: [PATCH 0807/1937] release 2014.12.12.3

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 61c106e1a..995bae3d4 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.12.2'
+__version__ = '2014.12.12.3'

From ff815fe65aab83db7e11251db5eafc76504adf05 Mon Sep 17 00:00:00 2001
From: Mark Schreiber <mark7@alumni.cmu.edu>
Date: Thu, 10 Jul 2014 20:11:11 -0700
Subject: [PATCH 0808/1937] Download playlist items in reverse order

Series of videos are typically uploaded to YouTube playlists in
chronological order.  By default, these videos are downloaded
latest-to-earliest; this is great for seeing the latest videos in a
series, but prevents streaming video in the order that the videos were
produced.  Add an option to download videos in reverse order,
earliest-to-latest.

Conflicts:
	youtube_dl/YoutubeDL.py
	youtube_dl/__init__.py
---
 README.md               | 1 +
 youtube_dl/YoutubeDL.py | 4 ++++
 youtube_dl/__init__.py  | 1 +
 youtube_dl/options.py   | 4 ++++
 4 files changed, 10 insertions(+)

diff --git a/README.md b/README.md
index b3af18912..e41ee8985 100644
--- a/README.md
+++ b/README.md
@@ -75,6 +75,7 @@ which means you can modify it, redistribute it or use it however you like.
 ## Video Selection:
     --playlist-start NUMBER          playlist video to start at (default is 1)
     --playlist-end NUMBER            playlist video to end at (default is last)
+    --playlist-reverse               Download playlist videos in reverse order
     --match-title REGEX              download only matching titles (regex or
                                      caseless sub-string)
     --reject-title REGEX             skip download for matching titles (regex or
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e7abb7e0b..578c8daf2 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -124,6 +124,7 @@ class YoutubeDL(object):
     nooverwrites:      Prevent overwriting files.
     playliststart:     Playlist item to start at.
     playlistend:       Playlist item to end at.
+    playlistreverse:   Download playlist items in reverse order.
     matchtitle:        Download only matching titles.
     rejecttitle:       Reject downloads for matching titles.
     logger:            Log messages to a logging.Logger instance.
@@ -670,6 +671,9 @@ class YoutubeDL(object):
                     "[%s] playlist %s: Downloading %d videos" %
                     (ie_result['extractor'], playlist, n_entries))
 
+            if self.params.get('playlistreverse', False):
+                entries = entries[::-1]
+
             for i, entry in enumerate(entries, 1):
                 self.to_screen('[download] Downloading video #%s of %s' % (i, n_entries))
                 extra = {
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 77b3384a0..70c4f25b1 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -249,6 +249,7 @@ def _real_main(argv=None):
         'progress_with_newline': opts.progress_with_newline,
         'playliststart': opts.playliststart,
         'playlistend': opts.playlistend,
+        'playlistreverse': opts.playlist_reverse,
         'noplaylist': opts.noplaylist,
         'logtostderr': opts.outtmpl == '-',
         'consoletitle': opts.consoletitle,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 041ca83d8..7654d1a4e 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -349,6 +349,10 @@ def parseOpts(overrideArguments=None):
         '--test',
         action='store_true', dest='test', default=False,
         help=optparse.SUPPRESS_HELP)
+    downloader.add_option(
+        '--playlist-reverse',
+        action='store_true',
+        help='Download playlist videos in reverse order')
 
     workarounds = optparse.OptionGroup(parser, 'Workarounds')
     workarounds.add_option(

From 8bdcb436f9bf90b90ddd9b93defff644760ebf02 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 17:06:52 +0100
Subject: [PATCH 0809/1937] [test_unicode_literals] Fix test

---
 test/helper.py                |  4 +++-
 test/test_unicode_literals.py | 14 ++++++++++++--
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 9a7f0746e..8a820526a 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -161,7 +161,9 @@ def assertRegexpMatches(self, text, regexp, msg=None):
     else:
         m = re.match(regexp, text)
         if not m:
-            note = 'Regexp didn\'t match: %r not found in %r' % (regexp, text)
+            note = 'Regexp didn\'t match: %r not found' % (regexp)
+            if len(text) < 1000:
+                note += ' in %r' % text
             if msg is None:
                 msg = note
             else:
diff --git a/test/test_unicode_literals.py b/test/test_unicode_literals.py
index 2cc431b0b..d3cba869b 100644
--- a/test/test_unicode_literals.py
+++ b/test/test_unicode_literals.py
@@ -1,5 +1,11 @@
 from __future__ import unicode_literals
 
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
 import io
 import os
 import re
@@ -14,6 +20,9 @@ IGNORED_FILES = [
 ]
 
 
+from helper import assertRegexpMatches
+
+
 class TestUnicodeLiterals(unittest.TestCase):
     def test_all_files(self):
         for dirpath, _, filenames in os.walk(rootDir):
@@ -29,9 +38,10 @@ class TestUnicodeLiterals(unittest.TestCase):
 
                 if "'" not in code and '"' not in code:
                     continue
-                self.assertRegexpMatches(
+                assertRegexpMatches(
+                    self,
                     code,
-                    r'(?:#.*\n*)?from __future__ import (?:[a-z_]+,\s*)*unicode_literals',
+                    r'(?:(?:#.*?|\s*)\n)*from __future__ import (?:[a-z_]+,\s*)*unicode_literals',
                     'unicode_literals import  missing in %s' % fn)
 
                 m = re.search(r'(?<=\s)u[\'"](?!\)|,|$)', code)

From 7c7a0d395c2eb60beb6ddf2fab19b89cae6a2cdc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 17:07:39 +0100
Subject: [PATCH 0810/1937] Remove unused imports

---
 youtube_dl/extractor/cnet.py            | 2 --
 youtube_dl/extractor/goldenmoustache.py | 3 ---
 2 files changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py
index a6f1f3a91..3145b3051 100644
--- a/youtube_dl/extractor/cnet.py
+++ b/youtube_dl/extractor/cnet.py
@@ -2,12 +2,10 @@
 from __future__ import unicode_literals
 
 import json
-import re
 
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    int_or_none,
 )
 
 
diff --git a/youtube_dl/extractor/goldenmoustache.py b/youtube_dl/extractor/goldenmoustache.py
index 4a07b5558..0fb509724 100644
--- a/youtube_dl/extractor/goldenmoustache.py
+++ b/youtube_dl/extractor/goldenmoustache.py
@@ -1,9 +1,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-)
 
 
 class GoldenMoustacheIE(InfoExtractor):

From 58c1f6f0a708eb2b084e773d7aa74bfe37d80c6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Dec 2014 22:10:32 +0600
Subject: [PATCH 0811/1937] [nbc] Fix extraction (Closes #4441)

---
 youtube_dl/extractor/nbc.py | 36 +++++++++++++++++++++++++-----------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 7b5449031..bf5132721 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -12,23 +12,37 @@ from ..utils import (
 
 
 class NBCIE(InfoExtractor):
-    _VALID_URL = r'http://www\.nbc\.com/[^/]+/video/[^/]+/(?P<id>n?\d+)'
+    _VALID_URL = r'http://www\.nbc\.com/(?:[^/]+/)+(?P<id>n?\d+)'
 
-    _TEST = {
-        'url': 'http://www.nbc.com/chicago-fire/video/i-am-a-firefighter/2734188',
-        # md5 checksum is not stable
-        'info_dict': {
-            'id': 'bTmnLCvIbaaH',
-            'ext': 'flv',
-            'title': 'I Am a Firefighter',
-            'description': 'An emergency puts Dawson\'sf irefighter skills to the ultimate test in this four-part digital series.',
+    _TESTS = [
+        {
+            'url': 'http://www.nbc.com/chicago-fire/video/i-am-a-firefighter/2734188',
+            # md5 checksum is not stable
+            'info_dict': {
+                'id': 'bTmnLCvIbaaH',
+                'ext': 'flv',
+                'title': 'I Am a Firefighter',
+                'description': 'An emergency puts Dawson\'sf irefighter skills to the ultimate test in this four-part digital series.',
+            },
         },
-    }
+        {
+            'url': 'http://www.nbc.com/the-tonight-show/episodes/176',
+            'info_dict': {
+                'id': 'XwU9KZkp98TH',
+                'ext': 'flv',
+                'title': 'Ricky Gervais, Steven Van Zandt, ILoveMakonnen',
+                'description': 'A brand new episode of The Tonight Show welcomes Ricky Gervais, Steven Van Zandt and ILoveMakonnen.',
+            },
+            'skip': 'Only works from US',
+        },
+    ]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        theplatform_url = self._search_regex('class="video-player video-player-full" data-mpx-url="(.*?)"', webpage, 'theplatform url')
+        theplatform_url = self._search_regex(
+            '(?:class="video-player video-player-full" data-mpx-url|class="player" src)="(.*?)"',
+            webpage, 'theplatform url').replace('_no_endcard', '')
         if theplatform_url.startswith('//'):
             theplatform_url = 'http:' + theplatform_url
         return self.url_result(theplatform_url)

From 2865cf0419a4c92133053f128934c64df0dccda9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 17:11:53 +0100
Subject: [PATCH 0812/1937] Deprecate --auto-number (Closes #2704)

---
 youtube_dl/options.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 7654d1a4e..21c452141 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -484,10 +484,6 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option(
         '--id', default=False,
         action='store_true', dest='useid', help='use only video ID in file name')
-    filesystem.add_option(
-        '-A', '--auto-number',
-        action='store_true', dest='autonumber', default=False,
-        help='number downloaded files starting from 00000')
     filesystem.add_option(
         '-o', '--output',
         dest='outtmpl', metavar='TEMPLATE',
@@ -515,6 +511,10 @@ def parseOpts(overrideArguments=None):
         '--restrict-filenames',
         action='store_true', dest='restrictfilenames', default=False,
         help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames')
+    filesystem.add_option(
+        '-A', '--auto-number',
+        action='store_true', dest='autonumber', default=False,
+        help='[deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] number downloaded files starting from 00000')
     filesystem.add_option(
         '-t', '--title',
         action='store_true', dest='usetitle', default=False,

From cc8c9281e6313c8b108f20df0ea633445c78bf2d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 17:17:09 +0100
Subject: [PATCH 0813/1937] [downloader/common] Do not use classic int division

---
 youtube_dl/downloader/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 6e44c88c9..2a566eabe 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -160,7 +160,7 @@ class FileDownloader(object):
             return
         speed = float(byte_counter) / elapsed
         if speed > rate_limit:
-            time.sleep(max((byte_counter / rate_limit) - elapsed, 0))
+            time.sleep(max((byte_counter // rate_limit) - elapsed, 0))
 
     def temp_name(self, filename):
         """Returns a temporary filename for the given filename."""

From 2101f5d4ccd5e568a2b860596e67d01364b9078b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 17:18:22 +0100
Subject: [PATCH 0814/1937] release 2014.12.12.4

---
 README.md             | 6 ++++--
 youtube_dl/version.py | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index e41ee8985..f10f06ee8 100644
--- a/README.md
+++ b/README.md
@@ -75,7 +75,6 @@ which means you can modify it, redistribute it or use it however you like.
 ## Video Selection:
     --playlist-start NUMBER          playlist video to start at (default is 1)
     --playlist-end NUMBER            playlist video to end at (default is last)
-    --playlist-reverse               Download playlist videos in reverse order
     --match-title REGEX              download only matching titles (regex or
                                      caseless sub-string)
     --reject-title REGEX             skip download for matching titles (regex or
@@ -114,12 +113,12 @@ which means you can modify it, redistribute it or use it however you like.
                                      size. By default, the buffer size is
                                      automatically resized from an initial value
                                      of SIZE.
+    --playlist-reverse               Download playlist videos in reverse order
 
 ## Filesystem Options:
     -a, --batch-file FILE            file containing URLs to download ('-' for
                                      stdin)
     --id                             use only video ID in file name
-    -A, --auto-number                number downloaded files starting from 00000
     -o, --output TEMPLATE            output filename template. Use %(title)s to
                                      get the title, %(uploader)s for the
                                      uploader name, %(uploader_id)s for the
@@ -153,6 +152,9 @@ which means you can modify it, redistribute it or use it however you like.
     --restrict-filenames             Restrict filenames to only ASCII
                                      characters, and avoid "&" and spaces in
                                      filenames
+    -A, --auto-number                [deprecated; use  -o
+                                     "%(autonumber)s-%(title)s.%(ext)s" ] number
+                                     downloaded files starting from 00000
     -t, --title                      [deprecated] use title in file name
                                      (default)
     -l, --literal                    [deprecated] alias of --title
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 995bae3d4..65290b43d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.12.3'
+__version__ = '2014.12.12.4'

From 57e086dcea6a9981c4943b6a281c0c7f806efaf8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 17:24:05 +0100
Subject: [PATCH 0815/1937] [ebaumsworld] Modernize

---
 youtube_dl/extractor/ebaumsworld.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ebaumsworld.py b/youtube_dl/extractor/ebaumsworld.py
index 63c2549d3..b6bfd2b2d 100644
--- a/youtube_dl/extractor/ebaumsworld.py
+++ b/youtube_dl/extractor/ebaumsworld.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -20,8 +18,7 @@ class EbaumsWorldIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         config = self._download_xml(
             'http://www.ebaumsworld.com/video/player/%s' % video_id, video_id)
         video_url = config.find('file').text

From 146323a7f8bf434de91298e48a81a8e0ff689244 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 17:39:33 +0100
Subject: [PATCH 0816/1937] [groupon] Add extractor (Fixes #4386)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/groupon.py  | 50 ++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 youtube_dl/extractor/groupon.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6866f06d4..104dce1eb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -159,6 +159,7 @@ from .googlesearch import GoogleSearchIE
 from .gorillavid import GorillaVidIE
 from .goshgay import GoshgayIE
 from .grooveshark import GroovesharkIE
+from .groupon import GrouponIE
 from .hark import HarkIE
 from .heise import HeiseIE
 from .helsinki import HelsinkiIE
diff --git a/youtube_dl/extractor/groupon.py b/youtube_dl/extractor/groupon.py
new file mode 100644
index 000000000..8b9e0e2f8
--- /dev/null
+++ b/youtube_dl/extractor/groupon.py
@@ -0,0 +1,50 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class GrouponIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.groupon\.com/deals/(?P<id>[^?#]+)'
+
+    _TEST = {
+        'url': 'https://www.groupon.com/deals/bikram-yoga-huntington-beach-2#ooid=tubGNycTo_9Uxg82uESj4i61EYX8nyuf',
+        'info_dict': {
+            'id': 'bikram-yoga-huntington-beach-2',
+            'title': '$49 for 10 Yoga Classes or One Month of Unlimited Classes at Bikram Yoga Huntington Beach ($180 Value)',
+            'description': 'Studio kept at 105 degrees and 40% humidity with anti-microbial and anti-slip Flotex flooring; certified instructors',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': 'tubGNycTo_9Uxg82uESj4i61EYX8nyuf',
+                'ext': 'mp4',
+                'title': 'Bikram Yoga Huntington Beach | Orange County',
+            },
+        }],
+        'params': {
+            'skip_download': 'HLS',
+        }
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        webpage = self._download_webpage(url, playlist_id)
+
+        payload = self._parse_json(self._search_regex(
+            r'var\s+payload\s*=\s*(.*?);\n', webpage, 'payload'), playlist_id)
+        videos = payload['carousel'].get('dealVideos', [])
+        entries = []
+        for v in videos:
+            if v.get('provider') != 'OOYALA':
+                self.report_warning(
+                    '%s: Unsupported video provider %s, skipping video' %
+                    (playlist_id, v.get('provider')))
+                continue
+            entries.append(self.url_result('ooyala:%s' % v['media']))
+
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'entries': entries,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+        }

From a099965badd87efb31c18603de6d350bbf5b1a24 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 17:40:27 +0100
Subject: [PATCH 0817/1937] release 2014.12.12.5

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 65290b43d..eea06b9d1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.12.4'
+__version__ = '2014.12.12.5'

From 07bc9a35308f2ac3ae21de5c53380bc4d01aa72c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 12 Dec 2014 22:44:16 +0600
Subject: [PATCH 0818/1937] [nowvideo] Add .li domain (Closes #4453)

---
 youtube_dl/extractor/nowvideo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nowvideo.py b/youtube_dl/extractor/nowvideo.py
index ecb38de2d..dec09cdfe 100644
--- a/youtube_dl/extractor/nowvideo.py
+++ b/youtube_dl/extractor/nowvideo.py
@@ -7,7 +7,7 @@ class NowVideoIE(NovaMovIE):
     IE_NAME = 'nowvideo'
     IE_DESC = 'NowVideo'
 
-    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:ch|sx|eu|at|ag|co)'}
+    _VALID_URL = NovaMovIE._VALID_URL_TEMPLATE % {'host': 'nowvideo\.(?:ch|sx|eu|at|ag|co|li)'}
 
     _HOST = 'www.nowvideo.ch'
 

From 7a60322abf5bfc72b8a6de62f031b1b3ecb52847 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 17:52:50 +0100
Subject: [PATCH 0819/1937] release 2014.12.12.6

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index eea06b9d1..24d1c08f6 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.12.5'
+__version__ = '2014.12.12.6'

From 1c985da0cabfa4b9e93617f743185c4123ea07bb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 18:25:58 +0100
Subject: [PATCH 0820/1937] release 2014.12.12.7

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 24d1c08f6..da959ea85 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.12.6'
+__version__ = '2014.12.12.7'

From c3f3b29b92dd46524dcbf9f4b0d957eaa0d0db0c Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 12 Dec 2014 20:22:24 +0200
Subject: [PATCH 0821/1937] [rtp] Add new extractor (Closes #4382)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/rtp.py      | 57 ++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+)
 create mode 100644 youtube_dl/extractor/rtp.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 104dce1eb..447833c01 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -323,6 +323,7 @@ from .roxwel import RoxwelIE
 from .rtbf import RTBFIE
 from .rtlnl import RtlXlIE
 from .rtlnow import RTLnowIE
+from .rtp import RTPIE
 from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE
 from .ruhd import RUHDIE
diff --git a/youtube_dl/extractor/rtp.py b/youtube_dl/extractor/rtp.py
new file mode 100644
index 000000000..0bed1f100
--- /dev/null
+++ b/youtube_dl/extractor/rtp.py
@@ -0,0 +1,57 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..utils import js_to_json
+
+
+class RTPIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?rtp\.pt/play/p(?P<program_id>[0-9]+)/e(?P<id>[0-9]+)/?'
+    _TEST = {
+        'url': 'http://www.rtp.pt/play/p405/e174042/paixoes-cruzadas',
+        'info_dict': {
+            'id': '174042',
+            'ext': 'mp3',
+            'title': 'Paixões Cruzadas',
+            'description': 'As paixões musicais de António Cartaxo e António Macedo',
+            'thumbnail': 're:^https?://.*\.jpg',
+        },
+        'params': {
+            'skip_download': True,  # RTMP download
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        title = self._html_search_meta(
+            'twitter:title', webpage, display_name='title', fatal=True)
+        description = self._html_search_meta('description', webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        player_config = self._search_regex(
+            r'(?s)RTPPLAY\.player\.newPlayer\(\s*(\{.*?\})\s*\)', webpage, 'player config')
+        config = json.loads(js_to_json(player_config))
+
+        path, ext = config.get('file').rsplit('.', 1)
+        formats = [{
+            'app': config.get('application'),
+            'play_path': '{ext:s}:{path:s}'.format(ext=ext, path=path),
+            'page_url': url,
+            'url': 'rtmp://{streamer:s}/{application:s}'.format(**config),
+            'rtmp_live': config.get('live', False),
+            'ext': ext,
+            'vcodec': config.get('type') == 'audio' and 'none' or None,
+            'player_url': 'http://programas.rtp.pt/play/player.swf?v3',
+        }]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'description': description,
+            'thumbnail': thumbnail,
+        }

From 4237ba10dc46ffd0c95fbe2d077521da98726fe4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 19:44:25 +0100
Subject: [PATCH 0822/1937] [pornotube] Adapt to new interface

---
 youtube_dl/extractor/pornotube.py | 102 ++++++++++++++++++++----------
 1 file changed, 70 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/pornotube.py b/youtube_dl/extractor/pornotube.py
index 5253aa3d3..34735c51e 100644
--- a/youtube_dl/extractor/pornotube.py
+++ b/youtube_dl/extractor/pornotube.py
@@ -1,56 +1,94 @@
 from __future__ import unicode_literals
 
-import re
+import json
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+)
 from ..utils import (
-    compat_urllib_parse,
-
-    unified_strdate,
+    int_or_none,
 )
 
 
 class PornotubeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:\w+\.)?pornotube\.com(/c/(?P<channel>[0-9]+))?(/m/(?P<videoid>[0-9]+))(/(?P<title>.+))$'
+    _VALID_URL = r'https?://(?:\w+\.)?pornotube\.com/(?:[^?#]*?)/video/(?P<id>[0-9]+)'
     _TEST = {
-        'url': 'http://pornotube.com/c/173/m/1689755/Marilyn-Monroe-Bathing',
-        'md5': '374dd6dcedd24234453b295209aa69b6',
+        'url': 'http://www.pornotube.com/orientation/straight/video/4964/title/weird-hot-and-wet-science',
+        'md5': '60fc5a4f0d93a97968fc7999d98260c9',
         'info_dict': {
-            'id': '1689755',
-            'ext': 'flv',
-            'upload_date': '20090708',
-            'title': 'Marilyn-Monroe-Bathing',
-            'age_limit': 18
+            'id': '4964',
+            'ext': 'mp4',
+            'upload_date': '20141203',
+            'title': 'Weird Hot and Wet Science',
+            'description': 'md5:a8304bef7ef06cb4ab476ca6029b01b0',
+            'categories': ['Adult Humor', 'Blondes'],
+            'uploader': 'Alpha Blue Archives',
+            'thumbnail': 're:^https?://.*\\.jpg$',
+            'timestamp': 1417582800,
+            'age_limit': 18,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
+        video_id = self._match_id(url)
 
-        video_id = mobj.group('videoid')
-        video_title = mobj.group('title')
+        # Fetch origin token
+        js_config = self._download_webpage(
+            'http://www.pornotube.com/assets/src/app/config.js', video_id,
+            note='Download JS config')
+        originAuthenticationSpaceKey = self._search_regex(
+            r"constant\('originAuthenticationSpaceKey',\s*'([^']+)'",
+            js_config, 'originAuthenticationSpaceKey')
 
-        # Get webpage content
-        webpage = self._download_webpage(url, video_id)
+        # Fetch actual token
+        token_req_data = {
+            'authenticationSpaceKey': originAuthenticationSpaceKey,
+            'credentials': 'Clip Application',
+        }
+        token_req = compat_urllib_request.Request(
+            'https://api.aebn.net/auth/v1/token/primal',
+            data=json.dumps(token_req_data).encode('utf-8'))
+        token_req.add_header('Content-Type', 'application/json')
+        token_req.add_header('Origin', 'http://www.pornotube.com')
+        token_answer = self._download_json(
+            token_req, video_id, note='Requesting primal token')
+        token = token_answer['tokenKey']
 
-        # Get the video URL
-        VIDEO_URL_RE = r'url: "(?P<url>http://video[0-9].pornotube.com/.+\.flv)",'
-        video_url = self._search_regex(VIDEO_URL_RE, webpage, 'video url')
-        video_url = compat_urllib_parse.unquote(video_url)
+        # Get video URL
+        delivery_req = compat_urllib_request.Request(
+            'https://api.aebn.net/delivery/v1/clips/%s/MP4' % video_id)
+        delivery_req.add_header('Authorization', token)
+        delivery_info = self._download_json(
+            delivery_req, video_id, note='Downloading delivery information')
+        video_url = delivery_info['mediaUrl']
 
-        # Get the uploaded date
-        VIDEO_UPLOADED_RE = r'<div class="video_added_by">Added (?P<date>[0-9\/]+) by'
-        upload_date = self._html_search_regex(VIDEO_UPLOADED_RE, webpage, 'upload date', fatal=False)
-        if upload_date:
-            upload_date = unified_strdate(upload_date)
-        age_limit = self._rta_search(webpage)
+        # Get additional info (title etc.)
+        info_req = compat_urllib_request.Request(
+            'https://api.aebn.net/content/v1/clips/%s?expand='
+            'title,description,primaryImageNumber,startSecond,endSecond,'
+            'movie.title,movie.MovieId,movie.boxCoverFront,movie.stars,'
+            'movie.studios,stars.name,studios.name,categories.name,'
+            'clipActive,movieActive,publishDate,orientations' % video_id)
+        info_req.add_header('Authorization', token)
+        info = self._download_json(
+            info_req, video_id, note='Downloading metadata')
+
+        timestamp = int_or_none(info.get('publishDate'), scale=1000)
+        uploader = info.get('studios', [{}])[0].get('name')
+        movie_id = info['movie']['movieId']
+        thumbnail = 'http://pic.aebn.net/dis/t/%s/%s_%08d.jpg' % (
+            movie_id, movie_id, info['primaryImageNumber'])
+        categories = [c['name'] for c in info.get('categories')]
 
         return {
             'id': video_id,
             'url': video_url,
-            'upload_date': upload_date,
-            'title': video_title,
-            'ext': 'flv',
-            'format': 'flv',
-            'age_limit': age_limit,
+            'title': info['title'],
+            'description': info.get('description'),
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'thumbnail': thumbnail,
+            'categories': categories,
+            'age_limit': 18,
         }

From dc65a213fd73a385470fe1d8f7ad48a174d11bd4 Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Fri, 12 Dec 2014 19:58:44 +0100
Subject: [PATCH 0823/1937] comediansincarsgettingcoffee.com support

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ccc.py      | 46 ++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 youtube_dl/extractor/ccc.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 447833c01..bd40e9ea5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -48,6 +48,7 @@ from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .cbs import CBSIE
 from .cbsnews import CBSNewsIE
+from .ccc import ComCarCoffIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chilloutzone import ChilloutzoneIE
diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
new file mode 100644
index 000000000..809725f12
--- /dev/null
+++ b/youtube_dl/extractor/ccc.py
@@ -0,0 +1,46 @@
+# encoding: utf-8
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate,
+)
+
+class ComCarCoffIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?comediansincarsgettingcoffee\.com/(?P<id>[a-z0-9\-]+)/?'
+    _TESTS = [
+        {
+            'url': 'http://comediansincarsgettingcoffee.com/miranda-sings-happy-thanksgiving-miranda/',
+            'info_dict': {
+                'id': 'miranda-sings-happy-thanksgiving-miranda',
+                'upload_date': '20141127',
+                'title': 'Happy Thanksgiving Miranda',
+                'description': 'Jerry Seinfeld and his special guest Miranda Sings cruise around town in search of coffee, complaining and apologizing along the way.',
+                'thumbnail': 'http://ccc.crackle.com/images/s5e4_thumb.jpg',
+            },
+        }
+    ]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        full_data = json.loads(self._search_regex(
+            r'<script type="application/json" id="videoData">(?P<json>.+?)</script>',
+            webpage, 'json'))
+
+        video_id = full_data['activeVideo']['video']
+        video_data = full_data['videos'][video_id]
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': video_data['title'],
+            'description': video_data['description'],
+            # XXX: the original datum is a full ISO timestamp... why convert it to a worse format?
+            'upload_date': unified_strdate(video_data['pubDate']),
+            'thumbnail': video_data['images']['thumb'],
+            # XXX: what do we do with video_data['images']['poster']?
+            'formats': self._extract_m3u8_formats(video_data['mediaUrl'], video_id),
+        }

From e9404524cc108d2a054a9aaeaecc0ac88f451676 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 20:10:15 +0100
Subject: [PATCH 0824/1937] [ninegag] Test for additional properties

---
 youtube_dl/extractor/ninegag.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 16a02ad79..7f842b5c2 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -23,6 +23,9 @@ class NineGagIE(InfoExtractor):
             "ext": "mp4",
             "description": "This 3-minute video will make you smile and then make you feel untalented and insignificant. Anyway, you should share this awesomeness. (Thanks, Dino!)",
             "title": "\"People Are Awesome 2013\" Is Absolutely Awesome",
+            'uploader_id': 'UCdEH6EjDKwtTe-sO2f0_1XA',
+            'uploader': 'CompilationChannel',
+            'upload_date': '20131110',
             "view_count": int,
             "thumbnail": "re:^https?://",
         },
@@ -35,6 +38,9 @@ class NineGagIE(InfoExtractor):
             'display_id': 'alternate-banned-opening-scene-of-gravity',
             "description": "While Gravity was a pretty awesome movie already, YouTuber Krishna Shenoi came up with a way to improve upon it, introducing a much better solution to Sandra Bullock's seemingly endless tumble in space. The ending is priceless.",
             'title': "Banned Opening Scene Of \"Gravity\" That Changes The Whole Movie",
+            'uploader': 'Krishna Shenoi',
+            'upload_date': '20140401',
+            'uploader_id': 'krishnashenoi93',
         },
     }]
 

From 90d71d3f088bbb787795aa6018eda21e2a821e7d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 20:11:23 +0100
Subject: [PATCH 0825/1937] [ooyala] Remove test md5sums

---
 youtube_dl/extractor/ooyala.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py
index f17a52858..d5b05c18f 100644
--- a/youtube_dl/extractor/ooyala.py
+++ b/youtube_dl/extractor/ooyala.py
@@ -16,7 +16,6 @@ class OoyalaIE(InfoExtractor):
         {
             # From http://it.slashdot.org/story/13/04/25/178216/recovering-data-from-broken-hard-drives-and-ssds-video
             'url': 'http://player.ooyala.com/player.js?embedCode=pxczE2YjpfHfn1f3M-ykG_AmJRRn0PD8',
-            'md5': '3f5cceb3a7bf461d6c29dc466cf8033c',
             'info_dict': {
                 'id': 'pxczE2YjpfHfn1f3M-ykG_AmJRRn0PD8',
                 'ext': 'mp4',
@@ -26,7 +25,6 @@ class OoyalaIE(InfoExtractor):
         }, {
             # Only available for ipad
             'url': 'http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0',
-            'md5': '4b9754921fddb68106e48c142e2a01e6',
             'info_dict': {
                 'id': 'x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0',
                 'ext': 'mp4',

From dc5596ff54860dc470579c17f4b072266f44cf5c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 20:32:02 +0100
Subject: [PATCH 0826/1937] [comcarcoff] (#4454)

---
 youtube_dl/extractor/__init__.py   |  2 +-
 youtube_dl/extractor/ccc.py        | 46 --------------------------
 youtube_dl/extractor/comcarcoff.py | 52 ++++++++++++++++++++++++++++++
 3 files changed, 53 insertions(+), 47 deletions(-)
 delete mode 100644 youtube_dl/extractor/ccc.py
 create mode 100644 youtube_dl/extractor/comcarcoff.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bd40e9ea5..119ec2044 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -48,7 +48,6 @@ from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .cbs import CBSIE
 from .cbsnews import CBSNewsIE
-from .ccc import ComCarCoffIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chilloutzone import ChilloutzoneIE
@@ -66,6 +65,7 @@ from .cnn import (
 )
 from .collegehumor import CollegeHumorIE
 from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
+from .comcarcoff import ComCarCoffIE
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
 from .criterion import CriterionIE
diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
deleted file mode 100644
index 809725f12..000000000
--- a/youtube_dl/extractor/ccc.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# encoding: utf-8
-import re
-import json
-
-from .common import InfoExtractor
-from ..utils import (
-    unified_strdate,
-)
-
-class ComCarCoffIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?comediansincarsgettingcoffee\.com/(?P<id>[a-z0-9\-]+)/?'
-    _TESTS = [
-        {
-            'url': 'http://comediansincarsgettingcoffee.com/miranda-sings-happy-thanksgiving-miranda/',
-            'info_dict': {
-                'id': 'miranda-sings-happy-thanksgiving-miranda',
-                'upload_date': '20141127',
-                'title': 'Happy Thanksgiving Miranda',
-                'description': 'Jerry Seinfeld and his special guest Miranda Sings cruise around town in search of coffee, complaining and apologizing along the way.',
-                'thumbnail': 'http://ccc.crackle.com/images/s5e4_thumb.jpg',
-            },
-        }
-    ]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-
-        full_data = json.loads(self._search_regex(
-            r'<script type="application/json" id="videoData">(?P<json>.+?)</script>',
-            webpage, 'json'))
-
-        video_id = full_data['activeVideo']['video']
-        video_data = full_data['videos'][video_id]
-
-        return {
-            'id': video_id,
-            'display_id': display_id,
-            'title': video_data['title'],
-            'description': video_data['description'],
-            # XXX: the original datum is a full ISO timestamp... why convert it to a worse format?
-            'upload_date': unified_strdate(video_data['pubDate']),
-            'thumbnail': video_data['images']['thumb'],
-            # XXX: what do we do with video_data['images']['poster']?
-            'formats': self._extract_m3u8_formats(video_data['mediaUrl'], video_id),
-        }
diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
new file mode 100644
index 000000000..46953c12e
--- /dev/null
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -0,0 +1,52 @@
+# encoding: utf-8
+import json
+
+from .common import InfoExtractor
+from ..utils import parse_iso8601
+
+
+class ComCarCoffIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?comediansincarsgettingcoffee\.com/(?P<id>[a-z0-9\-]+)/?'
+    _TESTS = [{
+        'url': 'http://comediansincarsgettingcoffee.com/miranda-sings-happy-thanksgiving-miranda/',
+        'info_dict': {
+            'id': 'miranda-sings-happy-thanksgiving-miranda',
+            'ext': 'mp4',
+            'upload_date': '20141127',
+            'timestamp': 1417107600,
+            'title': 'Happy Thanksgiving Miranda',
+            'description': 'Jerry Seinfeld and his special guest Miranda Sings cruise around town in search of coffee, complaining and apologizing along the way.',
+            'thumbnail': 'http://ccc.crackle.com/images/s5e4_thumb.jpg',
+        },
+        'params': {
+            'skip_download': 'requires ffmpeg',
+        }
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        full_data = json.loads(self._search_regex(
+            r'<script type="application/json" id="videoData">(?P<json>.+?)</script>',
+            webpage, 'full data json'))
+
+        video_id = full_data['activeVideo']['video']
+        video_data = full_data['videos'][video_id]
+        thumbnails = [{
+            'url': video_data['images']['thumb'],
+        }, {
+            'url': video_data['images']['poster'],
+        }]
+        formats = self._extract_m3u8_formats(
+            video_data['mediaUrl'], video_id, ext='mp4')
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': video_data['title'],
+            'description': video_data.get('description'),
+            'timestamp': parse_iso8601(video_data.get('pubDate')),
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }

From 3c864e930dd7533b404d03872d078e038883ac3d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 20:35:17 +0100
Subject: [PATCH 0827/1937] [comcarcoff] Adapt
 c62159ea91a04ef82560472b254aef1cc9f70a11

---
 youtube_dl/extractor/comcarcoff.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index 46953c12e..67724139c 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -6,7 +6,7 @@ from ..utils import parse_iso8601
 
 
 class ComCarCoffIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?comediansincarsgettingcoffee\.com/(?P<id>[a-z0-9\-]+)/?'
+    _VALID_URL = r'http://(?:www\.)?comediansincarsgettingcoffee\.com/(?P<id>[a-z0-9\-]*)'
     _TESTS = [{
         'url': 'http://comediansincarsgettingcoffee.com/miranda-sings-happy-thanksgiving-miranda/',
         'info_dict': {
@@ -25,6 +25,8 @@ class ComCarCoffIE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
+        if not display_id:
+            display_id = 'comediansincarsgettingcoffee.com'
         webpage = self._download_webpage(url, display_id)
 
         full_data = json.loads(self._search_regex(

From 6a1b20de2a562f76501ae96721957f43804d8e61 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 20:37:28 +0100
Subject: [PATCH 0828/1937] [urort] Modernize

---
 youtube_dl/extractor/urort.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/urort.py b/youtube_dl/extractor/urort.py
index 5d06fcc9e..468aa7600 100644
--- a/youtube_dl/extractor/urort.py
+++ b/youtube_dl/extractor/urort.py
@@ -33,13 +33,11 @@ class UrortIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        playlist_id = mobj.group('id')
+        playlist_id = self._match_id(url)
 
         fstr = compat_urllib_parse.quote("InternalBandUrl eq '%s'" % playlist_id)
         json_url = 'http://urort.p3.no/breeze/urort/TrackDtos?$filter=' + fstr
         songs = self._download_json(json_url, playlist_id)
-        print(songs[0])
 
         entries = [{
             'id': '%d-%s' % (s['BandId'], s['$id']),

From 19bf2b4e88a1d71f5d617a6e9728a9c2687a8df8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 20:37:58 +0100
Subject: [PATCH 0829/1937] [comcarcoff] Add unicode_literals declaration

---
 youtube_dl/extractor/comcarcoff.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index 67724139c..a2211ee44 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -1,4 +1,6 @@
 # encoding: utf-8
+from __future__ import unicode_literals
+
 import json
 
 from .common import InfoExtractor

From baa7081d68996377e44225c74a1ec05e801617a2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 12 Dec 2014 20:55:18 +0100
Subject: [PATCH 0830/1937] [urort] Update to new multi-format protocol

---
 youtube_dl/extractor/urort.py | 37 +++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/urort.py b/youtube_dl/extractor/urort.py
index 468aa7600..249a34c86 100644
--- a/youtube_dl/extractor/urort.py
+++ b/youtube_dl/extractor/urort.py
@@ -18,11 +18,10 @@ class UrortIE(InfoExtractor):
         'url': 'https://urort.p3.no/#!/Band/Gerilja',
         'md5': '5ed31a924be8a05e47812678a86e127b',
         'info_dict': {
-            'id': '33124-4',
+            'id': '33124-24',
             'ext': 'mp3',
             'title': 'The Bomb',
             'thumbnail': 're:^https?://.+\.jpg',
-            'like_count': int,
             'uploader': 'Gerilja',
             'uploader_id': 'Gerilja',
             'upload_date': '20100323',
@@ -36,20 +35,28 @@ class UrortIE(InfoExtractor):
         playlist_id = self._match_id(url)
 
         fstr = compat_urllib_parse.quote("InternalBandUrl eq '%s'" % playlist_id)
-        json_url = 'http://urort.p3.no/breeze/urort/TrackDtos?$filter=' + fstr
+        json_url = 'http://urort.p3.no/breeze/urort/TrackDTOViews?$filter=%s&$orderby=Released%%20desc&$expand=Tags%%2CFiles' % fstr
         songs = self._download_json(json_url, playlist_id)
-
-        entries = [{
-            'id': '%d-%s' % (s['BandId'], s['$id']),
-            'title': s['Title'],
-            'url': s['TrackUrl'],
-            'ext': 'mp3',
-            'uploader_id': playlist_id,
-            'uploader': s.get('BandName', playlist_id),
-            'like_count': s.get('LikeCount'),
-            'thumbnail': 'http://urort.p3.no/cloud/images/%s' % s['Image'],
-            'upload_date': unified_strdate(s.get('Released')),
-        } for s in songs]
+        entries = []
+        for s in songs:
+            formats = [{
+                'tbr': f.get('Quality'),
+                'ext': f['FileType'],
+                'format_id': '%s-%s' % (f['FileType'], f.get('Quality', '')),
+                'url': 'http://p3urort.blob.core.windows.net/tracks/%s' % f['FileRef'],
+                'preference': 3 if f['FileType'] == 'mp3' else 2,
+            } for f in s['Files']]
+            self._sort_formats(formats)
+            e = {
+                'id': '%d-%s' % (s['BandId'], s['$id']),
+                'title': s['Title'],
+                'uploader_id': playlist_id,
+                'uploader': s.get('BandName', playlist_id),
+                'thumbnail': 'http://urort.p3.no/cloud/images/%s' % s['Image'],
+                'upload_date': unified_strdate(s.get('Released')),
+                'formats': formats,
+            }
+            entries.append(e)
 
         return {
             '_type': 'playlist',

From 23d3608c6b5b50cbbc81314d18824c4951f8af27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 12 Dec 2014 22:23:54 +0100
Subject: [PATCH 0831/1937] [youtube:channel] Fix extraction (fixes #4435)

It uses now the same pagination system as playlists
---
 youtube_dl/extractor/youtube.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 7b6179a2a..a7bdce72f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1269,8 +1269,6 @@ class YoutubeTopListIE(YoutubePlaylistIE):
 class YoutubeChannelIE(InfoExtractor):
     IE_DESC = 'YouTube.com channels'
     _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/(?P<id>[0-9A-Za-z_-]+)'
-    _MORE_PAGES_INDICATOR = 'yt-uix-load-more'
-    _MORE_PAGES_URL = 'https://www.youtube.com/c4_browse_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
     IE_NAME = 'youtube:channel'
     _TESTS = [{
         'note': 'paginated channel',
@@ -1307,20 +1305,27 @@ class YoutubeChannelIE(InfoExtractor):
             return self.playlist_result(entries, channel_id)
 
         def _entries():
+            more_widget_html = content_html = channel_page
             for pagenum in itertools.count(1):
-                url = self._MORE_PAGES_URL % (pagenum, channel_id)
-                page = self._download_json(
-                    url, channel_id, note='Downloading page #%s' % pagenum,
-                    transform_source=uppercase_escape)
 
-                ids_in_page = self.extract_videos_from_page(page['content_html'])
+                ids_in_page = self.extract_videos_from_page(content_html)
                 for video_id in ids_in_page:
                     yield self.url_result(
                         video_id, 'Youtube', video_id=video_id)
 
-                if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
+                mobj = re.search(
+                    r'data-uix-load-more-href="/?(?P<more>[^"]+)"',
+                    more_widget_html)
+                if not mobj:
                     break
 
+                more = self._download_json(
+                    'https://youtube.com/%s' % mobj.group('more'), channel_id,
+                    'Downloading page #%s' % (pagenum + 1),
+                    transform_source=uppercase_escape)
+                content_html = more['content_html']
+                more_widget_html = more['load_more_widget_html']
+
         return self.playlist_result(_entries(), channel_id)
 
 

From 26c06f0c513347edae6b1005b875a634ec386d19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 12 Dec 2014 22:26:50 +0100
Subject: [PATCH 0832/1937] [youtube:playlist] Remove unused property

---
 youtube_dl/extractor/youtube.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a7bdce72f..823d6aaf3 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1072,7 +1072,6 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
                         ((?:PL|LL|EC|UU|FL|RD)[0-9A-Za-z-_]{10,})
                      )"""
     _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
-    _MORE_PAGES_INDICATOR = r'data-link-type="next"'
     _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)'
     IE_NAME = 'youtube:playlist'
     _TESTS = [{

From 7668a2c5cba305bf7dc41b14149fa7f55d4dffd6 Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Fri, 12 Dec 2014 20:49:50 +0100
Subject: [PATCH 0833/1937] [comcarcoff] add webpage_url datum

---
 youtube_dl/extractor/comcarcoff.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/comcarcoff.py b/youtube_dl/extractor/comcarcoff.py
index 67724139c..b62be56da 100644
--- a/youtube_dl/extractor/comcarcoff.py
+++ b/youtube_dl/extractor/comcarcoff.py
@@ -51,4 +51,5 @@ class ComCarCoffIE(InfoExtractor):
             'timestamp': parse_iso8601(video_data.get('pubDate')),
             'thumbnails': thumbnails,
             'formats': formats,
+            'webpage_url': 'http://comediansincarsgettingcoffee.com/%s' % (video_data.get('urlSlug', video_data.get('slug'))),
         }

From 0db261ba567cb5370455d67c4398e11e5e2119f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 12 Dec 2014 23:27:25 +0100
Subject: [PATCH 0834/1937] [utils] make_HTTPS_handler:  Use
 ssl.create_default_context in Python 2.7.9

The new features in the ssl module have been backported from 3.4, see https://docs.python.org/dev/whatsnew/2.7.html#pep-466-network-security-enhancements-for-python-2-7
---
 youtube_dl/utils.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 51a822e4f..83a45e702 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -388,7 +388,13 @@ def formatSeconds(secs):
 
 
 def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
-    if sys.version_info < (3, 2):
+    if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
+        context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+        context.options &= ~ssl.OP_NO_SSLv3  # Allow older, not-as-secure SSLv3
+        if opts_no_check_certificate:
+            context.verify_mode = ssl.CERT_NONE
+        return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
+    elif sys.version_info < (3, 2):
         import httplib
 
         class HTTPSConnectionV3(httplib.HTTPSConnection):
@@ -409,12 +415,6 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
             def https_open(self, req):
                 return self.do_open(HTTPSConnectionV3, req)
         return HTTPSHandlerV3(**kwargs)
-    elif hasattr(ssl, 'create_default_context'):  # Python >= 3.4
-        context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
-        context.options &= ~ssl.OP_NO_SSLv3  # Allow older, not-as-secure SSLv3
-        if opts_no_check_certificate:
-            context.verify_mode = ssl.CERT_NONE
-        return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
     else:  # Python < 3.4
         context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
         context.verify_mode = (ssl.CERT_NONE

From 293f0f39ce955b9aa8284d461a0444e27491c392 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 12 Dec 2014 23:43:25 +0100
Subject: [PATCH 0835/1937] [utils] make_HTTPS_handler: Remove try/except block
 that would always raise an exception

This code is only run for Python < 3.4, where context.load_default_certs doesn't exist
---
 youtube_dl/utils.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 83a45e702..ec34dcef9 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -421,10 +421,6 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
                                if opts_no_check_certificate
                                else ssl.CERT_REQUIRED)
         context.set_default_verify_paths()
-        try:
-            context.load_default_certs()
-        except AttributeError:
-            pass  # Python < 3.4
         return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
 
 

From 5e1912cfc102f457fb9cb2472fb93c973cb68732 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 11:02:24 +0100
Subject: [PATCH 0836/1937] [5min] Remove helper method and modernize

Previously, other extractor would go call a private(!) helper method. Instead, just hardcode the 5min:video_id format - it's not if that would ever change.
---
 youtube_dl/extractor/aol.py      | 49 ++++++++++++++++----------------
 youtube_dl/extractor/engadget.py |  8 ++----
 youtube_dl/extractor/fivemin.py  | 15 ++++------
 3 files changed, 32 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/aol.py b/youtube_dl/extractor/aol.py
index 47f8e4157..185ee3693 100644
--- a/youtube_dl/extractor/aol.py
+++ b/youtube_dl/extractor/aol.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .fivemin import FiveMinIE
 
 
 class AolIE(InfoExtractor):
@@ -42,31 +41,31 @@ class AolIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-
         playlist_id = mobj.group('playlist_id')
-        if playlist_id and not self._downloader.params.get('noplaylist'):
-            self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
+        if not playlist_id or self._downloader.params.get('noplaylist'):
+            return self.url_result('5min:%s' % video_id)
 
-            webpage = self._download_webpage(url, playlist_id)
-            title = self._html_search_regex(
-                r'<h1 class="video-title[^"]*">(.+?)</h1>', webpage, 'title')
-            playlist_html = self._search_regex(
-                r"(?s)<ul\s+class='video-related[^']*'>(.*?)</ul>", webpage,
-                'playlist HTML')
-            entries = [{
-                '_type': 'url',
-                'url': 'aol-video:%s' % m.group('id'),
-                'ie_key': 'Aol',
-            } for m in re.finditer(
-                r"<a\s+href='.*videoid=(?P<id>[0-9]+)'\s+class='video-thumb'>",
-                playlist_html)]
+        self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
 
-            return {
-                '_type': 'playlist',
-                'id': playlist_id,
-                'display_id': mobj.group('playlist_display_id'),
-                'title': title,
-                'entries': entries,
-            }
+        webpage = self._download_webpage(url, playlist_id)
+        title = self._html_search_regex(
+            r'<h1 class="video-title[^"]*">(.+?)</h1>', webpage, 'title')
+        playlist_html = self._search_regex(
+            r"(?s)<ul\s+class='video-related[^']*'>(.*?)</ul>", webpage,
+            'playlist HTML')
+        entries = [{
+            '_type': 'url',
+            'url': 'aol-video:%s' % m.group('id'),
+            'ie_key': 'Aol',
+        } for m in re.finditer(
+            r"<a\s+href='.*videoid=(?P<id>[0-9]+)'\s+class='video-thumb'>",
+            playlist_html)]
+
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'display_id': mobj.group('playlist_display_id'),
+            'title': title,
+            'entries': entries,
+        }
 
-        return FiveMinIE._build_result(video_id)
diff --git a/youtube_dl/extractor/engadget.py b/youtube_dl/extractor/engadget.py
index 92ada81d2..4ea37ebd9 100644
--- a/youtube_dl/extractor/engadget.py
+++ b/youtube_dl/extractor/engadget.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .fivemin import FiveMinIE
 from ..utils import (
     url_basename,
 )
@@ -27,11 +26,10 @@ class EngadgetIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         if video_id is not None:
-            return FiveMinIE._build_result(video_id)
+            return self.url_result('5min:%s' % video_id)
         else:
             title = url_basename(url)
             webpage = self._download_webpage(url, title)
@@ -39,5 +37,5 @@ class EngadgetIE(InfoExtractor):
             return {
                 '_type': 'playlist',
                 'title': title,
-                'entries': [FiveMinIE._build_result(id) for id in ids]
+                'entries': [self.url_result('5min:%s' % vid) for vid in ids]
             }
diff --git a/youtube_dl/extractor/fivemin.py b/youtube_dl/extractor/fivemin.py
index f9c127ce6..5b24b921c 100644
--- a/youtube_dl/extractor/fivemin.py
+++ b/youtube_dl/extractor/fivemin.py
@@ -1,11 +1,11 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
     compat_urllib_parse,
+)
+from ..utils import (
     ExtractorError,
 )
 
@@ -13,7 +13,7 @@ from ..utils import (
 class FiveMinIE(InfoExtractor):
     IE_NAME = '5min'
     _VALID_URL = r'''(?x)
-        (?:https?://[^/]*?5min\.com/Scripts/PlayerSeed\.js\?(.*?&)?playList=|
+        (?:https?://[^/]*?5min\.com/Scripts/PlayerSeed\.js\?(?:.*?&)?playList=|
             5min:)
         (?P<id>\d+)
         '''
@@ -41,13 +41,8 @@ class FiveMinIE(InfoExtractor):
         },
     ]
 
-    @classmethod
-    def _build_result(cls, video_id):
-        return cls.url_result('5min:%s' % video_id, cls.ie_key())
-
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         embed_url = 'https://embed.5min.com/playerseed/?playList=%s' % video_id
         embed_page = self._download_webpage(embed_url, video_id,
                                             'Downloading embed page')

From 20e35880bf7413c5085c1ade96af0fc6e56d84d1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 12:14:44 +0100
Subject: [PATCH 0837/1937] [streamcz] Update extractor

---
 youtube_dl/extractor/streamcz.py | 78 +++++++++++++++++---------------
 1 file changed, 41 insertions(+), 37 deletions(-)

diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index 73efe9542..778704839 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -5,14 +5,16 @@ import re
 import json
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_str,
+)
 from ..utils import (
     int_or_none,
-    compat_str,
 )
 
 
 class StreamCZIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?stream\.cz/.+/(?P<videoid>.+)'
+    _VALID_URL = r'https?://(?:www\.)?stream\.cz/.+/(?P<id>[0-9]+)'
 
     _TESTS = [{
         'url': 'http://www.stream.cz/peklonataliri/765767-ecka-pro-deti',
@@ -21,61 +23,63 @@ class StreamCZIE(InfoExtractor):
             'id': '765767',
             'ext': 'mp4',
             'title': 'Peklo na talíři: Éčka pro děti',
-            'description': 'md5:49ace0df986e95e331d0fe239d421519',
-            'thumbnail': 'http://im.stream.cz/episode/52961d7e19d423f8f06f0100',
+            'description': 'Taška s grónskou pomazánkou a další pekelnosti ZDE',
+            'thumbnail': 're:^http://im.stream.cz/episode/52961d7e19d423f8f06f0100',
             'duration': 256,
         },
     }, {
         'url': 'http://www.stream.cz/blanik/10002447-tri-roky-pro-mazanka',
-        'md5': '246272e753e26bbace7fcd9deca0650c',
+        'md5': 'e54a254fb8b871968fd8403255f28589',
         'info_dict': {
             'id': '10002447',
             'ext': 'mp4',
             'title': 'Kancelář Blaník: Tři roky pro Mazánka',
-            'description': 'md5:9177695a8b756a0a8ab160de4043b392',
-            'thumbnail': 'http://im.stream.cz/episode/537f838c50c11f8d21320000',
+            'description': 'md5:3862a00ba7bf0b3e44806b544032c859',
+            'thumbnail': 're:^http://im.stream.cz/episode/537f838c50c11f8d21320000',
             'duration': 368,
         },
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
-
-        webpage = self._download_webpage(url, video_id)
-
-        data = self._html_search_regex(r'Stream\.Data\.Episode\((.+?)\);', webpage, 'stream data')
-
-        jsonData = json.loads(data)
+        video_id = self._match_id(url)
+        data = self._download_json(
+            'http://www.stream.cz/API/episode/%s' % video_id, video_id)
 
         formats = []
-        for video in jsonData['instances']:
-            for video_format in video['instances']:
-                format_id = video_format['quality']
-
-                if format_id == '240p':
-                    quality = 0
-                elif format_id == '360p':
-                    quality = 1
-                elif format_id == '480p':
-                    quality = 2
-                elif format_id == '720p':
-                    quality = 3
-
+        for quality, video in enumerate(data['video_qualities']):
+            for f in video['formats']:
+                typ = f['type'].partition('/')[2]
+                qlabel = video.get('quality_label')
                 formats.append({
-                    'format_id': '%s-%s' % (video_format['type'].split('/')[1], format_id),
-                    'url': video_format['source'],
+                    'format_note': '%s-%s' % (qlabel, typ) if qlabel else typ,
+                    'format_id': '%s-%s' % (typ, f['quality']),
+                    'url': f['source'],
+                    'height': int_or_none(f['quality'].rstrip('p')),
                     'quality': quality,
                 })
-
         self._sort_formats(formats)
 
+        image = data.get('image')
+        if image:
+            thumbnail = self._proto_relative_url(
+                image.replace('{width}', '1240').replace('{height}', '697'),
+                scheme='http:',
+            )
+        else:
+            thumbnail = None
+
+        stream = data.get('_embedded', {}).get('stream:show', {}).get('name')
+        if stream:
+            title = '%s: %s' % (stream, data['name'])
+        else:
+            title = data['name']
+
         return {
-            'id': compat_str(jsonData['episode_id']),
-            'title': self._og_search_title(webpage),
-            'thumbnail': jsonData['episode_image_original_url'].replace('//', 'http://'),
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
             'formats': formats,
-            'description': self._og_search_description(webpage),
-            'duration': int_or_none(jsonData['duration']),
-            'view_count': int_or_none(jsonData['stats_total']),
+            'description': data.get('web_site_text'),
+            'duration': int_or_none(data.get('duration')),
+            'view_count': int_or_none(data.get('views')),
         }

From 1cc79574fc5df21bf35dccf61eac0e9e75ed8d20 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 12:24:42 +0100
Subject: [PATCH 0838/1937] Fix imports and general cleanup
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

· Import from compat what comes from compat. Yes, some names are available in utils too, but that's an implementation detail.
· Use _match_id consistently whenever possible
· Fix some outdated tests
· Use consistent valid URL (always match the whole protocol, no ^ at start required)
· Use modern test definitions
---
 youtube_dl/downloader/common.py              |  2 +-
 youtube_dl/downloader/f4m.py                 |  4 ++-
 youtube_dl/downloader/hls.py                 |  4 ++-
 youtube_dl/downloader/http.py                |  5 +--
 youtube_dl/downloader/rtmp.py                |  2 +-
 youtube_dl/extractor/allocine.py             |  5 +--
 youtube_dl/extractor/aol.py                  |  1 -
 youtube_dl/extractor/appletrailers.py        |  2 +-
 youtube_dl/extractor/auengine.py             |  2 +-
 youtube_dl/extractor/bambuser.py             |  2 +-
 youtube_dl/extractor/bandcamp.py             |  4 ++-
 youtube_dl/extractor/bet.py                  |  2 +-
 youtube_dl/extractor/bilibili.py             |  7 ++--
 youtube_dl/extractor/brightcove.py           | 19 +++++-----
 youtube_dl/extractor/ceskatelevize.py        |  4 ++-
 youtube_dl/extractor/comedycentral.py        |  4 ++-
 youtube_dl/extractor/condenast.py            |  6 ++--
 youtube_dl/extractor/crunchyroll.py          |  6 ++--
 youtube_dl/extractor/dailymotion.py          | 10 +++---
 youtube_dl/extractor/daum.py                 |  2 +-
 youtube_dl/extractor/ehow.py                 | 11 +++---
 youtube_dl/extractor/eighttracks.py          |  2 +-
 youtube_dl/extractor/escapist.py             |  5 +--
 youtube_dl/extractor/everyonesmixtape.py     |  4 ++-
 youtube_dl/extractor/extremetube.py          |  8 +++--
 youtube_dl/extractor/fc2.py                  | 13 ++++---
 youtube_dl/extractor/firedrive.py            | 11 +++---
 youtube_dl/extractor/fourtube.py             | 14 ++++----
 youtube_dl/extractor/franceculture.py        |  2 +-
 youtube_dl/extractor/francetv.py             | 12 ++++---
 youtube_dl/extractor/gamespot.py             |  4 ++-
 youtube_dl/extractor/gdcvault.py             |  2 +-
 youtube_dl/extractor/golem.py                |  4 ++-
 youtube_dl/extractor/googlesearch.py         |  2 +-
 youtube_dl/extractor/gorillavid.py           |  8 ++---
 youtube_dl/extractor/hostingbulk.py          |  8 ++---
 youtube_dl/extractor/hypem.py                | 11 +++---
 youtube_dl/extractor/imdb.py                 |  2 +-
 youtube_dl/extractor/infoq.py                |  7 ++--
 youtube_dl/extractor/internetvideoarchive.py |  4 ++-
 youtube_dl/extractor/iprima.py               |  4 ++-
 youtube_dl/extractor/ivi.py                  |  4 ++-
 youtube_dl/extractor/keezmovies.py           |  7 ++--
 youtube_dl/extractor/livestream.py           |  4 ++-
 youtube_dl/extractor/lynda.py                |  6 ++--
 youtube_dl/extractor/malemotion.py           | 38 +++++++-------------
 youtube_dl/extractor/metacafe.py             |  4 ++-
 youtube_dl/extractor/mit.py                  |  4 ++-
 youtube_dl/extractor/mitele.py               | 13 ++++---
 youtube_dl/extractor/mixcloud.py             |  4 ++-
 youtube_dl/extractor/moevideo.py             |  6 ++--
 youtube_dl/extractor/mofosex.py              |  6 ++--
 youtube_dl/extractor/moniker.py              |  7 ++--
 youtube_dl/extractor/mooshare.py             | 10 +++---
 youtube_dl/extractor/motorsport.py           | 10 +++---
 youtube_dl/extractor/movieclips.py           |  4 ++-
 youtube_dl/extractor/mtv.py                  |  4 ++-
 youtube_dl/extractor/myspass.py              |  5 +--
 youtube_dl/extractor/naver.py                |  8 +++--
 youtube_dl/extractor/nbc.py                  |  4 ++-
 youtube_dl/extractor/nfb.py                  | 14 ++++----
 youtube_dl/extractor/nfl.py                  |  4 ++-
 youtube_dl/extractor/niconico.py             | 10 +++---
 youtube_dl/extractor/noco.py                 | 14 ++++----
 youtube_dl/extractor/nosvideo.py             |  7 ++--
 youtube_dl/extractor/novamov.py              |  4 ++-
 youtube_dl/extractor/nuvid.py                |  9 ++---
 youtube_dl/extractor/photobucket.py          |  5 +--
 youtube_dl/extractor/played.py               |  7 ++--
 youtube_dl/extractor/playfm.py               |  4 ++-
 youtube_dl/extractor/playvid.py              | 24 ++++++-------
 youtube_dl/extractor/pornhub.py              |  8 +++--
 youtube_dl/extractor/promptfile.py           |  8 +++--
 youtube_dl/extractor/prosiebensat1.py        |  4 ++-
 youtube_dl/extractor/quickvid.py             |  4 ++-
 youtube_dl/extractor/rai.py                  |  4 ++-
 youtube_dl/extractor/rts.py                  |  4 ++-
 youtube_dl/extractor/rutube.py               | 13 ++++---
 youtube_dl/extractor/screencast.py           | 11 +++---
 youtube_dl/extractor/shared.py               | 38 +++++++++++---------
 youtube_dl/extractor/sharesix.py             |  4 ++-
 youtube_dl/extractor/sina.py                 |  2 +-
 youtube_dl/extractor/slideshare.py           |  4 ++-
 youtube_dl/extractor/smotri.py               |  4 ++-
 youtube_dl/extractor/sockshare.py            | 15 ++++----
 youtube_dl/extractor/soundcloud.py           |  5 +--
 youtube_dl/extractor/spankwire.py            |  8 +++--
 youtube_dl/extractor/sportdeutschland.py     |  4 ++-
 youtube_dl/extractor/streamcloud.py          |  2 +-
 youtube_dl/extractor/streamcz.py             |  6 ----
 youtube_dl/extractor/tapely.py               |  8 +++--
 youtube_dl/extractor/ted.py                  |  2 +-
 youtube_dl/extractor/theplatform.py          |  4 ++-
 youtube_dl/extractor/tlc.py                  |  2 +-
 youtube_dl/extractor/tube8.py                |  4 ++-
 youtube_dl/extractor/tutv.py                 |  8 ++---
 youtube_dl/extractor/twitch.py               |  4 ++-
 youtube_dl/extractor/udemy.py                |  4 ++-
 youtube_dl/extractor/urort.py                |  6 ++--
 youtube_dl/extractor/ustream.py              |  2 +-
 youtube_dl/extractor/vbox7.py                | 12 +++----
 youtube_dl/extractor/veehd.py                |  9 ++---
 youtube_dl/extractor/veoh.py                 |  4 ++-
 youtube_dl/extractor/vevo.py                 |  4 ++-
 youtube_dl/extractor/videodetective.py       |  9 ++---
 youtube_dl/extractor/videomega.py            | 10 +++---
 youtube_dl/extractor/vk.py                   | 11 +++---
 youtube_dl/extractor/vodlocker.py            |  6 ++--
 youtube_dl/extractor/vube.py                 |  4 ++-
 youtube_dl/extractor/vuclip.py               |  9 ++---
 youtube_dl/extractor/wdr.py                  |  5 +--
 youtube_dl/extractor/wistia.py               |  8 ++---
 youtube_dl/extractor/xbef.py                 |  9 ++---
 youtube_dl/extractor/xnxx.py                 |  9 ++---
 youtube_dl/extractor/xtube.py                |  8 +++--
 youtube_dl/extractor/xvideos.py              | 22 ++++--------
 youtube_dl/extractor/yahoo.py                |  6 ++--
 youtube_dl/extractor/ynet.py                 |  2 +-
 youtube_dl/extractor/youporn.py              |  5 +--
 youtube_dl/swfinterp.py                      |  2 +-
 120 files changed, 443 insertions(+), 384 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 2a566eabe..8181bca09 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -5,8 +5,8 @@ import re
 import sys
 import time
 
+from ..compat import compat_str
 from ..utils import (
-    compat_str,
     encodeFilename,
     format_bytes,
     timeconvert,
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 7cd22c504..00f3a026c 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -9,10 +9,12 @@ import xml.etree.ElementTree as etree
 
 from .common import FileDownloader
 from .http import HttpFD
+from ..compat import (
+    compat_urlparse,
+)
 from ..utils import (
     struct_pack,
     struct_unpack,
-    compat_urlparse,
     format_bytes,
     encodeFilename,
     sanitize_open,
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index ad26cfa40..5bb0f3cfd 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -6,9 +6,11 @@ import subprocess
 
 from ..postprocessor.ffmpeg import FFmpegPostProcessor
 from .common import FileDownloader
-from ..utils import (
+from ..compat import (
     compat_urlparse,
     compat_urllib_request,
+)
+from ..utils import (
     check_executable,
     encodeFilename,
 )
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 224962e86..e68f20c9f 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -4,11 +4,12 @@ import os
 import time
 
 from .common import FileDownloader
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
     compat_urllib_error,
+)
+from ..utils import (
     ContentTooShortError,
-
     encodeFilename,
     sanitize_open,
     format_bytes,
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 58ae2005c..575912675 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -7,9 +7,9 @@ import sys
 import time
 
 from .common import FileDownloader
+from ..compat import compat_str
 from ..utils import (
     check_executable,
-    compat_str,
     encodeFilename,
     format_bytes,
     get_exe_version,
diff --git a/youtube_dl/extractor/allocine.py b/youtube_dl/extractor/allocine.py
index 398e93bfb..623aeaf34 100644
--- a/youtube_dl/extractor/allocine.py
+++ b/youtube_dl/extractor/allocine.py
@@ -5,10 +5,9 @@ import re
 import json
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
-    compat_str,
     qualities,
-    determine_ext,
 )
 
 
@@ -75,9 +74,7 @@ class AllocineIE(InfoExtractor):
                     'format_id': format_id,
                     'quality': quality(format_id),
                     'url': v,
-                    'ext': determine_ext(v),
                 })
-
         self._sort_formats(formats)
 
         return {
diff --git a/youtube_dl/extractor/aol.py b/youtube_dl/extractor/aol.py
index 185ee3693..b51eafc45 100644
--- a/youtube_dl/extractor/aol.py
+++ b/youtube_dl/extractor/aol.py
@@ -68,4 +68,3 @@ class AolIE(InfoExtractor):
             'title': title,
             'entries': entries,
         }
-
diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 0c01fa1a1..7cd0482c7 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -4,8 +4,8 @@ import re
 import json
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
-    compat_urlparse,
     int_or_none,
 )
 
diff --git a/youtube_dl/extractor/auengine.py b/youtube_dl/extractor/auengine.py
index 1c765532a..014a21952 100644
--- a/youtube_dl/extractor/auengine.py
+++ b/youtube_dl/extractor/auengine.py
@@ -3,8 +3,8 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse
 from ..utils import (
-    compat_urllib_parse,
     determine_ext,
     ExtractorError,
 )
diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index 1ca0b7cf2..98e1443ab 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -5,7 +5,7 @@ import json
 import itertools
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
 )
 
diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index acddbc8f1..9fb770cb1 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -4,9 +4,11 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
     compat_urlparse,
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/bet.py b/youtube_dl/extractor/bet.py
index c1fc433f7..003e50002 100644
--- a/youtube_dl/extractor/bet.py
+++ b/youtube_dl/extractor/bet.py
@@ -1,8 +1,8 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse
 from ..utils import (
-    compat_urllib_parse,
     xpath_text,
     xpath_with_ns,
     int_or_none,
diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 0d5889f5d..241b904a9 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -4,8 +4,8 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_parse_qs
 from ..utils import (
-    compat_parse_qs,
     ExtractorError,
     int_or_none,
     unified_strdate,
@@ -29,10 +29,9 @@ class BiliBiliIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+
         video_code = self._search_regex(
             r'(?s)<div itemprop="video".*?>(.*?)</div>', webpage, 'video code')
 
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index bf18a97e0..1eca00470 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -6,20 +6,21 @@ import json
 import xml.etree.ElementTree
 
 from .common import InfoExtractor
-from ..utils import (
-    compat_urllib_parse,
-    find_xpath_attr,
-    fix_xml_ampersands,
-    compat_urlparse,
-    compat_str,
-    compat_urllib_request,
+from ..compat import (
     compat_parse_qs,
+    compat_str,
+    compat_urllib_parse,
     compat_urllib_parse_urlparse,
-
+    compat_urllib_request,
+    compat_urlparse,
+)
+from ..utils import (
     determine_ext,
     ExtractorError,
-    unsmuggle_url,
+    find_xpath_attr,
+    fix_xml_ampersands,
     unescapeHTML,
+    unsmuggle_url,
 )
 
 
diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 97feb6704..2f866f3ef 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -4,10 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
     compat_urllib_parse_urlparse,
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 2e3ef3fda..48e2410b6 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .mtv import MTVServicesInfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
     compat_urllib_parse,
+)
+from ..utils import (
     ExtractorError,
     float_or_none,
     unified_strdate,
diff --git a/youtube_dl/extractor/condenast.py b/youtube_dl/extractor/condenast.py
index 7a7e79360..3db4db4e4 100644
--- a/youtube_dl/extractor/condenast.py
+++ b/youtube_dl/extractor/condenast.py
@@ -5,12 +5,14 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
-    orderedSet,
     compat_urllib_parse_urlparse,
     compat_urlparse,
 )
+from ..utils import (
+    orderedSet,
+)
 
 
 class CondeNastIE(InfoExtractor):
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index d7e2b841e..8f1ea02e7 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -10,10 +10,12 @@ import xml.etree.ElementTree
 from hashlib import sha1
 from math import pow, sqrt, floor
 from .subtitles import SubtitlesInfoExtractor
-from ..utils import (
-    ExtractorError,
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
+    ExtractorError,
     bytes_to_intlist,
     intlist_to_bytes,
     unified_strdate,
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 936c13cd6..cf5841a7c 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -8,13 +8,15 @@ import itertools
 from .common import InfoExtractor
 from .subtitles import SubtitlesInfoExtractor
 
-from ..utils import (
-    compat_urllib_request,
+from ..compat import (
     compat_str,
+    compat_urllib_request,
+)
+from ..utils import (
+    ExtractorError,
+    int_or_none,
     orderedSet,
     str_to_int,
-    int_or_none,
-    ExtractorError,
     unescapeHTML,
 )
 
diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index 45d66e2e6..c6b813f58 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -5,7 +5,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
 )
 
diff --git a/youtube_dl/extractor/ehow.py b/youtube_dl/extractor/ehow.py
index b766e17f2..9cb1bf301 100644
--- a/youtube_dl/extractor/ehow.py
+++ b/youtube_dl/extractor/ehow.py
@@ -1,8 +1,6 @@
 from __future__ import unicode_literals
 
-import re
-
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
 )
 from .common import InfoExtractor
@@ -24,11 +22,10 @@ class EHowIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        video_url = self._search_regex(r'(?:file|source)=(http[^\'"&]*)',
-                                       webpage, 'video URL')
+        video_url = self._search_regex(
+            r'(?:file|source)=(http[^\'"&]*)', webpage, 'video URL')
         final_url = compat_urllib_parse.unquote(video_url)
         uploader = self._html_search_meta('uploader', webpage)
         title = self._og_search_title(webpage).replace(' | eHow', '')
diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index f4c1e2a72..a30a1f330 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -6,7 +6,7 @@ import random
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
 )
 
diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 476fc22b9..e240cb859 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -3,9 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
-
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/everyonesmixtape.py b/youtube_dl/extractor/everyonesmixtape.py
index d237a8281..d872d828f 100644
--- a/youtube_dl/extractor/everyonesmixtape.py
+++ b/youtube_dl/extractor/everyonesmixtape.py
@@ -3,8 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
index aacbf1414..36ba33128 100644
--- a/youtube_dl/extractor/extremetube.py
+++ b/youtube_dl/extractor/extremetube.py
@@ -3,16 +3,18 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urllib_parse,
+)
+from ..utils import (
     str_to_int,
 )
 
 
 class ExtremeTubeIE(InfoExtractor):
-    _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>extremetube\.com/.*?video/.+?(?P<videoid>[0-9]+))(?:[/?&]|$)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>extremetube\.com/.*?video/.+?(?P<id>[0-9]+))(?:[/?&]|$)'
     _TESTS = [{
         'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
         'md5': '1fb9228f5e3332ec8c057d6ac36f33e0',
@@ -31,7 +33,7 @@ class ExtremeTubeIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
+        video_id = mobj.group('id')
         url = 'http://www.' + mobj.group('url')
 
         req = compat_urllib_request.Request(url)
diff --git a/youtube_dl/extractor/fc2.py b/youtube_dl/extractor/fc2.py
index 6f5d23559..81ceace53 100644
--- a/youtube_dl/extractor/fc2.py
+++ b/youtube_dl/extractor/fc2.py
@@ -1,19 +1,20 @@
 #! -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import re
 import hashlib
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
+from ..compat import (
     compat_urllib_request,
     compat_urlparse,
 )
+from ..utils import (
+    ExtractorError,
+)
 
 
 class FC2IE(InfoExtractor):
-    _VALID_URL = r'^http://video\.fc2\.com/((?P<lang>[^/]+)/)?content/(?P<id>[^/]+)'
+    _VALID_URL = r'^http://video\.fc2\.com/(?:[^/]+/)?content/(?P<id>[^/]+)'
     IE_NAME = 'fc2'
     _TEST = {
         'url': 'http://video.fc2.com/en/content/20121103kUan1KHs',
@@ -26,9 +27,7 @@ class FC2IE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         self._downloader.cookiejar.clear_session_cookies()  # must clear
 
diff --git a/youtube_dl/extractor/firedrive.py b/youtube_dl/extractor/firedrive.py
index af439ccfe..3191116d9 100644
--- a/youtube_dl/extractor/firedrive.py
+++ b/youtube_dl/extractor/firedrive.py
@@ -4,11 +4,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
+from ..utils import (
+    ExtractorError,
+)
 
 
 class FiredriveIE(InfoExtractor):
@@ -28,11 +30,8 @@ class FiredriveIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         url = 'http://firedrive.com/file/%s' % video_id
-
         webpage = self._download_webpage(url, video_id)
 
         if re.search(self._FILE_DELETED_REGEX, webpage) is not None:
diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index b22ce2acb..7187e0752 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -3,12 +3,14 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
-    unified_strdate,
-    str_to_int,
-    parse_duration,
+)
+from ..utils import (
     clean_html,
+    parse_duration,
+    str_to_int,
+    unified_strdate,
 )
 
 
@@ -31,9 +33,7 @@ class FourTubeIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage_url = 'http://www.4tube.com/videos/' + video_id
         webpage = self._download_webpage(webpage_url, video_id)
 
diff --git a/youtube_dl/extractor/franceculture.py b/youtube_dl/extractor/franceculture.py
index 898e0dda7..0c2972162 100644
--- a/youtube_dl/extractor/franceculture.py
+++ b/youtube_dl/extractor/franceculture.py
@@ -5,7 +5,7 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_parse_qs,
     compat_urlparse,
 )
diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index e0420a48f..bbc760a49 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -6,13 +6,15 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
-    compat_urlparse,
-    ExtractorError,
-    clean_html,
-    parse_duration,
+from ..compat import (
     compat_urllib_parse_urlparse,
+    compat_urlparse,
+)
+from ..utils import (
+    clean_html,
+    ExtractorError,
     int_or_none,
+    parse_duration,
 )
 
 
diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index d570e3f6a..47373e215 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -4,9 +4,11 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urlparse,
+)
+from ..utils import (
     unescapeHTML,
 )
 
diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index de14ae1fb..d453ec010 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
diff --git a/youtube_dl/extractor/golem.py b/youtube_dl/extractor/golem.py
index 53714f47f..2bfb99040 100644
--- a/youtube_dl/extractor/golem.py
+++ b/youtube_dl/extractor/golem.py
@@ -2,8 +2,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urlparse,
+)
+from ..utils import (
     determine_ext,
 )
 
diff --git a/youtube_dl/extractor/googlesearch.py b/youtube_dl/extractor/googlesearch.py
index 469e1f935..498304cb2 100644
--- a/youtube_dl/extractor/googlesearch.py
+++ b/youtube_dl/extractor/googlesearch.py
@@ -4,7 +4,7 @@ import itertools
 import re
 
 from .common import SearchInfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
 )
 
diff --git a/youtube_dl/extractor/gorillavid.py b/youtube_dl/extractor/gorillavid.py
index 1ac1da856..ae24aff84 100644
--- a/youtube_dl/extractor/gorillavid.py
+++ b/youtube_dl/extractor/gorillavid.py
@@ -4,11 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    determine_ext,
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
+    ExtractorError,
     int_or_none,
 )
 
@@ -106,7 +107,6 @@ class GorillaVidIE(InfoExtractor):
         formats = [{
             'format_id': 'sd',
             'url': video_url,
-            'ext': determine_ext(video_url),
             'quality': 1,
         }]
 
diff --git a/youtube_dl/extractor/hostingbulk.py b/youtube_dl/extractor/hostingbulk.py
index 8e812b669..704d0285d 100644
--- a/youtube_dl/extractor/hostingbulk.py
+++ b/youtube_dl/extractor/hostingbulk.py
@@ -4,9 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+)
 from ..utils import (
     ExtractorError,
-    compat_urllib_request,
     int_or_none,
     urlencode_postdata,
 )
@@ -30,9 +32,7 @@ class HostingBulkIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         url = 'http://hostingbulk.com/{0:}.html'.format(video_id)
 
         # Custom request with cookie to set language to English, so our file
diff --git a/youtube_dl/extractor/hypem.py b/youtube_dl/extractor/hypem.py
index 6d0d847c6..aa0724a02 100644
--- a/youtube_dl/extractor/hypem.py
+++ b/youtube_dl/extractor/hypem.py
@@ -1,20 +1,20 @@
 from __future__ import unicode_literals
 
 import json
-import re
 import time
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
-
+)
+from ..utils import (
     ExtractorError,
 )
 
 
 class HypemIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?hypem\.com/track/([^/]+)/([^/]+)'
+    _VALID_URL = r'http://(?:www\.)?hypem\.com/track/(?P<id>[^/]+)/'
     _TEST = {
         'url': 'http://hypem.com/track/1v6ga/BODYWORK+-+TAME',
         'md5': 'b9cc91b5af8995e9f0c1cee04c575828',
@@ -27,8 +27,7 @@ class HypemIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        track_id = mobj.group(1)
+        track_id = self._match_id(url)
 
         data = {'ax': 1, 'ts': time.time()}
         data_encoded = compat_urllib_parse.urlencode(data)
diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index f2c1c10f5..13a53a0cb 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -4,7 +4,7 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urlparse,
 )
 
diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index e76dd222d..f25f43664 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -1,10 +1,9 @@
 from __future__ import unicode_literals
 
 import base64
-import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
 )
 
@@ -24,9 +23,7 @@ class InfoQIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         video_title = self._html_search_regex(r'<title>(.*?)</title>', webpage, 'title')
diff --git a/youtube_dl/extractor/internetvideoarchive.py b/youtube_dl/extractor/internetvideoarchive.py
index 1e4799187..c813d4b82 100644
--- a/youtube_dl/extractor/internetvideoarchive.py
+++ b/youtube_dl/extractor/internetvideoarchive.py
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urlparse,
     compat_urllib_parse,
+)
+from ..utils import (
     xpath_with_ns,
 )
 
diff --git a/youtube_dl/extractor/iprima.py b/youtube_dl/extractor/iprima.py
index 4247d6391..8529bedfc 100644
--- a/youtube_dl/extractor/iprima.py
+++ b/youtube_dl/extractor/iprima.py
@@ -6,8 +6,10 @@ from random import random
 from math import floor
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index f0fba1adb..7a400323d 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -5,8 +5,10 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/keezmovies.py b/youtube_dl/extractor/keezmovies.py
index 75b63cffb..97dcb518a 100644
--- a/youtube_dl/extractor/keezmovies.py
+++ b/youtube_dl/extractor/keezmovies.py
@@ -4,7 +4,7 @@ import os
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urllib_parse,
@@ -15,7 +15,7 @@ from ..aes import (
 
 
 class KeezMoviesIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:www\.)?keezmovies\.com/video/.+?(?P<videoid>[0-9]+)(?:[/?&]|$)'
+    _VALID_URL = r'https?://(?:www\.)?keezmovies\.com/video/.+?(?P<id>[0-9]+)(?:[/?&]|$)'
     _TEST = {
         'url': 'http://www.keezmovies.com/video/petite-asian-lady-mai-playing-in-bathtub-1214711',
         'file': '1214711.mp4',
@@ -27,8 +27,7 @@ class KeezMoviesIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
+        video_id = self._match_id(url)
 
         req = compat_urllib_request.Request(url)
         req.add_header('Cookie', 'age_verified=1')
diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 03c4691c6..5247c6f58 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -4,10 +4,12 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
     compat_urllib_parse_urlparse,
     compat_urlparse,
+)
+from ..utils import (
     ExtractorError,
     find_xpath_attr,
     int_or_none,
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 2160d6cb0..26e84970d 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -5,12 +5,14 @@ import json
 
 from .subtitles import SubtitlesInfoExtractor
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
+    compat_str,
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
     int_or_none,
-    compat_str,
 )
 
 
diff --git a/youtube_dl/extractor/malemotion.py b/youtube_dl/extractor/malemotion.py
index 1abf6e4f8..0b85a59d1 100644
--- a/youtube_dl/extractor/malemotion.py
+++ b/youtube_dl/extractor/malemotion.py
@@ -1,43 +1,33 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
 )
 
 
 class MalemotionIE(InfoExtractor):
-    _VALID_URL = r'^(?:https?://)?malemotion\.com/video/(.+?)\.(?P<id>.+?)(#|$)'
+    _VALID_URL = r'https?://malemotion\.com/video/(.+?)\.(?P<id>.+?)(#|$)'
     _TEST = {
-        'url': 'http://malemotion.com/video/bien-dur.10ew',
-        'file': '10ew.mp4',
-        'md5': 'b3cc49f953b107e4a363cdff07d100ce',
+        'url': 'http://malemotion.com/video/bete-de-concours.ltc',
+        'md5': '3013e53a0afbde2878bc39998c33e8a5',
         'info_dict': {
-            "title": "Bien dur",
-            "age_limit": 18,
+            'id': 'ltc',
+            'ext': 'mp4',
+            'title': 'Bête de Concours',
+            'age_limit': 18,
         },
-        'skip': 'This video has been deleted.'
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group("id")
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        self.report_extraction(video_id)
-
-        # Extract video URL
-        video_url = compat_urllib_parse.unquote(
-            self._search_regex(r'<source type="video/mp4" src="(.+?)"', webpage, 'video URL'))
-
-        # Extract title
+        video_url = compat_urllib_parse.unquote(self._search_regex(
+            r'<source type="video/mp4" src="(.+?)"', webpage, 'video URL'))
         video_title = self._html_search_regex(
             r'<title>(.*?)</title', webpage, 'title')
-
-        # Extract video thumbnail
         video_thumbnail = self._search_regex(
             r'<video .+?poster="(.+?)"', webpage, 'thumbnail', fatal=False)
 
@@ -47,14 +37,12 @@ class MalemotionIE(InfoExtractor):
             'format_id': 'mp4',
             'preference': 1,
         }]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'formats': formats,
-            'uploader': None,
-            'upload_date': None,
             'title': video_title,
             'thumbnail': video_thumbnail,
-            'description': None,
             'age_limit': 18,
         }
diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py
index 858c1c0c3..8bc333b02 100644
--- a/youtube_dl/extractor/metacafe.py
+++ b/youtube_dl/extractor/metacafe.py
@@ -3,10 +3,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_parse_qs,
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     determine_ext,
     ExtractorError,
     int_or_none,
diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index 807b1dc89..d354702d1 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -5,8 +5,10 @@ import json
 
 from .common import InfoExtractor
 from .youtube import YoutubeIE
-from ..utils import (
+from ..compat import (
     compat_urlparse,
+)
+from ..utils import (
     clean_html,
     ExtractorError,
     get_element_by_id,
diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 6691521e5..256758323 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -1,12 +1,13 @@
 from __future__ import unicode_literals
 
-import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urlparse,
+)
+from ..utils import (
     get_element_by_attribute,
     parse_duration,
     strip_jsonp,
@@ -15,7 +16,7 @@ from ..utils import (
 
 class MiTeleIE(InfoExtractor):
     IE_NAME = 'mitele.es'
-    _VALID_URL = r'http://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<episode>[^/]+)/'
+    _VALID_URL = r'http://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<id>[^/]+)/'
 
     _TEST = {
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
@@ -31,12 +32,10 @@ class MiTeleIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        episode = mobj.group('episode')
+        episode = self._match_id(url)
         webpage = self._download_webpage(url, episode)
         embed_data_json = self._search_regex(
-            r'MSV\.embedData\[.*?\]\s*=\s*({.*?});', webpage, 'embed data',
-            flags=re.DOTALL
+            r'(?s)MSV\.embedData\[.*?\]\s*=\s*({.*?});', webpage, 'embed data',
         ).replace('\'', '"')
         embed_data = json.loads(embed_data_json)
 
diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 55cc33a3e..07d194562 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -3,8 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
+)
+from ..utils import (
     ExtractorError,
     HEADRequest,
     int_or_none,
diff --git a/youtube_dl/extractor/moevideo.py b/youtube_dl/extractor/moevideo.py
index 2ff79b9b8..184f9c2c9 100644
--- a/youtube_dl/extractor/moevideo.py
+++ b/youtube_dl/extractor/moevideo.py
@@ -5,10 +5,12 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
+    ExtractorError,
     int_or_none,
 )
 
diff --git a/youtube_dl/extractor/mofosex.py b/youtube_dl/extractor/mofosex.py
index d658647e6..2cec12d35 100644
--- a/youtube_dl/extractor/mofosex.py
+++ b/youtube_dl/extractor/mofosex.py
@@ -4,7 +4,7 @@ import os
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urllib_parse,
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class MofosexIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:www\.)?(?P<url>mofosex\.com/videos/(?P<videoid>[0-9]+)/.*?\.html)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>mofosex\.com/videos/(?P<id>[0-9]+)/.*?\.html)'
     _TEST = {
         'url': 'http://www.mofosex.com/videos/5018/japanese-teen-music-video.html',
         'md5': '1b2eb47ac33cc75d4a80e3026b613c5a',
@@ -26,7 +26,7 @@ class MofosexIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
+        video_id = mobj.group('id')
         url = 'http://www.' + mobj.group('url')
 
         req = compat_urllib_request.Request(url)
diff --git a/youtube_dl/extractor/moniker.py b/youtube_dl/extractor/moniker.py
index 1c4f589cc..5de719bdc 100644
--- a/youtube_dl/extractor/moniker.py
+++ b/youtube_dl/extractor/moniker.py
@@ -5,7 +5,7 @@ import os.path
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
@@ -37,10 +37,9 @@ class MonikerIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         orig_webpage = self._download_webpage(url, video_id)
+
         fields = re.findall(r'type="hidden" name="(.+?)"\s* value="?(.+?)">', orig_webpage)
         data = dict(fields)
 
diff --git a/youtube_dl/extractor/mooshare.py b/youtube_dl/extractor/mooshare.py
index 34a4bec3a..9f2853fa3 100644
--- a/youtube_dl/extractor/mooshare.py
+++ b/youtube_dl/extractor/mooshare.py
@@ -4,11 +4,13 @@ import re
 import time
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
+from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
 )
+from ..utils import (
+    ExtractorError,
+)
 
 
 class MooshareIE(InfoExtractor):
@@ -43,9 +45,7 @@ class MooshareIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         page = self._download_webpage(url, video_id, 'Downloading page')
 
         if re.search(r'>Video Not Found or Deleted<', page) is not None:
diff --git a/youtube_dl/extractor/motorsport.py b/youtube_dl/extractor/motorsport.py
index 7c0ec6a12..f5ca74e97 100644
--- a/youtube_dl/extractor/motorsport.py
+++ b/youtube_dl/extractor/motorsport.py
@@ -3,13 +3,14 @@ from __future__ import unicode_literals
 
 import hashlib
 import json
-import re
 import time
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_parse_qs,
     compat_str,
+)
+from ..utils import (
     int_or_none,
 )
 
@@ -32,10 +33,9 @@ class MotorsportIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('id')
-
+        display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
+
         flashvars_code = self._html_search_regex(
             r'<embed id="player".*?flashvars="([^"]+)"', webpage, 'flashvars')
         flashvars = compat_parse_qs(flashvars_code)
diff --git a/youtube_dl/extractor/movieclips.py b/youtube_dl/extractor/movieclips.py
index 456807dd1..04e17d055 100644
--- a/youtube_dl/extractor/movieclips.py
+++ b/youtube_dl/extractor/movieclips.py
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_str,
+)
 from ..utils import (
     ExtractorError,
-    compat_str,
     clean_html,
 )
 
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index b482d6d4d..5ebc78033 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
     find_xpath_attr,
     fix_xml_ampersands,
diff --git a/youtube_dl/extractor/myspass.py b/youtube_dl/extractor/myspass.py
index 51e540814..5b9b9fbcd 100644
--- a/youtube_dl/extractor/myspass.py
+++ b/youtube_dl/extractor/myspass.py
@@ -2,9 +2,10 @@ from __future__ import unicode_literals
 import os.path
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse_urlparse,
-
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py
index fbe34defd..c10405f04 100644
--- a/youtube_dl/extractor/naver.py
+++ b/youtube_dl/extractor/naver.py
@@ -4,8 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
+)
+from ..utils import (
     ExtractorError,
     clean_html,
 )
@@ -26,9 +28,9 @@ class NaverIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+
         m_id = re.search(r'var rmcPlayer = new nhn.rmcnmv.RMCVideoPlayer\("(.+?)", "(.+?)"',
                          webpage)
         if m_id is None:
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index bf5132721..690c46b6a 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -4,8 +4,10 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
+)
+from ..utils import (
     ExtractorError,
     find_xpath_attr,
 )
diff --git a/youtube_dl/extractor/nfb.py b/youtube_dl/extractor/nfb.py
index 7ce1d481d..ea077254b 100644
--- a/youtube_dl/extractor/nfb.py
+++ b/youtube_dl/extractor/nfb.py
@@ -1,9 +1,7 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
 )
@@ -12,7 +10,7 @@ from ..utils import (
 class NFBIE(InfoExtractor):
     IE_NAME = 'nfb'
     IE_DESC = 'National Film Board of Canada'
-    _VALID_URL = r'https?://(?:www\.)?(nfb|onf)\.ca/film/(?P<id>[\da-z_-]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:nfb|onf)\.ca/film/(?P<id>[\da-z_-]+)'
 
     _TEST = {
         'url': 'https://www.nfb.ca/film/qallunaat_why_white_people_are_funny',
@@ -32,10 +30,10 @@ class NFBIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        page = self._download_webpage('https://www.nfb.ca/film/%s' % video_id, video_id, 'Downloading film page')
+        video_id = self._match_id(url)
+        page = self._download_webpage(
+            'https://www.nfb.ca/film/%s' % video_id, video_id,
+            'Downloading film page')
 
         uploader_id = self._html_search_regex(r'<a class="director-link" href="/explore-all-directors/([^/]+)/"',
                                               page, 'director id', fatal=False)
diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index cc7c921c3..606e2294e 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -4,9 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse_urlparse,
+)
 from ..utils import (
     ExtractorError,
-    compat_urllib_parse_urlparse,
     int_or_none,
     remove_end,
 )
diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 1d9c1a096..4c1890416 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -5,14 +5,16 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
     compat_urlparse,
-    unified_strdate,
-    parse_duration,
-    int_or_none,
+)
+from ..utils import (
     ExtractorError,
+    int_or_none,
+    parse_duration,
+    unified_strdate,
 )
 
 
diff --git a/youtube_dl/extractor/noco.py b/youtube_dl/extractor/noco.py
index 7d2ff7b9a..251e6da07 100644
--- a/youtube_dl/extractor/noco.py
+++ b/youtube_dl/extractor/noco.py
@@ -6,13 +6,15 @@ import time
 import hashlib
 
 from .common import InfoExtractor
-from ..utils import (
-    compat_urllib_request,
-    compat_urllib_parse,
-    ExtractorError,
-    clean_html,
-    unified_strdate,
+from ..compat import (
     compat_str,
+    compat_urllib_parse,
+    compat_urllib_request,
+)
+from ..utils import (
+    clean_html,
+    ExtractorError,
+    unified_strdate,
 )
 
 
diff --git a/youtube_dl/extractor/nosvideo.py b/youtube_dl/extractor/nosvideo.py
index f3be8f552..f5ef856db 100644
--- a/youtube_dl/extractor/nosvideo.py
+++ b/youtube_dl/extractor/nosvideo.py
@@ -4,9 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+)
 from ..utils import (
     ExtractorError,
-    compat_urllib_request,
     urlencode_postdata,
     xpath_text,
     xpath_with_ns,
@@ -32,8 +34,7 @@ class NosVideoIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         fields = {
             'id': video_id,
diff --git a/youtube_dl/extractor/novamov.py b/youtube_dl/extractor/novamov.py
index 38d05e466..04d779890 100644
--- a/youtube_dl/extractor/novamov.py
+++ b/youtube_dl/extractor/novamov.py
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urlparse,
+)
 from ..utils import (
     ExtractorError,
-    compat_urlparse
 )
 
 
diff --git a/youtube_dl/extractor/nuvid.py b/youtube_dl/extractor/nuvid.py
index 449c8a6a3..57928f2ae 100644
--- a/youtube_dl/extractor/nuvid.py
+++ b/youtube_dl/extractor/nuvid.py
@@ -3,15 +3,17 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+)
 from ..utils import (
     parse_duration,
     unified_strdate,
-    compat_urllib_request,
 )
 
 
 class NuvidIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:www|m)\.nuvid\.com/video/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www|m)\.nuvid\.com/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://m.nuvid.com/video/1310741/',
         'md5': 'eab207b7ac4fccfb4e23c86201f11277',
@@ -26,8 +28,7 @@ class NuvidIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         formats = []
 
diff --git a/youtube_dl/extractor/photobucket.py b/youtube_dl/extractor/photobucket.py
index b4389e0b6..c66db3cdc 100644
--- a/youtube_dl/extractor/photobucket.py
+++ b/youtube_dl/extractor/photobucket.py
@@ -4,16 +4,17 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..utils import compat_urllib_parse
+from ..compat import compat_urllib_parse
 
 
 class PhotobucketIE(InfoExtractor):
     _VALID_URL = r'http://(?:[a-z0-9]+\.)?photobucket\.com/.*(([\?\&]current=)|_)(?P<id>.*)\.(?P<ext>(flv)|(mp4))'
     _TEST = {
         'url': 'http://media.photobucket.com/user/rachaneronas/media/TiredofLinkBuildingTryBacklinkMyDomaincom_zpsc0c3b9fa.mp4.html?filters[term]=search&filters[primary]=videos&filters[secondary]=images&sort=1&o=0',
-        'file': 'zpsc0c3b9fa.mp4',
         'md5': '7dabfb92b0a31f6c16cebc0f8e60ff99',
         'info_dict': {
+            'id': 'zpsc0c3b9fa',
+            'ext': 'mp4',
             'timestamp': 1367669341,
             'upload_date': '20130504',
             'uploader': 'rachaneronas',
diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
index 17880471d..449d4836c 100644
--- a/youtube_dl/extractor/played.py
+++ b/youtube_dl/extractor/played.py
@@ -5,11 +5,13 @@ import re
 import os.path
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
+from ..utils import (
+    ExtractorError,
+)
 
 
 class PlayedIE(InfoExtractor):
@@ -28,7 +30,6 @@ class PlayedIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         orig_webpage = self._download_webpage(url, video_id)
 
         m_error = re.search(
diff --git a/youtube_dl/extractor/playfm.py b/youtube_dl/extractor/playfm.py
index ebc046804..9576aed0e 100644
--- a/youtube_dl/extractor/playfm.py
+++ b/youtube_dl/extractor/playfm.py
@@ -4,9 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
     float_or_none,
     int_or_none,
diff --git a/youtube_dl/extractor/playvid.py b/youtube_dl/extractor/playvid.py
index cd3905acb..c3e667e9e 100644
--- a/youtube_dl/extractor/playvid.py
+++ b/youtube_dl/extractor/playvid.py
@@ -3,31 +3,31 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    clean_html,
+from ..compat import (
     compat_urllib_parse,
 )
+from ..utils import (
+    clean_html,
+    ExtractorError,
+)
 
 
 class PlayvidIE(InfoExtractor):
-    _VALID_URL = r'^https?://www\.playvid\.com/watch(\?v=|/)(?P<id>.+?)(?:#|$)'
+    _VALID_URL = r'https?://www\.playvid\.com/watch(\?v=|/)(?P<id>.+?)(?:#|$)'
     _TEST = {
-        'url': 'http://www.playvid.com/watch/agbDDi7WZTV',
-        'md5': '44930f8afa616efdf9482daf4fe53e1e',
+        'url': 'http://www.playvid.com/watch/RnmBNgtrrJu',
+        'md5': 'ffa2f6b2119af359f544388d8c01eb6c',
         'info_dict': {
-            'id': 'agbDDi7WZTV',
+            'id': 'RnmBNgtrrJu',
             'ext': 'mp4',
-            'title': 'Michelle Lewin in Miami Beach',
-            'duration': 240,
+            'title': 'md5:9256d01c6317e3f703848b5906880dc8',
+            'duration': 82,
             'age_limit': 18,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         m_error = re.search(
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 2ca15b717..634142d0d 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -4,10 +4,12 @@ import os
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
+    compat_urllib_parse,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
-    compat_urllib_parse,
+)
+from ..utils import (
     str_to_int,
 )
 from ..aes import (
@@ -16,7 +18,7 @@ from ..aes import (
 
 
 class PornHubIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:www\.)?pornhub\.com/view_video\.php\?viewkey=(?P<id>[0-9a-f]+)'
+    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/view_video\.php\?viewkey=(?P<id>[0-9a-f]+)'
     _TEST = {
         'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
         'md5': '882f488fa1f0026f023f33576004a2ed',
diff --git a/youtube_dl/extractor/promptfile.py b/youtube_dl/extractor/promptfile.py
index 7fcde086c..f536e6e6c 100644
--- a/youtube_dl/extractor/promptfile.py
+++ b/youtube_dl/extractor/promptfile.py
@@ -4,12 +4,14 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    determine_ext,
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
+from ..utils import (
+    determine_ext,
+    ExtractorError,
+)
 
 
 class PromptFileIE(InfoExtractor):
diff --git a/youtube_dl/extractor/prosiebensat1.py b/youtube_dl/extractor/prosiebensat1.py
index 1262793c8..385681d06 100644
--- a/youtube_dl/extractor/prosiebensat1.py
+++ b/youtube_dl/extractor/prosiebensat1.py
@@ -5,8 +5,10 @@ import re
 
 from hashlib import sha1
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
+)
+from ..utils import (
     unified_strdate,
 )
 
diff --git a/youtube_dl/extractor/quickvid.py b/youtube_dl/extractor/quickvid.py
index 3bc78060d..af7d76cf4 100644
--- a/youtube_dl/extractor/quickvid.py
+++ b/youtube_dl/extractor/quickvid.py
@@ -3,8 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urlparse,
+)
+from ..utils import (
     determine_ext,
     int_or_none,
 )
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 2d39ecfe4..aa26b7e0b 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -3,10 +3,12 @@ from __future__ import unicode_literals
 import re
 
 from .subtitles import SubtitlesInfoExtractor
+from ..compat import (
+    compat_urllib_parse,
+)
 from ..utils import (
     parse_duration,
     unified_strdate,
-    compat_urllib_parse,
 )
 
 
diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index dc59a5e5c..5e84c1098 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -4,12 +4,14 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_str,
+)
 from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
     unescapeHTML,
-    compat_str,
 )
 
 
diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index 6941d96fb..b72b5a586 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -5,10 +5,12 @@ import re
 import itertools
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
-    unified_strdate,
+)
+from ..utils import (
     ExtractorError,
+    unified_strdate,
 )
 
 
@@ -36,9 +38,7 @@ class RutubeIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         video = self._download_json(
             'http://rutube.ru/api/video/%s/?format=json' % video_id,
             video_id, 'Downloading video JSON')
@@ -114,8 +114,7 @@ class RutubeMovieIE(RutubeChannelIE):
     _PAGE_TEMPLATE = 'http://rutube.ru/api/metainfo/tv/%s/video?page=%s&format=json'
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        movie_id = mobj.group('id')
+        movie_id = self._match_id(url)
         movie = self._download_json(
             self._MOVIE_TEMPLATE % movie_id, movie_id,
             'Downloading movie JSON')
diff --git a/youtube_dl/extractor/screencast.py b/youtube_dl/extractor/screencast.py
index c145f6fc7..dfd897ba3 100644
--- a/youtube_dl/extractor/screencast.py
+++ b/youtube_dl/extractor/screencast.py
@@ -1,14 +1,14 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
+from ..compat import (
     compat_parse_qs,
     compat_urllib_request,
 )
+from ..utils import (
+    ExtractorError,
+)
 
 
 class ScreencastIE(InfoExtractor):
@@ -57,8 +57,7 @@ class ScreencastIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         video_url = self._html_search_regex(
diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index fdc31603a..26ced716e 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -4,10 +4,12 @@ import re
 import base64
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_parse,
+    compat_urllib_request,
+)
 from ..utils import (
     ExtractorError,
-    compat_urllib_request,
-    compat_urllib_parse,
     int_or_none,
 )
 
@@ -26,26 +28,30 @@ class SharedIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
 
-        page = self._download_webpage(url, video_id)
+        if '>File does not exist<' in webpage:
+            raise ExtractorError(
+                'Video %s does not exist' % video_id, expected=True)
 
-        if re.search(r'>File does not exist<', page) is not None:
-            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
-
-        download_form = dict(re.findall(r'<input type="hidden" name="([^"]+)" value="([^"]*)"', page))
-
-        request = compat_urllib_request.Request(url, compat_urllib_parse.urlencode(download_form))
+        download_form = dict(re.findall(
+            r'<input type="hidden" name="([^"]+)" value="([^"]*)"', webpage))
+        request = compat_urllib_request.Request(
+            url, compat_urllib_parse.urlencode(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
-        video_page = self._download_webpage(request, video_id, 'Downloading video page')
+        video_page = self._download_webpage(
+            request, video_id, 'Downloading video page')
 
-        video_url = self._html_search_regex(r'data-url="([^"]+)"', video_page, 'video URL')
-        title = base64.b64decode(self._html_search_meta('full:title', page, 'title')).decode('utf-8')
-        filesize = int_or_none(self._html_search_meta('full:size', page, 'file size', fatal=False))
+        video_url = self._html_search_regex(
+            r'data-url="([^"]+)"', video_page, 'video URL')
+        title = base64.b64decode(self._html_search_meta(
+            'full:title', webpage, 'title')).decode('utf-8')
+        filesize = int_or_none(self._html_search_meta(
+            'full:size', webpage, 'file size', fatal=False))
         thumbnail = self._html_search_regex(
-            r'data-poster="([^"]+)"', video_page, 'thumbnail', fatal=False, default=None)
+            r'data-poster="([^"]+)"', video_page, 'thumbnail', default=None)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/sharesix.py b/youtube_dl/extractor/sharesix.py
index 7531e8325..ac3e3adf2 100644
--- a/youtube_dl/extractor/sharesix.py
+++ b/youtube_dl/extractor/sharesix.py
@@ -4,9 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     parse_duration,
 )
 
diff --git a/youtube_dl/extractor/sina.py b/youtube_dl/extractor/sina.py
index 5eadbb7ea..a63d126d4 100644
--- a/youtube_dl/extractor/sina.py
+++ b/youtube_dl/extractor/sina.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
 )
diff --git a/youtube_dl/extractor/slideshare.py b/youtube_dl/extractor/slideshare.py
index 5864b9936..e7d776e7b 100644
--- a/youtube_dl/extractor/slideshare.py
+++ b/youtube_dl/extractor/slideshare.py
@@ -4,8 +4,10 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urlparse,
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 646af3cc9..d031fe401 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -7,9 +7,11 @@ import hashlib
 import uuid
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
     int_or_none,
     unified_strdate,
diff --git a/youtube_dl/extractor/sockshare.py b/youtube_dl/extractor/sockshare.py
index c663e56d4..7d3c0e937 100644
--- a/youtube_dl/extractor/sockshare.py
+++ b/youtube_dl/extractor/sockshare.py
@@ -1,13 +1,16 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from ..utils import (
-    ExtractorError,
+import re
+
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
-    determine_ext,
 )
-import re
+from ..utils import (
+    determine_ext,
+    ExtractorError,
+)
 
 from .common import InfoExtractor
 
@@ -27,9 +30,7 @@ class SockshareIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         url = 'http://sockshare.com/file/%s' % video_id
         webpage = self._download_webpage(url, video_id)
 
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index ab9483d2d..5d60c4939 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -5,11 +5,12 @@ import re
 import itertools
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
     compat_urlparse,
     compat_urllib_parse,
-
+)
+from ..utils import (
     ExtractorError,
     int_or_none,
     unified_strdate,
diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 94602e89e..b936202f6 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -3,12 +3,14 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
+    compat_urllib_parse,
     compat_urllib_parse_urlparse,
     compat_urllib_request,
-    compat_urllib_parse,
-    unified_strdate,
+)
+from ..utils import (
     str_to_int,
+    unified_strdate,
 )
 from ..aes import aes_decrypt_text
 
diff --git a/youtube_dl/extractor/sportdeutschland.py b/youtube_dl/extractor/sportdeutschland.py
index 057ef5251..2f57f5b7c 100644
--- a/youtube_dl/extractor/sportdeutschland.py
+++ b/youtube_dl/extractor/sportdeutschland.py
@@ -4,8 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
+)
+from ..utils import (
     parse_iso8601,
 )
 
diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dl/extractor/streamcloud.py
index c1178f26d..38176498d 100644
--- a/youtube_dl/extractor/streamcloud.py
+++ b/youtube_dl/extractor/streamcloud.py
@@ -5,7 +5,7 @@ import re
 import time
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index 778704839..c3ceb5f76 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -1,13 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-import re
-import json
-
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-)
 from ..utils import (
     int_or_none,
 )
diff --git a/youtube_dl/extractor/tapely.py b/youtube_dl/extractor/tapely.py
index 283e11350..f1f43d0a7 100644
--- a/youtube_dl/extractor/tapely.py
+++ b/youtube_dl/extractor/tapely.py
@@ -4,10 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    clean_html,
+from ..compat import (
     compat_urllib_request,
+)
+from ..utils import (
+    clean_html,
+    ExtractorError,
     float_or_none,
     parse_iso8601,
 )
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 72160503c..944177426 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -5,7 +5,7 @@ import re
 
 from .subtitles import SubtitlesInfoExtractor
 
-from ..utils import (
+from ..compat import (
     compat_str,
 )
 
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index e2653d62d..af6ef0033 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -4,8 +4,10 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
+)
+from ..utils import (
     determine_ext,
     ExtractorError,
     xpath_with_ns,
diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/tlc.py
index 66d159e99..9f9e388c5 100644
--- a/youtube_dl/extractor/tlc.py
+++ b/youtube_dl/extractor/tlc.py
@@ -5,7 +5,7 @@ import re
 from .common import InfoExtractor
 from .brightcove import BrightcoveIE
 from .discovery import DiscoveryIE
-from ..utils import compat_urlparse
+from ..compat import compat_urlparse
 
 
 class TlcIE(DiscoveryIE):
diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index 64a1e9030..d73ad3762 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -4,9 +4,11 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
+)
+from ..utils import (
     int_or_none,
     str_to_int,
 )
diff --git a/youtube_dl/extractor/tutv.py b/youtube_dl/extractor/tutv.py
index d516b6427..4de0aac52 100644
--- a/youtube_dl/extractor/tutv.py
+++ b/youtube_dl/extractor/tutv.py
@@ -1,10 +1,9 @@
 from __future__ import unicode_literals
 
 import base64
-import re
 
 from .common import InfoExtractor
-from ..utils import compat_parse_qs
+from ..compat import compat_parse_qs
 
 
 class TutvIE(InfoExtractor):
@@ -20,10 +19,9 @@ class TutvIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+
         internal_id = self._search_regex(r'codVideo=([0-9]+)', webpage, 'internal video ID')
 
         data_content = self._download_webpage(
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 397d167e8..715f9930c 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -5,9 +5,11 @@ import itertools
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
     parse_iso8601,
 )
diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py
index 5271611ac..4667ed83b 100644
--- a/youtube_dl/extractor/udemy.py
+++ b/youtube_dl/extractor/udemy.py
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/urort.py b/youtube_dl/extractor/urort.py
index 249a34c86..8872cfcb2 100644
--- a/youtube_dl/extractor/urort.py
+++ b/youtube_dl/extractor/urort.py
@@ -1,11 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
+)
+from ..utils import (
     unified_strdate,
 )
 
diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 53dc3a496..68d03b999 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urlparse,
 )
 
diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index 455b6d9da..dd026748d 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -1,19 +1,18 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
-
+)
+from ..utils import (
     ExtractorError,
 )
 
 
 class Vbox7IE(InfoExtractor):
-    _VALID_URL = r'http://(www\.)?vbox7\.com/play:(?P<id>[^/]+)'
+    _VALID_URL = r'http://(?:www\.)?vbox7\.com/play:(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://vbox7.com/play:249bb972c2',
         'md5': '99f65c0c9ef9b682b97313e052734c3f',
@@ -25,8 +24,7 @@ class Vbox7IE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         redirect_page, urlh = self._download_webpage_handle(url, video_id)
         new_location = self._search_regex(r'window\.location = \'(.*)\';',
diff --git a/youtube_dl/extractor/veehd.py b/youtube_dl/extractor/veehd.py
index 94647d1c8..815f58468 100644
--- a/youtube_dl/extractor/veehd.py
+++ b/youtube_dl/extractor/veehd.py
@@ -4,10 +4,12 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urlparse,
-    get_element_by_id,
+)
+from ..utils import (
     clean_html,
+    get_element_by_id,
 )
 
 
@@ -26,8 +28,7 @@ class VeeHDIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         # VeeHD seems to send garbage on the first request.
         # See https://github.com/rg3/youtube-dl/issues/2102
diff --git a/youtube_dl/extractor/veoh.py b/youtube_dl/extractor/veoh.py
index a7953a7e7..01e258e32 100644
--- a/youtube_dl/extractor/veoh.py
+++ b/youtube_dl/extractor/veoh.py
@@ -4,8 +4,10 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
+)
+from ..utils import (
     int_or_none,
     ExtractorError,
 )
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index c912c3cbe..43f6b029d 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -4,8 +4,10 @@ import re
 import xml.etree.ElementTree
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
+)
+from ..utils import (
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/videodetective.py b/youtube_dl/extractor/videodetective.py
index ac6c25537..0ffc7ff7d 100644
--- a/youtube_dl/extractor/videodetective.py
+++ b/youtube_dl/extractor/videodetective.py
@@ -1,10 +1,8 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from .internetvideoarchive import InternetVideoArchiveIE
-from ..utils import compat_urlparse
 
 
 class VideoDetectiveIE(InfoExtractor):
@@ -17,13 +15,12 @@ class VideoDetectiveIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'KICK-ASS 2',
             'description': 'md5:65ba37ad619165afac7d432eaded6013',
-            'duration': 135,
+            'duration': 138,
         },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         og_video = self._og_search_video_url(webpage)
         query = compat_urlparse.urlparse(og_video).query
diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 29c4e0101..7a78f0d26 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -1,11 +1,11 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
+)
+from ..utils import (
     remove_start,
 )
 
@@ -27,9 +27,7 @@ class VideoMegaIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         url = 'http://videomega.tv/iframe.php?ref={0:}'.format(video_id)
         webpage = self._download_webpage(url, video_id)
 
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index ca6b0d5b3..542e9198a 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -5,14 +5,17 @@ import re
 import json
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_str,
+    compat_urllib_parse,
+    compat_urllib_request,
+)
 from ..utils import (
     ExtractorError,
-    compat_urllib_request,
-    compat_urllib_parse,
-    compat_str,
+    orderedSet,
     unescapeHTML,
     unified_strdate,
-    orderedSet)
+)
 
 
 class VKIE(InfoExtractor):
diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py
index affef6507..1c0966a79 100644
--- a/youtube_dl/extractor/vodlocker.py
+++ b/youtube_dl/extractor/vodlocker.py
@@ -2,8 +2,9 @@
 from __future__ import unicode_literals
 
 import re
+
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
@@ -24,8 +25,7 @@ class VodlockerIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         fields = dict(re.findall(r'''(?x)<input\s+
diff --git a/youtube_dl/extractor/vube.py b/youtube_dl/extractor/vube.py
index 1b2f731e9..405cb9db4 100644
--- a/youtube_dl/extractor/vube.py
+++ b/youtube_dl/extractor/vube.py
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_str,
+)
 from ..utils import (
     int_or_none,
-    compat_str,
     ExtractorError,
 )
 
diff --git a/youtube_dl/extractor/vuclip.py b/youtube_dl/extractor/vuclip.py
index ec3c010ad..c3fde53f5 100644
--- a/youtube_dl/extractor/vuclip.py
+++ b/youtube_dl/extractor/vuclip.py
@@ -3,8 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse_urlparse,
+)
+from ..utils import (
     ExtractorError,
     parse_duration,
     qualities,
@@ -25,10 +27,9 @@ class VuClipIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+
         ad_m = re.search(
             r'''value="No.*?" onClick="location.href='([^"']+)'"''', webpage)
         if ad_m:
diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 93a6e6454..8e25ecf28 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -4,9 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_parse_qs,
     compat_urlparse,
+)
+from ..utils import (
     determine_ext,
     unified_strdate,
 )
@@ -141,7 +143,6 @@ class WDRMobileIE(InfoExtractor):
             'title': mobj.group('title'),
             'age_limit': int(mobj.group('age_limit')),
             'url': url,
-            'ext': determine_ext(url),
             'user_agent': 'mobile',
         }
 
diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index 748443f81..13a079151 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -1,9 +1,8 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import ExtractorError, compat_urllib_request
+from ..compat import compat_urllib_request
+from ..utils import ExtractorError
 
 
 class WistiaIE(InfoExtractor):
@@ -22,8 +21,7 @@ class WistiaIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         request = compat_urllib_request.Request(self._API_URL.format(video_id))
         request.add_header('Referer', url)  # Some videos require this.
diff --git a/youtube_dl/extractor/xbef.py b/youtube_dl/extractor/xbef.py
index 1b4e88365..80c48c37d 100644
--- a/youtube_dl/extractor/xbef.py
+++ b/youtube_dl/extractor/xbef.py
@@ -1,9 +1,7 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
 )
 
@@ -23,10 +21,9 @@ class XBefIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+
         title = self._html_search_regex(
             r'<h1[^>]*>(.*?)</h1>', webpage, 'title')
 
diff --git a/youtube_dl/extractor/xnxx.py b/youtube_dl/extractor/xnxx.py
index 53ed7ef5a..79ed6c744 100644
--- a/youtube_dl/extractor/xnxx.py
+++ b/youtube_dl/extractor/xnxx.py
@@ -1,10 +1,8 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
 )
 
@@ -23,10 +21,7 @@ class XNXXIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        # Get webpage content
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         video_url = self._search_regex(r'flv_url=(.*?)&amp;',
diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 38448e7c0..f9d98b83f 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -4,15 +4,17 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_request,
+)
+from ..utils import (
     parse_duration,
     str_to_int,
 )
 
 
 class XTubeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?P<url>xtube\.com/watch\.php\?v=(?P<videoid>[^/?&]+))'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>xtube\.com/watch\.php\?v=(?P<id>[^/?&]+))'
     _TEST = {
         'url': 'http://www.xtube.com/watch.php?v=kVTUy_G222_',
         'md5': '092fbdd3cbe292c920ef6fc6a8a9cdab',
@@ -29,7 +31,7 @@ class XTubeIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
+        video_id = mobj.group('id')
         url = 'http://www.' + mobj.group('url')
 
         req = compat_urllib_request.Request(url)
diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index 7e0044824..2a45dc574 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -3,15 +3,17 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse,
-    ExtractorError,
+)
+from ..utils import (
     clean_html,
+    ExtractorError,
 )
 
 
 class XVideosIE(InfoExtractor):
-    _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)'
+    _VALID_URL = r'https?://(?:www\.)?xvideos\.com/video(?P<id>[0-9]+)(?:.*)'
     _TEST = {
         'url': 'http://www.xvideos.com/video4588838/biker_takes_his_girl',
         'md5': '4b46ae6ea5e6e9086e714d883313c0c9',
@@ -24,37 +26,25 @@ class XVideosIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        self.report_extraction(video_id)
-
         mobj = re.search(r'<h1 class="inlineError">(.+?)</h1>', webpage)
         if mobj:
             raise ExtractorError('%s said: %s' % (self.IE_NAME, clean_html(mobj.group(1))), expected=True)
 
-        # Extract video URL
         video_url = compat_urllib_parse.unquote(
             self._search_regex(r'flv_url=(.+?)&', webpage, 'video URL'))
-
-        # Extract title
         video_title = self._html_search_regex(
             r'<title>(.*?)\s+-\s+XVID', webpage, 'title')
-
-        # Extract video thumbnail
         video_thumbnail = self._search_regex(
             r'url_bigthumb=(.+?)&amp', webpage, 'thumbnail', fatal=False)
 
         return {
             'id': video_id,
             'url': video_url,
-            'uploader': None,
-            'upload_date': None,
             'title': video_title,
             'ext': 'flv',
             'thumbnail': video_thumbnail,
-            'description': None,
             'age_limit': 18,
         }
diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 0fdb12243..031226f27 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -6,11 +6,13 @@ import json
 import re
 
 from .common import InfoExtractor, SearchInfoExtractor
-from ..utils import (
-    ExtractorError,
+from ..compat import (
     compat_urllib_parse,
     compat_urlparse,
+)
+from ..utils import (
     clean_html,
+    ExtractorError,
     int_or_none,
 )
 
diff --git a/youtube_dl/extractor/ynet.py b/youtube_dl/extractor/ynet.py
index 7b621a9e3..894678a23 100644
--- a/youtube_dl/extractor/ynet.py
+++ b/youtube_dl/extractor/ynet.py
@@ -5,7 +5,7 @@ import re
 import json
 
 from .common import InfoExtractor
-from ..utils import compat_urllib_parse
+from ..compat import compat_urllib_parse
 
 
 class YnetIE(InfoExtractor):
diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index d9c06a2ee..107c9ac36 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -6,10 +6,11 @@ import re
 import sys
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
-
+)
+from ..utils import (
     ExtractorError,
     unescapeHTML,
     unified_strdate,
diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index 2bd264b30..e60505ace 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -4,8 +4,8 @@ import collections
 import io
 import zlib
 
+from .compat import compat_str
 from .utils import (
-    compat_str,
     ExtractorError,
     struct_unpack,
 )

From 7129bed51bc2452524c69874023ee4a0b351a046 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 12:35:13 +0100
Subject: [PATCH 0839/1937] [keek] Modernize and extract uploader

---
 youtube_dl/extractor/keek.py | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index 5d679e88d..e94e2f8ad 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -6,29 +6,36 @@ from .common import InfoExtractor
 
 
 class KeekIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?keek\.com/(?:!|\w+/keeks/)(?P<videoID>\w+)'
+    _VALID_URL = r'https?://(?:www\.)?keek\.com/(?:!|\w+/keeks/)(?P<id>\w+)'
     IE_NAME = 'keek'
     _TEST = {
         'url': 'https://www.keek.com/ytdl/keeks/NODfbab',
-        'file': 'NODfbab.mp4',
-        'md5': '9b0636f8c0f7614afa4ea5e4c6e57e83',
+        'md5': '09c5c109067536c1cec8bac8c21fea05',
         'info_dict': {
-            'uploader': 'ytdl',
+            'id': 'NODfbab',
+            'ext': 'mp4',
+            'uploader': 'youtube-dl project',
+            'uploader_id': 'ytdl',
             'title': 'test chars: "\'/\\\u00e4<>This is a test video for youtube-dl.For more information, contact phihag@phihag.de .',
         },
     }
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('videoID')
+        video_id = self._match_id(url)
 
         video_url = 'http://cdn.keek.com/keek/video/%s' % video_id
         thumbnail = 'http://cdn.keek.com/keek/thumbnail/%s/w100/h75' % video_id
         webpage = self._download_webpage(url, video_id)
 
-        uploader = self._html_search_regex(
-            r'<div class="user-name-and-bio">[\S\s]+?<h2>(?P<uploader>.+?)</h2>',
-            webpage, 'uploader', fatal=False)
+        raw_desc = self._html_search_meta('description', webpage)
+        if raw_desc:
+            uploader = self._html_search_regex(
+                r'Watch (.*?)\s+\(', raw_desc, 'uploader', fatal=False)
+            uploader_id = self._html_search_regex(
+                r'Watch .*?\(@(.+?)\)', raw_desc, 'uploader_id', fatal=False)
+        else:
+            uploader = None
+            uploader_id = None
 
         return {
             'id': video_id,
@@ -36,5 +43,6 @@ class KeekIE(InfoExtractor):
             'ext': 'mp4',
             'title': self._og_search_title(webpage),
             'thumbnail': thumbnail,
-            'uploader': uploader
+            'uploader': uploader,
+            'uploader_id': uploader_id,
         }

From 89e36657ccd382249c55701b00296b3ed78d29e8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 12:36:46 +0100
Subject: [PATCH 0840/1937] [keek] remove unused import

---
 youtube_dl/extractor/keek.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/keek.py b/youtube_dl/extractor/keek.py
index e94e2f8ad..c0956ba09 100644
--- a/youtube_dl/extractor/keek.py
+++ b/youtube_dl/extractor/keek.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 

From a6620ac28df7ddbe39f09575d8e9f7fe73398a00 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 12:41:31 +0100
Subject: [PATCH 0841/1937] [orf] Modernize

---
 youtube_dl/extractor/orf.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 572a234ad..a6e722bf5 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -20,21 +20,23 @@ class ORFTVthekIE(InfoExtractor):
     _VALID_URL = r'https?://tvthek\.orf\.at/(?:programs/.+?/episodes|topics/.+?|program/[^/]+)/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://tvthek.orf.at/program/matinee-Was-Sie-schon-immer-ueber-Klassik-wissen-wollten/7317210/Was-Sie-schon-immer-ueber-Klassik-wissen-wollten/7319746/Was-Sie-schon-immer-ueber-Klassik-wissen-wollten/7319747',
-        'file': '7319747.mp4',
-        'md5': 'bd803c5d8c32d3c64a0ea4b4eeddf375',
-        'info_dict': {
-            'title': 'Was Sie schon immer über Klassik wissen wollten',
-            'description': 'md5:0ddf0d5f0060bd53f744edaa5c2e04a4',
-            'duration': 3508,
-            'upload_date': '20140105',
-        },
+        'url': 'http://tvthek.orf.at/program/Aufgetischt/2745173/Aufgetischt-Mit-der-Steirischen-Tafelrunde/8891389',
+        'playlist': [{
+            'md5': '2942210346ed779588f428a92db88712',
+            'info_dict': {
+                'id': '8896777',
+                'ext': 'mp4',
+                'title': 'Aufgetischt: Mit der Steirischen Tafelrunde',
+                'description': 'md5:c1272f0245537812d4e36419c207b67d',
+                'duration': 2668,
+                'upload_date': '20141208',
+            },
+        }],
         'skip': 'Blocked outside of Austria',
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        playlist_id = mobj.group('id')
+        playlist_id = self._match_id(url)
         webpage = self._download_webpage(url, playlist_id)
 
         data_json = self._search_regex(
@@ -120,9 +122,7 @@ class ORFOE1IE(InfoExtractor):
     _VALID_URL = r'http://oe1\.orf\.at/programm/(?P<id>[0-9]+)'
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        show_id = mobj.group('id')
-
+        show_id = self._match_id(url)
         data = self._download_json(
             'http://oe1.orf.at/programm/%s/konsole' % show_id,
             show_id

From 1557ed153c42176335f2ef069661ef2c98f16ce4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 13 Dec 2014 12:39:58 +0100
Subject: [PATCH 0842/1937] [test_unicode_literals] Import from test.helper

---
 test/test_unicode_literals.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_unicode_literals.py b/test/test_unicode_literals.py
index d3cba869b..19813e034 100644
--- a/test/test_unicode_literals.py
+++ b/test/test_unicode_literals.py
@@ -20,7 +20,7 @@ IGNORED_FILES = [
 ]
 
 
-from helper import assertRegexpMatches
+from test.helper import assertRegexpMatches
 
 
 class TestUnicodeLiterals(unittest.TestCase):

From 8085fc15ccb161d96eee607bf21dfdb36b890b20 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Dec 2014 18:42:29 +0600
Subject: [PATCH 0843/1937] [adultswim] Improve segment duration extraction

---
 youtube_dl/extractor/adultswim.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 39e4ca296..502a9c25a 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -7,6 +7,8 @@ import json
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    xpath_text,
+    float_or_none,
 )
 
 
@@ -128,7 +130,8 @@ class AdultSwimIE(InfoExtractor):
                 segment_url, segment_title,
                 'Downloading segment information', 'Unable to download segment information')
 
-            segment_duration = idoc.find('.//trt').text.strip()
+            segment_duration = float_or_none(
+                xpath_text(idoc, './/trt', 'segment duration').strip())
 
             formats = []
             file_els = idoc.findall('.//files/file')

From 04c95441871775dfdaa0ea3eeca0a5d23f270f2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 13 Dec 2014 18:47:34 +0600
Subject: [PATCH 0844/1937] [bbccouk] Fix vpid warning

---
 youtube_dl/extractor/bbccouk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 01c02d360..2d2f742ae 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -209,7 +209,7 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
         webpage = self._download_webpage(url, group_id, 'Downloading video page')
 
         programme_id = self._search_regex(
-            r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False)
+            r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False, default=None)
         if programme_id:
             player = self._download_json(
                 'http://www.bbc.co.uk/iplayer/episode/%s.json' % group_id,

From 1fa174692aae06406c9524fec392a131f10c68fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Dec 2014 02:00:54 +0600
Subject: [PATCH 0845/1937] [bandcamp:album] Make path optional (Closes #4461)

---
 youtube_dl/extractor/bandcamp.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 9fb770cb1..bcb2821cd 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -106,7 +106,7 @@ class BandcampIE(InfoExtractor):
 
 class BandcampAlbumIE(InfoExtractor):
     IE_NAME = 'Bandcamp:album'
-    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<title>[^?#]+))'
+    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<title>[^?#]+))?'
 
     _TESTS = [{
         'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
@@ -141,6 +141,12 @@ class BandcampAlbumIE(InfoExtractor):
             'title': 'Hierophany of the Open Grave',
         },
         'playlist_mincount': 9,
+    }, {
+        'url': 'http://dotscale.bandcamp.com',
+        'info_dict': {
+            'title': 'Loom',
+        },
+        'playlist_mincount': 7,
     }]
 
     def _real_extract(self, url):

From 4a0132c570e29075e377c8b70d7f61018d1e4479 Mon Sep 17 00:00:00 2001
From: Mathias Rav <rav@cs.au.dk>
Date: Sat, 13 Dec 2014 21:58:52 +0100
Subject: [PATCH 0846/1937] [Restudy] Add new extractor for restudy.dk

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/restudy.py  | 41 ++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 youtube_dl/extractor/restudy.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 119ec2044..3ae7a8a52 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -316,6 +316,7 @@ from .radiofrance import RadioFranceIE
 from .rai import RaiIE
 from .rbmaradio import RBMARadioIE
 from .redtube import RedTubeIE
+from .restudy import RestudyIE
 from .reverbnation import ReverbNationIE
 from .ringtv import RingTVIE
 from .ro220 import Ro220IE
diff --git a/youtube_dl/extractor/restudy.py b/youtube_dl/extractor/restudy.py
new file mode 100644
index 000000000..56a6c0f93
--- /dev/null
+++ b/youtube_dl/extractor/restudy.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class RestudyIE(InfoExtractor):
+    _VALID_URL = r'https://www.restudy.dk/video/play/id/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://www.restudy.dk/video/play/id/1637',
+        # MD5 sum of first 10241 bytes of the video file, as reported by
+        # head -c 10241 Leiden-frosteffekt-1637.mp4 | md5sum
+        'md5': '4e755c4287f292a1fe5363834a683818',
+        'info_dict': {
+            'id': '1637',
+            'ext': 'mp4',
+            'title': 'Leiden-frosteffekt',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        xml_url = (
+            'https://www.restudy.dk/awsmedia/SmilDirectory/video_%s.xml'
+            % video_id)
+        xml = self._download_webpage(xml_url, video_id)
+
+        base = self._search_regex(
+            r'<meta base="([^"]+)', xml, 'meta base')
+        # TODO: Provide multiple video qualities instead of forcing highest
+        filename = self._search_regex(
+            r'<video src="mp4:([^"]+_high\.mp4)', xml, 'filename')
+        url = '%s%s' % (base, filename)
+        title = self._og_search_title(webpage)
+        return {
+            'id': video_id,
+            'title': title,
+            'url': url,
+            'protocol': 'rtmp',
+        }

From ac265bef1e685706c0be89237117e2df5f796ecb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Dec 2014 03:41:00 +0600
Subject: [PATCH 0847/1937] [restudy] Simplify and extract all formats

---
 youtube_dl/extractor/restudy.py | 37 ++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/restudy.py b/youtube_dl/extractor/restudy.py
index 56a6c0f93..b17c2bfc0 100644
--- a/youtube_dl/extractor/restudy.py
+++ b/youtube_dl/extractor/restudy.py
@@ -5,37 +5,36 @@ from .common import InfoExtractor
 
 
 class RestudyIE(InfoExtractor):
-    _VALID_URL = r'https://www.restudy.dk/video/play/id/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?restudy\.dk/video/play/id/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'https://www.restudy.dk/video/play/id/1637',
-        # MD5 sum of first 10241 bytes of the video file, as reported by
-        # head -c 10241 Leiden-frosteffekt-1637.mp4 | md5sum
-        'md5': '4e755c4287f292a1fe5363834a683818',
         'info_dict': {
             'id': '1637',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Leiden-frosteffekt',
+            'description': 'Denne video er et eksperiment med flydende kvælstof.',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        xml_url = (
-            'https://www.restudy.dk/awsmedia/SmilDirectory/video_%s.xml'
-            % video_id)
-        xml = self._download_webpage(xml_url, video_id)
 
-        base = self._search_regex(
-            r'<meta base="([^"]+)', xml, 'meta base')
-        # TODO: Provide multiple video qualities instead of forcing highest
-        filename = self._search_regex(
-            r'<video src="mp4:([^"]+_high\.mp4)', xml, 'filename')
-        url = '%s%s' % (base, filename)
-        title = self._og_search_title(webpage)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(webpage).strip()
+        description = self._og_search_description(webpage).strip()
+
+        formats = self._extract_smil_formats(
+            'https://www.restudy.dk/awsmedia/SmilDirectory/video_%s.xml' % video_id,
+            video_id)
+
         return {
             'id': video_id,
             'title': title,
-            'url': url,
-            'protocol': 'rtmp',
+            'description': description,
+            'formats': formats,
         }

From 2564300e55ec30754cbc416aaa134cbb0da1a5f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Dec 2014 03:42:42 +0600
Subject: [PATCH 0848/1937] Credit @Mortal for restudy (#4463)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index bfa00f91b..5d45e02fc 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -92,3 +92,4 @@ Tithen-Firion
 Zack Fernandes
 cryptonaut
 Adrian Kretz
+Mathias Rav

From da4d4191a9037ef439ef3f93f02935c67dfcee85 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 23:05:22 +0100
Subject: [PATCH 0849/1937] Merge branch 'master' of github.com:rg3/youtube-dl

---
 test/test_unicode_literals.py     | 2 +-
 youtube_dl/downloader/f4m.py      | 1 +
 youtube_dl/extractor/adultswim.py | 5 ++++-
 youtube_dl/extractor/bbccouk.py   | 2 +-
 4 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/test/test_unicode_literals.py b/test/test_unicode_literals.py
index d3cba869b..19813e034 100644
--- a/test/test_unicode_literals.py
+++ b/test/test_unicode_literals.py
@@ -20,7 +20,7 @@ IGNORED_FILES = [
 ]
 
 
-from helper import assertRegexpMatches
+from test.helper import assertRegexpMatches
 
 
 class TestUnicodeLiterals(unittest.TestCase):
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 00f3a026c..ef3e0d5f4 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -233,6 +233,7 @@ class F4mFD(FileDownloader):
                 'continuedl': True,
                 'quiet': True,
                 'noprogress': True,
+                'ratelimit': self.params.get('ratelimit', None),
                 'test': self.params.get('test', False),
             }
         )
diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 39e4ca296..502a9c25a 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -7,6 +7,8 @@ import json
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    xpath_text,
+    float_or_none,
 )
 
 
@@ -128,7 +130,8 @@ class AdultSwimIE(InfoExtractor):
                 segment_url, segment_title,
                 'Downloading segment information', 'Unable to download segment information')
 
-            segment_duration = idoc.find('.//trt').text.strip()
+            segment_duration = float_or_none(
+                xpath_text(idoc, './/trt', 'segment duration').strip())
 
             formats = []
             file_els = idoc.findall('.//files/file')
diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 01c02d360..2d2f742ae 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -209,7 +209,7 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
         webpage = self._download_webpage(url, group_id, 'Downloading video page')
 
         programme_id = self._search_regex(
-            r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False)
+            r'"vpid"\s*:\s*"([\da-z]{8})"', webpage, 'vpid', fatal=False, default=None)
         if programme_id:
             player = self._download_json(
                 'http://www.bbc.co.uk/iplayer/episode/%s.json' % group_id,

From 051c46256b47bae1bd0f07b5dad2b4ad041506d0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 23:13:48 +0100
Subject: [PATCH 0850/1937] release 2014.12.13

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index da959ea85..1e47c55a0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.12.7'
+__version__ = '2014.12.13'

From a23669220a6d45bfc01d5cf9b047eba42e772003 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 23:27:21 +0100
Subject: [PATCH 0851/1937] [utils] Make ssl work on Python 2.7.8

---
 youtube_dl/utils.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ec34dcef9..ac66f3de0 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -393,8 +393,14 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
         context.options &= ~ssl.OP_NO_SSLv3  # Allow older, not-as-secure SSLv3
         if opts_no_check_certificate:
             context.verify_mode = ssl.CERT_NONE
-        return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
-    elif sys.version_info < (3, 2):
+        try:
+            return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
+        except TypeError:
+            # Python 2.7.8
+            # (create_default_context present but HTTPSHandler has no context=)
+            pass
+
+    if sys.version_info < (3, 2):
         import httplib
 
         class HTTPSConnectionV3(httplib.HTTPSConnection):

From 2128b696b87ed1b44aac0ee31deb94b0b777f7db Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 23:44:50 +0100
Subject: [PATCH 0852/1937] [utils] Do not make an exception for SSLv3

SSLv3 is terminally vulnerable to POODLE; web browsers are currently deprecating/removing it.
Closes #4459, fixes #4294
---
 youtube_dl/utils.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ac66f3de0..5e92bcc71 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -390,7 +390,6 @@ def formatSeconds(secs):
 def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
     if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
         context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
-        context.options &= ~ssl.OP_NO_SSLv3  # Allow older, not-as-secure SSLv3
         if opts_no_check_certificate:
             context.verify_mode = ssl.CERT_NONE
         try:

From 4d144be8b0b93c052648e8137ffd961deee4516e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 23:50:04 +0100
Subject: [PATCH 0853/1937] [bandcamp:album] Do not match plain Bandcamp URLs
 (#4461)

The _VALID_URL 1fa174692aae06406c9524fec392a131f10c68fe is to broad, since it matches everything beginning with bandcamp.com.
---
 youtube_dl/extractor/bandcamp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index bcb2821cd..b45d68a61 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -106,7 +106,7 @@ class BandcampIE(InfoExtractor):
 
 class BandcampAlbumIE(InfoExtractor):
     IE_NAME = 'Bandcamp:album'
-    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<title>[^?#]+))?'
+    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<title>[^?#]+)|/?(?:$|[?#]))'
 
     _TESTS = [{
         'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',

From 6f18cc9abc081c715596dc3616dd8362a427a962 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 13 Dec 2014 23:51:57 +0100
Subject: [PATCH 0854/1937] release 2014.12.13.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1e47c55a0..e502affe7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.13'
+__version__ = '2014.12.13.1'

From bad5c1a3037e7630d599a002fa80790a446cfe77 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 14 Dec 2014 00:13:07 +0100
Subject: [PATCH 0855/1937] [rtp] Also match e-id-less URLs (#4382)

---
 youtube_dl/extractor/rtp.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtp.py b/youtube_dl/extractor/rtp.py
index 0bed1f100..54f67a52f 100644
--- a/youtube_dl/extractor/rtp.py
+++ b/youtube_dl/extractor/rtp.py
@@ -8,8 +8,8 @@ from ..utils import js_to_json
 
 
 class RTPIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?rtp\.pt/play/p(?P<program_id>[0-9]+)/e(?P<id>[0-9]+)/?'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?rtp\.pt/play/p(?P<program_id>[0-9]+)/(?P<id>[^/?#]+)/?'
+    _TESTS = [{
         'url': 'http://www.rtp.pt/play/p405/e174042/paixoes-cruzadas',
         'info_dict': {
             'id': '174042',
@@ -21,7 +21,10 @@ class RTPIE(InfoExtractor):
         'params': {
             'skip_download': True,  # RTMP download
         },
-    }
+    }, {
+        'url': 'http://www.rtp.pt/play/p831/a-quimica-das-coisas',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 4927de4f865eea988b92f37df40f3522c024f6e6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 14 Dec 2014 00:13:17 +0100
Subject: [PATCH 0856/1937] release 2014.12.14

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e502affe7..605ba5cf4 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.13.1'
+__version__ = '2014.12.14'

From 60620368d75346212e9eb3ec6e3e7f2c630a5b0e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 14 Dec 2014 00:43:34 +0100
Subject: [PATCH 0857/1937] [youtube] Fix player ID detection

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 823d6aaf3..3319d6bff 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -478,7 +478,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
     def _extract_signature_function(self, video_id, player_url, example_sig):
         id_m = re.match(
-            r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.(?P<ext>[a-z]+)$',
+            r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.(?P<ext>[a-z]+)$',
             player_url)
         if not id_m:
             raise ExtractorError('Cannot identify player %r' % player_url)

From 408b5839b186ffaea09a67e584b8da33525ed967 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 14 Dec 2014 17:59:25 +0200
Subject: [PATCH 0858/1937] [yesjapan] Add new extractor (Closes #4466)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/yesjapan.py | 62 ++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+)
 create mode 100644 youtube_dl/extractor/yesjapan.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3ae7a8a52..caf8c04f1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -510,6 +510,7 @@ from .yahoo import (
     YahooIE,
     YahooSearchIE,
 )
+from .yesjapan import YesJapanIE
 from .ynet import YnetIE
 from .youjizz import YouJizzIE
 from .youku import YoukuIE
diff --git a/youtube_dl/extractor/yesjapan.py b/youtube_dl/extractor/yesjapan.py
new file mode 100644
index 000000000..fec1ad1ac
--- /dev/null
+++ b/youtube_dl/extractor/yesjapan.py
@@ -0,0 +1,62 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    HEADRequest,
+    get_element_by_attribute,
+    parse_iso8601,
+)
+
+
+class YesJapanIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?yesjapan\.com/video/(?P<slug>[A-Za-z0-9\-]*)_(?P<id>[A-Za-z0-9]+)\.html'
+    _TEST = {
+        'url': 'http://www.yesjapan.com/video/japanese-in-5-20-wa-and-ga-particle-usages_726497834.html',
+        'md5': 'f0be416314e5be21a12b499b330c21cf',
+        'info_dict': {
+            'id': '726497834',
+            'title': 'Japanese in 5! #20 - WA And GA Particle Usages',
+            'description': 'This should clear up some issues most students of Japanese encounter with WA and GA....',
+            'ext': 'mp4',
+            'timestamp': 1416391590,
+            'upload_date': '20141119',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        title = self._og_search_title(webpage)
+        video_url = self._og_search_video_url(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        timestamp = None
+        submit_info = get_element_by_attribute('class', 'pm-submit-data', webpage)
+        if submit_info:
+            timestamp = parse_iso8601(self._search_regex(
+                r'datetime="([^"]+)"', webpage, 'upload date', fatal=False, default=None))
+
+        # attempt to resolve the final URL in order to get a proper extension
+        redirect_req = HEADRequest(video_url)
+        req = self._request_webpage(
+            redirect_req, video_id, note='Resolving final URL', errnote='Could not resolve final URL', fatal=False)
+        if req:
+            video_url = req.geturl()
+
+        formats = [{
+            'format_id': 'sd',
+            'url': video_url,
+        }]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'description': description,
+            'timestamp': timestamp,
+            'thumbnail': thumbnail,
+        }

From 4a4fbfc9670cb3d2e5f0bc486c83ef1e6eb21ad3 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 14 Dec 2014 18:03:05 +0200
Subject: [PATCH 0859/1937] [yesjapan] Look for datetime inside `submit_info`

Oops..
---
 youtube_dl/extractor/yesjapan.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yesjapan.py b/youtube_dl/extractor/yesjapan.py
index fec1ad1ac..112a6c030 100644
--- a/youtube_dl/extractor/yesjapan.py
+++ b/youtube_dl/extractor/yesjapan.py
@@ -38,7 +38,7 @@ class YesJapanIE(InfoExtractor):
         submit_info = get_element_by_attribute('class', 'pm-submit-data', webpage)
         if submit_info:
             timestamp = parse_iso8601(self._search_regex(
-                r'datetime="([^"]+)"', webpage, 'upload date', fatal=False, default=None))
+                r'datetime="([^"]+)"', submit_info, 'upload date', fatal=False, default=None))
 
         # attempt to resolve the final URL in order to get a proper extension
         redirect_req = HEADRequest(video_url)

From a0ab29f8a10e2a97f87182a85a6309ae61ec27b0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 14 Dec 2014 21:55:57 +0100
Subject: [PATCH 0860/1937] Add offlinetest make target

---
 Makefile | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index d5b6e4307..2a32a6c0c 100644
--- a/Makefile
+++ b/Makefile
@@ -35,13 +35,27 @@ install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtu
 	install -d $(DESTDIR)$(SYSCONFDIR)/fish/completions
 	install -m 644 youtube-dl.fish $(DESTDIR)$(SYSCONFDIR)/fish/completions/youtube-dl.fish
 
+codetest:
+	PYFLAKES_OUT=$$(pyflakes youtube_dl | grep -v youtube_dl/extractor/__init__.py); \
+	if test -n "$$PYFLAKES_OUT"; then \
+		echo "$$PYFLAKES_OUT"; \
+		exit 1; \
+	fi
+	pep8 . --ignore E501
+
 test:
 	#nosetests --with-coverage --cover-package=youtube_dl --cover-html --verbose --processes 4 test
 	nosetests --verbose test
+	$(MAKE) codetest
+
+ot: offlinetest
+
+offlinetest: codetest
+	nosetests --verbose test --exclude test_download --exclude test_age_restriction --exclude test_subtitles
 
 tar: youtube-dl.tar.gz
 
-.PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion
+.PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion ot offlinetest codetest
 
 pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 youtube-dl.fish
 

From 6cbf345f28f5950fb274c70be12f5d91679bf787 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 14 Dec 2014 21:56:12 +0100
Subject: [PATCH 0861/1937] Remove test/write_info_json

This is now covered by every single test_download testcase anyways :)
---
 test/test_write_info_json.py | 76 ------------------------------------
 1 file changed, 76 deletions(-)
 delete mode 100644 test/test_write_info_json.py

diff --git a/test/test_write_info_json.py b/test/test_write_info_json.py
deleted file mode 100644
index 0396ef262..000000000
--- a/test/test_write_info_json.py
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-from __future__ import unicode_literals
-
-# Allow direct execution
-import os
-import sys
-import unittest
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from test.helper import get_params
-
-
-import io
-import json
-
-import youtube_dl.YoutubeDL
-import youtube_dl.extractor
-
-
-class YoutubeDL(youtube_dl.YoutubeDL):
-    def __init__(self, *args, **kwargs):
-        super(YoutubeDL, self).__init__(*args, **kwargs)
-        self.to_stderr = self.to_screen
-
-params = get_params({
-    'writeinfojson': True,
-    'skip_download': True,
-    'writedescription': True,
-})
-
-
-TEST_ID = 'BaW_jenozKc'
-INFO_JSON_FILE = TEST_ID + '.info.json'
-DESCRIPTION_FILE = TEST_ID + '.mp4.description'
-EXPECTED_DESCRIPTION = '''test chars:  "'/\ä↭𝕐
-test URL: https://github.com/rg3/youtube-dl/issues/1892
-
-This is a test video for youtube-dl.
-
-For more information, contact phihag@phihag.de .'''
-
-
-class TestInfoJSON(unittest.TestCase):
-    def setUp(self):
-        # Clear old files
-        self.tearDown()
-
-    def test_info_json(self):
-        ie = youtube_dl.extractor.YoutubeIE()
-        ydl = YoutubeDL(params)
-        ydl.add_info_extractor(ie)
-        ydl.download([TEST_ID])
-        self.assertTrue(os.path.exists(INFO_JSON_FILE))
-        with io.open(INFO_JSON_FILE, 'r', encoding='utf-8') as jsonf:
-            jd = json.load(jsonf)
-        self.assertEqual(jd['upload_date'], '20121002')
-        self.assertEqual(jd['description'], EXPECTED_DESCRIPTION)
-        self.assertEqual(jd['id'], TEST_ID)
-        self.assertEqual(jd['extractor'], 'youtube')
-        self.assertEqual(jd['title'], '''youtube-dl test video "'/\ä↭𝕐''')
-        self.assertEqual(jd['uploader'], 'Philipp Hagemeister')
-
-        self.assertTrue(os.path.exists(DESCRIPTION_FILE))
-        with io.open(DESCRIPTION_FILE, 'r', encoding='utf-8') as descf:
-            descr = descf.read()
-        self.assertEqual(descr, EXPECTED_DESCRIPTION)
-
-    def tearDown(self):
-        if os.path.exists(INFO_JSON_FILE):
-            os.remove(INFO_JSON_FILE)
-        if os.path.exists(DESCRIPTION_FILE):
-            os.remove(DESCRIPTION_FILE)
-
-if __name__ == '__main__':
-    unittest.main()

From cae97f6521a846392e665d8743d4c000fb7d7173 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 14 Dec 2014 21:59:59 +0100
Subject: [PATCH 0862/1937] Improve and test ffmpeg version detection

---
 test/test_utils.py  | 38 +++++++++++++++++++++++++-------------
 youtube_dl/utils.py | 17 ++++++++++++-----
 2 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index d42df6d96..dd49a6d17 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -16,39 +16,40 @@ import json
 import xml.etree.ElementTree
 
 from youtube_dl.utils import (
+    args_to_str,
     clean_html,
     DateRange,
+    detect_exe_version,
     encodeFilename,
+    escape_rfc3986,
+    escape_url,
     find_xpath_attr,
     fix_xml_ampersands,
-    orderedSet,
-    OnDemandPagedList,
     InAdvancePagedList,
+    intlist_to_bytes,
+    js_to_json,
+    limit_length,
+    OnDemandPagedList,
+    orderedSet,
     parse_duration,
+    parse_filesize,
+    parse_iso8601,
     read_batch_urls,
     sanitize_filename,
     shell_quote,
     smuggle_url,
     str_to_int,
+    strip_jsonp,
     struct_unpack,
     timeconvert,
     unescapeHTML,
     unified_strdate,
     unsmuggle_url,
+    uppercase_escape,
     url_basename,
     urlencode_postdata,
-    xpath_with_ns,
-    parse_iso8601,
-    strip_jsonp,
-    uppercase_escape,
-    limit_length,
-    escape_rfc3986,
-    escape_url,
-    js_to_json,
-    intlist_to_bytes,
-    args_to_str,
-    parse_filesize,
     version_tuple,
+    xpath_with_ns,
 )
 
 
@@ -390,5 +391,16 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(version_tuple('10.23.344'), (10, 23, 344))
         self.assertEqual(version_tuple('10.1-6'), (10, 1, 6))  # avconv style
 
+    def test_detect_exe_version(self):
+        self.assertEqual(detect_exe_version('''ffmpeg version 1.2.1
+built on May 27 2013 08:37:26 with gcc 4.7 (Debian 4.7.3-4)
+configuration: --prefix=/usr --extra-'''), '1.2.1')
+        self.assertEqual(detect_exe_version('''ffmpeg version N-63176-g1fb4685
+built on May 15 2014 22:09:06 with gcc 4.8.2 (GCC)'''), 'N-63176-g1fb4685')
+        self.assertEqual(detect_exe_version('''X server found. dri2 connection failed!
+Trying to open render node...
+Success at /dev/dri/renderD128.
+ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 5e92bcc71..f9938616d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1262,18 +1262,25 @@ def check_executable(exe, args=[]):
 
 
 def get_exe_version(exe, args=['--version'],
-                    version_re=r'version\s+([0-9._-a-zA-Z]+)',
-                    unrecognized='present'):
+                    version_re=None, unrecognized='present'):
     """ Returns the version of the specified executable,
     or False if the executable is not present """
     try:
-        out, err = subprocess.Popen(
+        out, _ = subprocess.Popen(
             [exe] + args,
             stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
     except OSError:
         return False
-    firstline = out.partition(b'\n')[0].decode('ascii', 'ignore')
-    m = re.search(version_re, firstline)
+    if isinstance(out, bytes):  # Python 2.x
+        out = out.decode('ascii', 'ignore')
+    return detect_exe_version(out, version_re, unrecognized)
+
+
+def detect_exe_version(output, version_re=None, unrecognized='present'):
+    assert isinstance(output, compat_str)
+    if version_re is None:
+        version_re = r'version\s+([-0-9._a-zA-Z]+)'
+    m = re.search(version_re, output)
     if m:
         return m.group(1)
     else:

From 39f594d660cd86f04905d3eead91c5e2f16b0e66 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Dec 2014 00:59:23 +0100
Subject: [PATCH 0863/1937] [Makefile] Ensure that offline test really is
 offline

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 2a32a6c0c..6fb820c2a 100644
--- a/Makefile
+++ b/Makefile
@@ -51,7 +51,7 @@ test:
 ot: offlinetest
 
 offlinetest: codetest
-	nosetests --verbose test --exclude test_download --exclude test_age_restriction --exclude test_subtitles
+	nosetests --verbose test --exclude test_download --exclude test_age_restriction --exclude test_subtitles --exclude test_write_annotations
 
 tar: youtube-dl.tar.gz
 

From 4f026fafbc6cc4eac10a5f89b9375b44d64083c9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Dec 2014 01:06:25 +0100
Subject: [PATCH 0864/1937] [YoutubeDL] Make postprocessors declarative

Instead of having to configure PPs in code, this allows us and embedding programs not to worry about imports or finer details, similarly to how we handle IEs.
---
 README.md                            | 28 +++++++++++
 youtube_dl/YoutubeDL.py              | 19 +++++++-
 youtube_dl/__init__.py               | 71 +++++++++++++++-------------
 youtube_dl/postprocessor/__init__.py |  9 +++-
 youtube_dl/postprocessor/ffmpeg.py   |  4 +-
 5 files changed, 93 insertions(+), 38 deletions(-)

diff --git a/README.md b/README.md
index f10f06ee8..edfcfc223 100644
--- a/README.md
+++ b/README.md
@@ -537,6 +537,34 @@ From a Python program, you can embed youtube-dl in a more powerful fashion, like
 
 Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
+Here's a more complete example of a program that only outputs errors, and downloads/converts the video as mp3:
+
+
+    import youtube_dl
+
+
+    class MyLogger(object):
+        def debug(self, msg):
+            pass
+
+        def warning(self, msg):
+            pass
+
+        def error(self, msg):
+            print(msg)
+
+    ydl_opts = {
+        'format': 'bestaudio/best',
+        'postprocessors': [{
+            'key': 'FFmpegExtractAudio',
+            'preferredcodec': 'mp3',
+            'preferredquality': '64',
+        }],
+        'logger': MyLogger(),
+    }
+    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
+        ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
+
 # BUGS
 
 Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues> . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the irc channel #youtube-dl on freenode.
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 578c8daf2..6acfd8cf9 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -27,6 +27,7 @@ from .compat import (
     compat_cookiejar,
     compat_expanduser,
     compat_http_client,
+    compat_kwargs,
     compat_str,
     compat_urllib_error,
     compat_urllib_request,
@@ -67,7 +68,11 @@ from .cache import Cache
 from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader
 from .downloader.rtmp import rtmpdump_version
-from .postprocessor import FFmpegMergerPP, FFmpegPostProcessor
+from .postprocessor import (
+    FFmpegMergerPP,
+    FFmpegPostProcessor,
+    get_postprocessor,
+)
 from .version import __version__
 
 
@@ -176,6 +181,11 @@ class YoutubeDL(object):
     extract_flat:      Do not resolve URLs, return the immediate result.
                        Pass in 'in_playlist' to only show this behavior for
                        playlist items.
+    postprocessors:    A list of dictionaries, each with an entry
+                       key:  The name of the postprocessor. See
+                             youtube_dl/postprocessor/__init__.py for a list.
+                       as well as any further keyword arguments for the
+                       postprocessor.
 
     The following parameters are not used by YoutubeDL itself, they are used by
     the FileDownloader:
@@ -256,6 +266,13 @@ class YoutubeDL(object):
             self.print_debug_header()
             self.add_default_info_extractors()
 
+        for pp_def_raw in self.params.get('postprocessors', []):
+            pp_class = get_postprocessor(pp_def_raw['key'])
+            pp_def = dict(pp_def_raw)
+            del pp_def['key']
+            pp = pp_class(self, **compat_kwargs(pp_def))
+            self.add_post_processor(pp)
+
     def warn_if_short_id(self, argv):
         # short YouTube ID starting with dash?
         idxs = [
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 70c4f25b1..e79320323 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -40,16 +40,6 @@ from .downloader import (
 )
 from .extractor import gen_extractors
 from .YoutubeDL import YoutubeDL
-from .postprocessor import (
-    AtomicParsleyPP,
-    FFmpegAudioFixPP,
-    FFmpegMetadataPP,
-    FFmpegVideoConvertor,
-    FFmpegExtractAudioPP,
-    FFmpegEmbedSubtitlePP,
-    XAttrMetadataPP,
-    ExecAfterDownloadPP,
-)
 
 
 def _real_main(argv=None):
@@ -212,6 +202,43 @@ def _real_main(argv=None):
     any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
     download_archive_fn = compat_expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 
+    # PostProcessors
+    postprocessors = []
+    # Add the metadata pp first, the other pps will copy it
+    if opts.addmetadata:
+        postprocessors.append({'key': 'FFmpegMetadata'})
+    if opts.extractaudio:
+        postprocessors.append({
+            'key': 'FFmpegExtractAudio',
+            'preferredcodec': opts.audioformat,
+            'preferredquality': opts.audioquality,
+            'nopostoverwrites': opts.nopostoverwrites,
+        })
+    if opts.recodevideo:
+        postprocessors.append({
+            'key': 'FFmpegVideoConvertor',
+            'preferedformat': opts.recodevideo,
+        })
+    if opts.embedsubtitles:
+        postprocessors.append({
+            'key': 'FFmpegEmbedSubtitle',
+            'subtitlesformat': opts.subtitlesformat,
+        })
+    if opts.xattrs:
+        postprocessors.append({'key': 'XAttrMetadata'})
+    if opts.embedthumbnail:
+        if not opts.addmetadata:
+            postprocessors.append({'key': 'FFmpegAudioFix'})
+        postprocessors.append({'key': 'AtomicParsley'})
+    # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
+    # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
+    if opts.exec_cmd:
+        postprocessors.append({
+            'key': 'ExecAfterDownload',
+            'verboseOutput': opts.verbose,
+            'exec_cmd': opts.exec_cmd,
+        })
+
     ydl_opts = {
         'usenetrc': opts.usenetrc,
         'username': opts.username,
@@ -297,32 +324,10 @@ def _real_main(argv=None):
         'encoding': opts.encoding,
         'exec_cmd': opts.exec_cmd,
         'extract_flat': opts.extract_flat,
+        'postprocessors': postprocessors,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
-        # PostProcessors
-        # Add the metadata pp first, the other pps will copy it
-        if opts.addmetadata:
-            ydl.add_post_processor(FFmpegMetadataPP())
-        if opts.extractaudio:
-            ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
-        if opts.recodevideo:
-            ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
-        if opts.embedsubtitles:
-            ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
-        if opts.xattrs:
-            ydl.add_post_processor(XAttrMetadataPP())
-        if opts.embedthumbnail:
-            if not opts.addmetadata:
-                ydl.add_post_processor(FFmpegAudioFixPP())
-            ydl.add_post_processor(AtomicParsleyPP())
-
-        # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
-        # So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
-        if opts.exec_cmd:
-            ydl.add_post_processor(ExecAfterDownloadPP(
-                verboseOutput=opts.verbose, exec_cmd=opts.exec_cmd))
-
         # Update version
         if opts.update_self:
             update_self(ydl.to_screen, opts.verbose)
diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index fb367ebe4..7f505b58e 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -8,11 +8,16 @@ from .ffmpeg import (
     FFmpegExtractAudioPP,
     FFmpegMergerPP,
     FFmpegMetadataPP,
-    FFmpegVideoConvertor,
+    FFmpegVideoConvertorPP,
 )
 from .xattrpp import XAttrMetadataPP
 from .execafterdownload import ExecAfterDownloadPP
 
+
+def get_postprocessor(key):
+    return globals()[key + 'PP']
+
+
 __all__ = [
     'AtomicParsleyPP',
     'ExecAfterDownloadPP',
@@ -22,6 +27,6 @@ __all__ = [
     'FFmpegMergerPP',
     'FFmpegMetadataPP',
     'FFmpegPostProcessor',
-    'FFmpegVideoConvertor',
+    'FFmpegVideoConvertorPP',
     'XAttrMetadataPP',
 ]
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 965ded4c1..048525efc 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -236,9 +236,9 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
         return self._nopostoverwrites, information
 
 
-class FFmpegVideoConvertor(FFmpegPostProcessor):
+class FFmpegVideoConvertorPP(FFmpegPostProcessor):
     def __init__(self, downloader=None, preferedformat=None):
-        super(FFmpegVideoConvertor, self).__init__(downloader)
+        super(FFmpegVideoConvertorPP, self).__init__(downloader)
         self._preferedformat = preferedformat
 
     def run(self, information):

From 71b640cc5b2f15a6913a720b589bdd3ed318c154 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Dec 2014 01:26:18 +0100
Subject: [PATCH 0865/1937] [YoutubeDL] Add declarative version of progress
 hooks

---
 README.md                       |  9 ++++++++-
 youtube_dl/YoutubeDL.py         | 24 ++++++++++++++++++++++--
 youtube_dl/downloader/common.py | 17 ++---------------
 3 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/README.md b/README.md
index edfcfc223..bd35a8214 100644
--- a/README.md
+++ b/README.md
@@ -537,7 +537,7 @@ From a Python program, you can embed youtube-dl in a more powerful fashion, like
 
 Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
-Here's a more complete example of a program that only outputs errors, and downloads/converts the video as mp3:
+Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 
 
     import youtube_dl
@@ -553,6 +553,12 @@ Here's a more complete example of a program that only outputs errors, and downlo
         def error(self, msg):
             print(msg)
 
+
+    def my_hook(d):
+        if d['status'] == 'finished':
+            print('Done downloading, now converting ...')
+
+
     ydl_opts = {
         'format': 'bestaudio/best',
         'postprocessors': [{
@@ -561,6 +567,7 @@ Here's a more complete example of a program that only outputs errors, and downlo
             'preferredquality': '64',
         }],
         'logger': MyLogger(),
+        'progress_hooks': [my_hook],
     }
     with youtube_dl.YoutubeDL(ydl_opts) as ydl:
         ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6acfd8cf9..bea1ef83c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -182,10 +182,27 @@ class YoutubeDL(object):
                        Pass in 'in_playlist' to only show this behavior for
                        playlist items.
     postprocessors:    A list of dictionaries, each with an entry
-                       key:  The name of the postprocessor. See
-                             youtube_dl/postprocessor/__init__.py for a list.
+                       * key:  The name of the postprocessor. See
+                               youtube_dl/postprocessor/__init__.py for a list.
                        as well as any further keyword arguments for the
                        postprocessor.
+    progress_hooks:    A list of functions that get called on download
+                       progress, with a dictionary with the entries
+                       * filename: The final filename
+                       * status: One of "downloading" and "finished"
+
+                       The dict may also have some of the following entries:
+
+                       * downloaded_bytes: Bytes on disk
+                       * total_bytes: Size of the whole file, None if unknown
+                       * tmpfilename: The filename we're currently writing to
+                       * eta: The estimated time in seconds, None if unknown
+                       * speed: The download speed in bytes/second, None if
+                                unknown
+
+                       Progress hooks are guaranteed to be called at least once
+                       (with status "finished") if the download is successful.
+
 
     The following parameters are not used by YoutubeDL itself, they are used by
     the FileDownloader:
@@ -273,6 +290,9 @@ class YoutubeDL(object):
             pp = pp_class(self, **compat_kwargs(pp_def))
             self.add_post_processor(pp)
 
+        for ph in self.params.get('progress_hooks', []):
+            self.add_progress_hook(ph)
+
     def warn_if_short_id(self, argv):
         # short YouTube ID starting with dash?
         idxs = [
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 8181bca09..d3e0b0110 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -305,19 +305,6 @@ class FileDownloader(object):
             ph(status)
 
     def add_progress_hook(self, ph):
-        """ ph gets called on download progress, with a dictionary with the entries
-        * filename: The final filename
-        * status: One of "downloading" and "finished"
-
-        It can also have some of the following entries:
-
-        * downloaded_bytes: Bytes on disks
-        * total_bytes: Total bytes, None if unknown
-        * tmpfilename: The filename we're currently writing to
-        * eta: The estimated time in seconds, None if unknown
-        * speed: The download speed in bytes/second, None if unknown
-
-        Hooks are guaranteed to be called at least once (with status "finished")
-        if the download is successful.
-        """
+        # See YoutubeDl.py (search for progress_hooks) for a description of
+        # this interface
         self._progress_hooks.append(ph)

From 8acb83d9937211abe7194161be863194ae445d95 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Dec 2014 01:34:39 +0100
Subject: [PATCH 0866/1937] [README] Make example audio sound not that horrible
 ;)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index bd35a8214..843e01c87 100644
--- a/README.md
+++ b/README.md
@@ -564,7 +564,7 @@ Here's a more complete example of a program that outputs only errors (and a shor
         'postprocessors': [{
             'key': 'FFmpegExtractAudio',
             'preferredcodec': 'mp3',
-            'preferredquality': '64',
+            'preferredquality': '192',
         }],
         'logger': MyLogger(),
         'progress_hooks': [my_hook],

From e97a534f13ef9bf3d8e9007b80344ca43b8f0a20 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Dec 2014 01:36:46 +0100
Subject: [PATCH 0867/1937] release 2014.12.15

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 605ba5cf4..4d3adbf61 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.14'
+__version__ = '2014.12.15'

From 1bbe317508fa151336034eda4809d0fea1684667 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 15 Dec 2014 03:31:54 +0200
Subject: [PATCH 0868/1937] [mooshare] Modernize

---
 youtube_dl/extractor/mooshare.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mooshare.py b/youtube_dl/extractor/mooshare.py
index 9f2853fa3..4c5a4e673 100644
--- a/youtube_dl/extractor/mooshare.py
+++ b/youtube_dl/extractor/mooshare.py
@@ -64,8 +64,7 @@ class MooshareIE(InfoExtractor):
             'http://mooshare.biz/%s' % video_id, compat_urllib_parse.urlencode(download_form))
         request.add_header('Content-Type', 'application/x-www-form-urlencoded')
 
-        self.to_screen('%s: Waiting for timeout' % video_id)
-        time.sleep(5)
+        self._sleep(5, video_id)
 
         video_page = self._download_webpage(request, video_id, 'Downloading video page')
 

From 6f9b54933f1380423258fa7fe478e002ca97027b Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 15 Dec 2014 03:32:17 +0200
Subject: [PATCH 0869/1937] [streamcloud] Modernize

---
 youtube_dl/extractor/streamcloud.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dl/extractor/streamcloud.py
index 38176498d..fd0cd3c0c 100644
--- a/youtube_dl/extractor/streamcloud.py
+++ b/youtube_dl/extractor/streamcloud.py
@@ -40,8 +40,7 @@ class StreamcloudIE(InfoExtractor):
             ''', orig_webpage)
         post = compat_urllib_parse.urlencode(fields)
 
-        self.to_screen('%s: Waiting for timeout' % video_id)
-        time.sleep(12)
+        self._sleep(12, video_id)
         headers = {
             b'Content-Type': b'application/x-www-form-urlencoded',
         }

From 3013bbb27d79ba22dc432b1081acccef96b54931 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 15 Dec 2014 08:24:50 +0100
Subject: [PATCH 0870/1937] Remove unused imports

---
 youtube_dl/extractor/mooshare.py    | 1 -
 youtube_dl/extractor/streamcloud.py | 1 -
 2 files changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/mooshare.py b/youtube_dl/extractor/mooshare.py
index 4c5a4e673..7603af5e2 100644
--- a/youtube_dl/extractor/mooshare.py
+++ b/youtube_dl/extractor/mooshare.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import re
-import time
 
 from .common import InfoExtractor
 from ..compat import (
diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dl/extractor/streamcloud.py
index fd0cd3c0c..d4e134015 100644
--- a/youtube_dl/extractor/streamcloud.py
+++ b/youtube_dl/extractor/streamcloud.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import time
 
 from .common import InfoExtractor
 from ..compat import (

From 99a0f9824a2b6890998d5d668213974907f9c2de Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Dec 2014 11:11:52 +0100
Subject: [PATCH 0871/1937] [README] Highlight code examples

---
 README.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 843e01c87..80ef3dcca 100644
--- a/README.md
+++ b/README.md
@@ -529,17 +529,19 @@ youtube-dl makes the best effort to be a good command-line program, and thus sho
 
 From a Python program, you can embed youtube-dl in a more powerful fashion, like this:
 
+    ```python
     import youtube_dl
 
     ydl_opts = {}
     with youtube_dl.YoutubeDL(ydl_opts) as ydl:
         ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
+    ```
 
 Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 
-
+    ```python
     import youtube_dl
 
 
@@ -571,6 +573,7 @@ Here's a more complete example of a program that outputs only errors (and a shor
     }
     with youtube_dl.YoutubeDL(ydl_opts) as ydl:
         ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
+    ```
 
 # BUGS
 

From caff59499c17f1701ca9a6c654d001d70456b210 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 15 Dec 2014 11:14:06 +0100
Subject: [PATCH 0872/1937] [README] Fix code rendering

---
 README.md | 62 +++++++++++++++++++++++++++----------------------------
 1 file changed, 31 insertions(+), 31 deletions(-)

diff --git a/README.md b/README.md
index 80ef3dcca..ecc7302c8 100644
--- a/README.md
+++ b/README.md
@@ -529,51 +529,51 @@ youtube-dl makes the best effort to be a good command-line program, and thus sho
 
 From a Python program, you can embed youtube-dl in a more powerful fashion, like this:
 
-    ```python
-    import youtube_dl
+```python
+import youtube_dl
 
-    ydl_opts = {}
-    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
-        ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
-    ```
+ydl_opts = {}
+with youtube_dl.YoutubeDL(ydl_opts) as ydl:
+    ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
+```
 
 Most likely, you'll want to use various options. For a list of what can be done, have a look at [youtube_dl/YoutubeDL.py](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/YoutubeDL.py#L69). For a start, if you want to intercept youtube-dl's output, set a `logger` object.
 
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 
-    ```python
-    import youtube_dl
+```python
+import youtube_dl
 
 
-    class MyLogger(object):
-        def debug(self, msg):
-            pass
+class MyLogger(object):
+    def debug(self, msg):
+        pass
 
-        def warning(self, msg):
-            pass
+    def warning(self, msg):
+        pass
 
-        def error(self, msg):
-            print(msg)
+    def error(self, msg):
+        print(msg)
 
 
-    def my_hook(d):
-        if d['status'] == 'finished':
-            print('Done downloading, now converting ...')
+def my_hook(d):
+    if d['status'] == 'finished':
+        print('Done downloading, now converting ...')
 
 
-    ydl_opts = {
-        'format': 'bestaudio/best',
-        'postprocessors': [{
-            'key': 'FFmpegExtractAudio',
-            'preferredcodec': 'mp3',
-            'preferredquality': '192',
-        }],
-        'logger': MyLogger(),
-        'progress_hooks': [my_hook],
-    }
-    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
-        ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
-    ```
+ydl_opts = {
+    'format': 'bestaudio/best',
+    'postprocessors': [{
+        'key': 'FFmpegExtractAudio',
+        'preferredcodec': 'mp3',
+        'preferredquality': '192',
+    }],
+    'logger': MyLogger(),
+    'progress_hooks': [my_hook],
+}
+with youtube_dl.YoutubeDL(ydl_opts) as ydl:
+    ydl.download(['http://www.youtube.com/watch?v=BaW_jenozKc'])
+```
 
 # BUGS
 

From 4b7df0d30cfe62ab25f462974c2ed0fc5a82eb3f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 15 Dec 2014 19:19:15 +0100
Subject: [PATCH 0873/1937] [youtube:playlist] Work around buggy playlists
 (fixes #4449)

They show a "Load more" button, but they don't have more videos.
The continuation url in the json file was a link to itself, so we ended up in an infinite loop.
---
 youtube_dl/extractor/youtube.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3319d6bff..aaa07b5eb 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1128,6 +1128,13 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'info_dict': {
             'title': 'JODA7',
         }
+    }, {
+        'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
+        'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
+        'info_dict': {
+                'title': 'Uploads from Interstellar Movie',
+        },
+        'playlist_mincout': 21,
     }]
 
     def _real_initialize(self):
@@ -1212,6 +1219,10 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
                 'Downloading page #%s' % page_num,
                 transform_source=uppercase_escape)
             content_html = more['content_html']
+            if not content_html.strip():
+                # Some webpages show a "Load more" button but they don't
+                # have more videos
+                break
             more_widget_html = more['load_more_widget_html']
 
         playlist_title = self._html_search_regex(

From d86007873ef721819241ecd84258e9cce02ec961 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 00:22:23 +0100
Subject: [PATCH 0874/1937] [YoutubeDL] Document where details for format can
 be found

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index bea1ef83c..71c01facb 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -121,7 +121,7 @@ class YoutubeDL(object):
     dump_single_json:  Force printing the info_dict of the whole playlist
                        (or video) as a single JSON line.
     simulate:          Do not download the video files.
-    format:            Video format code.
+    format:            Video format code. See options.py for more information.
     format_limit:      Highest quality format to try.
     outtmpl:           Template for output names.
     restrictfilenames: Do not allow "&" and spaces in file names

From ccbfaa83b032587750cf466802d3f9ad162253fe Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 00:22:38 +0100
Subject: [PATCH 0875/1937] [devscripts/make_contributing] Switch to optparse
 (Fixes #4483)

---
 devscripts/make_contributing.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/devscripts/make_contributing.py b/devscripts/make_contributing.py
index 5fa8cf851..5e454a429 100755
--- a/devscripts/make_contributing.py
+++ b/devscripts/make_contributing.py
@@ -1,20 +1,20 @@
 #!/usr/bin/env python
 from __future__ import unicode_literals
 
-import argparse
 import io
+import optparse
 import re
 
 
 def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        'INFILE', help='README.md file name to read from')
-    parser.add_argument(
-        'OUTFILE', help='CONTRIBUTING.md file name to write to')
-    args = parser.parse_args()
+    parser = optparse.OptionParser(usage='%prog INFILE OUTFILE')
+    options, args = parser.parse_args()
+    if len(args) != 2:
+        parser.error('Expected an input and an output filename')
 
-    with io.open(args.INFILE, encoding='utf-8') as inf:
+    infile, outfile = args
+
+    with io.open(infile, encoding='utf-8') as inf:
         readme = inf.read()
 
     bug_text = re.search(
@@ -25,7 +25,7 @@ def main():
 
     out = bug_text + dev_text
 
-    with io.open(args.OUTFILE, 'w', encoding='utf-8') as outf:
+    with io.open(outfile, 'w', encoding='utf-8') as outf:
         outf.write(out)
 
 if __name__ == '__main__':

From 3940450878480cf12a12072a03db2c641c63c250 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 00:24:30 +0100
Subject: [PATCH 0876/1937] release 2014.12.16

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4d3adbf61..fb1bcdee1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.15'
+__version__ = '2014.12.16'

From 734ea11e3c57ca4df7d9cfc475b1b99b56c18034 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 00:37:42 +0100
Subject: [PATCH 0877/1937] Drop hash character in downloader output (#4484)

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 71c01facb..8a6bcd51c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -712,7 +712,7 @@ class YoutubeDL(object):
                 entries = entries[::-1]
 
             for i, entry in enumerate(entries, 1):
-                self.to_screen('[download] Downloading video #%s of %s' % (i, n_entries))
+                self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
                 extra = {
                     'n_entries': n_entries,
                     'playlist': playlist,

From 0647084f394ddbe440b012bdf30e1a4e00dcbebe Mon Sep 17 00:00:00 2001
From: Tailszefox <tails@tailszefox.no-ip.com>
Date: Tue, 16 Dec 2014 15:46:04 +0100
Subject: [PATCH 0878/1937] [Allocine] Support for more URLs

---
 youtube_dl/extractor/allocine.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/allocine.py b/youtube_dl/extractor/allocine.py
index 623aeaf34..023e06fc6 100644
--- a/youtube_dl/extractor/allocine.py
+++ b/youtube_dl/extractor/allocine.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class AllocineIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?allocine\.fr/(?P<typ>article|video|film)/(fichearticle_gen_carticle=|player_gen_cmedia=|fichefilm_gen_cfilm=)(?P<id>[0-9]+)(?:\.html)?'
+    _VALID_URL = r'https?://(?:www\.)?allocine\.fr/(?P<typ>article|video|film)/(fichearticle_gen_carticle=|player_gen_cmedia=|fichefilm_gen_cfilm=|video-)(?P<id>[0-9]+)(?:\.html)?'
 
     _TESTS = [{
         'url': 'http://www.allocine.fr/article/fichearticle_gen_carticle=18635087.html',

From bb18d787b5c0ded4865090e8022338db063520b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 16 Dec 2014 15:48:01 +0100
Subject: [PATCH 0879/1937] [aljazeera] Add extractor (closes #4487)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/aljazeera.py | 32 +++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 youtube_dl/extractor/aljazeera.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index caf8c04f1..b091505db 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -5,6 +5,7 @@ from .academicearth import AcademicEarthCourseIE
 from .addanime import AddAnimeIE
 from .adultswim import AdultSwimIE
 from .aftonbladet import AftonbladetIE
+from .aljazeera import AlJazeeraIE
 from .anitube import AnitubeIE
 from .anysex import AnySexIE
 from .aol import AolIE
diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
new file mode 100644
index 000000000..ace5ec19d
--- /dev/null
+++ b/youtube_dl/extractor/aljazeera.py
@@ -0,0 +1,32 @@
+from .common import InfoExtractor
+
+
+class AlJazeeraIE(InfoExtractor):
+    _VALID_URL = r'http://www\.aljazeera\.com/programmes/.*?/(?P<id>[^/]+)\.html'
+
+    _TEST = {
+        'url': 'http://www.aljazeera.com/programmes/the-slum/2014/08/deliverance-201482883754237240.html',
+        'info_dict': {
+            'id': '3792260579001',
+            'ext': 'mp4',
+            'title': 'The Slum - Episode 1: Deliverance',
+            'description': 'As a birth attendant advocating for family planning, Remy is on the frontline of Tondo\'s battle with overcrowding.',
+            'uploader': 'Al Jazeera English',
+        },
+        'add_ie': ['Brightcove'],
+    }
+
+    def _real_extract(self, url):
+        program_name = self._match_id(url)
+        webpage = self._download_webpage(url, program_name)
+        brightcove_id = self._search_regex(
+            r'RenderPagesVideo\(\'(.+?)\'', webpage, 'brightcove id')
+
+        return {
+            '_type': 'url',
+            'url':
+                'http://c.brightcove.com/services/viewer/federated_f9?'
+                '&playerKey=AQ~~%2CAAAAmtVJIFk~%2CTVGOQ5ZTwJbeMWnq5d_H4MOM57xfzApc'
+                '&%40videoPlayer={0}'.format(brightcove_id),
+            'ie_key': 'Brightcove',
+        }

From 84d84211acba32a2a9bfb8729eeffd05cd556918 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 15:59:31 +0100
Subject: [PATCH 0880/1937] [youtube:feeds] (Fixes #4486)

---
 youtube_dl/extractor/youtube.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index aaa07b5eb..100ecbc0a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1566,9 +1566,11 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
         feed_entries = []
         paging = 0
         for i in itertools.count(1):
-            info = self._download_json(self._FEED_TEMPLATE % paging,
-                                       '%s feed' % self._FEED_NAME,
-                                       'Downloading page %s' % i)
+            info = self._download_json(
+                self._FEED_TEMPLATE % paging,
+                '%s feed' % self._FEED_NAME,
+                'Downloading page %s' % i,
+                transform_source=uppercase_escape)
             feed_html = info.get('feed_html') or info.get('content_html')
             load_more_widget_html = info.get('load_more_widget_html') or feed_html
             m_ids = re.finditer(r'"/watch\?v=(.*?)["&]', feed_html)

From 9209fe38784dbb4d4c045537919f3dc50e02227a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Dec 2014 21:03:10 +0600
Subject: [PATCH 0881/1937] [allocine] Add test for new URL format

---
 youtube_dl/extractor/allocine.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/allocine.py b/youtube_dl/extractor/allocine.py
index 023e06fc6..7d65b8193 100644
--- a/youtube_dl/extractor/allocine.py
+++ b/youtube_dl/extractor/allocine.py
@@ -44,6 +44,9 @@ class AllocineIE(InfoExtractor):
             'description': 'md5:71742e3a74b0d692c7fce0dd2017a4ac',
             'thumbnail': 're:http://.*\.jpg',
         },
+    }, {
+        'url': 'http://www.allocine.fr/video/video-19550147/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From ac78b5e97be3827c1aa7e8dfa4a9f8f1a9cb8122 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 16:03:57 +0100
Subject: [PATCH 0882/1937] release 2014.12.16.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index fb1bcdee1..dacc47a78 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.16'
+__version__ = '2014.12.16.1'

From b50e3bc67fd864bbb14ed4ccf3f0d29e1ff3e1f0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 16:33:23 +0100
Subject: [PATCH 0883/1937] [README] Add table of contents (Closes #4458)

---
 README.md                     | 14 ++++++++++++--
 devscripts/prepare_manpage.py | 15 +++++++++++++--
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index ecc7302c8..b4afe17d2 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,15 @@
 youtube-dl - download videos from youtube.com or other video platforms
 
-# SYNOPSIS
-**youtube-dl** [OPTIONS] URL [URL...]
+- [INSTALLATION](#installation)
+- [DESCRIPTION](#description)
+- [OPTIONS](#options)
+- [CONFIGURATION](#configuration)
+- [OUTPUT TEMPLATE](#output-template)
+- [VIDEO SELECTION](#video-selection)
+- [FAQ](#faq)
+- [DEVELOPER INSTRUCTIONS](#developer-instructions)
+- [BUGS](#bugs)
+- [COPYRIGHT](#copyright)
 
 # INSTALLATION
 
@@ -34,6 +42,8 @@ YouTube.com and a few more sites. It requires the Python interpreter, version
 your Unix box, on Windows or on Mac OS X. It is released to the public domain,
 which means you can modify it, redistribute it or use it however you like.
 
+    youtube-dl [OPTIONS] URL [URL...]
+
 # OPTIONS
     -h, --help                       print this help text and exit
     --version                        print program version and exit
diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py
index f66bebfea..7ece37754 100644
--- a/devscripts/prepare_manpage.py
+++ b/devscripts/prepare_manpage.py
@@ -11,8 +11,19 @@ README_FILE = os.path.join(ROOT_DIR, 'README.md')
 with io.open(README_FILE, encoding='utf-8') as f:
     readme = f.read()
 
-PREFIX = '%YOUTUBE-DL(1)\n\n# NAME\n'
-readme = re.sub(r'(?s)# INSTALLATION.*?(?=# DESCRIPTION)', '', readme)
+PREFIX = '''%YOUTUBE-DL(1)
+
+# NAME
+
+youtube\-dl \- download videos from youtube.com or other video platforms
+
+# SYNOPSIS
+
+**youtube-dl** \[OPTIONS\] URL [URL...]
+
+'''
+readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme)
+readme = re.sub(r'\s+youtube-dl \[OPTIONS\] URL \[URL\.\.\.\]', '', readme)
 readme = PREFIX + readme
 
 if sys.version_info < (3, 0):

From ec05fee43acc49f4aded253e7072683b268a7965 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 16:38:26 +0100
Subject: [PATCH 0884/1937] [brightcove] Add shorter URL scheme for other
 extractors

---
 youtube_dl/extractor/aljazeera.py  | 9 +++++----
 youtube_dl/extractor/brightcove.py | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
index ace5ec19d..380d429d6 100644
--- a/youtube_dl/extractor/aljazeera.py
+++ b/youtube_dl/extractor/aljazeera.py
@@ -24,9 +24,10 @@ class AlJazeeraIE(InfoExtractor):
 
         return {
             '_type': 'url',
-            'url':
-                'http://c.brightcove.com/services/viewer/federated_f9?'
-                '&playerKey=AQ~~%2CAAAAmtVJIFk~%2CTVGOQ5ZTwJbeMWnq5d_H4MOM57xfzApc'
-                '&%40videoPlayer={0}'.format(brightcove_id),
+            'url': (
+                'brightcove:'
+                'playerKey=AQ~~%2CAAAAmtVJIFk~%2CTVGOQ5ZTwJbeMWnq5d_H4MOM57xfzApc'
+                '&%40videoPlayer={0}'.format(brightcove_id)
+            ),
             'ie_key': 'Brightcove',
         }
diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 1eca00470..003152c4e 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -25,7 +25,7 @@ from ..utils import (
 
 
 class BrightcoveIE(InfoExtractor):
-    _VALID_URL = r'https?://.*brightcove\.com/(services|viewer).*?\?(?P<query>.*)'
+    _VALID_URL = r'(?:https?://.*brightcove\.com/(services|viewer).*?\?|brightcove:)(?P<query>.*)'
     _FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
 
     _TESTS = [

From 13d27a42cc8db567ddaaa233c90c3ae68e7f8f0b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 16:45:28 +0100
Subject: [PATCH 0885/1937] [orf:tvthek] Add support for topic URLs (Fixes
 #4474)

---
 youtube_dl/extractor/orf.py | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index a6e722bf5..4fed83bd6 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -17,9 +17,9 @@ from ..utils import (
 class ORFTVthekIE(InfoExtractor):
     IE_NAME = 'orf:tvthek'
     IE_DESC = 'ORF TVthek'
-    _VALID_URL = r'https?://tvthek\.orf\.at/(?:programs/.+?/episodes|topics/.+?|program/[^/]+)/(?P<id>\d+)'
+    _VALID_URL = r'https?://tvthek\.orf\.at/(?:programs/.+?/episodes|topics?/.+?|program/[^/]+)/(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://tvthek.orf.at/program/Aufgetischt/2745173/Aufgetischt-Mit-der-Steirischen-Tafelrunde/8891389',
         'playlist': [{
             'md5': '2942210346ed779588f428a92db88712',
@@ -32,8 +32,21 @@ class ORFTVthekIE(InfoExtractor):
                 'upload_date': '20141208',
             },
         }],
-        'skip': 'Blocked outside of Austria',
-    }
+        'skip': 'Blocked outside of Austria / Germany',
+    }, {
+        'url': 'http://tvthek.orf.at/topic/Im-Wandel-der-Zeit/8002126/Best-of-Ingrid-Thurnher/7982256',
+        'playlist': [{
+            'md5': '68f543909aea49d621dfc7703a11cfaf',
+            'info_dict': {
+                'id': '7982259',
+                'ext': 'mp4',
+                'title': 'Best of Ingrid Thurnher',
+                'upload_date': '20140527',
+                'description': 'Viele Jahre war Ingrid Thurnher das "Gesicht" der ZIB 2. Vor ihrem Wechsel zur ZIB 2 im jahr 1995 moderierte sie unter anderem "Land und Leute", "Österreich-Bild" und "Niederösterreich heute".',
+            }
+        }],
+        '_skip': 'Blocked outside of Austria / Germany',
+    }]
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
@@ -45,7 +58,9 @@ class ORFTVthekIE(InfoExtractor):
 
         def get_segments(all_data):
             for data in all_data:
-                if data['name'] == 'Tracker::EPISODE_DETAIL_PAGE_OVER_PROGRAM':
+                if data['name'] in (
+                        'Tracker::EPISODE_DETAIL_PAGE_OVER_PROGRAM',
+                        'Tracker::EPISODE_DETAIL_PAGE_OVER_TOPIC'):
                     return data['values']['segments']
 
         sdata = get_segments(all_data)

From 964b190350f217c2e3f157130072ee667b299b24 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 16 Dec 2014 16:45:35 +0100
Subject: [PATCH 0886/1937] release 2014.12.16.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index dacc47a78..768dc7599 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.16.1'
+__version__ = '2014.12.16.2'

From 2cb5b03e53eefca4610a86e7c07bd2e0564b6dda Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 16 Dec 2014 20:33:23 +0100
Subject: [PATCH 0887/1937] [test/test_unicode_literals] Remove duplicated
 imports

---
 test/test_unicode_literals.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/test/test_unicode_literals.py b/test/test_unicode_literals.py
index 19813e034..7f816698e 100644
--- a/test/test_unicode_literals.py
+++ b/test/test_unicode_literals.py
@@ -7,9 +7,7 @@ import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import io
-import os
 import re
-import unittest
 
 rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 

From ee6dfe8308c967573a9d12d8dbbc1533cec45f51 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 16 Dec 2014 19:38:56 +0100
Subject: [PATCH 0888/1937] Use flake8 instead of pyflakes and pep8

It combines both tools
---
 Makefile | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 6fb820c2a..524c5fcda 100644
--- a/Makefile
+++ b/Makefile
@@ -36,12 +36,11 @@ install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtu
 	install -m 644 youtube-dl.fish $(DESTDIR)$(SYSCONFDIR)/fish/completions/youtube-dl.fish
 
 codetest:
-	PYFLAKES_OUT=$$(pyflakes youtube_dl | grep -v youtube_dl/extractor/__init__.py); \
-	if test -n "$$PYFLAKES_OUT"; then \
-		echo "$$PYFLAKES_OUT"; \
+	FLAKE8_OUT=$$(flake8 --ignore E501 . | grep -v youtube_dl/extractor/__init__.py); \
+	if test -n "$$FLAKE8_OUT"; then \
+		echo "$$FLAKE8_OUT"; \
 		exit 1; \
 	fi
-	pep8 . --ignore E501
 
 test:
 	#nosetests --with-coverage --cover-package=youtube_dl --cover-html --verbose --processes 4 test

From dc9a441bfa8b2767748a8e3d138ae765e0d7b0cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 16 Dec 2014 19:54:22 +0100
Subject: [PATCH 0889/1937] Move flake8 configuration to setup.cfg

It will be used when calling flake8 from any directory in the project
---
 Makefile  | 6 +-----
 setup.cfg | 4 ++++
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 524c5fcda..71c8553d8 100644
--- a/Makefile
+++ b/Makefile
@@ -36,11 +36,7 @@ install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtu
 	install -m 644 youtube-dl.fish $(DESTDIR)$(SYSCONFDIR)/fish/completions/youtube-dl.fish
 
 codetest:
-	FLAKE8_OUT=$$(flake8 --ignore E501 . | grep -v youtube_dl/extractor/__init__.py); \
-	if test -n "$$FLAKE8_OUT"; then \
-		echo "$$FLAKE8_OUT"; \
-		exit 1; \
-	fi
+	flake8 .
 
 test:
 	#nosetests --with-coverage --cover-package=youtube_dl --cover-html --verbose --processes 4 test
diff --git a/setup.cfg b/setup.cfg
index e57d130e3..8659b5069 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,2 +1,6 @@
 [wheel]
 universal = True
+
+[flake8]
+exclude = youtube_dl/extractor/__init__.py
+ignore = E501

From a16af518735cb0301731900914c84f26823e9372 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 16 Dec 2014 20:38:59 +0100
Subject: [PATCH 0890/1937] flake8: Add more ignored files

* setup.py: the '__version__' variable is not defined in the script, it is loadded from youtube_dl/version.py
* devscripts/buildserver.py: Produces a lot of messages
---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 8659b5069..898defb9d 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -2,5 +2,5 @@
 universal = True
 
 [flake8]
-exclude = youtube_dl/extractor/__init__.py
+exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,setup.py
 ignore = E501

From 876bef5937526c6132fc22fab461d41bbf79014b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 00:04:24 +0100
Subject: [PATCH 0891/1937] [mit] Modernize

---
 youtube_dl/extractor/mit.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index d354702d1..78787e8f1 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -17,7 +17,7 @@ from ..utils import (
 
 class TechTVMITIE(InfoExtractor):
     IE_NAME = 'techtv.mit.edu'
-    _VALID_URL = r'https?://techtv\.mit\.edu/(videos|embeds)/(?P<id>\d+)'
+    _VALID_URL = r'https?://techtv\.mit\.edu/(?:videos|embeds)/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://techtv.mit.edu/videos/25418-mit-dna-learning-center-set',
@@ -31,8 +31,7 @@ class TechTVMITIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         raw_page = self._download_webpage(
             'http://techtv.mit.edu/videos/%s' % video_id, video_id)
         clean_page = re.compile(r'<!--.*?-->', re.S).sub('', raw_page)
@@ -106,7 +105,7 @@ class OCWMITIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Lecture 7: Multiple Discrete Random Variables: Expectations, Conditioning, Independence',
                 'description': 'In this lecture, the professor discussed multiple random variables, expectations, and binomial distribution.',
-                #'subtitles': 'http://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-041-probabilistic-systems-analysis-and-applied-probability-fall-2010/video-lectures/lecture-7-multiple-variables-expectations-independence/MIT6_041F11_lec07_300k.mp4.srt'
+                # 'subtitles': 'http://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-041-probabilistic-systems-analysis-and-applied-probability-fall-2010/video-lectures/lecture-7-multiple-variables-expectations-independence/MIT6_041F11_lec07_300k.mp4.srt'
             }
         },
         {
@@ -116,7 +115,7 @@ class OCWMITIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Session 1: Introduction to Derivatives',
                 'description': 'This section contains lecture video excerpts, lecture notes, an interactive mathlet with supporting documents, and problem solving videos.',
-                #'subtitles': 'http://ocw.mit.edu//courses/mathematics/18-01sc-single-variable-calculus-fall-2010/ocw-18.01-f07-lec01_300k.SRT'
+                # 'subtitles': 'http://ocw.mit.edu//courses/mathematics/18-01sc-single-variable-calculus-fall-2010/ocw-18.01-f07-lec01_300k.SRT'
             }
         }
     ]

From 7af808a5ef4cc17ca30c8c4636660589e0e915ec Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 00:06:41 +0100
Subject: [PATCH 0892/1937] Improve code style

---
 youtube_dl/downloader/f4m.py        | 2 +-
 youtube_dl/downloader/rtmp.py       | 2 +-
 youtube_dl/extractor/crunchyroll.py | 1 -
 youtube_dl/extractor/cspan.py       | 1 -
 youtube_dl/extractor/tenplay.py     | 1 -
 youtube_dl/extractor/youtube.py     | 4 ++--
 youtube_dl/utils.py                 | 2 +-
 7 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index ef3e0d5f4..f9f6f3e73 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -203,7 +203,7 @@ def write_flv_header(stream, metadata):
     stream.write(b'\x00\x00\x00\x00\x00\x00\x00')
     stream.write(metadata)
     # Magic numbers extracted from the output files produced by AdobeHDS.php
-    #(https://github.com/K-S-V/Scripts)
+    # (https://github.com/K-S-V/Scripts)
     stream.write(b'\x00\x00\x01\x73')
 
 
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 575912675..5346cb9a0 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -185,7 +185,7 @@ class RtmpFD(FileDownloader):
             cursize = os.path.getsize(encodeFilename(tmpfilename))
             if prevsize == cursize and retval == RD_FAILED:
                 break
-             # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
+            # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
             if prevsize == cursize and retval == RD_INCOMPLETE and cursize > 1024:
                 self.to_screen('[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
                 retval = RD_SUCCESS
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 8f1ea02e7..2c7756eb6 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -32,7 +32,6 @@ class CrunchyrollIE(SubtitlesInfoExtractor):
     _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
     _TEST = {
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
-        #'md5': 'b1639fd6ddfaa43788c85f6d1dddd412',
         'info_dict': {
             'id': '645513',
             'ext': 'flv',
diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 541106684..955119d40 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -27,7 +27,6 @@ class CSpanIE(InfoExtractor):
         'url': 'http://www.c-span.org/video/?c4486943/cspan-international-health-care-models',
         # For whatever reason, the served video alternates between
         # two different ones
-        #'md5': 'dbb0f047376d457f2ab8b3929cbb2d0c',
         'info_dict': {
             'id': '340723',
             'ext': 'mp4',
diff --git a/youtube_dl/extractor/tenplay.py b/youtube_dl/extractor/tenplay.py
index 81ba169fb..466155ef8 100644
--- a/youtube_dl/extractor/tenplay.py
+++ b/youtube_dl/extractor/tenplay.py
@@ -8,7 +8,6 @@ class TenPlayIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?ten(play)?\.com\.au/.+'
     _TEST = {
         'url': 'http://tenplay.com.au/ten-insider/extra/season-2013/tenplay-tv-your-way',
-        #'md5': 'd68703d9f73dc8fccf3320ab34202590',
         'info_dict': {
             'id': '2695695426001',
             'ext': 'flv',
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 100ecbc0a..550e18733 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -527,8 +527,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 return 's[%s%s%s]' % (starts, ends, steps)
 
             step = None
-            start = '(Never used)'  # Quelch pyflakes warnings - start will be
-                                    # set as soon as step is set
+            # Quelch pyflakes warnings - start will be set when step is set
+            start = '(Never used)'
             for i, prev in zip(idxs[1:], idxs[:-1]):
                 if step is not None:
                     if i - prev == step:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f9938616d..43b7c94ba 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -363,7 +363,7 @@ def encodeArgument(s):
     if not isinstance(s, compat_str):
         # Legacy code that uses byte strings
         # Uncomment the following line after fixing all post processors
-        #assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
+        # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
         s = s.decode('ascii')
     return encodeFilename(s, True)
 

From 37381350f8f6a785df0aae75bef09d628fdb1b1a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 00:08:04 +0100
Subject: [PATCH 0893/1937] [aljazeera] Add unicode_literals marker

---
 youtube_dl/extractor/aljazeera.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
index 380d429d6..612708e25 100644
--- a/youtube_dl/extractor/aljazeera.py
+++ b/youtube_dl/extractor/aljazeera.py
@@ -1,3 +1,5 @@
+from __future__ import unicode_literals
+
 from .common import InfoExtractor
 
 

From b1ccbed3d43b10369ba779d8873ed6b85767f3c1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 00:26:04 +0100
Subject: [PATCH 0894/1937] [nhl] Allow upper-case video IDs (Fixes #4494)

---
 youtube_dl/extractor/nhl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index b2f40344f..d3a4fc513 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -54,7 +54,7 @@ class NHLBaseInfoExtractor(InfoExtractor):
 
 class NHLIE(NHLBaseInfoExtractor):
     IE_NAME = 'nhl.com'
-    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/console(?:\?(?:.*?[?&])?)id=(?P<id>[0-9a-z-]+)'
+    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/console(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
 
     _TESTS = [{
         'url': 'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614',

From 6e1b9395c69910cefdb4fb080855292cf4b6964a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 10:53:12 +0100
Subject: [PATCH 0895/1937] [screencastomatic] Add new extractor (Fixes #4497)

---
 youtube_dl/extractor/__init__.py         |  1 +
 youtube_dl/extractor/screencastomatic.py | 50 ++++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 youtube_dl/extractor/screencastomatic.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b091505db..c30cbc926 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -342,6 +342,7 @@ from .savefrom import SaveFromIE
 from .sbs import SBSIE
 from .scivee import SciVeeIE
 from .screencast import ScreencastIE
+from .screencastomatic import ScreencastOMaticIE
 from .screenwavemedia import CinemassacreIE, ScreenwaveMediaIE, TeamFourIE
 from .servingsys import ServingSysIE
 from .sexu import SexuIE
diff --git a/youtube_dl/extractor/screencastomatic.py b/youtube_dl/extractor/screencastomatic.py
new file mode 100644
index 000000000..1dafc4c31
--- /dev/null
+++ b/youtube_dl/extractor/screencastomatic.py
@@ -0,0 +1,50 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    ExtractorError,
+    js_to_json,
+)
+
+
+class ScreencastOMaticIE(InfoExtractor):
+    _VALID_URL = r'https?://screencast-o-matic\.com/watch/(?P<id>[0-9a-zA-Z]+)'
+    _TEST = {
+        'url': 'http://screencast-o-matic.com/watch/c2lD3BeOPl',
+        'md5': '483583cb80d92588f15ccbedd90f0c18',
+        'info_dict': {
+            'id': 'c2lD3BeOPl',
+            'ext': 'mp4',
+            'title': 'Welcome to 3-4 Philosophy @ DECV!',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'as the title says! also: some general info re 1) VCE philosophy and 2) distance learning.',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        setup_js = self._search_regex(
+            r"(?s)jwplayer\('mp4Player'\).setup\((\{.*?\})\);",
+            webpage, 'setup code')
+        data = self._parse_json(setup_js, video_id, transform_source=js_to_json)
+        modes = data['modes']
+        try:
+            video_data = next(
+                m for m in data['modes'] if m.get('type') == 'html5')
+        except StopIteration:
+            raise ExtractorError('Could not find any video entries!')
+        video_url = compat_urlparse.urljoin(url, video_data['config']['file'])
+        thumbnail = data.get('image')
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+            'url': video_url,
+            'ext': 'mp4',
+            'thumbnail': thumbnail,
+        }

From d6a8160902c2f316b3772648d6eec6bfcb095471 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 10:53:17 +0100
Subject: [PATCH 0896/1937] release 2014.12.17

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 768dc7599..a87e2dd1d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.16.2'
+__version__ = '2014.12.17'

From 27d67ea2ba3a61718e61787117832f1c1eb60b87 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 11:29:35 +0100
Subject: [PATCH 0897/1937] [comedycentral] Match URLs with a second ID (fixes
 #4499)

---
 youtube_dl/extractor/comedycentral.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 48e2410b6..8d27af5e5 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -50,7 +50,7 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
                           )|
                           (?P<interview>
                               extended-interviews/(?P<interID>[0-9a-z]+)/(?:playlist_tds_extended_)?(?P<interview_title>.*?)(/.*?)?)))
-                     (?:[?#].*|$)'''
+                     '''
     _TESTS = [{
         'url': 'http://thedailyshow.cc.com/watch/thu-december-13-2012/kristen-stewart',
         'md5': '4e2f5cb088a83cd8cdb7756132f9739d',
@@ -83,6 +83,9 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
     }, {
         'url': 'http://thedailyshow.cc.com/video-playlists/npde3s/the-daily-show-19088-highlights',
         'only_matching': True,
+    }, {
+        'url': 'http://thedailyshow.cc.com/video-playlists/t6d9sg/the-daily-show-20038-highlights/be3cwo',
+        'only_matching': True,
     }, {
         'url': 'http://thedailyshow.cc.com/special-editions/2l8fdb/special-edition---a-look-back-at-food',
         'only_matching': True,

From a91c9b15e3b2b2b3f2f9385b18ae7d75b802e72f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 11:29:52 +0100
Subject: [PATCH 0898/1937] release 2014.12.17.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a87e2dd1d..6f04cc2d0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.17'
+__version__ = '2014.12.17.1'

From fdc80008100ebcca2a1456d6e8a7064e65f6e69f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 11:38:59 +0100
Subject: [PATCH 0899/1937] [downloader] Handle a file ./- (Fixes #4498)

---
 youtube_dl/downloader/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index d3e0b0110..584bde732 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -285,7 +285,7 @@ class FileDownloader(object):
         Return True on success and False otherwise
         """
         # Check file already present
-        if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
+        if filename != '-' and self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
             self.report_file_already_downloaded(filename)
             self._hook_progress({
                 'filename': filename,

From 7642c08763cda0cadeb812393ed3d1b2c8ecb99a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 11:39:25 +0100
Subject: [PATCH 0900/1937] release 2014.12.17.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6f04cc2d0..1420af746 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.17.1'
+__version__ = '2014.12.17.2'

From c33bcf2051cac1580d5161a188acc3efd54734ea Mon Sep 17 00:00:00 2001
From: AndroKev <AndroKev@users.noreply.github.com>
Date: Wed, 17 Dec 2014 13:05:19 +0100
Subject: [PATCH 0901/1937] only add video-id to archive, when successful

Example:
no space left--> youtube-dl adds the id to archive, but the video isn't fully downloaded
---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8a6bcd51c..82acfe282 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1145,12 +1145,12 @@ class YoutubeDL(object):
 
             if success:
                 try:
+                    self.record_download_archive(info_dict)
                     self.post_process(filename, info_dict)
                 except (PostProcessingError) as err:
                     self.report_error('postprocessing: %s' % str(err))
                     return
 
-        self.record_download_archive(info_dict)
 
     def download(self, url_list):
         """Download a given list of URLs."""

From cd58dc3e56b2efa955bda6ced7105df2bb790e63 Mon Sep 17 00:00:00 2001
From: AndroKev <AndroKev@users.noreply.github.com>
Date: Wed, 17 Dec 2014 13:21:22 +0100
Subject: [PATCH 0902/1937] Update YoutubeDL.py

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 82acfe282..30f1cd1ee 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1145,11 +1145,11 @@ class YoutubeDL(object):
 
             if success:
                 try:
-                    self.record_download_archive(info_dict)
                     self.post_process(filename, info_dict)
                 except (PostProcessingError) as err:
                     self.report_error('postprocessing: %s' % str(err))
                     return
+                self.record_download_archive(info_dict)
 
 
     def download(self, url_list):

From 0e4cb4f406337d3b8879cabb556bde1ec6742b74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 17 Dec 2014 14:55:27 +0100
Subject: [PATCH 0903/1937] YoutubeDL: style fix

---
 youtube_dl/YoutubeDL.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 30f1cd1ee..14ee30e55 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1151,7 +1151,6 @@ class YoutubeDL(object):
                     return
                 self.record_download_archive(info_dict)
 
-
     def download(self, url_list):
         """Download a given list of URLs."""
         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)

From e54a3a2f01d2dec2c1d65d3064dd35e7e62cbf40 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 17 Dec 2014 14:56:30 +0100
Subject: [PATCH 0904/1937] [screencastomatic] Remove unused variable

---
 youtube_dl/extractor/screencastomatic.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/screencastomatic.py b/youtube_dl/extractor/screencastomatic.py
index 1dafc4c31..05337421c 100644
--- a/youtube_dl/extractor/screencastomatic.py
+++ b/youtube_dl/extractor/screencastomatic.py
@@ -31,7 +31,6 @@ class ScreencastOMaticIE(InfoExtractor):
             r"(?s)jwplayer\('mp4Player'\).setup\((\{.*?\})\);",
             webpage, 'setup code')
         data = self._parse_json(setup_js, video_id, transform_source=js_to_json)
-        modes = data['modes']
         try:
             video_data = next(
                 m for m in data['modes'] if m.get('type') == 'html5')

From 748ec6672533d80881475186ed292caea9f8e8c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Dec 2014 20:20:40 +0600
Subject: [PATCH 0905/1937] [theplatform] Extract captions (Closes #4495)

---
 youtube_dl/extractor/theplatform.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index af6ef0033..110ed976d 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 import json
 
-from .common import InfoExtractor
+from .subtitles import SubtitlesInfoExtractor
 from ..compat import (
     compat_str,
 )
@@ -16,7 +16,7 @@ from ..utils import (
 _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language'})
 
 
-class ThePlatformIE(InfoExtractor):
+class ThePlatformIE(SubtitlesInfoExtractor):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/[^/]+/
            (?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)?
@@ -66,6 +66,20 @@ class ThePlatformIE(InfoExtractor):
         info_json = self._download_webpage(info_url, video_id)
         info = json.loads(info_json)
 
+        subtitles = {}
+        captions = info.get('captions')
+        if isinstance(captions, list):
+            for caption in captions:
+                lang, src = caption.get('lang'), caption.get('src')
+                if lang and src:
+                    subtitles[lang] = src
+
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, subtitles)
+            return
+
+        subtitles = self.extract_subtitles(video_id, subtitles)
+
         head = meta.find(_x('smil:head'))
         body = meta.find(_x('smil:body'))
 
@@ -117,6 +131,7 @@ class ThePlatformIE(InfoExtractor):
         return {
             'id': video_id,
             'title': info['title'],
+            'subtitles': subtitles,
             'formats': formats,
             'description': info['description'],
             'thumbnail': info['defaultThumbnailUrl'],

From 60e5428925f7299182b4b327b3e88908e33eca1c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 15:36:16 +0100
Subject: [PATCH 0906/1937] [flake8] Ignore build/ directory

That directory is temporarily generated when building for PyPi and may be present if something goes wrong with the upload.
---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 898defb9d..13dcd8af6 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -2,5 +2,5 @@
 universal = True
 
 [flake8]
-exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,setup.py
+exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,setup.py,build
 ignore = E501

From 5f627b4448f3276089978177a550624bbd5e945f Mon Sep 17 00:00:00 2001
From: Petr Kutalek <petr.kutalek@forers.com>
Date: Wed, 17 Dec 2014 15:52:54 +0100
Subject: [PATCH 0907/1937] [dvtv] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/dvtv.py     | 63 ++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 youtube_dl/extractor/dvtv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c30cbc926..79f6f6f13 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -88,6 +88,7 @@ from .dotsub import DotsubIE
 from .dreisat import DreiSatIE
 from .drtuber import DrTuberIE
 from .drtv import DRTVIE
+from .dvtv import DVTVIE
 from .dump import DumpIE
 from .defense import DefenseGouvFrIE
 from .discovery import DiscoveryIE
diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dl/extractor/dvtv.py
new file mode 100644
index 000000000..a52bb3469
--- /dev/null
+++ b/youtube_dl/extractor/dvtv.py
@@ -0,0 +1,63 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+import re
+import json
+from .common import InfoExtractor
+from ..utils import (
+	ExtractorError,
+	js_to_json,
+	unescapeHTML
+)
+
+
+class DVTVIE(InfoExtractor):
+	IE_NAME = 'dvtv'
+	IE_DESC = 'http://video.aktualne.cz/dvtv/'
+
+	_VALID_URL = r'http://video\.aktualne\.cz/dvtv/(?P<id>[a-z0-9-]+/r~[0-9a-f]{32})/?'
+
+	_TESTS = [{
+		'url': 'http://video.aktualne.cz/dvtv/vondra-o-ceskem-stoleti-pri-pohledu-na-havla-mi-bylo-trapne/r~e5efe9ca855511e4833a0025900fea04/',
+		'md5': '75800f964fa0f82939a2914563301f72',
+		'info_dict': {
+			'id': 'e5efe9ca855511e4833a0025900fea04',
+			'ext': 'webm',
+			'title': 'Vondra o Českém století: Při pohledu na Havla mi bylo trapně'
+		}
+	}, {
+		'url': 'http://video.aktualne.cz/dvtv/stropnicky-policie-vrbetice-preventivne-nekontrolovala/r~82ed4322849211e4a10c0025900fea04/',
+		'md5': 'd50455195a67a94c57f931360cc68a1b',
+		'info_dict': {
+			'id': '82ed4322849211e4a10c0025900fea04',
+			'ext': 'webm',
+			'title': 'Stropnický: Policie Vrbětice preventivně nekontrolovala'
+		}
+	}]
+
+	def _real_extract(self, url):
+		video_id = self._match_id(url)
+		webpage = self._download_webpage(url, video_id)
+
+		code = self._search_regex(r'embedData[0-9a-f]{32}\[\'asset\'\] = (\{.+?\});', webpage, 'video JSON', flags=re.DOTALL)
+		payload = self._parse_json(code, video_id, transform_source=js_to_json)
+		formats = []
+		for source in payload['sources']:
+			formats.append({
+				'url': source['file'],
+				'ext': source['type'][6:],
+				'format': '%s %s' % (source['type'][6:], source['label']),
+				'format_id': '%s-%s' % (source['type'][6:], source['label']),
+				'resolution': source['label'],
+				'fps': 25,
+				'preference': -1 if source['type'][6:] == 'mp4' and source['label'] == '720p' else -2
+			})
+
+		return {
+			'id': video_id[-32:],
+			'display_id': video_id[:-35],
+			'title': unescapeHTML(payload['title']),
+			'thumbnail': 'http:%s' % payload['image'],
+			'formats': formats
+		}

From b9465395cbc222fc9609b7881098c7246ee93989 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 23:18:06 +0100
Subject: [PATCH 0908/1937] [dvtv] PEP8 and correct format sorting (#4502)

---
 youtube_dl/extractor/dvtv.py | 96 ++++++++++++++++++------------------
 1 file changed, 48 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dl/extractor/dvtv.py
index a52bb3469..af552831c 100644
--- a/youtube_dl/extractor/dvtv.py
+++ b/youtube_dl/extractor/dvtv.py
@@ -2,62 +2,62 @@
 
 from __future__ import unicode_literals
 
-import re
-import json
 from .common import InfoExtractor
 from ..utils import (
-	ExtractorError,
-	js_to_json,
-	unescapeHTML
+    js_to_json,
+    unescapeHTML
 )
 
 
 class DVTVIE(InfoExtractor):
-	IE_NAME = 'dvtv'
-	IE_DESC = 'http://video.aktualne.cz/dvtv/'
+    IE_NAME = 'dvtv'
+    IE_DESC = 'http://video.aktualne.cz/dvtv/'
 
-	_VALID_URL = r'http://video\.aktualne\.cz/dvtv/(?P<id>[a-z0-9-]+/r~[0-9a-f]{32})/?'
+    _VALID_URL = r'http://video\.aktualne\.cz/dvtv/(?P<id>[a-z0-9-]+/r~[0-9a-f]{32})/?'
 
-	_TESTS = [{
-		'url': 'http://video.aktualne.cz/dvtv/vondra-o-ceskem-stoleti-pri-pohledu-na-havla-mi-bylo-trapne/r~e5efe9ca855511e4833a0025900fea04/',
-		'md5': '75800f964fa0f82939a2914563301f72',
-		'info_dict': {
-			'id': 'e5efe9ca855511e4833a0025900fea04',
-			'ext': 'webm',
-			'title': 'Vondra o Českém století: Při pohledu na Havla mi bylo trapně'
-		}
-	}, {
-		'url': 'http://video.aktualne.cz/dvtv/stropnicky-policie-vrbetice-preventivne-nekontrolovala/r~82ed4322849211e4a10c0025900fea04/',
-		'md5': 'd50455195a67a94c57f931360cc68a1b',
-		'info_dict': {
-			'id': '82ed4322849211e4a10c0025900fea04',
-			'ext': 'webm',
-			'title': 'Stropnický: Policie Vrbětice preventivně nekontrolovala'
-		}
-	}]
+    _TESTS = [{
+        'url': 'http://video.aktualne.cz/dvtv/vondra-o-ceskem-stoleti-pri-pohledu-na-havla-mi-bylo-trapne/r~e5efe9ca855511e4833a0025900fea04/',
+        'md5': '75800f964fa0f82939a2914563301f72',
+        'info_dict': {
+            'id': 'e5efe9ca855511e4833a0025900fea04',
+            'ext': 'webm',
+            'title': 'Vondra o Českém století: Při pohledu na Havla mi bylo trapně'
+        }
+    }, {
+        'url': 'http://video.aktualne.cz/dvtv/stropnicky-policie-vrbetice-preventivne-nekontrolovala/r~82ed4322849211e4a10c0025900fea04/',
+        'md5': '6388f1941b48537dbd28791f712af8bf',
+        'info_dict': {
+            'id': '82ed4322849211e4a10c0025900fea04',
+            'ext': 'mp4',
+            'title': 'Stropnický: Policie Vrbětice preventivně nekontrolovala'
+        }
+    }]
 
-	def _real_extract(self, url):
-		video_id = self._match_id(url)
-		webpage = self._download_webpage(url, video_id)
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
 
-		code = self._search_regex(r'embedData[0-9a-f]{32}\[\'asset\'\] = (\{.+?\});', webpage, 'video JSON', flags=re.DOTALL)
-		payload = self._parse_json(code, video_id, transform_source=js_to_json)
-		formats = []
-		for source in payload['sources']:
-			formats.append({
-				'url': source['file'],
-				'ext': source['type'][6:],
-				'format': '%s %s' % (source['type'][6:], source['label']),
-				'format_id': '%s-%s' % (source['type'][6:], source['label']),
-				'resolution': source['label'],
-				'fps': 25,
-				'preference': -1 if source['type'][6:] == 'mp4' and source['label'] == '720p' else -2
-			})
+        code = self._search_regex(
+            r'(?s)embedData[0-9a-f]{32}\[\'asset\'\] = (\{.+?\});',
+            webpage, 'video JSON')
+        payload = self._parse_json(code, video_id, transform_source=js_to_json)
+        formats = []
+        for source in payload['sources']:
+            ext = source['type'][6:]
+            formats.append({
+                'url': source['file'],
+                'ext': ext,
+                'format': '%s %s' % (ext, source['label']),
+                'format_id': '%s-%s' % (ext, source['label']),
+                'height': int(source['label'].rstrip('p')),
+                'fps': 25,
+            })
+        self._sort_formats(formats)
 
-		return {
-			'id': video_id[-32:],
-			'display_id': video_id[:-35],
-			'title': unescapeHTML(payload['title']),
-			'thumbnail': 'http:%s' % payload['image'],
-			'formats': formats
-		}
+        return {
+            'id': video_id[-32:],
+            'display_id': video_id[:-35],
+            'title': unescapeHTML(payload['title']),
+            'thumbnail': 'http:%s' % payload['image'],
+            'formats': formats
+        }

From 87c4c21e7583b8d6af04dda822768c226df4ca25 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 17 Dec 2014 23:37:57 +0100
Subject: [PATCH 0909/1937] Credit @petrkutalek for dvtv (#4502)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 5d45e02fc..a4fa76801 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -93,3 +93,4 @@ Zack Fernandes
 cryptonaut
 Adrian Kretz
 Mathias Rav
+Petr Kutalek

From a22524b00438fc7dcde1e041d7a576eb443b4743 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Dec 2014 21:25:42 +0600
Subject: [PATCH 0910/1937] [twitch] Add support for vods (Closes #4512)

---
 youtube_dl/extractor/twitch.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 715f9930c..b11a1d561 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -22,6 +22,7 @@ class TwitchIE(InfoExtractor):
     _VALID_URL = r"""(?x)^(?:http://)?(?:www\.)?twitch\.tv/
         (?:
             (?P<channelid>[^/]+)|
+            (?:(?:[^/]+)/v/(?P<vodid>[^/]+))|
             (?:(?:[^/]+)/b/(?P<videoid>[^/]+))|
             (?:(?:[^/]+)/c/(?P<chapterid>[^/]+))
         )
@@ -70,11 +71,24 @@ class TwitchIE(InfoExtractor):
     def _extract_media(self, item, item_id):
         ITEMS = {
             'a': 'video',
+            'v': 'vod',
             'c': 'chapter',
         }
         info = self._extract_info(self._download_json(
             '%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
             'Downloading %s info JSON' % ITEMS[item]))
+
+        if item == 'v':
+            access_token = self._download_json(
+                '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
+                'Downloading %s access token' % ITEMS[item])
+            formats = self._extract_m3u8_formats(
+                'http://usher.twitch.tv/vod/%s?nauth=%s&nauthsig=%s'
+                % (item_id, access_token['token'], access_token['sig']),
+                item_id, 'mp4')
+            info['formats'] = formats
+            return info
+
         response = self._download_json(
             '%s/api/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
             'Downloading %s playlist JSON' % ITEMS[item])
@@ -209,6 +223,8 @@ class TwitchIE(InfoExtractor):
             """
         elif mobj.group('videoid'):
             return self._extract_media('a', mobj.group('videoid'))
+        elif mobj.group('vodid'):
+            return self._extract_media('v', mobj.group('vodid'))
         elif mobj.group('channelid'):
             channel_id = mobj.group('channelid')
             info = self._download_json(

From c58843b3a15393a46e74c8c1301b828dfcb5002c Mon Sep 17 00:00:00 2001
From: Will Glynn <will@willglynn.com>
Date: Thu, 18 Dec 2014 22:28:13 -0600
Subject: [PATCH 0911/1937] [nerdcubed] Add new extractor

nerdcubed.co.uk describes videos in a single a feed.json file, providing
references to and metadata on >1300 YouTube videos spread across 3 main
channels as well as guest appareances on other channels via a single HTTP
request.

NerdCubedFeedIE transforms this feed into a youtube-dl playlist, preserving
information present in the upstream JSON (allowing zero-cost title/date
matches) and ultimately referencing the embedded YouTube videos.
---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/nerdcubed.py | 36 +++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 youtube_dl/extractor/nerdcubed.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 79f6f6f13..76f13bf52 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -264,6 +264,7 @@ from .nbc import (
 )
 from .ndr import NDRIE
 from .ndtv import NDTVIE
+from .nerdcubed import NerdCubedFeedIE
 from .newgrounds import NewgroundsIE
 from .newstube import NewstubeIE
 from .nfb import NFBIE
diff --git a/youtube_dl/extractor/nerdcubed.py b/youtube_dl/extractor/nerdcubed.py
new file mode 100644
index 000000000..9f2e678e0
--- /dev/null
+++ b/youtube_dl/extractor/nerdcubed.py
@@ -0,0 +1,36 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import datetime
+
+from .common import InfoExtractor
+
+
+class NerdCubedFeedIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?nerdcubed\.co\.uk/feed\.json'
+    _TEST = {
+        'url': 'http://www.nerdcubed.co.uk/feed.json',
+        'info_dict': {
+            'title': 'nerdcubed.co.uk feed',
+        },
+        'playlist_mincount': 1300,
+    }
+
+    def _real_extract(self, url):
+        feed = self._download_json(url, url, "Downloading NerdCubed JSON feed")
+        
+        entries = [{
+            '_type': 'url',
+            'title': feed_entry['title'],
+            'uploader': feed_entry['source']['name'] if feed_entry['source'] else None,
+            'upload_date': datetime.datetime.strptime(feed_entry['date'], '%Y-%m-%d').strftime('%Y%m%d'),
+            'url': "http://www.youtube.com/watch?v=" + feed_entry['youtube_id'],
+        } for feed_entry in feed]
+
+        return {
+            '_type': 'playlist',
+            'title': 'nerdcubed.co.uk feed',
+            'id': 'nerdcubed-feed',
+            'entries': entries,
+        }
+

From a2a4bae9298c77f5262ae45f6fc0fc8cb243d4af Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 19 Dec 2014 10:32:20 +0100
Subject: [PATCH 0912/1937] Credit @willglynn for nerdcubed (#4515)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index a4fa76801..6ea958fce 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -94,3 +94,4 @@ cryptonaut
 Adrian Kretz
 Mathias Rav
 Petr Kutalek
+Will Glynn

From 16ea8179681263d570e632d3ce523ed23ddbf204 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 19 Dec 2014 21:56:44 +0600
Subject: [PATCH 0913/1937] [xtube] Fix and modernize (Closes #4489)

---
 youtube_dl/extractor/xtube.py | 62 ++++++++++++++++++++---------------
 1 file changed, 35 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index f9d98b83f..cf74d4fd5 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -1,11 +1,11 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_request,
+    compat_urllib_parse,
 )
 from ..utils import (
     parse_duration,
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class XTubeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?P<url>xtube\.com/watch\.php\?v=(?P<id>[^/?&]+))'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>xtube\.com/watch\.php\?v=(?P<id>[^/?&#]+))'
     _TEST = {
         'url': 'http://www.xtube.com/watch.php?v=kVTUy_G222_',
         'md5': '092fbdd3cbe292c920ef6fc6a8a9cdab',
@@ -30,41 +30,49 @@ class XTubeIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        url = 'http://www.' + mobj.group('url')
+        video_id = self._match_id(url)
 
         req = compat_urllib_request.Request(url)
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 
-        video_title = self._html_search_regex(r'<p class="title">([^<]+)', webpage, 'title')
+        video_title = self._html_search_regex(
+            r'<p class="title">([^<]+)', webpage, 'title')
         video_uploader = self._html_search_regex(
-            r'so_s\.addVariable\("owner_u", "([^"]+)', webpage, 'uploader', fatal=False)
+            [r"var\s+contentOwnerId\s*=\s*'([^']+)",
+             r'By:\s*<a href="/community/profile\.php?user=([^"]+)'],
+            webpage, 'uploader', fatal=False)
         video_description = self._html_search_regex(
-            r'<p class="fieldsDesc">([^<]+)', webpage, 'description', fatal=False)
+            r'<p class="fieldsDesc">([^<]+)',
+            webpage, 'description', fatal=False)
         duration = parse_duration(self._html_search_regex(
-            r'<span class="bold">Runtime:</span> ([^<]+)</p>', webpage, 'duration', fatal=False))
-        view_count = self._html_search_regex(
-            r'<span class="bold">Views:</span> ([\d,\.]+)</p>', webpage, 'view count', fatal=False)
-        if view_count:
-            view_count = str_to_int(view_count)
-        comment_count = self._html_search_regex(
-            r'<div id="commentBar">([\d,\.]+) Comments</div>', webpage, 'comment count', fatal=False)
-        if comment_count:
-            comment_count = str_to_int(comment_count)
+            r'<span class="bold">Runtime:</span> ([^<]+)</p>',
+            webpage, 'duration', fatal=False))
+        view_count = str_to_int(self._html_search_regex(
+            r'<span class="bold">Views:</span> ([\d,\.]+)</p>',
+            webpage, 'view count', fatal=False))
+        comment_count = str_to_int(self._html_search_regex(
+            r'<div id="commentBar">([\d,\.]+) Comments</div>',
+            webpage, 'comment count', fatal=False))
 
-        player_quality_option = json.loads(self._html_search_regex(
-            r'playerQualityOption = ({.+?});', webpage, 'player quality option'))
-
-        QUALITIES = ['3gp', 'mp4_normal', 'mp4_high', 'flv', 'mp4_ultra', 'mp4_720', 'mp4_1080']
-        formats = [
-            {
-                'url': furl,
+        formats = []
+        for format_id, video_url in re.findall(
+                r'flashvars\.quality_(.+?)\s*=\s*"([^"]+)"', webpage):
+            fmt = {
+                'url': compat_urllib_parse.unquote(video_url),
                 'format_id': format_id,
-                'preference': QUALITIES.index(format_id) if format_id in QUALITIES else -1,
-            } for format_id, furl in player_quality_option.items()
-        ]
+            }
+            m = re.search(r'^(?P<height>\d+)[pP]', format_id)
+            if m:
+                fmt['height'] = int(m.group('height'))
+            formats.append(fmt)
+
+        if not formats:
+            video_url = compat_urllib_parse.unquote(self._search_regex(
+                r'flashvars\.video_url\s*=\s*"([^"]+)"',
+                webpage, 'video URL'))
+            formats.append({'url': video_url})
+
         self._sort_formats(formats)
 
         return {

From 1ff30d7b79504035bf8f642668361ca71d905ab7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 20 Dec 2014 18:30:56 +0600
Subject: [PATCH 0914/1937] [npo] Add support for streams (Closes #4276)

---
 youtube_dl/extractor/npo.py | 88 +++++++++++++++++++++++++++----------
 1 file changed, 65 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index ce31694a5..8da76ae45 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -9,6 +9,7 @@ from ..utils import (
     qualities,
     strip_jsonp,
     url_basename,
+    fix_xml_ampersands,
 )
 
 
@@ -51,7 +52,21 @@ class NPOIE(InfoExtractor):
                 'upload_date': '20130225',
                 'duration': 3000,
             },
-        }
+        },
+        {
+            'url': 'http://www.npo.nl/de-nieuwe-mens-deel-1/21-07-2010/WO_VPRO_043706',
+            'info_dict': {
+                'id': 'WO_VPRO_043706',
+                'ext': 'wmv',
+                'title': 'De nieuwe mens - Deel 1',
+                'description': 'md5:518ae51ba1293ffb80d8d8ce90b74e4b',
+                'duration': 4680,
+            },
+            'params': {
+                # mplayer mms download
+                'skip_download': True,
+            }
+        },
     ]
 
     def _real_extract(self, url):
@@ -74,31 +89,58 @@ class NPOIE(InfoExtractor):
         token = self._search_regex(r'npoplayer\.token = "(.+?)"', token_page, 'token')
 
         formats = []
-        quality = qualities(['adaptive', 'wmv_sb', 'h264_sb', 'wmv_bb', 'h264_bb', 'wvc1_std', 'h264_std'])
-        for format_id in metadata['pubopties']:
-            format_info = self._download_json(
-                'http://ida.omroep.nl/odi/?prid=%s&puboptions=%s&adaptive=yes&token=%s' % (video_id, format_id, token),
-                video_id, 'Downloading %s JSON' % format_id)
-            if format_info.get('error_code', 0) or format_info.get('errorcode', 0):
-                continue
-            streams = format_info.get('streams')
-            if streams:
-                video_info = self._download_json(
-                    streams[0] + '&type=json',
-                    video_id, 'Downloading %s stream JSON' % format_id)
-            else:
-                video_info = format_info
-            video_url = video_info.get('url')
-            if not video_url:
-                continue
-            if format_id == 'adaptive':
-                formats.extend(self._extract_m3u8_formats(video_url, video_id))
-            else:
+
+        pubopties = metadata.get('pubopties')
+        if pubopties:
+            quality = qualities(['adaptive', 'wmv_sb', 'h264_sb', 'wmv_bb', 'h264_bb', 'wvc1_std', 'h264_std'])
+            for format_id in pubopties:
+                format_info = self._download_json(
+                    'http://ida.omroep.nl/odi/?prid=%s&puboptions=%s&adaptive=yes&token=%s'
+                    % (video_id, format_id, token),
+                    video_id, 'Downloading %s JSON' % format_id)
+                if format_info.get('error_code', 0) or format_info.get('errorcode', 0):
+                    continue
+                streams = format_info.get('streams')
+                if streams:
+                    video_info = self._download_json(
+                        streams[0] + '&type=json',
+                        video_id, 'Downloading %s stream JSON' % format_id)
+                else:
+                    video_info = format_info
+                video_url = video_info.get('url')
+                if not video_url:
+                    continue
+                if format_id == 'adaptive':
+                    formats.extend(self._extract_m3u8_formats(video_url, video_id))
+                else:
+                    formats.append({
+                        'url': video_url,
+                        'format_id': format_id,
+                        'quality': quality(format_id),
+                    })
+
+        streams = metadata.get('streams')
+        if streams:
+            for i, stream in enumerate(streams):
+                stream_url = stream.get('url')
+                if not stream_url:
+                    continue
+                asx = self._download_xml(
+                    stream_url, video_id,
+                    'Downloading stream %d ASX playlist' % i,
+                    transform_source=fix_xml_ampersands)
+                ref = asx.find('./ENTRY/Ref')
+                if ref is None:
+                    continue
+                video_url = ref.get('href')
+                if not video_url:
+                    continue
                 formats.append({
                     'url': video_url,
-                    'format_id': format_id,
-                    'quality': quality(format_id),
+                    'ext': stream.get('formaat', 'asf'),
+                    'quality': stream.get('kwaliteit'),
                 })
+
         self._sort_formats(formats)
 
         return {

From 1d4247f64e4124476f807e9eeb58fd401ffcd8ae Mon Sep 17 00:00:00 2001
From: Petr Kutalek <petr.kutalek@forers.com>
Date: Sun, 21 Dec 2014 01:24:05 +0100
Subject: [PATCH 0915/1937] [dvtv] Add support for playlists

---
 youtube_dl/extractor/dvtv.py | 118 ++++++++++++++++++++++++++---------
 1 file changed, 89 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dl/extractor/dvtv.py
index af552831c..2449b03b5 100644
--- a/youtube_dl/extractor/dvtv.py
+++ b/youtube_dl/extractor/dvtv.py
@@ -2,62 +2,122 @@
 
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     js_to_json,
-    unescapeHTML
+    unescapeHTML,
+    ExtractorError
 )
 
 
 class DVTVIE(InfoExtractor):
     IE_NAME = 'dvtv'
-    IE_DESC = 'http://video.aktualne.cz/dvtv/'
+    IE_DESC = 'http://video.aktualne.cz/'
 
-    _VALID_URL = r'http://video\.aktualne\.cz/dvtv/(?P<id>[a-z0-9-]+/r~[0-9a-f]{32})/?'
+    _VALID_URL = r'http://video\.aktualne\.cz/.*/r~(?P<id>[0-9a-f]{32})/'
 
     _TESTS = [{
         'url': 'http://video.aktualne.cz/dvtv/vondra-o-ceskem-stoleti-pri-pohledu-na-havla-mi-bylo-trapne/r~e5efe9ca855511e4833a0025900fea04/',
-        'md5': '75800f964fa0f82939a2914563301f72',
+        'md5': '67cb83e4a955d36e1b5d31993134a0c2',
         'info_dict': {
-            'id': 'e5efe9ca855511e4833a0025900fea04',
-            'ext': 'webm',
+            'id': 'dc0768de855511e49e4b0025900fea04',
+            'ext': 'mp4',
             'title': 'Vondra o Českém století: Při pohledu na Havla mi bylo trapně'
         }
     }, {
         'url': 'http://video.aktualne.cz/dvtv/stropnicky-policie-vrbetice-preventivne-nekontrolovala/r~82ed4322849211e4a10c0025900fea04/',
         'md5': '6388f1941b48537dbd28791f712af8bf',
         'info_dict': {
-            'id': '82ed4322849211e4a10c0025900fea04',
+            'id': '72c02230849211e49f60002590604f2e',
             'ext': 'mp4',
             'title': 'Stropnický: Policie Vrbětice preventivně nekontrolovala'
         }
+    }, {
+        'url': 'http://video.aktualne.cz/dvtv/dvtv-16-12-2014-utok-talibanu-boj-o-kliniku-uprchlici/r~973eb3bc854e11e498be002590604f2e/',
+        'info_dict': {
+            'title': 'DVTV 16. 12. 2014: útok Talibanu, boj o kliniku, uprchlíci',
+            'id': '973eb3bc854e11e498be002590604f2e'
+        },
+        'playlist': [{
+            'md5': 'da7ca6be4935532241fa9520b3ad91e4',
+            'info_dict': {
+                'id': 'b0b40906854d11e4bdad0025900fea04',
+                'ext': 'mp4',
+                'title': 'Drtinová Veselovský TV 16. 12. 2014: Témata dne'
+            }
+        }, {
+            'md5': '5f7652a08b05009c1292317b449ffea2',
+            'info_dict': {
+                'id': '420ad9ec854a11e4bdad0025900fea04',
+                'ext': 'mp4',
+                'title': 'Školní masakr možná změní boj s Talibanem, říká novinářka'
+            }
+        }, {
+            'md5': '498eb9dfa97169f409126c617e2a3d64',
+            'info_dict': {
+                'id': '95d35580846a11e4b6d20025900fea04',
+                'ext': 'mp4',
+                'title': 'Boj o kliniku: Veřejný zájem, nebo právo na majetek?'
+            }
+        }, {
+            'md5': 'b8dc6b744844032dab6ba3781a7274b9',
+            'info_dict': {
+                'id': '6fe14d66853511e4833a0025900fea04',
+                'ext': 'mp4',
+                'title': 'Pánek: Odmítání syrských uprchlíků je ostudou české vlády'
+            }
+        }]
     }]
 
+    def _parse_video_metadata(self, js, video_id):
+        metadata = self._parse_json(js, video_id, transform_source=js_to_json)
+
+        formats = []
+        for video in metadata['sources']:
+            ext = video['type'][6:]
+            formats.append({
+                'url': video['file'],
+                'ext': ext,
+                'format': '%s %s' % (ext, video['label']),
+                'format_id': '%s-%s' % (ext, video['label']),
+                'height': int(video['label'].rstrip('p')),
+                'fps': 25,
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': metadata['mediaid'],
+            'title': unescapeHTML(metadata['title']),
+            'thumbnail': 'http:%s' % metadata['image'],
+            'formats': formats
+        }
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        code = self._search_regex(
+        # singe video
+        items =  re.findall(
             r'(?s)embedData[0-9a-f]{32}\[\'asset\'\] = (\{.+?\});',
-            webpage, 'video JSON')
-        payload = self._parse_json(code, video_id, transform_source=js_to_json)
-        formats = []
-        for source in payload['sources']:
-            ext = source['type'][6:]
-            formats.append({
-                'url': source['file'],
-                'ext': ext,
-                'format': '%s %s' % (ext, source['label']),
-                'format_id': '%s-%s' % (ext, source['label']),
-                'height': int(source['label'].rstrip('p')),
-                'fps': 25,
-            })
-        self._sort_formats(formats)
+            webpage)
 
-        return {
-            'id': video_id[-32:],
-            'display_id': video_id[:-35],
-            'title': unescapeHTML(payload['title']),
-            'thumbnail': 'http:%s' % payload['image'],
-            'formats': formats
-        }
+        if items:
+            return self._parse_video_metadata(items[0], video_id)
+
+        # playlist
+        items = re.findall(
+            r'(?s)BBX\.context\.assets\[\'[0-9a-f]{32}\'\]\.push\((\{.+?\})\);',
+            webpage)
+
+        if items:
+            return {
+                '_type': 'playlist',
+                'id': video_id,
+                'title': self._og_search_title(webpage),
+                'entries': [self._parse_video_metadata(i, video_id) for i in items]
+            }
+
+        raise ExtractorError('Could not find neither video nor playlist for requested ID')

From 081d6e47840fb923ed6f565fa56dd5def9b9a184 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Dec 2014 07:33:58 +0600
Subject: [PATCH 0916/1937] [dvtv] Simplify

---
 youtube_dl/extractor/dvtv.py | 36 +++++++++++++++++++-----------------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dl/extractor/dvtv.py
index 2449b03b5..5f8ac3e46 100644
--- a/youtube_dl/extractor/dvtv.py
+++ b/youtube_dl/extractor/dvtv.py
@@ -1,5 +1,4 @@
 # coding: utf-8
-
 from __future__ import unicode_literals
 
 import re
@@ -8,7 +7,7 @@ from .common import InfoExtractor
 from ..utils import (
     js_to_json,
     unescapeHTML,
-    ExtractorError
+    ExtractorError,
 )
 
 
@@ -16,7 +15,7 @@ class DVTVIE(InfoExtractor):
     IE_NAME = 'dvtv'
     IE_DESC = 'http://video.aktualne.cz/'
 
-    _VALID_URL = r'http://video\.aktualne\.cz/.*/r~(?P<id>[0-9a-f]{32})/'
+    _VALID_URL = r'http://video\.aktualne\.cz/(?:[^/]+/)+r~(?P<id>[0-9a-f]{32})'
 
     _TESTS = [{
         'url': 'http://video.aktualne.cz/dvtv/vondra-o-ceskem-stoleti-pri-pohledu-na-havla-mi-bylo-trapne/r~e5efe9ca855511e4833a0025900fea04/',
@@ -24,7 +23,7 @@ class DVTVIE(InfoExtractor):
         'info_dict': {
             'id': 'dc0768de855511e49e4b0025900fea04',
             'ext': 'mp4',
-            'title': 'Vondra o Českém století: Při pohledu na Havla mi bylo trapně'
+            'title': 'Vondra o Českém století: Při pohledu na Havla mi bylo trapně',
         }
     }, {
         'url': 'http://video.aktualne.cz/dvtv/stropnicky-policie-vrbetice-preventivne-nekontrolovala/r~82ed4322849211e4a10c0025900fea04/',
@@ -32,13 +31,13 @@ class DVTVIE(InfoExtractor):
         'info_dict': {
             'id': '72c02230849211e49f60002590604f2e',
             'ext': 'mp4',
-            'title': 'Stropnický: Policie Vrbětice preventivně nekontrolovala'
+            'title': 'Stropnický: Policie Vrbětice preventivně nekontrolovala',
         }
     }, {
         'url': 'http://video.aktualne.cz/dvtv/dvtv-16-12-2014-utok-talibanu-boj-o-kliniku-uprchlici/r~973eb3bc854e11e498be002590604f2e/',
         'info_dict': {
             'title': 'DVTV 16. 12. 2014: útok Talibanu, boj o kliniku, uprchlíci',
-            'id': '973eb3bc854e11e498be002590604f2e'
+            'id': '973eb3bc854e11e498be002590604f2e',
         },
         'playlist': [{
             'md5': 'da7ca6be4935532241fa9520b3ad91e4',
@@ -68,7 +67,10 @@ class DVTVIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Pánek: Odmítání syrských uprchlíků je ostudou české vlády'
             }
-        }]
+        }],
+    }, {
+        'url': 'http://video.aktualne.cz/v-cechach-poprve-zazni-zelenkova-zrestaurovana-mse/r~45b4b00483ec11e4883b002590604f2e/',
+        'only_matching': True,
     }]
 
     def _parse_video_metadata(self, js, video_id):
@@ -80,7 +82,6 @@ class DVTVIE(InfoExtractor):
             formats.append({
                 'url': video['file'],
                 'ext': ext,
-                'format': '%s %s' % (ext, video['label']),
                 'format_id': '%s-%s' % (ext, video['label']),
                 'height': int(video['label'].rstrip('p')),
                 'fps': 25,
@@ -91,25 +92,26 @@ class DVTVIE(InfoExtractor):
         return {
             'id': metadata['mediaid'],
             'title': unescapeHTML(metadata['title']),
-            'thumbnail': 'http:%s' % metadata['image'],
+            'thumbnail': self._proto_relative_url(metadata['image']),
             'formats': formats
         }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
 
-        # singe video
-        items =  re.findall(
-            r'(?s)embedData[0-9a-f]{32}\[\'asset\'\] = (\{.+?\});',
-            webpage)
+        # single video
+        item = self._search_regex(
+            r"(?s)embedData[0-9a-f]{32}\['asset'\]\s*=\s*(\{.+?\});",
+            webpage, 'video', default=None, fatal=False)
 
-        if items:
-            return self._parse_video_metadata(items[0], video_id)
+        if item:
+            return self._parse_video_metadata(item, video_id)
 
         # playlist
         items = re.findall(
-            r'(?s)BBX\.context\.assets\[\'[0-9a-f]{32}\'\]\.push\((\{.+?\})\);',
+            r"(?s)BBX\.context\.assets\['[0-9a-f]{32}'\]\.push\(({.+?})\);",
             webpage)
 
         if items:
@@ -120,4 +122,4 @@ class DVTVIE(InfoExtractor):
                 'entries': [self._parse_video_metadata(i, video_id) for i in items]
             }
 
-        raise ExtractorError('Could not find neither video nor playlist for requested ID')
+        raise ExtractorError('Could not find neither video nor playlist')

From 2c2a42587b1755624a75a4b748c59a24f8b146aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Dec 2014 07:38:55 +0600
Subject: [PATCH 0917/1937] [dvtv] Fix thumbnail scheme

---
 youtube_dl/extractor/dvtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dvtv.py b/youtube_dl/extractor/dvtv.py
index 5f8ac3e46..c1a4bc757 100644
--- a/youtube_dl/extractor/dvtv.py
+++ b/youtube_dl/extractor/dvtv.py
@@ -92,7 +92,7 @@ class DVTVIE(InfoExtractor):
         return {
             'id': metadata['mediaid'],
             'title': unescapeHTML(metadata['title']),
-            'thumbnail': self._proto_relative_url(metadata['image']),
+            'thumbnail': self._proto_relative_url(metadata['image'], 'http:'),
             'formats': formats
         }
 

From ee45625290c678c53b7ee86d2adba40847b79573 Mon Sep 17 00:00:00 2001
From: MaxReimann <max.reimann@student.hpi.uni-potsdam.de>
Date: Sun, 21 Dec 2014 11:01:28 +0100
Subject: [PATCH 0918/1937] Add extractor for teletask

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/teletask.py | 59 ++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 youtube_dl/extractor/teletask.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 76f13bf52..073ac1fab 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -405,6 +405,7 @@ from .ted import TEDIE
 from .telebruxelles import TeleBruxellesIE
 from .telecinco import TelecincoIE
 from .telemb import TeleMBIE
+from .teletask import TeleTaskIE
 from .tenplay import TenPlayIE
 from .testurl import TestURLIE
 from .tf1 import TF1IE
diff --git a/youtube_dl/extractor/teletask.py b/youtube_dl/extractor/teletask.py
new file mode 100644
index 000000000..1c42ac6f4
--- /dev/null
+++ b/youtube_dl/extractor/teletask.py
@@ -0,0 +1,59 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re 
+import datetime
+
+from .common import InfoExtractor
+
+
+class TeleTaskIE(InfoExtractor):
+    _VALID_URL = r'http?://(?:www\.)?tele-task\.de/archive/video/html5/(?P<id>[0-9]+)/'
+    _TEST = {
+        'url': 'http://www.tele-task.de/archive/video/html5/26168/', 
+        'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
+        'info_dict': {
+            'id': '26168',
+            'ext': 'mp4',
+            'title': 'Duplicate Detection',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'date': '20141218',
+            # TODO more properties, either as:
+            # * A value
+            # * MD5 checksum; start the string with md5:
+            # * A regular expression; start the string with re:
+            # * Any Python type (for example int or float)
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        
+        lecture_url = self._html_search_regex(
+                    r'href="([^"]+)" itemprop="name">', webpage, 'title')
+        lecture_id = re.search("([0-9]+)/",lecture_url).group(1)
+        overview_page = self._download_webpage("http://www.tele-task.de" + lecture_url, 
+            lecture_id)
+        
+        title = self._html_search_regex(
+            r'itemprop="name">([^"]+)</a>', webpage, 'title')
+        url = self._html_search_regex(
+            r'class="speaker".*?src="([^"]+)"', webpage, 'video_url', flags=re.DOTALL)
+        description = self._html_search_regex(
+            r'Description of the series:</p>([^"]+)</div>', overview_page, 
+            'description',flags=re.DOTALL)
+        
+        date = self._html_search_regex(
+            r'<td class="label">Date:</td><td>([^"]+)</td>', webpage, 'date')
+        date = datetime.datetime.strptime(date, '%d.%m.%Y').strftime('%Y%m%d')
+
+        
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'url': url,
+            'upload_date': date,
+        }
\ No newline at end of file

From 3f7421b71b0d7da917a92b6f37985bc26c5dc11d Mon Sep 17 00:00:00 2001
From: MaxReimann <max.reimann@student.hpi.uni-potsdam.de>
Date: Sun, 21 Dec 2014 11:13:59 +0100
Subject: [PATCH 0919/1937] fix test and remove lengthy description

---
 youtube_dl/extractor/teletask.py | 20 ++------------------
 1 file changed, 2 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/teletask.py b/youtube_dl/extractor/teletask.py
index 1c42ac6f4..6920b871e 100644
--- a/youtube_dl/extractor/teletask.py
+++ b/youtube_dl/extractor/teletask.py
@@ -11,18 +11,12 @@ class TeleTaskIE(InfoExtractor):
     _VALID_URL = r'http?://(?:www\.)?tele-task\.de/archive/video/html5/(?P<id>[0-9]+)/'
     _TEST = {
         'url': 'http://www.tele-task.de/archive/video/html5/26168/', 
-        'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
+        'md5': '290ef69fb2792e481169c3958dbfbd57',
         'info_dict': {
             'id': '26168',
             'ext': 'mp4',
             'title': 'Duplicate Detection',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'date': '20141218',
-            # TODO more properties, either as:
-            # * A value
-            # * MD5 checksum; start the string with md5:
-            # * A regular expression; start the string with re:
-            # * Any Python type (for example int or float)
+            'upload_date': '20141218',
         }
     }
 
@@ -30,19 +24,10 @@ class TeleTaskIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         
-        lecture_url = self._html_search_regex(
-                    r'href="([^"]+)" itemprop="name">', webpage, 'title')
-        lecture_id = re.search("([0-9]+)/",lecture_url).group(1)
-        overview_page = self._download_webpage("http://www.tele-task.de" + lecture_url, 
-            lecture_id)
-        
         title = self._html_search_regex(
             r'itemprop="name">([^"]+)</a>', webpage, 'title')
         url = self._html_search_regex(
             r'class="speaker".*?src="([^"]+)"', webpage, 'video_url', flags=re.DOTALL)
-        description = self._html_search_regex(
-            r'Description of the series:</p>([^"]+)</div>', overview_page, 
-            'description',flags=re.DOTALL)
         
         date = self._html_search_regex(
             r'<td class="label">Date:</td><td>([^"]+)</td>', webpage, 'date')
@@ -53,7 +38,6 @@ class TeleTaskIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'description': description,
             'url': url,
             'upload_date': date,
         }
\ No newline at end of file

From 33b53b60212ca28f589616f1acb650b323a18f11 Mon Sep 17 00:00:00 2001
From: MaxReimann <max.reimann@student.hpi.uni-potsdam.de>
Date: Sun, 21 Dec 2014 12:26:47 +0100
Subject: [PATCH 0920/1937] [teletask] Add new extractor

---
 youtube_dl/extractor/teletask.py | 62 ++++++++++++++++++++++----------
 1 file changed, 43 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/teletask.py b/youtube_dl/extractor/teletask.py
index 6920b871e..aa5535042 100644
--- a/youtube_dl/extractor/teletask.py
+++ b/youtube_dl/extractor/teletask.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
-
-import re 
+import re
 import datetime
 
 from .common import InfoExtractor
@@ -10,34 +9,59 @@ from .common import InfoExtractor
 class TeleTaskIE(InfoExtractor):
     _VALID_URL = r'http?://(?:www\.)?tele-task\.de/archive/video/html5/(?P<id>[0-9]+)/'
     _TEST = {
-        'url': 'http://www.tele-task.de/archive/video/html5/26168/', 
-        'md5': '290ef69fb2792e481169c3958dbfbd57',
+        'url': 'http://www.tele-task.de/archive/video/html5/26168/',
         'info_dict': {
-            'id': '26168',
-            'ext': 'mp4',
             'title': 'Duplicate Detection',
-            'upload_date': '20141218',
-        }
+        },
+        'playlist': [{
+            'md5': '290ef69fb2792e481169c3958dbfbd57',
+            'info_dict': {
+                'title': 'Duplicate Detection',
+                'upload_date': '20141218',
+                'id': 'speaker_26168',
+                'ext': 'mp4',
+            }
+        },
+            {
+            'md5': 'e1e7218c5f0e4790015a437fcf6c71b4',
+            'info_dict': {
+                'title': 'Duplicate Detection',
+                'upload_date': '20141218',
+                'id': 'slides_26168',
+                'ext': 'mp4',
+            }
+        }]
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        
+        lecture_id = self._match_id(url)
+        webpage = self._download_webpage(url, lecture_id)
+
         title = self._html_search_regex(
             r'itemprop="name">([^"]+)</a>', webpage, 'title')
-        url = self._html_search_regex(
-            r'class="speaker".*?src="([^"]+)"', webpage, 'video_url', flags=re.DOTALL)
-        
+        url_speaker = self._html_search_regex(
+            r'class="speaker".*?src="([^"]+)"', webpage, 'video_url_speaker', flags=re.DOTALL)
+        url_slides = self._html_search_regex(
+            r'class="slides".*?src="([^"]+)"', webpage, 'video_url_slides', flags=re.DOTALL)
         date = self._html_search_regex(
             r'<td class="label">Date:</td><td>([^"]+)</td>', webpage, 'date')
         date = datetime.datetime.strptime(date, '%d.%m.%Y').strftime('%Y%m%d')
 
-        
+        entries = [{
+            'title': title,
+            'upload_date': date,
+            'id': "speaker_"+lecture_id,
+            'url': url_speaker,
+        },
+            {
+            'title': title,
+            'upload_date': date,
+            'id': "slides_"+lecture_id,
+            'url': url_slides}]
 
         return {
-            'id': video_id,
+            '_type': "playlist",
+            'id': lecture_id,
             'title': title,
-            'url': url,
-            'upload_date': date,
-        }
\ No newline at end of file
+            'entries': entries,
+        }

From 3aa9176f082e2e2909826a61a80a0923b4bd88d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 12:33:30 +0100
Subject: [PATCH 0921/1937] [yahoo] Improve video id detection (fixes #4521)

---
 youtube_dl/extractor/yahoo.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 031226f27..2d9e311b0 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -12,6 +12,7 @@ from ..compat import (
 )
 from ..utils import (
     clean_html,
+    unescapeHTML,
     ExtractorError,
     int_or_none,
 )
@@ -116,6 +117,16 @@ class YahooIE(InfoExtractor):
                 'description': 'md5:1428185051cfd1949807ad4ff6d3686a',
                 'duration': 201,
             }
+        }, {
+            'url': 'https://www.yahoo.com/movies/v/true-story-trailer-173000497.html',
+            'md5': '989396ae73d20c6f057746fb226aa215',
+            'info_dict': {
+                'id': '071c4013-ce30-3a93-a5b2-e0413cd4a9d1',
+                'ext': 'mp4',
+                'title': '\'True Story\' Trailer',
+                'description': 'True Story',
+                'duration': 150,
+            },
         }, {
             'url': 'https://gma.yahoo.com/pizza-delivery-man-surprised-huge-tip-college-kids-195200785.html',
             'only_matching': True,
@@ -125,6 +136,7 @@ class YahooIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         display_id = mobj.group('display_id')
+        page_id = mobj.group('id')
         url = mobj.group('url')
         host = mobj.group('host')
         webpage = self._download_webpage(url, display_id)
@@ -149,6 +161,7 @@ class YahooIE(InfoExtractor):
                 r'YUI\.namespace\("Media"\)\.CONTENT_ID\s*=\s*"([^"]+)"',
                 r'root\.App\.Cache\.context\.videoCache\.curVideo = \{"([^"]+)"',
                 r'"first_videoid"\s*:\s*"([^"]+)"',
+                r'%s[^}]*"ccm_id"\s*:\s*"([^"]+)"' % re.escape(page_id),
             ]
             video_id = self._search_regex(CONTENT_ID_REGEXES, webpage, 'content ID')
         else:
@@ -211,7 +224,7 @@ class YahooIE(InfoExtractor):
         return {
             'id': video_id,
             'display_id': display_id,
-            'title': meta['title'],
+            'title': unescapeHTML(meta['title']),
             'formats': formats,
             'description': clean_html(meta['description']),
             'thumbnail': meta['thumbnail'] if meta.get('thumbnail') else self._og_search_thumbnail(webpage),

From 85fab7e47b4d4d69a8d105e1d3a07d8a0dd67685 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 14:47:12 +0100
Subject: [PATCH 0922/1937] [yahoo] Replace two tests

The first one returned an internal server error.
The other doesn't seem to contain a video anymore.
---
 youtube_dl/extractor/yahoo.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index 2d9e311b0..eed9d4325 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -56,14 +56,14 @@ class YahooIE(InfoExtractor):
             }
         },
         {
-            'url': 'https://tw.screen.yahoo.com/taipei-opinion-poll/選情站報-街頭民調-台北市篇-102823042.html',
-            'md5': '92a7fdd8a08783c68a174d7aa067dde8',
+            'url': 'https://tw.screen.yahoo.com/election-2014-askmayor/敢問市長-黃秀霜批賴清德-非常高傲-033009720.html',
+            'md5': '3a09cf59349cfaddae1797acc3c087fc',
             'info_dict': {
-                'id': '7a23b569-7bea-36cb-85b9-bd5301a0a1fb',
+                'id': 'cac903b3-fcf4-3c14-b632-643ab541712f',
                 'ext': 'mp4',
-                'title': '選情站報 街頭民調 台北市篇',
-                'description': '選情站報 街頭民調 台北市篇',
-                'duration': 429,
+                'title': '敢問市長／黃秀霜批賴清德「非常高傲」',
+                'description': '直言台南沒捷運 交通居五都之末',
+                'duration': 396,
             }
         },
         {
@@ -88,14 +88,14 @@ class YahooIE(InfoExtractor):
                 'duration': 121,
             }
         }, {
-            'url': 'https://ca.finance.yahoo.com/news/20-most-valuable-brands-world-112600775.html',
-            'md5': '3e401e4eed6325aa29d9b96125fd5b4f',
+            'url': 'https://ca.finance.yahoo.com/news/hackers-sony-more-trouble-well-154609075.html',
+            'md5': '226a895aae7e21b0129e2a2006fe9690',
             'info_dict': {
-                'id': 'c1b4c09c-8ed8-3b65-8b05-169c55358a83',
+                'id': 'e624c4bc-3389-34de-9dfc-025f74943409',
                 'ext': 'mp4',
-                'title': "Apple Is The World's Most Valuable Brand",
-                'description': 'md5:73eabc1a11c6f59752593b2ceefa1262',
-                'duration': 21,
+                'title': '\'The Interview\' TV Spot: War',
+                'description': 'The Interview',
+                'duration': 30,
             }
         }, {
             'url': 'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html',

From 653d14e2f9bee934ca491123f0461a37f770e9e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 14:47:44 +0100
Subject: [PATCH 0923/1937] [yahoo] Update extraction process

Their webpage uses now https://video.media.yql.yahoo.com/v1/video/sapi/streams/ for getting the video info.
---
 youtube_dl/extractor/yahoo.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index eed9d4325..f8e7041a0 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -176,17 +176,15 @@ class YahooIE(InfoExtractor):
         region = self._search_regex(
             r'\\?"region\\?"\s*:\s*\\?"([^"]+?)\\?"',
             webpage, 'region', fatal=False, default='US')
-        query = ('SELECT * FROM yahoo.media.video.streams WHERE id="%s"'
-                 ' AND plrs="86Gj0vCaSzV_Iuf6hNylf2" AND region="%s"'
-                 ' AND protocol="http"' % (video_id, region))
         data = compat_urllib_parse.urlencode({
-            'q': query,
-            'env': 'prod',
-            'format': 'json',
+            'protocol': 'http',
+            'region': region,
         })
+        query_url = (
+            'https://video.media.yql.yahoo.com/v1/video/sapi/streams/'
+            '{id}?{data}'.format(id=video_id, data=data))
         query_result = self._download_json(
-            'http://video.query.yahoo.com/v1/public/yql?' + data,
-            display_id, 'Downloading video info')
+            query_url, display_id, 'Downloading video info')
 
         info = query_result['query']['results']['mediaObj'][0]
         meta = info.get('meta')

From 1f809a856017dacc70462d743337b5197a0f980b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 15:22:30 +0100
Subject: [PATCH 0924/1937] [nerdcubed] Style fixes

---
 youtube_dl/extractor/nerdcubed.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nerdcubed.py b/youtube_dl/extractor/nerdcubed.py
index 9f2e678e0..efc903afa 100644
--- a/youtube_dl/extractor/nerdcubed.py
+++ b/youtube_dl/extractor/nerdcubed.py
@@ -18,7 +18,7 @@ class NerdCubedFeedIE(InfoExtractor):
 
     def _real_extract(self, url):
         feed = self._download_json(url, url, "Downloading NerdCubed JSON feed")
-        
+
         entries = [{
             '_type': 'url',
             'title': feed_entry['title'],
@@ -33,4 +33,3 @@ class NerdCubedFeedIE(InfoExtractor):
             'id': 'nerdcubed-feed',
             'entries': entries,
         }
-

From 3f1399de8ab73806aca5b418d1097feb5f06d091 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 15:26:00 +0100
Subject: [PATCH 0925/1937] [tmz] Fix test's thumbnail field

---
 youtube_dl/extractor/tmz.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tmz.py b/youtube_dl/extractor/tmz.py
index 827aa08a4..c5c6fdc51 100644
--- a/youtube_dl/extractor/tmz.py
+++ b/youtube_dl/extractor/tmz.py
@@ -15,7 +15,7 @@ class TMZIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Kim Kardashian\'s Boobs Unlock a Mystery!',
             'description': 'Did Kim Kardasain try to one-up Khloe by one-upping Kylie???  Or is she just showing off her amazing boobs?',
-            'thumbnail': 'http://cdnbakmi.kaltura.com/p/591531/sp/59153100/thumbnail/entry_id/0_okj015ty/version/100002/acv/182/width/640',
+            'thumbnail': r're:http://cdnbakmi\.kaltura\.com/.*thumbnail.*',
         }
     }
 

From a86cbf587685230d73bb19ff619a8327fbcc29c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 15:28:40 +0100
Subject: [PATCH 0926/1937] [rtp] Fix test's id field

---
 youtube_dl/extractor/rtp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtp.py b/youtube_dl/extractor/rtp.py
index 54f67a52f..7736cabba 100644
--- a/youtube_dl/extractor/rtp.py
+++ b/youtube_dl/extractor/rtp.py
@@ -12,7 +12,7 @@ class RTPIE(InfoExtractor):
     _TESTS = [{
         'url': 'http://www.rtp.pt/play/p405/e174042/paixoes-cruzadas',
         'info_dict': {
-            'id': '174042',
+            'id': 'e174042',
             'ext': 'mp3',
             'title': 'Paixões Cruzadas',
             'description': 'As paixões musicais de António Cartaxo e António Macedo',

From 85d253af6b10dda05fa9d3821606346c3b26b697 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 15:35:36 +0100
Subject: [PATCH 0927/1937] [internetvideoarchive] Update test's duration field

---
 youtube_dl/extractor/internetvideoarchive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/internetvideoarchive.py b/youtube_dl/extractor/internetvideoarchive.py
index c813d4b82..483cc6f9e 100644
--- a/youtube_dl/extractor/internetvideoarchive.py
+++ b/youtube_dl/extractor/internetvideoarchive.py
@@ -22,7 +22,7 @@ class InternetVideoArchiveIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'SKYFALL',
             'description': 'In SKYFALL, Bond\'s loyalty to M is tested as her past comes back to haunt her. As MI6 comes under attack, 007 must track down and destroy the threat, no matter how personal the cost.',
-            'duration': 149,
+            'duration': 152,
         },
     }
 

From 70f6796e7d644b78ed346163287a148cd72733e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 16:54:53 +0100
Subject: [PATCH 0928/1937] [telecinco] Rename 'episode' group to 'id' in the
 _VALID_URL regex

MiTeleIE now uses '_match_id'
---
 youtube_dl/extractor/telecinco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index 2a2fff5e1..be3f72df7 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -6,7 +6,7 @@ from .mitele import MiTeleIE
 
 class TelecincoIE(MiTeleIE):
     IE_NAME = 'telecinco.es'
-    _VALID_URL = r'https?://www\.telecinco\.es/[^/]+/[^/]+/[^/]+/(?P<episode>.*?)\.html'
+    _VALID_URL = r'https?://www\.telecinco\.es/[^/]+/[^/]+/[^/]+/(?P<id>.*?)\.html'
 
     _TEST = {
         'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html',

From c8f167823f483b4dd043e011722ccce9aa8223b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 16:57:07 +0100
Subject: [PATCH 0929/1937] [dbtv] Make sure the 'id' field is a string

---
 youtube_dl/extractor/dbtv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dbtv.py b/youtube_dl/extractor/dbtv.py
index 1d3e2ff08..212217625 100644
--- a/youtube_dl/extractor/dbtv.py
+++ b/youtube_dl/extractor/dbtv.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     float_or_none,
     int_or_none,
@@ -61,7 +62,7 @@ class DBTVIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': video['id'],
+            'id': compat_str(video['id']),
             'display_id': display_id,
             'title': video['title'],
             'description': clean_html(video['desc']),

From 5d3808524ddcb9e4b2f4cadf69fc590b1bd0a405 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 16:58:29 +0100
Subject: [PATCH 0930/1937] [extractor/common] Update docstring: replace
 FileDownloader with YoutubeDL

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d302fe45f..4b950e485 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -40,7 +40,7 @@ class InfoExtractor(object):
     information about the video (or videos) the URL refers to. This
     information includes the real video URL, the video title, author and
     others. The information is stored in a dictionary which is then
-    passed to the FileDownloader. The FileDownloader processes this
+    passed to the YoutubeDL. The YoutubeDL processes this
     information possibly downloading the video to the file system, among
     other possible outcomes.
 

From a8896c5ac202c17f60d5d68cb70f73ceceee52d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Dec 2014 00:58:15 +0600
Subject: [PATCH 0931/1937] [crunchyroll] Add .fr domain (#4537)

---
 youtube_dl/extractor/crunchyroll.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 2c7756eb6..354046a9e 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -29,8 +29,8 @@ from .common import InfoExtractor
 
 
 class CrunchyrollIE(SubtitlesInfoExtractor):
-    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.com/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
+    _TESTS = [{
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
         'info_dict': {
             'id': '645513',
@@ -46,7 +46,10 @@ class CrunchyrollIE(SubtitlesInfoExtractor):
             # rtmp
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'http://www.crunchyroll.fr/girl-friend-beta/episode-11-goodbye-la-mode-661697',
+        'only_matching': True,
+    }]
 
     _FORMAT_IDS = {
         '360': ('60', '106'),

From f00fd51dae685cee2406401b50b847878e9a9d2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 21 Dec 2014 20:49:14 +0100
Subject: [PATCH 0932/1937] Don't write the description file if
 info_dict['description'] is None (#3166)

---
 youtube_dl/YoutubeDL.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 14ee30e55..e2b823f66 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1016,13 +1016,13 @@ class YoutubeDL(object):
             descfn = filename + '.description'
             if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
                 self.to_screen('[info] Video description is already present')
+            elif info_dict.get('description') is None:
+                self.report_warning('There\'s no description to write.')
             else:
                 try:
                     self.to_screen('[info] Writing video description to: ' + descfn)
                     with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
                         descfile.write(info_dict['description'])
-                except (KeyError, TypeError):
-                    self.report_warning('There\'s no description to write.')
                 except (OSError, IOError):
                     self.report_error('Cannot write description file ' + descfn)
                     return

From 1362bbbb4bfbd9380211a66032dbe3fe09aa2173 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Dec 2014 22:05:47 +0600
Subject: [PATCH 0933/1937] [adobetv] Add extractor (Closes #4536)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/adobetv.py  | 70 ++++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 youtube_dl/extractor/adobetv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 76f13bf52..92aca503c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 from .abc import ABCIE
 from .academicearth import AcademicEarthCourseIE
 from .addanime import AddAnimeIE
+from .adobetv import AdobeTVIE
 from .adultswim import AdultSwimIE
 from .aftonbladet import AftonbladetIE
 from .aljazeera import AlJazeeraIE
diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
new file mode 100644
index 000000000..28e07f8b0
--- /dev/null
+++ b/youtube_dl/extractor/adobetv.py
@@ -0,0 +1,70 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_duration,
+    unified_strdate,
+    str_to_int,
+)
+
+
+class AdobeTVIE(InfoExtractor):
+    _VALID_URL = r'https?://tv\.adobe\.com/watch/[^/]+/(?P<id>[^/]+)'
+
+    _TEST = {
+        'url': 'http://tv.adobe.com/watch/the-complete-picture-with-julieanne-kost/quick-tip-how-to-draw-a-circle-around-an-object-in-photoshop/',
+        'md5': '9bc5727bcdd55251f35ad311ca74fa1e',
+        'info_dict': {
+            'id': 'quick-tip-how-to-draw-a-circle-around-an-object-in-photoshop',
+            'ext': 'mp4',
+            'title': 'Quick Tip - How to Draw a Circle Around an Object in Photoshop',
+            'description': 'md5:99ec318dc909d7ba2a1f2b038f7d2311',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'upload_date': '20110914',
+            'duration': 60,
+            'view_count': int,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        player = self._parse_json(
+            self._search_regex(r'html5player:\s*({.+?})\s*\n', webpage, 'player'),
+            video_id)
+
+        title = player.get('title') or self._search_regex(
+            r'data-title="([^"]+)"', webpage, 'title')
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        upload_date = unified_strdate(
+            self._html_search_meta('datepublished', webpage, 'upload date'))
+
+        duration = parse_duration(
+            self._html_search_meta('duration', webpage, 'duration')
+            or self._search_regex(r'Runtime:\s*(\d{2}:\d{2}:\d{2})', webpage, 'duration'))
+
+        view_count = str_to_int(self._search_regex(
+            r'<div class="views">\s*Views?:\s*([\d,.]+)\s*</div>',
+            webpage, 'view count'))
+
+        formats = [{
+            'url': source['src'],
+            'format_id': source.get('quality') or source['src'].split('-')[-1].split('.')[0] or None,
+            'tbr': source.get('bitrate'),
+        } for source in player['sources']]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'duration': duration,
+            'view_count': view_count,
+            'formats': formats,
+        }

From 9532d723715f1d1a08c126ba8af6940113b88d3b Mon Sep 17 00:00:00 2001
From: Adrian Kretz <adriankretz@gmail.com>
Date: Mon, 22 Dec 2014 18:40:36 +0100
Subject: [PATCH 0934/1937] [cnn] Add support for articles with videos (fixes
 #4541)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/cnn.py      | 25 +++++++++++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 92aca503c..ba12e3263 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -64,6 +64,7 @@ from .cnet import CNETIE
 from .cnn import (
     CNNIE,
     CNNBlogsIE,
+    CNNArticleIE,
 )
 from .collegehumor import CollegeHumorIE
 from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 81142ee41..cddcacdf1 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -127,3 +127,28 @@ class CNNBlogsIE(InfoExtractor):
             'url': cnn_url,
             'ie_key': CNNIE.ie_key(),
         }
+
+
+class CNNArticleIE(InfoExtractor):
+    _VALID_URL = r'https?://((edition|www)\.)?cnn\.com/(?!video/)'
+    _TEST = {
+        'url': 'http://www.cnn.com/2014/12/21/politics/obama-north-koreas-hack-not-war-but-cyber-vandalism/',
+        'md5': '275b326f85d80dff7592a9820f5dc887',
+        'info_dict': {
+            'id': 'bestoftv/2014/12/21/sotu-crowley-president-obama-north-korea-not-going-to-be-intimidated.cnn',
+            'ext': 'mp4',
+            'title': 'Obama: We\'re not going to be intimidated',
+            'description': 'md5:e735586f3dc936075fa654a4d91b21f9',
+            'upload_date': '20141220',
+        },
+        'add_ie': ['CNN'],
+    }
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(url, url_basename(url))
+        cnn_url = self._html_search_regex(r"video:\s*'([^']+)'", webpage, 'cnn url')
+        return {
+            '_type': 'url',
+            'url': r'http://cnn.com/video/?/video/' + cnn_url,
+            'ie_key': CNNIE.ie_key(),
+        }

From e0ae1814b112b100b73957fe1349d1569ec7a3dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 22 Dec 2014 22:24:19 +0100
Subject: [PATCH 0935/1937] [sportdeutschland] Fix extraction (fixes #4544)

---
 youtube_dl/extractor/sportdeutschland.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sportdeutschland.py b/youtube_dl/extractor/sportdeutschland.py
index 2f57f5b7c..1a57aebf1 100644
--- a/youtube_dl/extractor/sportdeutschland.py
+++ b/youtube_dl/extractor/sportdeutschland.py
@@ -60,9 +60,10 @@ class SportDeutschlandIE(InfoExtractor):
 
         categories = list(data.get('section', {}).get('tags', {}).values())
         asset = data['asset']
+        assets_info = self._download_json(asset['url'], video_id)
 
         formats = []
-        smil_url = asset['video']
+        smil_url = assets_info['video']
         if '.smil' in smil_url:
             m3u8_url = smil_url.replace('.smil', '.m3u8')
             formats.extend(

From 544dec62980f4e5733747c175b64c2837d5f6c0e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 Dec 2014 20:33:56 +0600
Subject: [PATCH 0936/1937] [smotri] Skip broken tests

---
 youtube_dl/extractor/smotri.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index d031fe401..baef3daa0 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -69,6 +69,7 @@ class SmotriIE(InfoExtractor):
             'params': {
                 'videopassword': 'qwerty',
             },
+            'skip': 'Video is not approved by moderator',
         },
         # age limit + video-password
         {
@@ -86,7 +87,8 @@ class SmotriIE(InfoExtractor):
             },
             'params': {
                 'videopassword': '333'
-            }
+            },
+            'skip': 'Video is not approved by moderator',
         },
         # swf player
         {

From 55f0cab3a38135060c967023accb5b5f3fa3dfb5 Mon Sep 17 00:00:00 2001
From: Tobias Bell <tobias.bell@gmail.com>
Date: Tue, 23 Dec 2014 22:09:21 +0100
Subject: [PATCH 0937/1937] [gameone] This fix resolves issue #4552

The duration metadata for certain episodes contained floating point
numbers instead of integers. Now only the integer part will be
interpreted. Also added a test for this
---
 youtube_dl/extractor/gameone.py | 50 ++++++++++++++++++++++-----------
 1 file changed, 34 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/gameone.py b/youtube_dl/extractor/gameone.py
index 3022f539d..da9d44791 100644
--- a/youtube_dl/extractor/gameone.py
+++ b/youtube_dl/extractor/gameone.py
@@ -21,21 +21,38 @@ RAW_MP4_URL = 'http://cdn.riptide-mtvn.com/'
 
 class GameOneIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?gameone\.de/tv/(?P<id>\d+)'
-    _TEST = {
-        'url': 'http://www.gameone.de/tv/288',
-        'md5': '136656b7fb4c9cb4a8e2d500651c499b',
-        'info_dict': {
-            'id': '288',
-            'ext': 'mp4',
-            'title': 'Game One - Folge 288',
-            'duration': 1238,
-            'thumbnail': 'http://s3.gameone.de/gameone/assets/video_metas/teaser_images/000/643/636/big/640x360.jpg',
-            'description': 'FIFA-Pressepokal 2014, Star Citizen, Kingdom Come: Deliverance, Project Cars, Schöner Trants Nerdquiz Folge 2 Runde 1',
-            'age_limit': 16,
-            'upload_date': '20140513',
-            'timestamp': 1399980122,
+    _TESTS = [
+        {
+            'url': 'http://www.gameone.de/tv/288',
+            'md5': '136656b7fb4c9cb4a8e2d500651c499b',
+            'info_dict': {
+                'id': '288',
+                'ext': 'mp4',
+                'title': 'Game One - Folge 288',
+                'duration': 1238,
+                'thumbnail': 'http://s3.gameone.de/gameone/assets/video_metas/teaser_images/000/643/636/big/640x360.jpg',
+                'description': 'FIFA-Pressepokal 2014, Star Citizen, Kingdom Come: Deliverance, Project Cars, Schöner Trants Nerdquiz Folge 2 Runde 1',
+                'age_limit': 16,
+                'upload_date': '20140513',
+                'timestamp': 1399980122,
+            }
+        },
+        {
+            'url': 'http://gameone.de/tv/220',
+            'md5': '5227ca74c4ae6b5f74c0510a7c48839e',
+            'info_dict': {
+                'id': '220',
+                'ext': 'mp4',
+                'upload_date': '20120918',
+                'description': 'Jet Set Radio HD, Tekken Tag Tournament 2, Source Filmmaker',
+                'timestamp': 1347971451,
+                'title': 'Game One - Folge 220',
+                'duration': 896,
+                'age_limit': 16,
+            }
         }
-    }
+
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -66,7 +83,7 @@ class GameOneIE(InfoExtractor):
             video_id,
             'Downloading media:content')
         rendition_items = content.findall('.//rendition')
-        duration = int(rendition_items[0].get('duration'))
+        duration = int(rendition_items[0].get('duration').split('.')[0])
         formats = [
             {
                 'url': re.sub(r'.*/(r2)', RAW_MP4_URL + r'\1', r.find('./src').text),
@@ -105,7 +122,8 @@ class GameOnePlaylistIE(InfoExtractor):
         webpage = self._download_webpage('http://www.gameone.de/tv', 'TV')
         max_id = max(map(int, re.findall(r'<a href="/tv/(\d+)"', webpage)))
         entries = [
-            self.url_result('http://www.gameone.de/tv/%d' % video_id, 'GameOne')
+            self.url_result('http://www.gameone.de/tv/%d' %
+                            video_id, 'GameOne')
             for video_id in range(max_id, 0, -1)]
 
         return {

From 39ac7c94354967b44588155b2d6752f780eaaf16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Dec 2014 19:18:59 +0600
Subject: [PATCH 0938/1937] [gameone] Extract duration as float

---
 youtube_dl/extractor/gameone.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/gameone.py b/youtube_dl/extractor/gameone.py
index da9d44791..75f180928 100644
--- a/youtube_dl/extractor/gameone.py
+++ b/youtube_dl/extractor/gameone.py
@@ -6,7 +6,9 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     xpath_with_ns,
-    parse_iso8601
+    parse_iso8601,
+    float_or_none,
+    int_or_none,
 )
 
 NAMESPACE_MAP = {
@@ -47,7 +49,7 @@ class GameOneIE(InfoExtractor):
                 'description': 'Jet Set Radio HD, Tekken Tag Tournament 2, Source Filmmaker',
                 'timestamp': 1347971451,
                 'title': 'Game One - Folge 220',
-                'duration': 896,
+                'duration': 896.62,
                 'age_limit': 16,
             }
         }
@@ -83,13 +85,13 @@ class GameOneIE(InfoExtractor):
             video_id,
             'Downloading media:content')
         rendition_items = content.findall('.//rendition')
-        duration = int(rendition_items[0].get('duration').split('.')[0])
+        duration = float_or_none(rendition_items[0].get('duration'))
         formats = [
             {
                 'url': re.sub(r'.*/(r2)', RAW_MP4_URL + r'\1', r.find('./src').text),
-                'width': int(r.get('width')),
-                'height': int(r.get('height')),
-                'tbr': int(r.get('bitrate')),
+                'width': int_or_none(r.get('width')),
+                'height': int_or_none(r.get('height')),
+                'tbr': int_or_none(r.get('bitrate')),
             }
             for r in rendition_items
         ]

From 5ac71f0b27fbbca11517e5592978a2fe7bfd9f34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Dec 2014 22:25:05 +0600
Subject: [PATCH 0939/1937] [sohu] Modernize and extract all formats and more
 metadata (Closes #4409, closes #2056, closes #3009)

---
 youtube_dl/extractor/sohu.py | 92 ++++++++++++++++++++----------------
 1 file changed, 52 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 07f514a46..c04791997 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -1,11 +1,10 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from .common import compat_str
 
 
 class SohuIE(InfoExtractor):
@@ -29,60 +28,73 @@ class SohuIE(InfoExtractor):
                 base_data_url = 'http://my.tv.sohu.com/play/videonew.do?vid='
             else:
                 base_data_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid='
-            data_url = base_data_url + str(vid_id)
-            data_json = self._download_webpage(
-                data_url, video_id,
-                note='Downloading JSON data for ' + str(vid_id))
-            return json.loads(data_json)
+
+            return self._download_json(
+                base_data_url + vid_id, video_id,
+                'Downloading JSON data for %s' % vid_id)
 
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         mytv = mobj.group('mytv') is not None
 
         webpage = self._download_webpage(url, video_id)
-        raw_title = self._html_search_regex(r'(?s)<title>(.+?)</title>',
-                                            webpage, 'video title')
+        raw_title = self._html_search_regex(
+            r'(?s)<title>(.+?)</title>',
+            webpage, 'video title')
         title = raw_title.partition('-')[0].strip()
 
-        vid = self._html_search_regex(r'var vid ?= ?["\'](\d+)["\']', webpage,
-                                      'video path')
-        data = _fetch_data(vid, mytv)
+        vid = self._html_search_regex(
+            r'var vid ?= ?["\'](\d+)["\']',
+            webpage, 'video path')
+        vid_data = _fetch_data(vid, mytv)
 
-        QUALITIES = ('ori', 'super', 'high', 'nor')
-        vid_ids = [data['data'][q + 'Vid']
-                   for q in QUALITIES
-                   if data['data'][q + 'Vid'] != 0]
-        if not vid_ids:
-            raise ExtractorError('No formats available for this video')
+        formats_json = {}
+        for format_id in ('nor', 'high', 'super', 'ori', 'h2644k', 'h2654k'):
+            vid_id = vid_data['data'].get('%sVid' % format_id)
+            if not vid_id:
+                continue
+            vid_id = compat_str(vid_id)
+            formats_json[format_id] = vid_data if vid == vid_id else _fetch_data(vid_id, mytv)
 
-        # For now, we just pick the highest available quality
-        vid_id = vid_ids[-1]
-
-        format_data = data if vid == vid_id else _fetch_data(vid_id, mytv)
-        part_count = format_data['data']['totalBlocks']
-        allot = format_data['allot']
-        prot = format_data['prot']
-        clipsURL = format_data['data']['clipsURL']
-        su = format_data['data']['su']
+        part_count = vid_data['data']['totalBlocks']
 
         playlist = []
         for i in range(part_count):
-            part_url = ('http://%s/?prot=%s&file=%s&new=%s' %
-                        (allot, prot, clipsURL[i], su[i]))
-            part_str = self._download_webpage(
-                part_url, video_id,
-                note='Downloading part %d of %d' % (i + 1, part_count))
+            formats = []
+            for format_id, format_data in formats_json.items():
+                allot = format_data['allot']
+                prot = format_data['prot']
 
-            part_info = part_str.split('|')
-            video_url = '%s%s?key=%s' % (part_info[0], su[i], part_info[3])
+                data = format_data['data']
+                clips_url = data['clipsURL']
+                su = data['su']
 
-            video_info = {
-                'id': '%s_part%02d' % (video_id, i + 1),
+                part_str = self._download_webpage(
+                    'http://%s/?prot=%s&file=%s&new=%s' %
+                    (allot, prot, clips_url[i], su[i]),
+                    video_id,
+                    'Downloading %s video URL part %d of %d'
+                    % (format_id, i + 1, part_count))
+
+                part_info = part_str.split('|')
+                video_url = '%s%s?key=%s' % (part_info[0], su[i], part_info[3])
+
+                formats.append({
+                    'url': video_url,
+                    'format_id': format_id,
+                    'filesize': data['clipsBytes'][i],
+                    'width': data['width'],
+                    'height': data['height'],
+                    'fps': data['fps'],
+                })
+            self._sort_formats(formats)
+
+            playlist.append({
+                'id': '%s_part%d' % (video_id, i + 1),
                 'title': title,
-                'url': video_url,
-                'ext': 'mp4',
-            }
-            playlist.append(video_info)
+                'duration': vid_data['data']['clipsDuration'][i],
+                'formats': formats,
+            })
 
         if len(playlist) == 1:
             info = playlist[0]

From 885e4384a1ad8b12037810824d15e9436134a148 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Dec 2014 23:26:57 +0600
Subject: [PATCH 0940/1937] [teletask] Simplify

---
 youtube_dl/extractor/teletask.py | 50 ++++++++++++--------------------
 1 file changed, 18 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/extractor/teletask.py b/youtube_dl/extractor/teletask.py
index aa5535042..e54145105 100644
--- a/youtube_dl/extractor/teletask.py
+++ b/youtube_dl/extractor/teletask.py
@@ -1,13 +1,13 @@
-# coding: utf-8
 from __future__ import unicode_literals
+
 import re
-import datetime
 
 from .common import InfoExtractor
+from ..utils import unified_strdate
 
 
 class TeleTaskIE(InfoExtractor):
-    _VALID_URL = r'http?://(?:www\.)?tele-task\.de/archive/video/html5/(?P<id>[0-9]+)/'
+    _VALID_URL = r'https?://(?:www\.)?tele-task\.de/archive/video/html5/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.tele-task.de/archive/video/html5/26168/',
         'info_dict': {
@@ -16,52 +16,38 @@ class TeleTaskIE(InfoExtractor):
         'playlist': [{
             'md5': '290ef69fb2792e481169c3958dbfbd57',
             'info_dict': {
+                'id': '26168-speaker',
+                'ext': 'mp4',
                 'title': 'Duplicate Detection',
                 'upload_date': '20141218',
-                'id': 'speaker_26168',
-                'ext': 'mp4',
             }
-        },
-            {
+        }, {
             'md5': 'e1e7218c5f0e4790015a437fcf6c71b4',
             'info_dict': {
+                'id': '26168-slides',
+                'ext': 'mp4',
                 'title': 'Duplicate Detection',
                 'upload_date': '20141218',
-                'id': 'slides_26168',
-                'ext': 'mp4',
             }
         }]
     }
 
     def _real_extract(self, url):
         lecture_id = self._match_id(url)
+
         webpage = self._download_webpage(url, lecture_id)
 
         title = self._html_search_regex(
-            r'itemprop="name">([^"]+)</a>', webpage, 'title')
-        url_speaker = self._html_search_regex(
-            r'class="speaker".*?src="([^"]+)"', webpage, 'video_url_speaker', flags=re.DOTALL)
-        url_slides = self._html_search_regex(
-            r'class="slides".*?src="([^"]+)"', webpage, 'video_url_slides', flags=re.DOTALL)
-        date = self._html_search_regex(
-            r'<td class="label">Date:</td><td>([^"]+)</td>', webpage, 'date')
-        date = datetime.datetime.strptime(date, '%d.%m.%Y').strftime('%Y%m%d')
+            r'itemprop="name">([^<]+)</a>', webpage, 'title')
+        upload_date = unified_strdate(self._html_search_regex(
+            r'Date:</td><td>([^<]+)</td>', webpage, 'date', fatal=False))
 
         entries = [{
+            'id': '%s-%s' % (lecture_id, format_id),
+            'url': video_url,
             'title': title,
-            'upload_date': date,
-            'id': "speaker_"+lecture_id,
-            'url': url_speaker,
-        },
-            {
-            'title': title,
-            'upload_date': date,
-            'id': "slides_"+lecture_id,
-            'url': url_slides}]
+            'upload_date': upload_date,
+        } for format_id, video_url in re.findall(
+            r'<video class="([^"]+)"[^>]*>\s*<source src="([^"]+)"', webpage)]
 
-        return {
-            '_type': "playlist",
-            'id': lecture_id,
-            'title': title,
-            'entries': entries,
-        }
+        return self.playlist_result(entries, lecture_id, title)

From a5424052000d6ab69412f380a96f1963f315a295 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Dec 2014 23:29:10 +0600
Subject: [PATCH 0941/1937] Credit @MaxReimann for teletask (#4533)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 6ea958fce..bb4d8b4d1 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -95,3 +95,4 @@ Adrian Kretz
 Mathias Rav
 Petr Kutalek
 Will Glynn
+Max Reimann

From 2c0b4752357b0102220874040934d0d124bed35b Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Fri, 26 Dec 2014 12:49:13 +0100
Subject: [PATCH 0942/1937] [sunporno] fix duration

---
 youtube_dl/extractor/sunporno.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dl/extractor/sunporno.py
index 263f09b46..6cfbc0160 100644
--- a/youtube_dl/extractor/sunporno.py
+++ b/youtube_dl/extractor/sunporno.py
@@ -39,7 +39,7 @@ class SunPornoIE(InfoExtractor):
             r'poster="([^"]+)"', webpage, 'thumbnail', fatal=False)
 
         duration = parse_duration(self._search_regex(
-            r'Duration:\s*(\d+:\d+)\s*<', webpage, 'duration', fatal=False))
+            r'Duration:[^\d]+(\d+:\d+)\s*<', webpage, 'duration', fatal=False))
 
         view_count = int_or_none(self._html_search_regex(
             r'class="views">\s*(\d+)\s*<', webpage, 'view count', fatal=False))

From 31424c126fb48acc49cd539bad6aa7193c7583d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Dec 2014 19:28:51 +0600
Subject: [PATCH 0943/1937] [sunporno] Modernize

---
 youtube_dl/extractor/sunporno.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dl/extractor/sunporno.py
index 6cfbc0160..8a333f1d2 100644
--- a/youtube_dl/extractor/sunporno.py
+++ b/youtube_dl/extractor/sunporno.py
@@ -28,23 +28,27 @@ class SunPornoIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, 'title')
-        description = self._html_search_meta('description', webpage, 'description')
+        title = self._html_search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title')
+        description = self._html_search_meta(
+            'description', webpage, 'description')
         thumbnail = self._html_search_regex(
             r'poster="([^"]+)"', webpage, 'thumbnail', fatal=False)
 
         duration = parse_duration(self._search_regex(
-            r'Duration:[^\d]+(\d+:\d+)\s*<', webpage, 'duration', fatal=False))
+            r'itemprop="duration">\s*(\d+:\d+)\s*<',
+            webpage, 'duration', fatal=False))
 
         view_count = int_or_none(self._html_search_regex(
-            r'class="views">\s*(\d+)\s*<', webpage, 'view count', fatal=False))
+            r'class="views">\s*(\d+)\s*<',
+            webpage, 'view count', fatal=False))
         comment_count = int_or_none(self._html_search_regex(
-            r'(\d+)</b> Comments?', webpage, 'comment count', fatal=False))
+            r'(\d+)</b> Comments?',
+            webpage, 'comment count', fatal=False))
 
         formats = []
         quality = qualities(['mp4', 'flv'])

From 6a1c4fbfcb57c138b8ace305127c2654844e8099 Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Fri, 26 Dec 2014 15:49:12 +0100
Subject: [PATCH 0944/1937] [hellporno] new extractor

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/hellporno.py | 56 +++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 youtube_dl/extractor/hellporno.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fd0ebffe3..de5206729 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -166,6 +166,7 @@ from .grooveshark import GroovesharkIE
 from .groupon import GrouponIE
 from .hark import HarkIE
 from .heise import HeiseIE
+from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
 from .hornbunny import HornBunnyIE
diff --git a/youtube_dl/extractor/hellporno.py b/youtube_dl/extractor/hellporno.py
new file mode 100644
index 000000000..06f85127f
--- /dev/null
+++ b/youtube_dl/extractor/hellporno.py
@@ -0,0 +1,56 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+class HellPornoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?hellporno\.com/videos/(?P<display_id>[^/]+)'
+    _TEST = {
+        'url': 'http://hellporno.com/videos/dixie-is-posing-with-naked-ass-very-erotic/',
+        'md5': '1fee339c610d2049699ef2aa699439f1',
+        'info_dict': {
+            'id': '149116',
+            'ext': 'mp4',
+            'title': 'Dixie is posing with naked ass very erotic',
+            'description': 'md5:5ba02cbf31eff820147b3cc25306d89a',
+            'categories': list,  # NSFW
+            'thumbnail': 're:https?://.*\.jpg$',
+            'age_limit': 18,
+        }
+    }
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(url, 'main')
+
+        video_id = self._html_search_regex(r'video_id:\s*\'([^\']+)\'', webpage, 'id')
+
+        video_url = self._html_search_regex(r'video_url:\s*\'([^\']+)\'', webpage, 'video_url')
+
+        ext = self._html_search_regex(r'postfix:\s*\'([^\']+)\'', webpage, 'ext')[1:]
+
+        title = self._html_search_regex(
+            r'<title>([^<]+)\s*-\s*Hell Porno</title>', webpage, 'title')
+
+        description = self._html_search_meta('description', webpage, 'description', fatal=False)
+
+        thumbnail = self._html_search_regex(
+            r'preview_url:\s*\'([^\']+)\'',
+            webpage, 'thumbnail', fatal=False)
+
+        categories_str = self._html_search_regex(
+            r'<meta name="keywords" content="([^"]+)"', webpage, 'categories', fatal=False)
+        categories = (
+            None if categories_str is None
+            else categories_str.split(','))
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'ext': ext,
+            'description': description,
+            'thumbnail': thumbnail,
+            'categories': categories,
+            'age_limit': 18,
+        }

From 4cda41ac7be1af03cb5fc4e9bbfcbf944925ab63 Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Fri, 26 Dec 2014 16:17:35 +0100
Subject: [PATCH 0945/1937] [alphaporno] new extractor

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/alphaporno.py | 54 ++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+)
 create mode 100644 youtube_dl/extractor/alphaporno.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fd0ebffe3..3fd0240c3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -7,6 +7,7 @@ from .adobetv import AdobeTVIE
 from .adultswim import AdultSwimIE
 from .aftonbladet import AftonbladetIE
 from .aljazeera import AlJazeeraIE
+from .alphaporno import AlphaPornoIE
 from .anitube import AnitubeIE
 from .anysex import AnySexIE
 from .aol import AolIE
diff --git a/youtube_dl/extractor/alphaporno.py b/youtube_dl/extractor/alphaporno.py
new file mode 100644
index 000000000..c8eaf837f
--- /dev/null
+++ b/youtube_dl/extractor/alphaporno.py
@@ -0,0 +1,54 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+class AlphaPornoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?alphaporno\.com/videos/(?P<display_id>[^/]+)'
+    _TEST = {
+        'url': 'http://www.alphaporno.com/videos/sensual-striptease-porn-with-samantha-alexandra/',
+        'md5': 'feb6d3bba8848cd54467a87ad34bd38e',
+        'info_dict': {
+            'id': '258807',
+            'ext': 'mp4',
+            'title': 'Sensual striptease porn with Samantha Alexandra - Striptease Porn',
+            'description': 'md5:c4447dc80e5be4c5f2711f7806e45424',
+            'categories': list,  # NSFW
+            'thumbnail': 're:https?://.*\.jpg$',
+            'age_limit': 18,
+        }
+    }
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(url, 'main')
+
+        video_id = self._html_search_regex(r'video_id:\s*\'([^\']+)\'', webpage, 'id')
+
+        video_url = self._html_search_regex(r'video_url:\s*\'([^\']+)\'', webpage, 'video_url')
+
+        ext = self._html_search_meta('encodingFormat', webpage, 'ext')[1:]
+
+        title = self._html_search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title')
+
+        description = self._html_search_meta('description', webpage, 'description', fatal=False)
+
+        thumbnail = self._html_search_meta('thumbnail', webpage, 'thumbnail', fatal=False)
+
+        categories_str = self._html_search_meta(
+            'keywords', webpage, 'categories', fatal=False)
+        categories = (
+            None if categories_str is None
+            else categories_str.split(','))
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'ext': ext,
+            'description': description,
+            'thumbnail': thumbnail,
+            'categories': categories,
+            'age_limit': 18,
+        }

From 461b00f34aabf6e75e0d53419919a8e6cc9cfd7a Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Fri, 26 Dec 2014 17:15:34 +0100
Subject: [PATCH 0946/1937] [eroprofile] new extractor

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/eroprofile.py | 53 ++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)
 create mode 100644 youtube_dl/extractor/eroprofile.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fd0ebffe3..39788712c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -108,6 +108,7 @@ from .elpais import ElPaisIE
 from .empflix import EMPFlixIE
 from .engadget import EngadgetIE
 from .eporner import EpornerIE
+from .eroprofile import EroProfileIE
 from .escapist import EscapistIE
 from .everyonesmixtape import EveryonesMixtapeIE
 from .exfm import ExfmIE
diff --git a/youtube_dl/extractor/eroprofile.py b/youtube_dl/extractor/eroprofile.py
new file mode 100644
index 000000000..f6745fc52
--- /dev/null
+++ b/youtube_dl/extractor/eroprofile.py
@@ -0,0 +1,53 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+class EroProfileIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?eroprofile\.com/m/videos/view/(?P<display_id>[^/]+)'
+    _TEST = {
+        'url': 'http://www.eroprofile.com/m/videos/view/sexy-babe-softcore',
+        'md5': 'c26f351332edf23e1ea28ce9ec9de32f',
+        'info_dict': {
+            'id': '3733775',
+            'ext': 'm4v',
+            'title': 'HD MOVIES - sexy babe softcore',
+            'description': 'md5:831ee50526c333eb4f6c1e58d382c295',
+            'categories': list,  # NSFW
+            'thumbnail': 're:https?://.*\.jpg',
+            'age_limit': 18,
+        }
+    }
+
+    def _real_extract(self, url):
+        webpage = self._download_webpage(url, 'main')
+
+        video_id = self._html_search_regex(r'glbUpdViews\s*\(\'\d*\',\'(\d+)\'', webpage, 'id')
+
+        video_url = self._html_search_regex(r'<source src="([^"]+)', webpage, 'video_url')
+
+        title = self._html_search_regex(
+            r'<title>([^<]+)\s*-\s*EroProfile</title>', webpage, 'title')
+
+        description = self._html_search_meta('description', webpage, 'description', fatal=False)
+
+        thumbnail = self._html_search_regex(
+          r'onclick="showVideoPlayer\(\)"><img src="([^"]+)', webpage, 'thumbnail', fatal=False)
+
+        categories_str = self._html_search_meta(
+            'keywords', webpage, 'categories', fatal=False)
+        categories = (
+            None if categories_str is None
+            else categories_str.split(','))
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'ext': 'm4v',
+            'description': description,
+            'thumbnail': thumbnail,
+            'categories': categories,
+            'age_limit': 18,
+        }

From f74b341dde36fe2f56ab52c92416eb3ff5ccecf3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Dec 2014 23:07:24 +0600
Subject: [PATCH 0947/1937] expect_info_dict actual-expected argument
 consistency

---
 test/helper.py        | 2 +-
 test/test_download.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 8a820526a..96d58b7c1 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -99,7 +99,7 @@ def gettestcases(include_onlymatching=False):
 md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
 
 
-def expect_info_dict(self, expected_dict, got_dict):
+def expect_info_dict(self, got_dict, expected_dict):
     for info_field, expected in expected_dict.items():
         if isinstance(expected, compat_str) and expected.startswith('re:'):
             got = got_dict.get(info_field)
diff --git a/test/test_download.py b/test/test_download.py
index a009aa475..412f3dbce 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -155,7 +155,7 @@ def generator(test_case):
             if is_playlist:
                 self.assertEqual(res_dict['_type'], 'playlist')
                 self.assertTrue('entries' in res_dict)
-                expect_info_dict(self, test_case.get('info_dict', {}), res_dict)
+                expect_info_dict(self, res_dict, test_case.get('info_dict', {}))
 
             if 'playlist_mincount' in test_case:
                 assertGreaterEqual(
@@ -204,7 +204,7 @@ def generator(test_case):
                 with io.open(info_json_fn, encoding='utf-8') as infof:
                     info_dict = json.load(infof)
 
-                expect_info_dict(self, tc.get('info_dict', {}), info_dict)
+                expect_info_dict(self, info_dict, tc.get('info_dict', {}))
         finally:
             try_rm_tcs_files()
             if is_playlist and res_dict is not None and res_dict.get('entries'):

From f9b9e88646fcad16501a8390c040f9dd2257dd15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 26 Dec 2014 23:32:41 +0600
Subject: [PATCH 0948/1937] [eroprofile] Simplify

---
 youtube_dl/extractor/eroprofile.py | 40 ++++++++++++------------------
 1 file changed, 16 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/eroprofile.py b/youtube_dl/extractor/eroprofile.py
index f6745fc52..79e2fbd39 100644
--- a/youtube_dl/extractor/eroprofile.py
+++ b/youtube_dl/extractor/eroprofile.py
@@ -1,53 +1,45 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
+
 class EroProfileIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?eroprofile\.com/m/videos/view/(?P<display_id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?eroprofile\.com/m/videos/view/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://www.eroprofile.com/m/videos/view/sexy-babe-softcore',
         'md5': 'c26f351332edf23e1ea28ce9ec9de32f',
         'info_dict': {
             'id': '3733775',
+            'display_id': 'sexy-babe-softcore',
             'ext': 'm4v',
-            'title': 'HD MOVIES - sexy babe softcore',
-            'description': 'md5:831ee50526c333eb4f6c1e58d382c295',
-            'categories': list,  # NSFW
+            'title': 'sexy babe softcore',
             'thumbnail': 're:https?://.*\.jpg',
             'age_limit': 18,
         }
     }
 
     def _real_extract(self, url):
-        webpage = self._download_webpage(url, 'main')
+        display_id = self._match_id(url)
 
-        video_id = self._html_search_regex(r'glbUpdViews\s*\(\'\d*\',\'(\d+)\'', webpage, 'id')
+        webpage = self._download_webpage(url, display_id)
 
-        video_url = self._html_search_regex(r'<source src="([^"]+)', webpage, 'video_url')
+        video_id = self._search_regex(
+            [r"glbUpdViews\s*\('\d*','(\d+)'", r'p/report/video/(\d+)'],
+            webpage, 'video id', default=None)
 
+        video_url = self._search_regex(
+            r'<source src="([^"]+)', webpage, 'video url')
         title = self._html_search_regex(
-            r'<title>([^<]+)\s*-\s*EroProfile</title>', webpage, 'title')
-
-        description = self._html_search_meta('description', webpage, 'description', fatal=False)
-
-        thumbnail = self._html_search_regex(
-          r'onclick="showVideoPlayer\(\)"><img src="([^"]+)', webpage, 'thumbnail', fatal=False)
-
-        categories_str = self._html_search_meta(
-            'keywords', webpage, 'categories', fatal=False)
-        categories = (
-            None if categories_str is None
-            else categories_str.split(','))
+            r'Title:</th><td>([^<]+)</td>', webpage, 'title')
+        thumbnail = self._search_regex(
+            r'onclick="showVideoPlayer\(\)"><img src="([^"]+)',
+            webpage, 'thumbnail', fatal=False)
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'url': video_url,
             'title': title,
-            'ext': 'm4v',
-            'description': description,
             'thumbnail': thumbnail,
-            'categories': categories,
             'age_limit': 18,
         }

From e82def52a97faf85eeeedcac1266c27a565e2f21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 27 Dec 2014 00:08:04 +0600
Subject: [PATCH 0949/1937] [alphaporno] Improve

---
 youtube_dl/extractor/alphaporno.py | 71 ++++++++++++++++++++----------
 1 file changed, 47 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/alphaporno.py b/youtube_dl/extractor/alphaporno.py
index c8eaf837f..c34719d1f 100644
--- a/youtube_dl/extractor/alphaporno.py
+++ b/youtube_dl/extractor/alphaporno.py
@@ -1,54 +1,77 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    parse_duration,
+    parse_filesize,
+    int_or_none,
+)
+
 
 class AlphaPornoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?alphaporno\.com/videos/(?P<display_id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?alphaporno\.com/videos/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://www.alphaporno.com/videos/sensual-striptease-porn-with-samantha-alexandra/',
         'md5': 'feb6d3bba8848cd54467a87ad34bd38e',
         'info_dict': {
             'id': '258807',
+            'display_id': 'sensual-striptease-porn-with-samantha-alexandra',
             'ext': 'mp4',
-            'title': 'Sensual striptease porn with Samantha Alexandra - Striptease Porn',
-            'description': 'md5:c4447dc80e5be4c5f2711f7806e45424',
-            'categories': list,  # NSFW
+            'title': 'Sensual striptease porn with Samantha Alexandra',
             'thumbnail': 're:https?://.*\.jpg$',
+            'timestamp': 1418694611,
+            'upload_date': '20141216',
+            'duration': 387,
+            'filesize_approx': 54120000,
+            'tbr': 1145,
+            'categories': list,
             'age_limit': 18,
         }
     }
 
     def _real_extract(self, url):
-        webpage = self._download_webpage(url, 'main')
+        display_id = self._match_id(url)
 
-        video_id = self._html_search_regex(r'video_id:\s*\'([^\']+)\'', webpage, 'id')
+        webpage = self._download_webpage(url, display_id)
 
-        video_url = self._html_search_regex(r'video_url:\s*\'([^\']+)\'', webpage, 'video_url')
+        video_id = self._search_regex(
+            r"video_id\s*:\s*'([^']+)'", webpage, 'video id', default=None)
 
-        ext = self._html_search_meta('encodingFormat', webpage, 'ext')[1:]
+        video_url = self._search_regex(
+            r"video_url\s*:\s*'([^']+)'", webpage, 'video url')
+        ext = self._html_search_meta(
+            'encodingFormat', webpage, 'ext', default='.mp4')[1:]
 
-        title = self._html_search_regex(
-            r'<title>([^<]+)</title>', webpage, 'title')
+        title = self._search_regex(
+            [r'<meta content="([^"]+)" itemprop="description">',
+             r'class="title" itemprop="name">([^<]+)<'],
+            webpage, 'title')
+        thumbnail = self._html_search_meta('thumbnail', webpage, 'thumbnail')
+        timestamp = parse_iso8601(self._html_search_meta(
+            'uploadDate', webpage, 'upload date'))
+        duration = parse_duration(self._html_search_meta(
+            'duration', webpage, 'duration'))
+        filesize_approx = parse_filesize(self._html_search_meta(
+            'contentSize', webpage, 'file size'))
+        bitrate = int_or_none(self._html_search_meta(
+            'bitrate', webpage, 'bitrate'))
+        categories = self._html_search_meta(
+            'keywords', webpage, 'categories', default='').split(',')
 
-        description = self._html_search_meta('description', webpage, 'description', fatal=False)
-
-        thumbnail = self._html_search_meta('thumbnail', webpage, 'thumbnail', fatal=False)
-
-        categories_str = self._html_search_meta(
-            'keywords', webpage, 'categories', fatal=False)
-        categories = (
-            None if categories_str is None
-            else categories_str.split(','))
+        age_limit = self._rta_search(webpage)
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'url': video_url,
-            'title': title,
             'ext': ext,
-            'description': description,
+            'title': title,
             'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+            'filesize_approx': filesize_approx,
+            'tbr': bitrate,
             'categories': categories,
-            'age_limit': 18,
+            'age_limit': age_limit,
         }

From 1b7b1d6eac4b40eddd0d574e5e5222f949704104 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Dec 2014 15:41:52 +0600
Subject: [PATCH 0950/1937] [arte.tv:+7] Make quality optional (Closes #4583)

---
 youtube_dl/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 219631b9b..f02539327 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -133,7 +133,7 @@ class ArteTVPlus7IE(InfoExtractor):
                 'width': int_or_none(f.get('width')),
                 'height': int_or_none(f.get('height')),
                 'tbr': int_or_none(f.get('bitrate')),
-                'quality': qfunc(f['quality']),
+                'quality': qfunc(f.get('quality')),
                 'source_preference': source_pref,
             }
 

From 5a1840305746b7242c8d8307e945b9f889603a8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 28 Dec 2014 15:42:29 +0600
Subject: [PATCH 0951/1937] [arte.tv] Fix typo

---
 youtube_dl/extractor/arte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index f02539327..929dd3cc5 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -37,7 +37,7 @@ class ArteTvIE(InfoExtractor):
             config_xml_url, video_id, note='Downloading configuration')
 
         formats = [{
-            'forma_id': q.attrib['quality'],
+            'format_id': q.attrib['quality'],
             # The playpath starts at 'mp4:', if we don't manually
             # split the url, rtmpdump will incorrectly parse them
             'url': q.text.split('mp4:', 1)[0],

From ff7a07d5c4d260a8b27fa7bf523c405725f9cb20 Mon Sep 17 00:00:00 2001
From: Johannes Knoedtel <johannes@dev-baron.de>
Date: Sun, 28 Dec 2014 20:04:21 +0100
Subject: [PATCH 0952/1937] [archiveorg] most metadata fields are optional

Example: https://archive.org/details/Cops1922
---
 youtube_dl/extractor/archiveorg.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/archiveorg.py b/youtube_dl/extractor/archiveorg.py
index 34ce8429b..ba94b5454 100644
--- a/youtube_dl/extractor/archiveorg.py
+++ b/youtube_dl/extractor/archiveorg.py
@@ -25,6 +25,12 @@ class ArchiveOrgIE(InfoExtractor):
         }
     }
 
+    def get_optional_metadata(self, data, field):
+        try:
+            return data['metadata'][field][0]
+        except KeyError:
+            return None
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -33,10 +39,12 @@ class ArchiveOrgIE(InfoExtractor):
         json_data = self._download_webpage(json_url, video_id)
         data = json.loads(json_data)
 
-        title = data['metadata']['title'][0]
-        description = data['metadata']['description'][0]
-        uploader = data['metadata']['creator'][0]
-        upload_date = unified_strdate(data['metadata']['date'][0])
+        title = self.get_optional_metadata(data, 'title')
+        description = self.get_optional_metadata(data, 'description')
+        uploader = self.get_optional_metadata(data, 'creator')
+        upload_date = self.get_optional_metadata(data, 'date')
+        if upload_date:
+            upload_date = unified_strdate(upload_date)
 
         formats = [
             {

From e8e28989eb8ab0dbb755a273cd1d79847494744b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Dec 2014 02:08:46 +0600
Subject: [PATCH 0953/1937] [archiveorg] Add test, simplify and modernize

---
 youtube_dl/extractor/archiveorg.py | 60 +++++++++++++++---------------
 1 file changed, 29 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/archiveorg.py b/youtube_dl/extractor/archiveorg.py
index ba94b5454..9fc35a42b 100644
--- a/youtube_dl/extractor/archiveorg.py
+++ b/youtube_dl/extractor/archiveorg.py
@@ -1,50 +1,48 @@
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
-from ..utils import (
-    unified_strdate,
-)
+from ..utils import unified_strdate
 
 
 class ArchiveOrgIE(InfoExtractor):
     IE_NAME = 'archive.org'
     IE_DESC = 'archive.org videos'
-    _VALID_URL = r'(?:https?://)?(?:www\.)?archive\.org/details/(?P<id>[^?/]+)(?:[?].*)?$'
-    _TEST = {
-        "url": "http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect",
-        'file': 'XD300-23_68HighlightsAResearchCntAugHumanIntellect.ogv',
+    _VALID_URL = r'https?://(?:www\.)?archive\.org/details/(?P<id>[^?/]+)(?:[?].*)?$'
+    _TESTS = [{
+        'url': 'http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
         'md5': '8af1d4cf447933ed3c7f4871162602db',
         'info_dict': {
-            "title": "1968 Demo - FJCC Conference Presentation Reel #1",
-            "description": "Reel 1 of 3: Also known as the \"Mother of All Demos\", Doug Engelbart's presentation at the Fall Joint Computer Conference in San Francisco, December 9, 1968 titled \"A Research Center for Augmenting Human Intellect.\" For this presentation, Doug and his team astonished the audience by not only relating their research, but demonstrating it live. This was the debut of the mouse, interactive computing, hypermedia, computer supported software engineering, video teleconferencing, etc. See also <a href=\"http://dougengelbart.org/firsts/dougs-1968-demo.html\" rel=\"nofollow\">Doug's 1968 Demo page</a> for more background, highlights, links, and the detailed paper published in this conference proceedings. Filmed on 3 reels: Reel 1 | <a href=\"http://www.archive.org/details/XD300-24_68HighlightsAResearchCntAugHumanIntellect\" rel=\"nofollow\">Reel 2</a> | <a href=\"http://www.archive.org/details/XD300-25_68HighlightsAResearchCntAugHumanIntellect\" rel=\"nofollow\">Reel 3</a>",
-            "upload_date": "19681210",
-            "uploader": "SRI International"
+            'id': 'XD300-23_68HighlightsAResearchCntAugHumanIntellect',
+            'ext': 'ogv',
+            'title': '1968 Demo - FJCC Conference Presentation Reel #1',
+            'description': 'md5:1780b464abaca9991d8968c877bb53ed',
+            'upload_date': '19681210',
+            'uploader': 'SRI International'
         }
-    }
-
-    def get_optional_metadata(self, data, field):
-        try:
-            return data['metadata'][field][0]
-        except KeyError:
-            return None
+    }, {
+        'url': 'https://archive.org/details/Cops1922',
+        'md5': '18f2a19e6d89af8425671da1cf3d4e04',
+        'info_dict': {
+            'id': 'Cops1922',
+            'ext': 'ogv',
+            'title': 'Buster Keaton\'s "Cops" (1922)',
+            'description': 'md5:70f72ee70882f713d4578725461ffcc3',
+        }
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         json_url = url + ('?' if '?' in url else '&') + 'output=json'
-        json_data = self._download_webpage(json_url, video_id)
-        data = json.loads(json_data)
+        data = self._download_json(json_url, video_id)
 
-        title = self.get_optional_metadata(data, 'title')
-        description = self.get_optional_metadata(data, 'description')
-        uploader = self.get_optional_metadata(data, 'creator')
-        upload_date = self.get_optional_metadata(data, 'date')
-        if upload_date:
-            upload_date = unified_strdate(upload_date)
+        def get_optional(data_dict, field):
+            return data_dict['metadata'].get(field, [None])[0]
+
+        title = get_optional(data, 'title')
+        description = get_optional(data, 'description')
+        uploader = get_optional(data, 'creator')
+        upload_date = unified_strdate(get_optional(data, 'date'))
 
         formats = [
             {

From ae6986fb895ceb72e084b587bc9d070c86848347 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Dec 2014 03:00:24 +0600
Subject: [PATCH 0954/1937] [bbccouk] Switch to new JSON playlist format
 (Closes #4588)

---
 youtube_dl/extractor/bbccouk.py | 95 +++++++++++++++++++++++----------
 1 file changed, 68 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 2d2f742ae..f690dc803 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -71,7 +71,20 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
                 'skip_download': True,
             },
             'skip': 'Currently BBC iPlayer TV programmes are available to play in the UK only',
-        },
+        }, {
+            'url': 'http://www.bbc.co.uk/programmes/b04v20dw',
+            'info_dict': {
+                'id': 'b04v209v',
+                'ext': 'flv',
+                'title': 'Pete Tong, The Essential New Tune Special',
+                'description': "Pete has a very special mix - all of 2014's Essential New Tunes!",
+                'duration': 10800,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            }
+        }
     ]
 
     def _extract_asx_playlist(self, connection, programme_id):
@@ -203,6 +216,59 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
 
         return formats, subtitles
 
+    def _download_playlist(self, playlist_id):
+        try:
+            playlist = self._download_json(
+                'http://www.bbc.co.uk/programmes/%s/playlist.json' % playlist_id,
+                playlist_id, 'Downloading playlist JSON')
+
+            version = playlist.get('defaultAvailableVersion')
+            if version:
+                smp_config = version['smpConfig']
+                title = smp_config['title']
+                description = smp_config['summary']
+                for item in smp_config['items']:
+                    kind = item['kind']
+                    if kind != 'programme' and kind != 'radioProgramme':
+                        continue
+                    programme_id = item.get('vpid')
+                    duration = int(item.get('duration'))
+                    formats, subtitles = self._download_media_selector(programme_id)
+                return programme_id, title, description, duration, formats, subtitles
+        except ExtractorError as ee:
+            if not isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404:
+                raise
+
+        # fallback to legacy playlist
+        playlist = self._download_xml(
+                'http://www.bbc.co.uk/iplayer/playlist/%s' % playlist_id,
+                playlist_id, 'Downloading legacy playlist XML')
+
+        no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
+        if no_items is not None:
+            reason = no_items.get('reason')
+            if reason == 'preAvailability':
+                msg = 'Episode %s is not yet available' % playlist_id
+            elif reason == 'postAvailability':
+                msg = 'Episode %s is no longer available' % playlist_id
+            elif reason == 'noMedia':
+                msg = 'Episode %s is not currently available' % playlist_id
+            else:
+                msg = 'Episode %s is not available: %s' % (playlist_id, reason)
+            raise ExtractorError(msg, expected=True)
+
+        for item in self._extract_items(playlist):
+            kind = item.get('kind')
+            if kind != 'programme' and kind != 'radioProgramme':
+                continue
+            title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
+            description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
+            programme_id = item.get('identifier')
+            duration = int(item.get('duration'))
+            formats, subtitles = self._download_media_selector(programme_id)
+
+        return programme_id, title, description, duration, formats, subtitles
+
     def _real_extract(self, url):
         group_id = self._match_id(url)
 
@@ -219,32 +285,7 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
             duration = player['duration']
             formats, subtitles = self._download_media_selector(programme_id)
         else:
-            playlist = self._download_xml(
-                'http://www.bbc.co.uk/iplayer/playlist/%s' % group_id,
-                group_id, 'Downloading playlist XML')
-
-            no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
-            if no_items is not None:
-                reason = no_items.get('reason')
-                if reason == 'preAvailability':
-                    msg = 'Episode %s is not yet available' % group_id
-                elif reason == 'postAvailability':
-                    msg = 'Episode %s is no longer available' % group_id
-                elif reason == 'noMedia':
-                    msg = 'Episode %s is not currently available' % group_id
-                else:
-                    msg = 'Episode %s is not available: %s' % (group_id, reason)
-                raise ExtractorError(msg, expected=True)
-
-            for item in self._extract_items(playlist):
-                kind = item.get('kind')
-                if kind != 'programme' and kind != 'radioProgramme':
-                    continue
-                title = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}title').text
-                description = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}summary').text
-                programme_id = item.get('identifier')
-                duration = int(item.get('duration'))
-                formats, subtitles = self._download_media_selector(programme_id)
+            programme_id, title, description, duration, formats, subtitles = self._download_playlist(group_id)
 
         if self._downloader.params.get('listsubtitles', False):
             self._list_available_subtitles(programme_id, subtitles)

From 513fd2a8723b640844d9fe7adaa8e14fe6746ec2 Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Mon, 29 Dec 2014 10:38:07 +0100
Subject: [PATCH 0955/1937] [hellporno] simplify

---
 youtube_dl/extractor/hellporno.py | 32 ++++++++++++++-----------------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/hellporno.py b/youtube_dl/extractor/hellporno.py
index 06f85127f..754322cdf 100644
--- a/youtube_dl/extractor/hellporno.py
+++ b/youtube_dl/extractor/hellporno.py
@@ -1,55 +1,51 @@
 from __future__ import unicode_literals
-
-import re
-
 from .common import InfoExtractor
 
+
 class HellPornoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?hellporno\.com/videos/(?P<display_id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?hellporno\.com/videos/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://hellporno.com/videos/dixie-is-posing-with-naked-ass-very-erotic/',
         'md5': '1fee339c610d2049699ef2aa699439f1',
         'info_dict': {
             'id': '149116',
+            'display_id': 'dixie-is-posing-with-naked-ass-very-erotic',
             'ext': 'mp4',
             'title': 'Dixie is posing with naked ass very erotic',
-            'description': 'md5:5ba02cbf31eff820147b3cc25306d89a',
-            'categories': list,  # NSFW
             'thumbnail': 're:https?://.*\.jpg$',
             'age_limit': 18,
         }
     }
 
     def _real_extract(self, url):
-        webpage = self._download_webpage(url, 'main')
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
 
-        video_id = self._html_search_regex(r'video_id:\s*\'([^\']+)\'', webpage, 'id')
+        video_id = self._html_search_regex(
+            r'video_id:\s*\'([^\']+)\'', webpage, 'id')
 
-        video_url = self._html_search_regex(r'video_url:\s*\'([^\']+)\'', webpage, 'video_url')
+        ext = self._html_search_regex(
+            r'postfix:\s*\'([^\']+)\'', webpage, 'ext')[1:]
 
-        ext = self._html_search_regex(r'postfix:\s*\'([^\']+)\'', webpage, 'ext')[1:]
+        video_url = self._html_search_regex(
+            r'video_url:\s*\'([^\']+)\'', webpage, 'video_url')
 
         title = self._html_search_regex(
             r'<title>([^<]+)\s*-\s*Hell Porno</title>', webpage, 'title')
 
-        description = self._html_search_meta('description', webpage, 'description', fatal=False)
-
         thumbnail = self._html_search_regex(
             r'preview_url:\s*\'([^\']+)\'',
             webpage, 'thumbnail', fatal=False)
 
-        categories_str = self._html_search_regex(
-            r'<meta name="keywords" content="([^"]+)"', webpage, 'categories', fatal=False)
-        categories = (
-            None if categories_str is None
-            else categories_str.split(','))
+        categories = self._html_search_meta(
+            'keywords', webpage, 'categories', default='').split(',')
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'url': video_url,
             'title': title,
             'ext': ext,
-            'description': description,
             'thumbnail': thumbnail,
             'categories': categories,
             'age_limit': 18,

From 0cc4f8e3850e41dc5e3f237315a20eac5eb99f46 Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Mon, 29 Dec 2014 11:31:22 +0100
Subject: [PATCH 0956/1937] [xxxymovies] new ectractor

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/xxxymovies.py | 63 ++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 youtube_dl/extractor/xxxymovies.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fd0ebffe3..c8a77616e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -512,6 +512,7 @@ from .xminus import XMinusIE
 from .xnxx import XNXXIE
 from .xvideos import XVideosIE
 from .xtube import XTubeUserIE, XTubeIE
+from .xxxymovies import XXXYMoviesIE
 from .yahoo import (
     YahooIE,
     YahooSearchIE,
diff --git a/youtube_dl/extractor/xxxymovies.py b/youtube_dl/extractor/xxxymovies.py
new file mode 100644
index 000000000..4a9144661
--- /dev/null
+++ b/youtube_dl/extractor/xxxymovies.py
@@ -0,0 +1,63 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_duration,
+    int_or_none,
+)
+
+
+class XXXYMoviesIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?xxxymovies\.com/videos/(?P<id>\d+)/(?P<display_id>[^/]+)'
+    _TEST = {
+        'url': 'http://xxxymovies.com/videos/138669/ecstatic-orgasm-sofcore/',
+        'md5': '810b1bdbbffff89dd13bdb369fe7be4b',
+        'info_dict': {
+            'id': '138669',
+            'display_id': 'ecstatic-orgasm-sofcore',
+            'ext': 'mp4',
+            'title': 'Ecstatic Orgasm Sofcore',
+            'duration': 931,
+            'age_limit': 18,
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_url = self._html_search_regex(
+            r"video_url\s*:\s*'([^']+)'", webpage, 'video URL')
+
+        title = self._html_search_regex(
+            r'<title>(.*?)\s*-\s*XXXYMovies.com</title>', webpage, 'title')
+
+        thumbnail = self._html_search_regex(
+            r'preview_url\s*:\s*\'(.*?)\'', webpage, 'thumbnail', fatal=False)
+
+        categories = self._html_search_meta(
+            'keywords', webpage, 'categories', default='').split(',')
+
+        duration = parse_duration(self._search_regex(
+            r'<span>Duration:</span>\s*(\d+:\d+)', webpage, 'duration', fatal=False))
+
+        view_count = int_or_none(self._html_search_regex(
+            r'<div class="video_views">\s*(\d+)', webpage, 'view count', fatal=False))
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'url': video_url,
+            'title': title,
+            'thumbnail': thumbnail,
+            'categories': categories,
+            'duration': duration,
+            'view_count': view_count,
+            'age_limit': 18,
+        }

From 7fc2cd819e10ca6c69cb3601c0bbe7d577256da7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Dec 2014 20:27:09 +0600
Subject: [PATCH 0957/1937] [cnn] Improve regexes and fix test

---
 youtube_dl/extractor/cnn.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index cddcacdf1..1bff005d6 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -11,14 +11,14 @@ from ..utils import (
 
 
 class CNNIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://((edition|www)\.)?cnn\.com/video/(data/.+?|\?)/
-        (?P<path>.+?/(?P<title>[^/]+?)(?:\.cnn(-ap)?|(?=&)))'''
+    _VALID_URL = r'''(?x)https?://(?:(?:edition|www)\.)?cnn\.com/video/(?:data/.+?|\?)/
+        (?P<path>.+?/(?P<title>[^/]+?)(?:\.cnn(?:-ap)?|(?=&)))'''
 
     _TESTS = [{
         'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',
         'md5': '3e6121ea48df7e2259fe73a0628605c4',
         'info_dict': {
-            'id': 'sports_2013_06_09_nadal-1-on-1.cnn',
+            'id': 'sports/2013/06/09/nadal-1-on-1.cnn',
             'ext': 'mp4',
             'title': 'Nadal wins 8th French Open title',
             'description': 'World Sport\'s Amanda Davies chats with 2013 French Open champion Rafael Nadal.',
@@ -130,7 +130,7 @@ class CNNBlogsIE(InfoExtractor):
 
 
 class CNNArticleIE(InfoExtractor):
-    _VALID_URL = r'https?://((edition|www)\.)?cnn\.com/(?!video/)'
+    _VALID_URL = r'https?://(?:(?:edition|www)\.)?cnn\.com/(?!video/)'
     _TEST = {
         'url': 'http://www.cnn.com/2014/12/21/politics/obama-north-koreas-hack-not-war-but-cyber-vandalism/',
         'md5': '275b326f85d80dff7592a9820f5dc887',
@@ -149,6 +149,6 @@ class CNNArticleIE(InfoExtractor):
         cnn_url = self._html_search_regex(r"video:\s*'([^']+)'", webpage, 'cnn url')
         return {
             '_type': 'url',
-            'url': r'http://cnn.com/video/?/video/' + cnn_url,
+            'url': 'http://cnn.com/video/?/video/' + cnn_url,
             'ie_key': CNNIE.ie_key(),
         }

From 6343a5f68e6dbf5a4fc1b0c7d0bec9e1ddc57143 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Dec 2014 21:05:21 +0600
Subject: [PATCH 0958/1937] [xxxymovies] Improve

---
 youtube_dl/extractor/xxxymovies.py | 36 ++++++++++++++++++++++--------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/xxxymovies.py b/youtube_dl/extractor/xxxymovies.py
index 4a9144661..5c8f17eb2 100644
--- a/youtube_dl/extractor/xxxymovies.py
+++ b/youtube_dl/extractor/xxxymovies.py
@@ -20,35 +20,51 @@ class XXXYMoviesIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Ecstatic Orgasm Sofcore',
             'duration': 931,
+            'categories': list,
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
             'age_limit': 18,
         }
     }
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-
         video_id = mobj.group('id')
         display_id = mobj.group('display_id')
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, display_id)
 
-        video_url = self._html_search_regex(
+        video_url = self._search_regex(
             r"video_url\s*:\s*'([^']+)'", webpage, 'video URL')
 
         title = self._html_search_regex(
-            r'<title>(.*?)\s*-\s*XXXYMovies.com</title>', webpage, 'title')
+            [r'<div class="block_header">\s*<h1>([^<]+)</h1>',
+             r'<title>(.*?)\s*-\s*XXXYMovies\.com</title>'],
+            webpage, 'title')
 
-        thumbnail = self._html_search_regex(
-            r'preview_url\s*:\s*\'(.*?)\'', webpage, 'thumbnail', fatal=False)
+        thumbnail = self._search_regex(
+            r"preview_url\s*:\s*'([^']+)'",
+            webpage, 'thumbnail', fatal=False)
 
         categories = self._html_search_meta(
             'keywords', webpage, 'categories', default='').split(',')
 
         duration = parse_duration(self._search_regex(
-            r'<span>Duration:</span>\s*(\d+:\d+)', webpage, 'duration', fatal=False))
+            r'<span>Duration:</span>\s*(\d+:\d+)',
+            webpage, 'duration', fatal=False))
 
         view_count = int_or_none(self._html_search_regex(
-            r'<div class="video_views">\s*(\d+)', webpage, 'view count', fatal=False))
+            r'<div class="video_views">\s*(\d+)',
+            webpage, 'view count', fatal=False))
+        like_count = int_or_none(self._search_regex(
+            r'>\s*Likes? <b>\((\d+)\)',
+            webpage, 'like count', fatal=False))
+        dislike_count = int_or_none(self._search_regex(
+            r'>\s*Dislike <b>\((\d+)\)</b>',
+            webpage, 'dislike count', fatal=False))
+
+        age_limit = self._rta_search(webpage)
 
         return {
             'id': video_id,
@@ -59,5 +75,7 @@ class XXXYMoviesIE(InfoExtractor):
             'categories': categories,
             'duration': duration,
             'view_count': view_count,
-            'age_limit': 18,
+            'like_count': like_count,
+            'dislike_count': dislike_count,
+            'age_limit': age_limit,
         }

From 355e41466d8edcb5b2457dfa0c5715ef87ff5832 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 29 Dec 2014 21:33:41 +0600
Subject: [PATCH 0959/1937] [hellporno] Extract all formats and improve

---
 youtube_dl/extractor/hellporno.py | 47 ++++++++++++++++++++++---------
 1 file changed, 33 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/hellporno.py b/youtube_dl/extractor/hellporno.py
index 754322cdf..7a1c75b65 100644
--- a/youtube_dl/extractor/hellporno.py
+++ b/youtube_dl/extractor/hellporno.py
@@ -1,5 +1,12 @@
 from __future__ import unicode_literals
+
+import re
+
 from .common import InfoExtractor
+from ..utils import (
+    js_to_json,
+    remove_end,
+)
 
 
 class HellPornoIE(InfoExtractor):
@@ -19,23 +26,36 @@ class HellPornoIE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
+
         webpage = self._download_webpage(url, display_id)
 
-        video_id = self._html_search_regex(
-            r'video_id:\s*\'([^\']+)\'', webpage, 'id')
+        title = remove_end(self._html_search_regex(
+            r'<title>([^<]+)</title>', webpage, 'title'), ' - Hell Porno')
 
-        ext = self._html_search_regex(
-            r'postfix:\s*\'([^\']+)\'', webpage, 'ext')[1:]
+        flashvars = self._parse_json(self._search_regex(
+            r'var\s+flashvars\s*=\s*({.+?});', webpage, 'flashvars'),
+            display_id, transform_source=js_to_json)
 
-        video_url = self._html_search_regex(
-            r'video_url:\s*\'([^\']+)\'', webpage, 'video_url')
+        video_id = flashvars.get('video_id')
+        thumbnail = flashvars.get('preview_url')
+        ext = flashvars.get('postfix', '.mp4')[1:]
 
-        title = self._html_search_regex(
-            r'<title>([^<]+)\s*-\s*Hell Porno</title>', webpage, 'title')
-
-        thumbnail = self._html_search_regex(
-            r'preview_url:\s*\'([^\']+)\'',
-            webpage, 'thumbnail', fatal=False)
+        formats = []
+        for video_url_key in ['video_url', 'video_alt_url']:
+            video_url = flashvars.get(video_url_key)
+            if not video_url:
+                continue
+            video_text = flashvars.get('%s_text' % video_url_key)
+            fmt = {
+                'url': video_url,
+                'ext': ext,
+                'format_id': video_text,
+            }
+            m = re.search(r'^(?P<height>\d+)[pP]', video_text)
+            if m:
+                fmt['height'] = int(m.group('height'))
+            formats.append(fmt)
+        self._sort_formats(formats)
 
         categories = self._html_search_meta(
             'keywords', webpage, 'categories', default='').split(',')
@@ -43,10 +63,9 @@ class HellPornoIE(InfoExtractor):
         return {
             'id': video_id,
             'display_id': display_id,
-            'url': video_url,
             'title': title,
-            'ext': ext,
             'thumbnail': thumbnail,
             'categories': categories,
             'age_limit': 18,
+            'formats': formats,
         }

From 429ddfd38dfb53aa05f0e2a77aeeb4ac5d20a17d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Dec 2014 01:50:28 +0600
Subject: [PATCH 0960/1937] [cnn] Add support for hln URL format (Closes #4595)

---
 youtube_dl/extractor/cnn.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 1bff005d6..93e8d0de3 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class CNNIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://(?:(?:edition|www)\.)?cnn\.com/video/(?:data/.+?|\?)/
-        (?P<path>.+?/(?P<title>[^/]+?)(?:\.cnn(?:-ap)?|(?=&)))'''
+        (?P<path>.+?/(?P<title>[^/]+?)(?:\.(?:cnn|hln)(?:-ap)?|(?=&)))'''
 
     _TESTS = [{
         'url': 'http://edition.cnn.com/video/?/video/sports/2013/06/09/nadal-1-on-1.cnn',
@@ -35,6 +35,16 @@ class CNNIE(InfoExtractor):
             "description": "A Georgia Tech student welcomes the incoming freshmen with an epic speech backed by music from \"2001: A Space Odyssey.\"",
             "upload_date": "20130821",
         }
+    }, {
+        'url': 'http://www.cnn.com/video/data/2.0/video/living/2014/12/22/growing-america-nashville-salemtown-board-episode-1.hln.html',
+        'md5': 'f14d02ebd264df951feb2400e2c25a1b',
+        'info_dict': {
+            'id': 'living/2014/12/22/growing-america-nashville-salemtown-board-episode-1.hln',
+            'ext': 'mp4',
+            'title': 'Nashville Ep. 1: Hand crafted skateboards',
+            'description': 'md5:e7223a503315c9f150acac52e76de086',
+            'upload_date': '20141222',
+        }
     }]
 
     def _real_extract(self, url):

From da3f7fb7f84e47de6aa0b29b16f78cb5bdf7d746 Mon Sep 17 00:00:00 2001
From: t0mm0 <dev@onairsoftware.co.uk>
Date: Sun, 28 Dec 2014 17:07:32 +0000
Subject: [PATCH 0961/1937] [hitbox] add extractor for hitbox vods

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/hitbox.py   | 104 +++++++++++++++++++++++++++++++
 2 files changed, 105 insertions(+)
 create mode 100644 youtube_dl/extractor/hitbox.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ea4faf2a6..3300dfeb4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -172,6 +172,7 @@ from .heise import HeiseIE
 from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
+from .hitbox import HitboxIE
 from .hornbunny import HornBunnyIE
 from .hostingbulk import HostingBulkIE
 from .hotnewhiphop import HotNewHipHopIE
diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
new file mode 100644
index 000000000..239da3cd4
--- /dev/null
+++ b/youtube_dl/extractor/hitbox.py
@@ -0,0 +1,104 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate,
+)
+
+
+class HitboxIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?hitbox\.tv/video/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://www.hitbox.tv/video/358062',
+        'info_dict': {
+            'id': '358062',
+            'title': 'Megaman',
+            'alt_title': 'Megaman',
+            'description': '',
+            'ext': 'mp4',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 3834,
+            'resolution': 'SD 480p',
+            'uploader_id': 'supergreatfriend',
+            'view_count': int,
+            'upload_date': '20141225',
+            'categories': [None],
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.hitbox.tv/video/203213',
+        'info_dict': {
+            'id': '203213',
+            'title': 'hitbox @ gamescom, Sub Button Hype extended, Giveaway - hitbox News Update with Oxy',
+            'alt_title': 'hitboxlive - Aug 9th #6',
+            'description': '',
+            'ext': 'mp4',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 215,
+            'resolution': 'HD 720p',
+            'uploader_id': 'hitboxlive',
+            'view_count': int,
+            'upload_date': '20140809',
+            'categories': ['Live Show'],
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        thumb_base = 'https://edge.sf.hitbox.tv'
+        metadata = self._download_json(
+            'https://www.hitbox.tv/api/media/video/%s' % (video_id), video_id
+        )
+
+        video_meta = metadata.get('video', [])[0]
+        title = video_meta.get('media_status')
+        alt_title = video_meta.get('media_title')
+        description = video_meta.get('media_description')
+        duration = int(float(video_meta.get('media_duration')))
+        uploader = video_meta.get('media_user_name')
+        views = int(video_meta.get('media_views'))
+        upload_date = unified_strdate(video_meta.get('media_date_added'))
+        categories = [video_meta.get('category_name')]
+        thumbs = [
+            {'url': thumb_base + video_meta.get('media_thumbnail'),
+             'width': 320,
+             'height': 180},
+            {'url': thumb_base + video_meta.get('media_thumbnail_large'),
+             'width': 768,
+             'height': 432},
+        ]
+
+        player_config = self._download_json(
+            'https://www.hitbox.tv/api/player/config/video/%s' % (video_id),
+            video_id
+        )
+
+        clip = player_config.get('clip')
+        video_url = clip.get('url')
+        res = clip.get('bitrates', [])[0].get('label')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'alt_title': alt_title,
+            'description': description,
+            'url': video_url,
+            'ext': 'mp4',
+            'thumbnails': thumbs,
+            'duration': duration,
+            'resolution': res,
+            'uploader_id': uploader,
+            'view_count': views,
+            'upload_date': upload_date,
+            'categories': categories,
+            'protocol': 'm3u8',
+        }

From e3947e2b7fe1cd81f841daa1c4dc2ca72af8aefe Mon Sep 17 00:00:00 2001
From: t0mm0 <dev@onairsoftware.co.uk>
Date: Mon, 29 Dec 2014 20:10:59 +0000
Subject: [PATCH 0962/1937] [hitbox] add support for live streams

---
 youtube_dl/extractor/__init__.py |   2 +-
 youtube_dl/extractor/hitbox.py   | 146 +++++++++++++++++++++----------
 2 files changed, 102 insertions(+), 46 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3300dfeb4..4f5a1ce18 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -172,7 +172,7 @@ from .heise import HeiseIE
 from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
-from .hitbox import HitboxIE
+from .hitbox import HitboxIE, HitboxLiveIE
 from .hornbunny import HornBunnyIE
 from .hostingbulk import HostingBulkIE
 from .hotnewhiphop import HotNewHipHopIE
diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index 239da3cd4..eab2749ec 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -1,5 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
+import re
 
 from .common import InfoExtractor
 from ..utils import (
@@ -9,33 +10,13 @@ from ..utils import (
 
 class HitboxIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?hitbox\.tv/video/(?P<id>[0-9]+)'
-    _TESTS = [{
-        'url': 'http://www.hitbox.tv/video/358062',
-        'info_dict': {
-            'id': '358062',
-            'title': 'Megaman',
-            'alt_title': 'Megaman',
-            'description': '',
-            'ext': 'mp4',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'duration': 3834,
-            'resolution': 'SD 480p',
-            'uploader_id': 'supergreatfriend',
-            'view_count': int,
-            'upload_date': '20141225',
-            'categories': [None],
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }, {
+    _TEST = {
         'url': 'http://www.hitbox.tv/video/203213',
         'info_dict': {
             'id': '203213',
             'title': 'hitbox @ gamescom, Sub Button Hype extended, Giveaway - hitbox News Update with Oxy',
             'alt_title': 'hitboxlive - Aug 9th #6',
-            'description': '',
+            'description': '\n',
             'ext': 'mp4',
             'thumbnail': 're:^https?://.*\.jpg$',
             'duration': 215,
@@ -49,24 +30,28 @@ class HitboxIE(InfoExtractor):
             # m3u8 download
             'skip_download': True,
         },
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
+    }
 
+    def _extract_metadata(self, url, video_id):
         thumb_base = 'https://edge.sf.hitbox.tv'
         metadata = self._download_json(
-            'https://www.hitbox.tv/api/media/video/%s' % (video_id), video_id
+            '%s/%s' % (url, video_id), video_id
         )
 
-        video_meta = metadata.get('video', [])[0]
+        date = 'media_live_since'
+        media_type = 'livestream'
+        if metadata.get('media_type') == 'video':
+            media_type = 'video'
+            date = 'media_date_added'
+
+        video_meta = metadata.get(media_type, [])[0]
         title = video_meta.get('media_status')
         alt_title = video_meta.get('media_title')
-        description = video_meta.get('media_description')
+        description = video_meta.get('media_description_md')
         duration = int(float(video_meta.get('media_duration')))
         uploader = video_meta.get('media_user_name')
         views = int(video_meta.get('media_views'))
-        upload_date = unified_strdate(video_meta.get('media_date_added'))
+        upload_date = unified_strdate(video_meta.get(date))
         categories = [video_meta.get('category_name')]
         thumbs = [
             {'url': thumb_base + video_meta.get('media_thumbnail'),
@@ -77,6 +62,28 @@ class HitboxIE(InfoExtractor):
              'height': 432},
         ]
 
+        return {
+            'id': video_id,
+            'title': title,
+            'alt_title': alt_title,
+            'description': description,
+            'ext': 'mp4',
+            'thumbnails': thumbs,
+            'duration': duration,
+            'uploader_id': uploader,
+            'view_count': views,
+            'upload_date': upload_date,
+            'categories': categories,
+        }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        metadata = self._extract_metadata(
+            'https://www.hitbox.tv/api/media/video',
+            video_id
+        )
+
         player_config = self._download_json(
             'https://www.hitbox.tv/api/player/config/video/%s' % (video_id),
             video_id
@@ -86,19 +93,68 @@ class HitboxIE(InfoExtractor):
         video_url = clip.get('url')
         res = clip.get('bitrates', [])[0].get('label')
 
-        return {
-            'id': video_id,
-            'title': title,
-            'alt_title': alt_title,
-            'description': description,
-            'url': video_url,
+        metadata['resolution'] = res
+        metadata['url'] = video_url
+        metadata['protocol'] = 'm3u8'
+
+        return metadata
+
+
+class HitboxLiveIE(HitboxIE):
+    _VALID_URL = r'https?://(?:www\.)?hitbox\.tv/(?!video)(?P<id>.+)'
+    _TEST = {
+        'url': 'http://www.hitbox.tv/dimak',
+        'info_dict': {
+            'id': 'dimak',
             'ext': 'mp4',
-            'thumbnails': thumbs,
-            'duration': duration,
-            'resolution': res,
-            'uploader_id': uploader,
-            'view_count': views,
-            'upload_date': upload_date,
-            'categories': categories,
-            'protocol': 'm3u8',
-        }
+            'description': str,
+            'upload_date': str,
+            'title': str,
+            'uploader_id': 'Dimak',
+        },
+        'params': {
+            # live
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        metadata = self._extract_metadata(
+            'https://www.hitbox.tv/api/media/live',
+            video_id
+        )
+
+        player_config = self._download_json(
+            'https://www.hitbox.tv/api/player/config/live/%s' % (video_id),
+            video_id
+        )
+
+        formats = []
+        cdns = player_config.get('cdns')
+        servers = []
+        for cdn in cdns:
+            base_url = cdn.get('netConnectionUrl')
+            host = re.search('.+\.([^\.]+\.[^\./]+)/.+', base_url).group(1)
+            if base_url not in servers:
+                servers.append(base_url)
+                for stream in cdn.get('bitrates'):
+                    label = stream.get('label')
+                    if label != 'Auto':
+                        formats.append({
+                            'url': '%s/%s' % (base_url, stream.get('url')),
+                            'ext': 'mp4',
+                            'vbr': stream.get('bitrate'),
+                            'resolution': label,
+                            'rtmp_live': True,
+                            'format_note': host,
+                            'page_url': url,
+                            'player_url': 'http://www.hitbox.tv/static/player/flowplayer/flowplayer.commercial-3.2.16.swf',
+                        })
+
+        self._sort_formats(formats)
+        metadata['formats'] = formats
+        metadata['is_live'] = True
+        metadata['title'] = self._live_title(metadata.get('title'))
+        return metadata

From 0c0a70f4c6839903b326d7d9074e93235defaa5b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Dec 2014 03:22:07 +0600
Subject: [PATCH 0963/1937] [hitbox] Minor changes

---
 youtube_dl/extractor/hitbox.py | 58 +++++++++++++++++++---------------
 1 file changed, 32 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/hitbox.py b/youtube_dl/extractor/hitbox.py
index eab2749ec..84bd7c080 100644
--- a/youtube_dl/extractor/hitbox.py
+++ b/youtube_dl/extractor/hitbox.py
@@ -1,14 +1,20 @@
 # coding: utf-8
 from __future__ import unicode_literals
+
 import re
 
 from .common import InfoExtractor
 from ..utils import (
-    unified_strdate,
+    clean_html,
+    parse_iso8601,
+    float_or_none,
+    int_or_none,
+    compat_str,
 )
 
 
 class HitboxIE(InfoExtractor):
+    IE_NAME = 'hitbox'
     _VALID_URL = r'https?://(?:www\.)?hitbox\.tv/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.hitbox.tv/video/203213',
@@ -16,13 +22,14 @@ class HitboxIE(InfoExtractor):
             'id': '203213',
             'title': 'hitbox @ gamescom, Sub Button Hype extended, Giveaway - hitbox News Update with Oxy',
             'alt_title': 'hitboxlive - Aug 9th #6',
-            'description': '\n',
+            'description': '',
             'ext': 'mp4',
             'thumbnail': 're:^https?://.*\.jpg$',
-            'duration': 215,
+            'duration': 215.1666,
             'resolution': 'HD 720p',
-            'uploader_id': 'hitboxlive',
+            'uploader': 'hitboxlive',
             'view_count': int,
+            'timestamp': 1407576133,
             'upload_date': '20140809',
             'categories': ['Live Show'],
         },
@@ -35,8 +42,7 @@ class HitboxIE(InfoExtractor):
     def _extract_metadata(self, url, video_id):
         thumb_base = 'https://edge.sf.hitbox.tv'
         metadata = self._download_json(
-            '%s/%s' % (url, video_id), video_id
-        )
+            '%s/%s' % (url, video_id), video_id)
 
         date = 'media_live_since'
         media_type = 'livestream'
@@ -47,11 +53,13 @@ class HitboxIE(InfoExtractor):
         video_meta = metadata.get(media_type, [])[0]
         title = video_meta.get('media_status')
         alt_title = video_meta.get('media_title')
-        description = video_meta.get('media_description_md')
-        duration = int(float(video_meta.get('media_duration')))
+        description = clean_html(
+            video_meta.get('media_description') or
+            video_meta.get('media_description_md'))
+        duration = float_or_none(video_meta.get('media_duration'))
         uploader = video_meta.get('media_user_name')
-        views = int(video_meta.get('media_views'))
-        upload_date = unified_strdate(video_meta.get(date))
+        views = int_or_none(video_meta.get('media_views'))
+        timestamp = parse_iso8601(video_meta.get(date), ' ')
         categories = [video_meta.get('category_name')]
         thumbs = [
             {'url': thumb_base + video_meta.get('media_thumbnail'),
@@ -70,9 +78,9 @@ class HitboxIE(InfoExtractor):
             'ext': 'mp4',
             'thumbnails': thumbs,
             'duration': duration,
-            'uploader_id': uploader,
+            'uploader': uploader,
             'view_count': views,
-            'upload_date': upload_date,
+            'timestamp': timestamp,
             'categories': categories,
         }
 
@@ -81,13 +89,11 @@ class HitboxIE(InfoExtractor):
 
         metadata = self._extract_metadata(
             'https://www.hitbox.tv/api/media/video',
-            video_id
-        )
+            video_id)
 
         player_config = self._download_json(
-            'https://www.hitbox.tv/api/player/config/video/%s' % (video_id),
-            video_id
-        )
+            'https://www.hitbox.tv/api/player/config/video/%s' % video_id,
+            video_id)
 
         clip = player_config.get('clip')
         video_url = clip.get('url')
@@ -101,16 +107,18 @@ class HitboxIE(InfoExtractor):
 
 
 class HitboxLiveIE(HitboxIE):
+    IE_NAME = 'hitbox:live'
     _VALID_URL = r'https?://(?:www\.)?hitbox\.tv/(?!video)(?P<id>.+)'
     _TEST = {
         'url': 'http://www.hitbox.tv/dimak',
         'info_dict': {
             'id': 'dimak',
             'ext': 'mp4',
-            'description': str,
-            'upload_date': str,
-            'title': str,
-            'uploader_id': 'Dimak',
+            'description': 'md5:c9f80fa4410bc588d7faa40003fc7d0e',
+            'timestamp': int,
+            'upload_date': compat_str,
+            'title': compat_str,
+            'uploader': 'Dimak',
         },
         'params': {
             # live
@@ -123,13 +131,11 @@ class HitboxLiveIE(HitboxIE):
 
         metadata = self._extract_metadata(
             'https://www.hitbox.tv/api/media/live',
-            video_id
-        )
+            video_id)
 
         player_config = self._download_json(
-            'https://www.hitbox.tv/api/player/config/live/%s' % (video_id),
-            video_id
-        )
+            'https://www.hitbox.tv/api/player/config/live/%s' % video_id,
+            video_id)
 
         formats = []
         cdns = player_config.get('cdns')

From beb95e778170895c1435f55b71e3d76cbd9d8bcf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ce=CC=81dric=20Luthi?= <cedric.luthi@gmail.com>
Date: Mon, 29 Dec 2014 22:58:14 +0100
Subject: [PATCH 0964/1937] [youtube] Fix videos with age gate and encrypted
 signatures

The `sts` value is available on the embed webpage, get it from there.

Fixes #4108.
---
 youtube_dl/extractor/youtube.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 550e18733..175e43272 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -766,11 +766,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             age_gate = True
             # We simulate the access to the video from www.youtube.com/v/{video_id}
             # this can be viewed without login into Youtube
+            url = proto + '://www.youtube.com/embed/%s' % video_id
+            embed_webpage = self._download_webpage(url, video_id, 'Downloading embed webpage')
             data = compat_urllib_parse.urlencode({
                 'video_id': video_id,
                 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
                 'sts': self._search_regex(
-                    r'"sts"\s*:\s*(\d+)', video_webpage, 'sts', default=''),
+                    r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
             })
             video_info_url = proto + '://www.youtube.com/get_video_info?' + data
             video_info_webpage = self._download_webpage(
@@ -968,11 +970,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 elif 's' in url_data:
                     encrypted_sig = url_data['s'][0]
 
-                    if not age_gate:
-                        jsplayer_url_json = self._search_regex(
-                            r'"assets":.+?"js":\s*("[^"]+")',
-                            video_webpage, 'JS player URL')
-                        player_url = json.loads(jsplayer_url_json)
+                    jsplayer_url_json = self._search_regex(
+                        r'"assets":.+?"js":\s*("[^"]+")',
+                        embed_webpage if age_gate else video_webpage, 'JS player URL')
+                    player_url = json.loads(jsplayer_url_json)
                     if player_url is None:
                         player_url_json = self._search_regex(
                             r'ytplayer\.config.*?"url"\s*:\s*("[^"]+")',

From a87bb090d95bdfe02429c37a2a10a4733ae69fc6 Mon Sep 17 00:00:00 2001
From: t0mm0 <dev@onairsoftware.co.uk>
Date: Mon, 29 Dec 2014 23:06:56 +0000
Subject: [PATCH 0965/1937] [daum] update 'full id' regex

fixes #4566
---
 youtube_dl/extractor/daum.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index c6b813f58..b4b3b795c 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -38,7 +38,7 @@ class DaumIE(InfoExtractor):
         canonical_url = 'http://tvpot.daum.net/v/%s' % video_id
         webpage = self._download_webpage(canonical_url, video_id)
         full_id = self._search_regex(
-            r'<iframe src="http://videofarm.daum.net/controller/video/viewer/Video.html\?.*?vid=(.+?)[&"]',
+            r'src=["\']http://videofarm.daum.net/controller/video/viewer/Video.html\?.*?vid=(.+?)[&"\']',
             webpage, 'full id')
         query = compat_urllib_parse.urlencode({'vid': full_id})
         info = self._download_xml(

From 1c57e7f1f496a6f1fad43c0b193ce534fc9ce67b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Dec 2014 16:55:53 +0600
Subject: [PATCH 0966/1937] [daum] Improve full_id regex

---
 youtube_dl/extractor/daum.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py
index b4b3b795c..934da765e 100644
--- a/youtube_dl/extractor/daum.py
+++ b/youtube_dl/extractor/daum.py
@@ -38,7 +38,7 @@ class DaumIE(InfoExtractor):
         canonical_url = 'http://tvpot.daum.net/v/%s' % video_id
         webpage = self._download_webpage(canonical_url, video_id)
         full_id = self._search_regex(
-            r'src=["\']http://videofarm.daum.net/controller/video/viewer/Video.html\?.*?vid=(.+?)[&"\']',
+            r'src=["\']http://videofarm\.daum\.net/controller/video/viewer/Video\.html\?.*?vid=(.+?)[&"\']',
             webpage, 'full id')
         query = compat_urllib_parse.urlencode({'vid': full_id})
         info = self._download_xml(

From 3ee08848dbc2314df39e6931c75920b406733868 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Dec 2014 17:12:12 +0600
Subject: [PATCH 0967/1937] Credit @0xced for #4598

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index bb4d8b4d1..29ce9e3e4 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -96,3 +96,4 @@ Mathias Rav
 Petr Kutalek
 Will Glynn
 Max Reimann
+Cédric Luthi

From fccae2b911970d0ffa97800b27e70b1937cd3058 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Dec 2014 17:26:21 +0600
Subject: [PATCH 0968/1937] [youtube] Add test for age-gate video with
 encrypted signature

---
 youtube_dl/extractor/youtube.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 175e43272..3da83e3a8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -418,6 +418,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'upload_date': '20140605',
             },
         },
+        # Age-gate video with encrypted signature
+        {
+            'url': 'http://www.youtube.com/watch?v=6kLq3WMV1nU',
+            'info_dict': {
+                'id': '6kLq3WMV1nU',
+                'ext': 'mp4',
+                'title': 'Dedication To My Ex (Miss That) (Lyric Video)',
+                'description': 'md5:33765bb339e1b47e7e72b5490139bb41',
+                'uploader': 'LloydVEVO',
+                'uploader_id': 'LloydVEVO',
+                'upload_date': '20110629',
+            },
+        },
         # video_info is None (https://github.com/rg3/youtube-dl/issues/4421)
         {
             'url': '__2ABJjxzNo',

From a349873226c873a2f3bea9f5dffe167053cb1666 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Dec 2014 22:28:07 +0600
Subject: [PATCH 0969/1937] [atresplayer] Add extractor (Closes #2341)

---
 youtube_dl/extractor/__init__.py    |   1 +
 youtube_dl/extractor/atresplayer.py | 111 ++++++++++++++++++++++++++++
 2 files changed, 112 insertions(+)
 create mode 100644 youtube_dl/extractor/atresplayer.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4f5a1ce18..c15786ad7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -25,6 +25,7 @@ from .arte import (
     ArteTVDDCIE,
     ArteTVEmbedIE,
 )
+from .atresplayer import AtresPlayerIE
 from .audiomack import AudiomackIE
 from .auengine import AUEngineIE
 from .azubu import AzubuIE
diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
new file mode 100644
index 000000000..7e987b2a0
--- /dev/null
+++ b/youtube_dl/extractor/atresplayer.py
@@ -0,0 +1,111 @@
+from __future__ import unicode_literals
+
+import time
+import hmac
+
+from .common import InfoExtractor
+from ..utils import (
+    compat_str,
+    compat_urllib_request,
+    int_or_none,
+    float_or_none,
+    xpath_text,
+    ExtractorError,
+)
+
+
+class AtresPlayerIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?atresplayer\.com/television/[^/]+/[^/]+/[^/]+/(?P<id>.+?)_\d+\.html'
+    _TESTS = [
+        {
+            'url': 'http://www.atresplayer.com/television/programas/el-club-de-la-comedia/temporada-4/capitulo-10-especial-solidario-nochebuena_2014122100174.html',
+            'md5': 'efd56753cda1bb64df52a3074f62e38a',
+            'info_dict': {
+                'id': 'capitulo-10-especial-solidario-nochebuena',
+                'ext': 'mp4',
+                'title': 'Especial Solidario de Nochebuena',
+                'description': 'md5:e2d52ff12214fa937107d21064075bf1',
+                'duration': 5527.6,
+                'thumbnail': 're:^https?://.*\.jpg$',
+            },
+        },
+        {
+            'url': 'http://www.atresplayer.com/television/series/el-secreto-de-puente-viejo/el-chico-de-los-tres-lunares/capitulo-977-29-12-14_2014122400174.html',
+            'only_matching': True,
+        },
+    ]
+
+    _USER_AGENT = 'Dalvik/1.6.0 (Linux; U; Android 4.3; GT-I9300 Build/JSS15J'
+    _MAGIC = 'QWtMLXs414Yo+c#_+Q#K@NN)'
+    _TIMESTAMP_SHIFT = 30000
+
+    _TIME_API_URL = 'http://servicios.atresplayer.com/api/admin/time.json'
+    _URL_VIDEO_TEMPLATE = 'https://servicios.atresplayer.com/api/urlVideo/{1}/{0}/{1}|{2}|{3}.json'
+    _PLAYER_URL_TEMPLATE = 'https://servicios.atresplayer.com/episode/getplayer.json?episodePk=%s'
+    _EPISODE_URL_TEMPLATE = 'http://www.atresplayer.com/episodexml/%s'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        episode_id = self._search_regex(
+            r'episode="([^"]+)"', webpage, 'episode id')
+
+        timestamp = int_or_none(self._download_webpage(
+            self._TIME_API_URL,
+            video_id, 'Downloading timestamp', fatal=False), 1000, time.time())
+        timestamp_shifted = compat_str(timestamp + self._TIMESTAMP_SHIFT)
+        token = hmac.new(self._MAGIC.encode('utf-8'), episode_id + timestamp_shifted).hexdigest()
+
+        formats = []
+        for fmt in ['windows', 'android_tablet']:
+            request = compat_urllib_request.Request(
+                self._URL_VIDEO_TEMPLATE.format(fmt, episode_id, timestamp_shifted, token))
+            request.add_header('Youtubedl-user-agent', self._USER_AGENT)
+
+            fmt_json = self._download_json(
+                request, video_id, 'Downloading %s video JSON' % fmt)
+
+            result = fmt_json.get('resultDes')
+            if result.lower() != 'ok':
+                raise ExtractorError(
+                    '%s returned error: %s' % (self.IE_NAME, result), expected=True)
+
+            for _, video_url in fmt_json['resultObject'].items():
+                if video_url.endswith('/Manifest'):
+                    formats.extend(self._extract_f4m_formats(video_url[:-9] + '/manifest.f4m', video_id))
+                else:
+                    formats.append({
+                        'url': video_url,
+                        'format_id': 'android',
+                        'preference': 1,
+                    })
+        self._sort_formats(formats)
+
+        player = self._download_json(
+            self._PLAYER_URL_TEMPLATE % episode_id,
+            episode_id)
+
+        path_data = player.get('pathData')
+
+        episode = self._download_xml(
+            self._EPISODE_URL_TEMPLATE % path_data,
+            video_id, 'Downloading episode XML')
+
+        duration = float_or_none(xpath_text(
+            episode, './media/asset/info/technical/contentDuration', 'duration'))
+
+        art = episode.find('./media/asset/info/art')
+        title = xpath_text(art, './name', 'title')
+        description = xpath_text(art, './description', 'description')
+        thumbnail = xpath_text(episode, './media/asset/files/background', 'thumbnail')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
+        }

From e83eebb12f984c1614204e53c09dc5124b52b45c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Dec 2014 22:46:23 +0600
Subject: [PATCH 0970/1937] [atresplayer] Fix python3 bug

---
 youtube_dl/extractor/atresplayer.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 7e987b2a0..72e83bfc2 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -56,7 +56,10 @@ class AtresPlayerIE(InfoExtractor):
             self._TIME_API_URL,
             video_id, 'Downloading timestamp', fatal=False), 1000, time.time())
         timestamp_shifted = compat_str(timestamp + self._TIMESTAMP_SHIFT)
-        token = hmac.new(self._MAGIC.encode('utf-8'), episode_id + timestamp_shifted).hexdigest()
+        token = hmac.new(
+            self._MAGIC.encode('ascii'),
+            (episode_id + timestamp_shifted).encode('utf-8')
+        ).hexdigest()
 
         formats = []
         for fmt in ['windows', 'android_tablet']:

From 416c7fcbce86324587afae11414c71ff603ad296 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 30 Dec 2014 19:35:35 +0100
Subject: [PATCH 0971/1937] Add documentation about supported sites (Fixes
 #4503)

---
 Makefile                          |   7 +-
 README.md                         |   8 +
 devscripts/make_supportedsites.py |  45 +++
 docs/supportedsites.md            | 500 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/generic.py   |   3 +-
 youtube_dl/utils.py               |   7 +
 6 files changed, 567 insertions(+), 3 deletions(-)
 create mode 100644 devscripts/make_supportedsites.py
 create mode 100644 docs/supportedsites.md

diff --git a/Makefile b/Makefile
index 71c8553d8..71470eedb 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
+all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
 	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json CONTRIBUTING.md.tmp
@@ -50,7 +50,7 @@ offlinetest: codetest
 
 tar: youtube-dl.tar.gz
 
-.PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion ot offlinetest codetest
+.PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion ot offlinetest codetest supportedsites
 
 pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 youtube-dl.fish
 
@@ -68,6 +68,9 @@ README.md: youtube_dl/*.py youtube_dl/*/*.py
 CONTRIBUTING.md: README.md
 	python devscripts/make_contributing.py README.md CONTRIBUTING.md
 
+supportedsites:
+	python devscripts/make_supportedsites.py docs/supportedsites.md
+
 README.txt: README.md
 	pandoc -f markdown -t plain README.md -o README.txt
 
diff --git a/README.md b/README.md
index b4afe17d2..93a2750c2 100644
--- a/README.md
+++ b/README.md
@@ -449,6 +449,14 @@ Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unz
 
 To run the exe you need to install first the [Microsoft Visual C++ 2008 Redistributable Package](http://www.microsoft.com/en-us/download/details.aspx?id=29).
 
+### How can I detect whether a given URL is supported by youtube-dl?
+
+For one, have a look at the [list of supported sites](docs/supportedsites). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
+
+It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which maches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
+
+If you want to find out, simply call youtube-dl. If you get no videos back, chances are the URL is either not referring to a video or unsupported. You can find out which by examining the output (if you run youtube-dl on the console) or catching an `UnsupportedError` exception if you run it from a Python program.
+
 # DEVELOPER INSTRUCTIONS
 
 Most users do not need to build youtube-dl and can [download the builds](http://rg3.github.io/youtube-dl/download.html) or get them from their distribution.
diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py
new file mode 100644
index 000000000..140010644
--- /dev/null
+++ b/devscripts/make_supportedsites.py
@@ -0,0 +1,45 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+import io
+import optparse
+import os
+import sys
+
+
+# Import youtube_dl
+ROOT_DIR = os.path.join(os.path.dirname(__file__), '..')
+sys.path.append(ROOT_DIR)
+import youtube_dl
+
+
+def main():
+    parser = optparse.OptionParser(usage='%prog OUTFILE.md')
+    options, args = parser.parse_args()
+    if len(args) != 1:
+        parser.error('Expected an output filename')
+
+    outfile, = args
+
+    def gen_ies_md(ies):
+        for ie in ies:
+            ie_md = '**{}**'.format(ie.IE_NAME)
+            ie_desc = getattr(ie, 'IE_DESC', None)
+            if ie_desc is False:
+                continue
+            if ie_desc is not None:
+                ie_md += ': {}'.format(ie.IE_DESC)
+            if not ie.working():
+                ie_md += ' (Currently broken)'
+            yield ie_md
+
+    ies = sorted(youtube_dl.gen_extractors(), key=lambda i: i.IE_NAME.lower())
+    out = '# Supported sites\n' + ''.join(
+        ' - ' + md + '\n'
+        for md in gen_ies_md(ies))
+
+    with io.open(outfile, 'w', encoding='utf-8') as outf:
+        outf.write(out)
+
+if __name__ == '__main__':
+    main()
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
new file mode 100644
index 000000000..dbbf4a797
--- /dev/null
+++ b/docs/supportedsites.md
@@ -0,0 +1,500 @@
+# Supported sites
+ - **1up.com**
+ - **220.ro**
+ - **24video**
+ - **3sat**
+ - **4tube**
+ - **56.com**
+ - **5min**
+ - **8tracks**
+ - **9gag**
+ - **abc.net.au**
+ - **AcademicEarth:Course**
+ - **AddAnime**
+ - **AdobeTV**
+ - **AdultSwim**
+ - **Aftonbladet**
+ - **AlJazeera**
+ - **Allocine**
+ - **anitube.se**
+ - **AnySex**
+ - **Aparat**
+ - **AppleTrailers**
+ - **archive.org**: archive.org videos
+ - **ARD**
+ - **ARD:mediathek**
+ - **arte.tv**
+ - **arte.tv:+7**
+ - **arte.tv:concert**
+ - **arte.tv:creative**
+ - **arte.tv:ddc**
+ - **arte.tv:embed**
+ - **arte.tv:future**
+ - **audiomack**
+ - **AUEngine**
+ - **Azubu**
+ - **bambuser**
+ - **bambuser:channel**
+ - **Bandcamp**
+ - **Bandcamp:album**
+ - **bbc.co.uk**: BBC iPlayer
+ - **Beeg**
+ - **BehindKink**
+ - **Bet**
+ - **Bild**: Bild.de
+ - **BiliBili**
+ - **blinkx**
+ - **blip.tv:user**
+ - **BlipTV**
+ - **Bloomberg**
+ - **Bpb**: Bundeszentrale für politische Bildung
+ - **BR**: Bayerischer Rundfunk Mediathek
+ - **Break**
+ - **Brightcove**
+ - **BuzzFeed**
+ - **BYUtv**
+ - **Canal13cl**
+ - **canalc2.tv**
+ - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
+ - **CBS**
+ - **CBSNews**: CBS News
+ - **CeskaTelevize**
+ - **channel9**: Channel 9
+ - **Chilloutzone**
+ - **Cinchcast**
+ - **Cinemassacre**
+ - **clipfish**
+ - **cliphunter**
+ - **Clipsyndicate**
+ - **Cloudy**
+ - **Clubic**
+ - **cmt.com**
+ - **CNET**
+ - **CNN**
+ - **CNNBlogs**
+ - **CollegeHumor**
+ - **ComCarCoff**
+ - **ComedyCentral**
+ - **ComedyCentralShows**: The Daily Show / The Colbert Report
+ - **CondeNast**: Condé Nast media group: Condé Nast, GQ, Glamour, Vanity Fair, Vogue, W Magazine, WIRED
+ - **Cracked**
+ - **Criterion**
+ - **Crunchyroll**
+ - **crunchyroll:playlist**
+ - **CSpan**: C-SPAN
+ - **culturebox.francetvinfo.fr**
+ - **dailymotion**
+ - **dailymotion:playlist**
+ - **dailymotion:user**
+ - **daum.net**
+ - **DBTV**
+ - **DeezerPlaylist**
+ - **defense.gouv.fr**
+ - **Discovery**
+ - **divxstage**: DivxStage
+ - **Dotsub**
+ - **Dropbox**
+ - **DrTuber**
+ - **DRTV**
+ - **Dump**
+ - **dvtv**: http://video.aktualne.cz/
+ - **EbaumsWorld**
+ - **eHow**
+ - **Einthusan**
+ - **eitb.tv**
+ - **EllenTV**
+ - **EllenTV:clips**
+ - **ElPais**: El País
+ - **EMPFlix**
+ - **Engadget**
+ - **Eporner**
+ - **Escapist**
+ - **EveryonesMixtape**
+ - **exfm**: ex.fm
+ - **ExpoTV**
+ - **ExtremeTube**
+ - **facebook**
+ - **faz.net**
+ - **fc2**
+ - **fernsehkritik.tv**
+ - **fernsehkritik.tv:postecke**
+ - **Firedrive**
+ - **Firstpost**
+ - **firsttv**: Видеоархив - Первый канал
+ - **Flickr**
+ - **Folketinget**: Folketinget (ft.dk; Danish parliament)
+ - **Foxgay**
+ - **FoxNews**
+ - **france2.fr:generation-quoi**
+ - **FranceCulture**
+ - **FranceInter**
+ - **francetv**: France 2, 3, 4, 5 and Ô
+ - **francetvinfo.fr**
+ - **Freesound**
+ - **freespeech.org**
+ - **FreeVideo**
+ - **FunnyOrDie**
+ - **Gamekings**
+ - **GameOne**
+ - **gameone:playlist**
+ - **GameSpot**
+ - **GameStar**
+ - **Gametrailers**
+ - **GDCVault**
+ - **generic**: Generic downloader that works on some sites
+ - **GiantBomb**
+ - **Glide**: Glide mobile video messages (glide.me)
+ - **Globo**
+ - **GodTube**
+ - **GoldenMoustache**
+ - **Golem**
+ - **GorillaVid**: GorillaVid.in, daclips.in, movpod.in and fastvideo.in
+ - **Goshgay**
+ - **Grooveshark**
+ - **Groupon**
+ - **Hark**
+ - **Heise**
+ - **Helsinki**: helsinki.fi
+ - **HentaiStigma**
+ - **HornBunny**
+ - **HostingBulk**
+ - **HotNewHipHop**
+ - **Howcast**
+ - **HowStuffWorks**
+ - **HuffPost**: Huffington Post
+ - **Hypem**
+ - **Iconosquare**
+ - **ign.com**
+ - **imdb**: Internet Movie Database trailers
+ - **imdb:list**: Internet Movie Database lists
+ - **Ina**
+ - **InfoQ**
+ - **Instagram**
+ - **instagram:user**: Instagram user profile
+ - **InternetVideoArchive**
+ - **IPrima**
+ - **ivi**: ivi.ru
+ - **ivi:compilation**: ivi.ru compilations
+ - **Izlesene**
+ - **JadoreCettePub**
+ - **JeuxVideo**
+ - **Jove**
+ - **jpopsuki.tv**
+ - **Jukebox**
+ - **Kankan**
+ - **keek**
+ - **KeezMovies**
+ - **KhanAcademy**
+ - **KickStarter**
+ - **kontrtube**: KontrTube.ru - Труба зовёт
+ - **KrasView**: Красвью
+ - **Ku6**
+ - **la7.tv**
+ - **Laola1Tv**
+ - **lifenews**: LIFE | NEWS
+ - **LiveLeak**
+ - **livestream**
+ - **livestream:original**
+ - **lrt.lt**
+ - **lynda**: lynda.com videos
+ - **lynda:course**: lynda.com online courses
+ - **m6**
+ - **macgamestore**: MacGameStore trailers
+ - **mailru**: Видео@Mail.Ru
+ - **Malemotion**
+ - **MDR**
+ - **metacafe**
+ - **Metacritic**
+ - **Mgoon**
+ - **Minhateca**
+ - **MinistryGrid**
+ - **mitele.es**
+ - **mixcloud**
+ - **MLB**
+ - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
+ - **Mofosex**
+ - **Mojvideo**
+ - **Moniker**: allmyvideos.net and vidspot.net
+ - **mooshare**: Mooshare.biz
+ - **Morningstar**: morningstar.com
+ - **Motherless**
+ - **Motorsport**: motorsport.com
+ - **MovieClips**
+ - **Moviezine**
+ - **movshare**: MovShare
+ - **MPORA**
+ - **MTV**
+ - **mtviggy.com**
+ - **mtvservices:embedded**
+ - **MuenchenTV**: münchen.tv
+ - **MusicPlayOn**
+ - **MusicVault**
+ - **muzu.tv**
+ - **MySpace**
+ - **MySpace:album**
+ - **MySpass**
+ - **myvideo**
+ - **MyVidster**
+ - **Naver**
+ - **NBA**
+ - **NBC**
+ - **NBCNews**
+ - **ndr**: NDR.de - Mediathek
+ - **NDTV**
+ - **NerdCubedFeed**
+ - **Newgrounds**
+ - **Newstube**
+ - **nfb**: National Film Board of Canada
+ - **nfl.com**
+ - **nhl.com**
+ - **nhl.com:videocenter**: NHL videocenter category
+ - **niconico**: ニコニコ動画
+ - **NiconicoPlaylist**
+ - **Noco**
+ - **Normalboots**
+ - **NosVideo**
+ - **novamov**: NovaMov
+ - **Nowness**
+ - **nowvideo**: NowVideo
+ - **npo.nl**
+ - **NRK**
+ - **NRKTV**
+ - **NTV**
+ - **Nuvid**
+ - **NYTimes**
+ - **ocw.mit.edu**
+ - **OktoberfestTV**
+ - **on.aol.com**
+ - **Ooyala**
+ - **orf:oe1**: Radio Österreich 1
+ - **orf:tvthek**: ORF TVthek
+ - **ORFFM4**: radio FM4
+ - **parliamentlive.tv**: UK parliament videos
+ - **Patreon**
+ - **PBS**
+ - **Phoenix**
+ - **Photobucket**
+ - **PlanetaPlay**
+ - **play.fm**
+ - **played.to**
+ - **Playvid**
+ - **plus.google**: Google Plus
+ - **pluzz.francetv.fr**
+ - **podomatic**
+ - **PornHd**
+ - **PornHub**
+ - **Pornotube**
+ - **PornoXO**
+ - **PromptFile**
+ - **prosiebensat1**: ProSiebenSat.1 Digital
+ - **Pyvideo**
+ - **QuickVid**
+ - **radio.de**
+ - **radiofrance**
+ - **Rai**
+ - **RBMARadio**
+ - **RedTube**
+ - **Restudy**
+ - **ReverbNation**
+ - **RingTV**
+ - **RottenTomatoes**
+ - **Roxwel**
+ - **RTBF**
+ - **RTLnow**
+ - **rtlxl.nl**
+ - **RTP**
+ - **RTS**: RTS.ch
+ - **rtve.es:alacarta**: RTVE a la carta
+ - **rtve.es:live**: RTVE.es live streams
+ - **RUHD**
+ - **rutube**: Rutube videos
+ - **rutube:channel**: Rutube channels
+ - **rutube:movie**: Rutube movies
+ - **rutube:person**: Rutube person videos
+ - **RUTV**: RUTV.RU
+ - **Sapo**: SAPO Vídeos
+ - **savefrom.net**
+ - **SBS**: sbs.com.au
+ - **SciVee**
+ - **screen.yahoo:search**: Yahoo screen search
+ - **Screencast**
+ - **ScreencastOMatic**
+ - **ScreenwaveMedia**
+ - **ServingSys**
+ - **Sexu**
+ - **SexyKarma**: Sexy Karma and Watch Indian Porn
+ - **Shared**
+ - **ShareSix**
+ - **Sina**
+ - **Slideshare**
+ - **Slutload**
+ - **smotri**: Smotri.com
+ - **smotri:broadcast**: Smotri.com broadcasts
+ - **smotri:community**: Smotri.com community videos
+ - **smotri:user**: Smotri.com user videos
+ - **Snotr**
+ - **Sockshare**
+ - **Sohu**
+ - **soundcloud**
+ - **soundcloud:playlist**
+ - **soundcloud:set**
+ - **soundcloud:user**
+ - **Soundgasm**
+ - **southpark.cc.com**
+ - **southpark.de**
+ - **Space**
+ - **Spankwire**
+ - **Spiegel**
+ - **Spiegel:Article**: Articles on spiegel.de
+ - **Spiegeltv**
+ - **Spike**
+ - **Sport5**
+ - **SportBox**
+ - **SportDeutschland**
+ - **SRMediathek**: Süddeutscher Rundfunk
+ - **stanfordoc**: Stanford Open ClassRoom
+ - **Steam**
+ - **streamcloud.eu**
+ - **StreamCZ**
+ - **SunPorno**
+ - **SWRMediathek**
+ - **Syfy**
+ - **SztvHu**
+ - **Tagesschau**
+ - **Tapely**
+ - **Tass**
+ - **teachertube**: teachertube.com videos
+ - **teachertube:user:collection**: teachertube.com user and collection videos
+ - **TeachingChannel**
+ - **Teamcoco**
+ - **TeamFour**
+ - **TechTalks**
+ - **techtv.mit.edu**
+ - **TED**
+ - **tegenlicht.vpro.nl**
+ - **TeleBruxelles**
+ - **telecinco.es**
+ - **TeleMB**
+ - **TenPlay**
+ - **TF1**
+ - **TheOnion**
+ - **ThePlatform**
+ - **TheSixtyOne**
+ - **ThisAV**
+ - **THVideo**
+ - **THVideoPlaylist**
+ - **tinypic**: tinypic.com videos
+ - **tlc.com**
+ - **tlc.de**
+ - **TMZ**
+ - **TNAFlix**
+ - **tou.tv**
+ - **Toypics**: Toypics user profile
+ - **ToypicsUser**: Toypics user profile
+ - **TrailerAddict** (Currently broken)
+ - **Trilulilu**
+ - **TruTube**
+ - **Tube8**
+ - **Tudou**
+ - **Tumblr**
+ - **TuneIn**
+ - **Turbo**
+ - **Tutv**
+ - **tv.dfb.de**
+ - **tvigle**: Интернет-телевидение Tvigle.ru
+ - **tvp.pl**
+ - **TVPlay**: TV3Play and related services
+ - **Twitch**
+ - **Ubu**
+ - **udemy**
+ - **udemy:course**
+ - **Unistra**
+ - **Urort**: NRK P3 Urørt
+ - **ustream**
+ - **ustream:channel**
+ - **Vbox7**
+ - **VeeHD**
+ - **Veoh**
+ - **Vesti**: Вести.Ru
+ - **Vevo**
+ - **VGTV**
+ - **vh1.com**
+ - **Vice**
+ - **Viddler**
+ - **video.google:search**: Google Video search
+ - **video.mit.edu**
+ - **VideoBam**
+ - **VideoDetective**
+ - **videofy.me**
+ - **videolectures.net**
+ - **VideoMega**
+ - **VideoPremium**
+ - **VideoTt**: video.tt - Your True Tube
+ - **videoweed**: VideoWeed
+ - **Vidme**
+ - **Vidzi**
+ - **viki**
+ - **vimeo**
+ - **vimeo:album**
+ - **vimeo:channel**
+ - **vimeo:group**
+ - **vimeo:likes**: Vimeo user likes
+ - **vimeo:review**: Review pages on vimeo
+ - **vimeo:user**
+ - **vimeo:watchlater**: Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)
+ - **Vimple**: Vimple.ru
+ - **Vine**
+ - **vine:user**
+ - **vk.com**
+ - **vk.com:user-videos**: vk.com:All of a user's videos
+ - **Vodlocker**
+ - **Vporn**
+ - **VRT**
+ - **vube**: Vube.com
+ - **VuClip**
+ - **vulture.com**
+ - **Walla**
+ - **WashingtonPost**
+ - **wat.tv**
+ - **WayOfTheMaster**
+ - **WDR**
+ - **wdr:mobile**
+ - **WDRMaus**: Sendung mit der Maus
+ - **Weibo**
+ - **Wimp**
+ - **Wistia**
+ - **WorldStarHipHop**
+ - **wrzuta.pl**
+ - **XBef**
+ - **XboxClips**
+ - **XHamster**
+ - **XMinus**
+ - **XNXX**
+ - **XTube**
+ - **XTubeUser**: XTube user profile
+ - **XVideos**
+ - **Yahoo**: Yahoo screen and movies
+ - **YesJapan**
+ - **Ynet**
+ - **YouJizz**
+ - **Youku**
+ - **YouPorn**
+ - **YourUpload**
+ - **youtube**: YouTube.com
+ - **youtube:channel**: YouTube.com channels
+ - **youtube:favorites**: YouTube.com favourite videos, ":ytfav" for short (requires authentication)
+ - **youtube:history**: Youtube watch history, ":ythistory" for short (requires authentication)
+ - **youtube:playlist**: YouTube.com playlists
+ - **youtube:recommended**: YouTube.com recommended videos, ":ytrec" for short (requires authentication)
+ - **youtube:search**: YouTube.com searches
+ - **youtube:search:date**: YouTube.com searches, newest videos first
+ - **youtube:search_url**: YouTube.com search URLs
+ - **youtube:show**: YouTube.com (multi-season) shows
+ - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
+ - **youtube:toplist**: YouTube.com top lists, "yttoplist:{channel}:{list title}" (Example: "yttoplist:music:Top Tracks")
+ - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
+ - **youtube:watch_later**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
+ - **ZDF**
+ - **ZDFChannel**
+ - **zingmp3:album**: mp3.zing.vn albums
+ - **zingmp3:song**: mp3.zing.vn songs
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2b4d8c62f..40b2791c7 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -23,6 +23,7 @@ from ..utils import (
     unescapeHTML,
     unified_strdate,
     unsmuggle_url,
+    UnsupportedError,
     url_basename,
 )
 from .brightcove import BrightcoveIE
@@ -1057,7 +1058,7 @@ class GenericIE(InfoExtractor):
                     'url': new_url,
                 }
         if not found:
-            raise ExtractorError('Unsupported URL: %s' % url)
+            raise UnsupportedError(url)
 
         entries = []
         for video_url in found:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 43b7c94ba..efbe64fb3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -464,6 +464,13 @@ class ExtractorError(Exception):
         return ''.join(traceback.format_tb(self.traceback))
 
 
+class UnsupportedError(ExtractorError):
+    def __init__(self, url):
+        super(UnsupportedError, self).__init__(
+            'Unsupported URL: %s' % url, expected=True)
+        self.url = url
+
+
 class RegexNotFoundError(ExtractorError):
     """Error when a regex didn't match"""
     pass

From 966ea3aebde5968c3ca1b655bac2e91f6bcea466 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 30 Dec 2014 23:41:29 +0100
Subject: [PATCH 0972/1937] [README] Typo / clarify FAQ

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 93a2750c2..915bcd0cd 100644
--- a/README.md
+++ b/README.md
@@ -453,9 +453,9 @@ To run the exe you need to install first the [Microsoft Visual C++ 2008 Redistri
 
 For one, have a look at the [list of supported sites](docs/supportedsites). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
 
-It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which maches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
+It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
 
-If you want to find out, simply call youtube-dl. If you get no videos back, chances are the URL is either not referring to a video or unsupported. You can find out which by examining the output (if you run youtube-dl on the console) or catching an `UnsupportedError` exception if you run it from a Python program.
+If you want to find out whether a given URL is supported, simply call youtube-dl with it. If you get no videos back, chances are the URL is either not referring to a video or unsupported. You can find out which by examining the output (if you run youtube-dl on the console) or catching an `UnsupportedError` exception if you run it from a Python program.
 
 # DEVELOPER INSTRUCTIONS
 

From 1931a73f392698d01d148f69e612391ad2d7138d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 31 Dec 2014 18:03:51 +0600
Subject: [PATCH 0973/1937] [echomsk] Add extractor (Closes #4600)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/echomsk.py  | 46 ++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 youtube_dl/extractor/echomsk.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c15786ad7..45b50792e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -99,6 +99,7 @@ from .discovery import DiscoveryIE
 from .divxstage import DivxStageIE
 from .dropbox import DropboxIE
 from .ebaumsworld import EbaumsWorldIE
+from .echomsk import EchoMskIE
 from .ehow import EHowIE
 from .eighttracks import EightTracksIE
 from .einthusan import EinthusanIE
diff --git a/youtube_dl/extractor/echomsk.py b/youtube_dl/extractor/echomsk.py
new file mode 100644
index 000000000..d2d94049d
--- /dev/null
+++ b/youtube_dl/extractor/echomsk.py
@@ -0,0 +1,46 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class EchoMskIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?echo\.msk\.ru/sounds/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.echo.msk.ru/sounds/1464134.html',
+        'md5': '2e44b3b78daff5b458e4dbc37f191f7c',
+        'info_dict': {
+            'id': '1464134',
+            'ext': 'mp3',
+            'title': 'Особое мнение - 29 декабря 2014, 19:08',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        audio_url = self._search_regex(
+            r'<a rel="mp3" href="([^"]+)">', webpage, 'audio URL')
+
+        title = self._html_search_regex(
+            r'<a href="/programs/[^"]+" target="_blank">([^<]+)</a>',
+            webpage, 'title')
+
+        air_date = self._html_search_regex(
+            r'(?s)<div class="date">(.+?)</div>',
+            webpage, 'date', fatal=False, default=None)
+
+        if air_date:
+            air_date = re.sub(r'(\s)\1+', r'\1', air_date)
+            if air_date:
+                title = '%s - %s' % (title, air_date)
+
+        return {
+            'id': video_id,
+            'url': audio_url,
+            'title': title,
+        }

From 7d900ef1bf131fc140d35d306500a5cec9565f89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 31 Dec 2014 14:15:16 +0100
Subject: [PATCH 0974/1937] [youtube] Add support for automatically translated
 subtitles (fixes #4555)

They have a manually uploaded subtitles track and YouTube can transtale it.
---
 test/test_subtitles.py          | 8 ++++++++
 youtube_dl/extractor/youtube.py | 5 +++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 7c4cd8218..d34565191 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -88,6 +88,14 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
         subtitles = self.getSubtitles()
         self.assertTrue(subtitles['it'] is not None)
 
+    def test_youtube_translated_subtitles(self):
+        # This video has a subtitles track, which can be translated
+        self.url = 'Ky9eprVWzlI'
+        self.DL.params['writeautomaticsub'] = True
+        self.DL.params['subtitleslangs'] = ['it']
+        subtitles = self.getSubtitles()
+        self.assertTrue(subtitles['it'] is not None)
+
     def test_youtube_nosubtitles(self):
         self.DL.expect_warning('video doesn\'t have subtitles')
         self.url = 'n5BB19UTcdA'
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3da83e3a8..224f1b041 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -662,10 +662,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             list_url = caption_url + '&' + list_params
             caption_list = self._download_xml(list_url, video_id)
             original_lang_node = caption_list.find('track')
-            if original_lang_node is None or original_lang_node.attrib.get('kind') != 'asr':
+            if original_lang_node is None:
                 self._downloader.report_warning('Video doesn\'t have automatic captions')
                 return {}
             original_lang = original_lang_node.attrib['lang_code']
+            caption_kind = original_lang_node.attrib.get('kind', '')
 
             sub_lang_list = {}
             for lang_node in caption_list.findall('target'):
@@ -675,7 +676,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                     'tlang': sub_lang,
                     'fmt': sub_format,
                     'ts': timestamp,
-                    'kind': 'asr',
+                    'kind': caption_kind,
                 })
                 sub_lang_list[sub_lang] = caption_url + '&' + params
             return sub_lang_list

From b8bc7a696ba15cbc6d45b20320d928340246b508 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 31 Dec 2014 19:40:35 +0600
Subject: [PATCH 0975/1937] [openfilm] Add extractor (Closes #4538)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/openfilm.py | 70 ++++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 youtube_dl/extractor/openfilm.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 45b50792e..6c5827f88 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -299,6 +299,7 @@ from .nytimes import NYTimesIE
 from .nuvid import NuvidIE
 from .oktoberfesttv import OktoberfestTVIE
 from .ooyala import OoyalaIE
+from .openfilm import OpenFilmIE
 from .orf import (
     ORFTVthekIE,
     ORFOE1IE,
diff --git a/youtube_dl/extractor/openfilm.py b/youtube_dl/extractor/openfilm.py
new file mode 100644
index 000000000..2249657eb
--- /dev/null
+++ b/youtube_dl/extractor/openfilm.py
@@ -0,0 +1,70 @@
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    compat_urllib_parse,
+    parse_age_limit,
+    int_or_none,
+)
+
+
+class OpenFilmIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)openfilm\.com/videos/(?P<id>.+)'
+    _TEST = {
+        'url': 'http://www.openfilm.com/videos/human-resources-remastered',
+        'md5': '42bcd88c2f3ec13b65edf0f8ad1cac37',
+        'info_dict': {
+            'id': '32736',
+            'display_id': 'human-resources-remastered',
+            'ext': 'mp4',
+            'title': 'Human Resources (Remastered)',
+            'description': 'Social Engineering in the 20th Century.',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 7164,
+            'timestamp': 1334756988,
+            'upload_date': '20120418',
+            'uploader_id': '41117',
+            'view_count': int,
+            'age_limit': 0,
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        player = compat_urllib_parse.unquote_plus(
+            self._og_search_video_url(webpage))
+
+        video = json.loads(self._search_regex(
+            r'\bp=({.+?})(?:&|$)', player, 'video JSON'))
+
+        video_url = '%s1.mp4' % video['location']
+        video_id = video.get('video_id')
+        display_id = video.get('alias') or display_id
+        title = video.get('title')
+        description = video.get('description')
+        thumbnail = video.get('main_thumb')
+        duration = int_or_none(video.get('duration'))
+        timestamp = parse_iso8601(video.get('dt_published'), ' ')
+        uploader_id = video.get('user_id')
+        view_count = int_or_none(video.get('views_count'))
+        age_limit = parse_age_limit(video.get('age_limit'))
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'url': video_url,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'timestamp': timestamp,
+            'uploader_id': uploader_id,
+            'view_count': view_count,
+            'age_limit': age_limit,
+        }

From 60e47a2699bea32f12a92e59d02d1ba0e9cbdec9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 31 Dec 2014 15:44:15 +0100
Subject: [PATCH 0976/1937] [youtube] Use '_download_xml' for getting the
 available subtitles

---
 youtube_dl/extractor/youtube.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 224f1b041..7f5aeb25b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -612,24 +612,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
     def _get_available_subtitles(self, video_id, webpage):
         try:
-            sub_list = self._download_webpage(
+            subs_doc = self._download_xml(
                 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
                 video_id, note=False)
         except ExtractorError as err:
             self._downloader.report_warning('unable to download video subtitles: %s' % compat_str(err))
             return {}
-        lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list)
 
         sub_lang_list = {}
-        for l in lang_list:
-            lang = l[1]
+        for track in subs_doc.findall('track'):
+            lang = track.attrib['lang_code']
             if lang in sub_lang_list:
                 continue
             params = compat_urllib_parse.urlencode({
                 'lang': lang,
                 'v': video_id,
                 'fmt': self._downloader.params.get('subtitlesformat', 'srt'),
-                'name': unescapeHTML(l[0]).encode('utf-8'),
+                'name': track.attrib['name'].encode('utf-8'),
             })
             url = 'https://www.youtube.com/api/timedtext?' + params
             sub_lang_list[lang] = url

From f86d543ebb6eafa1ca939a17601307bf29b88f6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 31 Dec 2014 17:24:14 +0100
Subject: [PATCH 0977/1937] [pbs] Catch geoblocking errors (closes #4516)

---
 youtube_dl/extractor/pbs.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py
index 6118ed5c2..afce732e1 100644
--- a/youtube_dl/extractor/pbs.py
+++ b/youtube_dl/extractor/pbs.py
@@ -4,6 +4,7 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     unified_strdate,
     US_RATINGS,
 )
@@ -151,6 +152,19 @@ class PBSIE(InfoExtractor):
         info_url = 'http://video.pbs.org/videoInfo/%s?format=json' % video_id
         info = self._download_json(info_url, display_id)
 
+        redirect_url = info['alternate_encoding']['url']
+        redirect_info = self._download_json(
+            redirect_url + '?format=json', display_id,
+            'Downloading video url info')
+        if redirect_info['status'] == 'error':
+            if redirect_info['http_code'] == 403:
+                message = (
+                    'The video is not available in your region due to '
+                    'right restrictions')
+            else:
+                message = redirect_info['message']
+            raise ExtractorError(message, expected=True)
+
         rating_str = info.get('rating')
         if rating_str is not None:
             rating_str = rating_str.rpartition('-')[2]
@@ -160,7 +174,7 @@ class PBSIE(InfoExtractor):
             'id': video_id,
             'display_id': display_id,
             'title': info['title'],
-            'url': info['alternate_encoding']['url'],
+            'url': redirect_info['url'],
             'ext': 'mp4',
             'description': info['program'].get('description'),
             'thumbnail': info.get('image_url'),

From 7b24bbdf49f4ad17a1cb134f9bf6a1075cb3f655 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 31 Dec 2014 23:59:16 +0600
Subject: [PATCH 0978/1937] [xboxclips] Fix extraction

---
 youtube_dl/extractor/xboxclips.py | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/xboxclips.py b/youtube_dl/extractor/xboxclips.py
index a9aa72e73..9cf867807 100644
--- a/youtube_dl/extractor/xboxclips.py
+++ b/youtube_dl/extractor/xboxclips.py
@@ -1,46 +1,42 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
-    parse_iso8601,
-    float_or_none,
     int_or_none,
+    parse_filesize,
+    unified_strdate,
 )
 
 
 class XboxClipsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?xboxclips\.com/video\.php\?.*vid=(?P<id>[\w-]{36})'
+    _VALID_URL = r'https?://(?:www\.)?xboxclips\.com/(?:video\.php\?.*vid=|[^/]+/)(?P<id>[\w-]{36})'
     _TEST = {
         'url': 'https://xboxclips.com/video.php?uid=2533274823424419&gamertag=Iabdulelah&vid=074a69a9-5faf-46aa-b93b-9909c1720325',
         'md5': 'fbe1ec805e920aeb8eced3c3e657df5d',
         'info_dict': {
             'id': '074a69a9-5faf-46aa-b93b-9909c1720325',
             'ext': 'mp4',
-            'title': 'Iabdulelah playing Upload Studio',
-            'filesize_approx': 28101836.8,
-            'timestamp': 1407388500,
+            'title': 'Iabdulelah playing Titanfall',
+            'filesize_approx': 26800000,
             'upload_date': '20140807',
             'duration': 56,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
         video_url = self._html_search_regex(
-            r'>Link: <a href="([^"]+)">', webpage, 'video URL')
+            r'>(?:Link|Download): <a href="([^"]+)">', webpage, 'video URL')
         title = self._html_search_regex(
             r'<title>XboxClips \| ([^<]+)</title>', webpage, 'title')
-        timestamp = parse_iso8601(self._html_search_regex(
+        upload_date = unified_strdate(self._html_search_regex(
             r'>Recorded: ([^<]+)<', webpage, 'upload date', fatal=False))
-        filesize = float_or_none(self._html_search_regex(
-            r'>Size: ([\d\.]+)MB<', webpage, 'file size', fatal=False), invscale=1024 * 1024)
+        filesize = parse_filesize(self._html_search_regex(
+            r'>Size: ([^<]+)<', webpage, 'file size', fatal=False))
         duration = int_or_none(self._html_search_regex(
             r'>Duration: (\d+) Seconds<', webpage, 'duration', fatal=False))
         view_count = int_or_none(self._html_search_regex(
@@ -50,7 +46,7 @@ class XboxClipsIE(InfoExtractor):
             'id': video_id,
             'url': video_url,
             'title': title,
-            'timestamp': timestamp,
+            'upload_date': upload_date,
             'filesize_approx': filesize,
             'duration': duration,
             'view_count': view_count,

From 079d1dcd8033b7c1470ad84899967d787cf1dec7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jan 2015 00:11:56 +0600
Subject: [PATCH 0979/1937] [tnaflix] Fix title extraction

---
 youtube_dl/extractor/tnaflix.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index 0ecd695f8..b9179ca80 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -12,7 +12,7 @@ from ..utils import (
 class TNAFlixIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?tnaflix\.com/(?P<cat_id>[\w-]+)/(?P<display_id>[\w-]+)/video(?P<id>\d+)'
 
-    _TITLE_REGEX = None
+    _TITLE_REGEX = r'<title>(.+?) - TNAFlix Porn Videos</title>'
     _DESCRIPTION_REGEX = r'<h3 itemprop="description">([^<]+)</h3>'
     _CONFIG_REGEX = r'flashvars\.config\s*=\s*escape\("([^"]+)"'
 

From 68f705cac53510dc272eebfe83694a2cbf2c2b46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jan 2015 00:12:41 +0600
Subject: [PATCH 0980/1937] [tnaflix] Make sure config URL has correct scheme

---
 youtube_dl/extractor/tnaflix.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tnaflix.py b/youtube_dl/extractor/tnaflix.py
index b9179ca80..d48cbbf14 100644
--- a/youtube_dl/extractor/tnaflix.py
+++ b/youtube_dl/extractor/tnaflix.py
@@ -49,8 +49,8 @@ class TNAFlixIE(InfoExtractor):
         if duration:
             duration = parse_duration(duration[1:])
 
-        cfg_url = self._html_search_regex(
-            self._CONFIG_REGEX, webpage, 'flashvars.config')
+        cfg_url = self._proto_relative_url(self._html_search_regex(
+            self._CONFIG_REGEX, webpage, 'flashvars.config'), 'http:')
 
         cfg_xml = self._download_xml(
             cfg_url, display_id, note='Downloading metadata',

From b7a7319c388e3a33851d87b589586abb6796a0e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jan 2015 00:26:19 +0600
Subject: [PATCH 0981/1937] [slideshare] Fix extraction

---
 youtube_dl/extractor/slideshare.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/slideshare.py b/youtube_dl/extractor/slideshare.py
index e7d776e7b..9f79ff5c1 100644
--- a/youtube_dl/extractor/slideshare.py
+++ b/youtube_dl/extractor/slideshare.py
@@ -30,7 +30,7 @@ class SlideshareIE(InfoExtractor):
         page_title = mobj.group('title')
         webpage = self._download_webpage(url, page_title)
         slideshare_obj = self._search_regex(
-            r'var slideshare_object =  ({.*?}); var user_info =',
+            r'var\s+slideshare_object\s*=\s*({.*?});\s*var\s+user_info\s*=',
             webpage, 'slideshare object')
         info = json.loads(slideshare_obj)
         if info['slideshow']['type'] != 'video':
@@ -41,7 +41,7 @@ class SlideshareIE(InfoExtractor):
         ext = info['jsplayer']['video_extension']
         video_url = compat_urlparse.urljoin(bucket, doc + '-SD.' + ext)
         description = self._html_search_regex(
-            r'<p\s+(?:style="[^"]*"\s+)?class=".*?description.*?"[^>]*>(.*?)</p>', webpage,
+            r'(?s)<p[^>]+itemprop="description"[^>]*>(.+?)</p>', webpage,
             'description', fatal=False)
 
         return {

From 6c6f1408f264f30fc4e2970e958cb7f9dc470d57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jan 2015 00:37:14 +0600
Subject: [PATCH 0982/1937] [extractor/common] Allow multiline content tags

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4b950e485..6e264f687 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -589,7 +589,7 @@ class InfoExtractor(object):
         if display_name is None:
             display_name = name
         return self._html_search_regex(
-            r'''(?ix)<meta
+            r'''(?isx)<meta
                     (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
                     [^>]+content=(["\'])(?P<content>.*?)\1''' % re.escape(name),
             html, display_name, fatal=fatal, group='content', **kwargs)

From 5878e6398ce7945ccbb141a38c815667f8a1fe9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jan 2015 00:37:57 +0600
Subject: [PATCH 0983/1937] [nrktv] Update tests' checksums

---
 youtube_dl/extractor/nrk.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 96f0ae1eb..43e8e619f 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -77,7 +77,7 @@ class NRKTVIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
-            'md5': '7b96112fbae1faf09a6f9ae1aff6cb84',
+            'md5': 'adf2c5454fa2bf032f47a9f8fb351342',
             'info_dict': {
                 'id': 'MUHH48000314',
                 'ext': 'flv',
@@ -89,7 +89,7 @@ class NRKTVIE(InfoExtractor):
         },
         {
             'url': 'http://tv.nrk.no/program/mdfp15000514',
-            'md5': 'af01795a31f1cf7265c8657534d8077b',
+            'md5': '383650ece2b25ecec996ad7b5bb2a384',
             'info_dict': {
                 'id': 'mdfp15000514',
                 'ext': 'flv',

From a1e9e6440f60f82b8d8552053219e1e8e3149c5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jan 2015 00:46:03 +0600
Subject: [PATCH 0984/1937] [moevideo] Skip removed video test

---
 youtube_dl/extractor/moevideo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/moevideo.py b/youtube_dl/extractor/moevideo.py
index 184f9c2c9..5a66302f6 100644
--- a/youtube_dl/extractor/moevideo.py
+++ b/youtube_dl/extractor/moevideo.py
@@ -52,7 +52,8 @@ class MoeVideoIE(InfoExtractor):
                 'height': 296,
                 'duration': 6027,
                 'filesize': 588257923,
-            }
+            },
+            'skip': 'Video has been removed',
         },
     ]
 

From 02ec32a1efc18a4f09b2422dc8c5bfb397a7e6ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Jan 2015 20:01:55 +0600
Subject: [PATCH 0985/1937] [ceskatelevize] Adapt to new API (Closes #4531)

---
 youtube_dl/extractor/ceskatelevize.py | 88 +++++++++++----------------
 1 file changed, 37 insertions(+), 51 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 2f866f3ef..ba8376338 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -11,6 +11,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    float_or_none,
 )
 
 
@@ -19,41 +20,33 @@ class CeskaTelevizeIE(InfoExtractor):
 
     _TESTS = [
         {
-            'url': 'http://www.ceskatelevize.cz/ivysilani/10532695142-prvni-republika/213512120230004-spanelska-chripka',
+            'url': 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220',
             'info_dict': {
-                'id': '213512120230004',
-                'ext': 'flv',
-                'title': 'První republika: Španělská chřipka',
-                'duration': 3107.4,
+                'id': '214411058091220',
+                'ext': 'mp4',
+                'title': 'Hyde Park Civilizace',
+                'description': 'Věda a současná civilizace. Interaktivní pořad - prostor pro vaše otázky a komentáře',
+                'thumbnail': 're:^https?://.*\.jpg',
+                'duration': 3350,
             },
             'params': {
-                'skip_download': True,  # requires rtmpdump
+                # m3u8 download
+                'skip_download': True,
             },
-            'skip': 'Works only from Czech Republic.',
-        },
-        {
-            'url': 'http://www.ceskatelevize.cz/ivysilani/1030584952-tsatsiki-maminka-a-policajt',
-            'info_dict': {
-                'id': '20138143440',
-                'ext': 'flv',
-                'title': 'Tsatsiki, maminka a policajt',
-                'duration': 6754.1,
-            },
-            'params': {
-                'skip_download': True,  # requires rtmpdump
-            },
-            'skip': 'Works only from Czech Republic.',
         },
         {
             'url': 'http://www.ceskatelevize.cz/ivysilani/10532695142-prvni-republika/bonus/14716-zpevacka-z-duparny-bobina',
             'info_dict': {
                 'id': '14716',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'První republika: Zpěvačka z Dupárny Bobina',
-                'duration': 90,
+                'description': 'Sága mapující atmosféru první republiky od r. 1918 do r. 1945.',
+                'thumbnail': 're:^https?://.*\.jpg',
+                'duration': 88.4,
             },
             'params': {
-                'skip_download': True,  # requires rtmpdump
+                # m3u8 download
+                'skip_download': True,
             },
         },
     ]
@@ -80,8 +73,9 @@ class CeskaTelevizeIE(InfoExtractor):
             'requestSource': 'iVysilani',
         }
 
-        req = compat_urllib_request.Request('http://www.ceskatelevize.cz/ivysilani/ajax/get-playlist-url',
-                                            data=compat_urllib_parse.urlencode(data))
+        req = compat_urllib_request.Request(
+            'http://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
+            data=compat_urllib_parse.urlencode(data))
 
         req.add_header('Content-type', 'application/x-www-form-urlencoded')
         req.add_header('x-addr', '127.0.0.1')
@@ -90,39 +84,31 @@ class CeskaTelevizeIE(InfoExtractor):
 
         playlistpage = self._download_json(req, video_id)
 
-        req = compat_urllib_request.Request(compat_urllib_parse.unquote(playlistpage['url']))
+        playlist_url = playlistpage['url']
+        if playlist_url == 'error_region':
+            raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
+
+        req = compat_urllib_request.Request(compat_urllib_parse.unquote(playlist_url))
         req.add_header('Referer', url)
 
-        playlist = self._download_xml(req, video_id)
+        playlist = self._download_json(req, video_id)
 
+        item = playlist['playlist'][0]
         formats = []
-        for i in playlist.find('smilRoot/body'):
-            if 'AD' not in i.attrib['id']:
-                base_url = i.attrib['base']
-                parsedurl = compat_urllib_parse_urlparse(base_url)
-                duration = i.attrib['duration']
-
-                for video in i.findall('video'):
-                    if video.attrib['label'] != 'AD':
-                        format_id = video.attrib['label']
-                        play_path = video.attrib['src']
-                        vbr = int(video.attrib['system-bitrate'])
-
-                        formats.append({
-                            'format_id': format_id,
-                            'url': base_url,
-                            'vbr': vbr,
-                            'play_path': play_path,
-                            'app': parsedurl.path[1:] + '?' + parsedurl.query,
-                            'rtmp_live': True,
-                            'ext': 'flv',
-                        })
-
+        for format_id, stream_url in item['streamUrls'].items():
+            formats.extend(self._extract_m3u8_formats(stream_url, video_id, 'mp4'))
         self._sort_formats(formats)
 
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        duration = float_or_none(item.get('duration'))
+        thumbnail = item.get('previewImageUrl')
+
         return {
             'id': episode_id,
-            'title': self._html_search_regex(r'<title>(.+?) — iVysílání — Česká televize</title>', webpage, 'title'),
-            'duration': float(duration),
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
             'formats': formats,
         }

From 9a237b776caa66fc6d1f6986c8f58599a1711aec Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 1 Jan 2015 21:41:42 +0100
Subject: [PATCH 0986/1937] release 2015.01.01

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1420af746..89707ceef 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2014.12.17.2'
+__version__ = '2015.01.01'

From a42419da42808f8b4a82efa4918c8d09afb8ff39 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 1 Jan 2015 22:01:26 +0100
Subject: [PATCH 0987/1937] [options] Upper-case options and URL in --help
 output

Hopefully, this reduces confusion as in #4610.
---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 21c452141..a018c1d71 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -109,7 +109,7 @@ def parseOpts(overrideArguments=None):
     kw = {
         'version': __version__,
         'formatter': fmt,
-        'usage': '%prog [options] url [url...]',
+        'usage': '%prog [OPTIONS] URL [URL...]',
         'conflict_handler': 'resolve',
     }
 

From 931e2d1d26f32c0ae3c1508b664207654529344d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 1 Jan 2015 22:15:46 +0100
Subject: [PATCH 0988/1937] [bbccouk] PEP8

---
 youtube_dl/extractor/bbccouk.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index f690dc803..53aea0ec9 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -241,8 +241,8 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
 
         # fallback to legacy playlist
         playlist = self._download_xml(
-                'http://www.bbc.co.uk/iplayer/playlist/%s' % playlist_id,
-                playlist_id, 'Downloading legacy playlist XML')
+            'http://www.bbc.co.uk/iplayer/playlist/%s' % playlist_id,
+            playlist_id, 'Downloading legacy playlist XML')
 
         no_items = playlist.find('./{http://bbc.co.uk/2008/emp/playlist}noItems')
         if no_items is not None:

From 9f435c5f1ce9566cf69b582b98b941d980d61262 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 1 Jan 2015 22:34:58 +0100
Subject: [PATCH 0989/1937] Add an extractor for common mistakes (#4610)

---
 youtube_dl/extractor/__init__.py       |  1 +
 youtube_dl/extractor/commonmistakes.py | 29 ++++++++++++++++++++++++++
 2 files changed, 30 insertions(+)
 create mode 100644 youtube_dl/extractor/commonmistakes.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6c5827f88..a95de54cb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -71,6 +71,7 @@ from .cnn import (
 from .collegehumor import CollegeHumorIE
 from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
 from .comcarcoff import ComCarCoffIE
+from .commonmistakes import CommonMistakesIE
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
 from .criterion import CriterionIE
diff --git a/youtube_dl/extractor/commonmistakes.py b/youtube_dl/extractor/commonmistakes.py
new file mode 100644
index 000000000..75c06903f
--- /dev/null
+++ b/youtube_dl/extractor/commonmistakes.py
@@ -0,0 +1,29 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class CommonMistakesIE(InfoExtractor):
+    IE_DESC = False  # Do not list
+    _VALID_URL = r'''(?x)
+        (?:url|URL)
+    '''
+
+    _TESTS = [{
+        'url': 'url',
+        'only_matching': True,
+    }, {
+        'url': 'URL',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        msg = (
+            'You\'ve asked youtube-dl to download the URL "%s". '
+            'That doesn\'t make any sense. '
+            'Simply remove the parameter in your command or configuration.'
+        ) % url
+        if self._downloader.params.get('verbose'):
+            msg += ' Add -v to the command line to see what arguments and configuration youtube-dl got.'
+        raise ExtractorError(msg, expected=True)

From 50a0f6df7e058c466d668d033a358b7e02a2f80f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 1 Jan 2015 22:47:21 +0100
Subject: [PATCH 0990/1937] [/__init__] Add another cute search example

---
 youtube_dl/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index e79320323..56f560d26 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -112,7 +112,7 @@ def _real_main(argv=None):
             if desc is False:
                 continue
             if hasattr(ie, 'SEARCH_KEY'):
-                _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny')
+                _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny', 'burping cow')
                 _COUNTS = ('', '5', '10', 'all')
                 desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
             compat_print(desc)

From 772fd5cc44c3682aed4a166936f7bf8ccccbbea4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 1 Jan 2015 23:44:39 +0100
Subject: [PATCH 0991/1937] [youtube] Add a pseudo-extractor for truncated
 YouTube video IDs (#4610)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/youtube.py  | 17 +++++++++++++++++
 2 files changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a95de54cb..9848ff611 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -545,6 +545,7 @@ from .youtube import (
     YoutubeShowIE,
     YoutubeSubscriptionsIE,
     YoutubeTopListIE,
+    YoutubeTruncatedIDIE,
     YoutubeTruncatedURLIE,
     YoutubeUserIE,
     YoutubeWatchLaterIE,
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 7f5aeb25b..8c7842ee8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1701,3 +1701,20 @@ class YoutubeTruncatedURLIE(InfoExtractor):
             '"http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
             ' or simply  youtube-dl BaW_jenozKc  .',
             expected=True)
+
+
+class YoutubeTruncatedIDIE(InfoExtractor):
+    IE_NAME = 'youtube:truncated_id'
+    IE_DESC = False  # Do not list
+    _VALID_URL = r'https?://(?:www\.)youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
+
+    _TESTS = [{
+        'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        raise ExtractorError(
+            'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),
+            expected=True)

From defaf19f5de839777cda1654d42b54850a78d8a7 Mon Sep 17 00:00:00 2001
From: Your Name <you@example.com>
Date: Fri, 2 Jan 2015 02:20:04 -0600
Subject: [PATCH 0992/1937] Push api updates to simplify audiomack, add support
 for albums

---
 youtube_dl/extractor/__init__.py  |   2 +-
 youtube_dl/extractor/audiomack.py | 110 ++++++++++++++++++++++--------
 2 files changed, 84 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 6c5827f88..e4c51f238 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -26,7 +26,7 @@ from .arte import (
     ArteTVEmbedIE,
 )
 from .atresplayer import AtresPlayerIE
-from .audiomack import AudiomackIE
+from .audiomack import AudiomackIE, AudiomackAlbumIE
 from .auengine import AUEngineIE
 from .azubu import AzubuIE
 from .bambuser import BambuserIE, BambuserChannelIE
diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 622b20989..b6f3039c7 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -9,20 +9,21 @@ import time
 
 
 class AudiomackIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?audiomack\.com/song/(?P<id>[\w/-]+)'
+    _VALID_URL = r'https?://(?:www\.)?audiomack\.com/(song)/(?P<id>[\w/-]+)'
     IE_NAME = 'audiomack'
     _TESTS = [
-        # hosted on audiomack
+        # audiomack
         {
             'url': 'http://www.audiomack.com/song/roosh-williams/extraordinary',
             'info_dict':
             {
-                'id': 'roosh-williams/extraordinary',
-                'ext': 'mp3',
-                'title': 'Roosh Williams - Extraordinary'
+                'id': '310086',
+                "ext": "mp3",
+                "artist": "Roosh Williams",
+                'title': 'Extraordinary'
             }
         },
-        # hosted on soundcloud via audiomack
+        # audiomack through soundcloud
         {
             'add_ie': ['Soundcloud'],
             'url': 'http://www.audiomack.com/song/xclusiveszone/take-kare',
@@ -37,33 +38,88 @@ class AudiomackIE(InfoExtractor):
         },
     ]
 
+    @staticmethod
+    def create_song_dictionary(api_response, album_url_tag, track_no=0):
+        # All keys are the same in audiomack api and InfoExtractor format
+        entry = {key: api_response[key] for key in ["title", "artist", "id", "url"] if key in api_response}
+        # Fudge values in the face of missing metadata
+        if "id" not in entry:
+            entry["id"] = track_no
+        if "title" not in entry:
+            entry["title"] = album_url_tag
+        return entry
+
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        # URLs end with [uploader name]/[uploader title]
+        # this title is whatever the user types in, and is rarely
+        # the proper song title.  Real metadata is in the api response
+        album_url_tag = self._match_id(url)
 
+        # Request the extended version of the api for extra fields like artist and title
         api_response = self._download_json(
-            "http://www.audiomack.com/api/music/url/song/%s?_=%d" % (
-                video_id, time.time()),
-            video_id)
+            "http://www.audiomack.com/api/music/url/song/%s?extended=1&_=%d" % (
+                album_url_tag, time.time()),
+            album_url_tag)
 
-        if "url" not in api_response:
-            raise ExtractorError("Unable to deduce api url of song")
-        realurl = api_response["url"]
+        # API is inconsistent with errors
+        if "url" not in api_response or not api_response["url"] or "error" in api_response:
+            raise ExtractorError("Invalid url %s", url)
 
         # Audiomack wraps a lot of soundcloud tracks in their branded wrapper
-        # - if so, pass the work off to the soundcloud extractor
-        if SoundcloudIE.suitable(realurl):
-            return {'_type': 'url', 'url': realurl, 'ie_key': 'Soundcloud'}
+        # if so, pass the work off to the soundcloud extractor
+        if SoundcloudIE.suitable(api_response["url"]):
+            return {'_type': 'url', 'url': api_response["url"], 'ie_key': 'Soundcloud'}
 
-        webpage = self._download_webpage(url, video_id)
-        artist = self._html_search_regex(
-            r'<span class="artist">(.*?)</span>', webpage, "artist")
-        songtitle = self._html_search_regex(
-            r'<h1 class="profile-title song-title"><span class="artist">.*?</span>(.*?)</h1>',
-            webpage, "title")
-        title = artist + " - " + songtitle
+        return self.create_song_dictionary(api_response, album_url_tag)
 
-        return {
-            'id': video_id,
-            'title': title,
-            'url': realurl,
+
+class AudiomackAlbumIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?audiomack\.com/album/(?P<id>[\w/-]+)'
+    IE_NAME = 'audiomack:album'
+    _TESTS = [
+        # Standard album playlist
+        {
+            'url': 'http://www.audiomack.com/album/flytunezcom/tha-tour-part-2-mixtape',
+            "playlist_count": 15,
+            'info_dict':
+            {
+                'id': "812251",
+                'title': "Tha Tour: Part 2 (Official Mixtape)"
+            }
+        },
+        # Album playlist ripped from fakeshoredrive with no metadata
+        {
+            "url": "http://www.audiomack.com/album/fakeshoredrive/ppp-pistol-p-project",
+            "playlist_count": 10
         }
+    ]
+
+    def _real_extract(self, url):
+        # URLs end with [uploader name]/[uploader title]
+        # this title is whatever the user types in, and is rarely
+        # the proper song title.  Real metadata is in the api response
+        album_url_tag = self._match_id(url)
+        result = {"_type": "playlist", "entries": []}
+        # There is no one endpoint for album metadata - instead it is included/repeated in each song's metadata
+        # Therefore we don't know how many songs the album has and must infi-loop until failure
+        track_no = 0
+        while True:
+            # Get song's metadata
+            api_response = self._download_json("http://www.audiomack.com/api/music/url/album/%s/%d?extended=1&_=%d"
+                                               % (album_url_tag, track_no, time.time()), album_url_tag)
+
+            # Total failure, only occurs when url is totally wrong
+            # Won't happen in middle of valid playlist (next case)
+            if "url" not in api_response or "error" in api_response:
+                raise ExtractorError("Invalid url for track %d of album url %s" % (track_no, url))
+            # URL is good but song id doesn't exist - usually means end of playlist
+            elif not api_response["url"]:
+                break
+            else:
+                # Pull out the album metadata and add to result (if it exists)
+                for resultkey, apikey in [("id", "album_id"), ("title", "album_title")]:
+                    if apikey in api_response and resultkey not in result:
+                        result[resultkey] = api_response[apikey]
+                result["entries"].append(AudiomackIE.create_song_dictionary(api_response, album_url_tag, track_no))
+            track_no += 1
+        return result

From 5b9aefef773463977504081faa1db88bd0e94300 Mon Sep 17 00:00:00 2001
From: dwemthy <dwemthy@users.noreply.github.com>
Date: Fri, 2 Jan 2015 11:52:48 +0000
Subject: [PATCH 0993/1937] [xhamster] Add HTTPS support

---
 youtube_dl/extractor/xhamster.py | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 6b37bcbc9..519bfc191 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 class XHamsterIE(InfoExtractor):
     """Information Extractor for xHamster"""
-    _VALID_URL = r'http://(?:.+?\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.+?)\.html(?:\?.*)?'
+    _VALID_URL = r'(?P<proto>https?)://(?:.+?\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.+?)\.html(?:\?.*)?'
     _TESTS = [
         {
             'url': 'http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html',
@@ -39,6 +39,30 @@ class XHamsterIE(InfoExtractor):
                 'duration': 200,
                 'age_limit': 18,
             }
+        },
+        {
+            'url': 'https://xhamster.com/movies/2272726/amber_slayed_by_the_knight.html',
+            'info_dict': {
+                'id': '2272726',
+                'ext': 'mp4',
+                'title': 'Amber slayed by the Knight',
+                'upload_date': '20131009',
+                'uploader_id': 'amberblank',
+                'duration': 149,
+                'age_limit': 18,
+            }
+        },
+        {
+            "url": "https://xhamster.com/movies/1444747/hotkinkyjo_amp_asian_girl_elbow_deep_anal_fisting_and_footing.html",
+            'info_dict': {
+                "id": "1444747",
+                'ext': 'mp4',
+                "title": "HOTKINKYJO & ASIAN GIRL ELBOW DEEP ANAL FISTING AND FOOTING",
+                "upload_date": "20120922",
+                "uploader_id": "alex1981",
+                "duration": 80,
+                "age_limit": 18,
+            }
         }
     ]
 
@@ -57,7 +81,8 @@ class XHamsterIE(InfoExtractor):
 
         video_id = mobj.group('id')
         seo = mobj.group('seo')
-        mrss_url = 'http://xhamster.com/movies/%s/%s.html' % (video_id, seo)
+        proto = mobj.group('proto')
+        mrss_url = '%s://xhamster.com/movies/%s/%s.html' % (proto, video_id, seo)
         webpage = self._download_webpage(mrss_url, video_id)
 
         title = self._html_search_regex(r'<title>(?P<title>.+?) - xHamster\.com</title>', webpage, 'title')

From f58487b392f7943c31e41b200bef8e97822529db Mon Sep 17 00:00:00 2001
From: Thijs Vermeir <thijsvermeir@gmail.com>
Date: Fri, 2 Jan 2015 13:13:18 +0100
Subject: [PATCH 0994/1937] [vier] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/vier.py     | 75 ++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)
 create mode 100644 youtube_dl/extractor/vier.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9848ff611..9ccd1b32e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -474,6 +474,7 @@ from .videott import VideoTtIE
 from .videoweed import VideoWeedIE
 from .vidme import VidmeIE
 from .vidzi import VidziIE
+from .vier import VierIE, VierVideosIE
 from .vimeo import (
     VimeoIE,
     VimeoAlbumIE,
diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
new file mode 100644
index 000000000..e3f1b4714
--- /dev/null
+++ b/youtube_dl/extractor/vier.py
@@ -0,0 +1,75 @@
+from .common import InfoExtractor
+from ..utils import escape_url
+
+import re
+
+class VierIE (InfoExtractor):
+  _VALID_URL = r'(?:http://)?www.vier.be/(?P<program>.*)/videos/(.+?)/(?P<id>\d*)'
+  _TEST = {
+      'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129',
+      'md5': 'bf48f4eb998cbde44ecd02fc42c51149',
+      'info_dict': {
+          'id': '16129',
+          'ext': 'mp4',
+          'title': 'Het wordt warm in De Moestuin',
+          'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...',
+      },
+  }
+
+  def _real_extract (self, url):
+    mobj = re.match (self._VALID_URL, url)
+
+    program = mobj.group ('program')
+    video_id = mobj.group ('id')
+
+    webpage = self._download_webpage (url, video_id)
+
+    title = self._html_search_regex(r'<meta property="og:title" content="(.+?)" />', webpage, u'title')
+    description = self._html_search_regex (r'<meta property="og:description" content="(.+?)" />', webpage, u'description')
+    vod_id =  self._html_search_regex(r'"filename" : "(.+?)"', webpage, u'playlist URL')
+    url = escape_url ("http://vod.streamcloud.be/vier_vod/mp4:_definst_/" + vod_id + ".mp4/playlist.m3u8")
+
+    return {
+        'id': video_id,
+        'title': title,
+        'description': description,
+        'formats': self._extract_m3u8_formats(url, video_id, 'mp4'),
+    }
+
+class VierVideosIE (InfoExtractor):
+  _VALID_URL = r'http://www.vier.be/(?P<program>.*)/videos(\?page=(?P<page>\d*))?$'
+  _TESTS = [{
+    'url': 'http://www.vier.be/demoestuin/videos',
+    'info_dict': {
+      'id': 'demoestuin page(0)',
+    },
+    'playlist_mincount': 20,
+  },
+  {
+    'url': 'http://www.vier.be/demoestuin/videos?page=6',
+    'info_dict': {
+      'id': 'demoestuin page(6)',
+    },
+    'playlist_mincount': 20,
+  }]
+
+  def _real_extract (self, url):
+    mobj = re.match (self._VALID_URL, url)
+
+    program = mobj.group ('program')
+    page = mobj.group ('page')
+    if page == None:
+      page = 0
+
+    videos_id = program + " page(" + str (page) + ")"
+    videos_page = self._download_webpage (url, videos_id, note='Retrieving videos page')
+
+    return {
+        '_type': 'playlist',
+        'id': videos_id,
+        'entries': [{
+            '_type': 'url',
+            'url': "http://www.vier.be/" + eurl[0],
+            'ie_key': 'Vier',
+          } for eurl in re.findall (r'<h3><a href="(.+?)">(.+?)</a></h3>', videos_page)]
+        }

From 9eb4f404cbd10268befa47a1bcf699f07a47f5de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 20:15:40 +0600
Subject: [PATCH 0995/1937] [vier] Simplify, add support for more URL formats,
 extract all playlist pages when page is not specified

---
 youtube_dl/extractor/vier.py | 169 ++++++++++++++++++++++-------------
 1 file changed, 106 insertions(+), 63 deletions(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index e3f1b4714..fe54ae475 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -1,75 +1,118 @@
-from .common import InfoExtractor
-from ..utils import escape_url
+# coding: utf-8
+from __future__ import unicode_literals
 
 import re
 
-class VierIE (InfoExtractor):
-  _VALID_URL = r'(?:http://)?www.vier.be/(?P<program>.*)/videos/(.+?)/(?P<id>\d*)'
-  _TEST = {
-      'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129',
-      'md5': 'bf48f4eb998cbde44ecd02fc42c51149',
-      'info_dict': {
-          'id': '16129',
-          'ext': 'mp4',
-          'title': 'Het wordt warm in De Moestuin',
-          'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...',
-      },
-  }
+from .common import InfoExtractor
 
-  def _real_extract (self, url):
-    mobj = re.match (self._VALID_URL, url)
 
-    program = mobj.group ('program')
-    video_id = mobj.group ('id')
+class VierIE(InfoExtractor):
+    IE_NAME = 'vier'
+    _VALID_URL = r'https?://(?:www\.)?vier\.be/(?:[^/]+/videos/(?P<display_id>[^/]+)(?:/(?P<id>\d+))?|video/v3/embed/(?P<embed_id>\d+))'
+    _TESTS = [{
+        'url': 'http://www.vier.be/planb/videos/het-wordt-warm-de-moestuin/16129',
+        'info_dict': {
+            'id': '16129',
+            'display_id': 'het-wordt-warm-de-moestuin',
+            'ext': 'mp4',
+            'title': 'Het wordt warm in De Moestuin',
+            'description': 'De vele uren werk eisen hun tol. Wim droomt van assistentie...',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'http://www.vier.be/planb/videos/mieren-herders-van-de-bladluizen',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.vier.be/video/v3/embed/16129',
+        'only_matching': True,
+    }]
 
-    webpage = self._download_webpage (url, video_id)
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        embed_id = mobj.group('embed_id')
+        display_id = mobj.group('display_id') or embed_id
 
-    title = self._html_search_regex(r'<meta property="og:title" content="(.+?)" />', webpage, u'title')
-    description = self._html_search_regex (r'<meta property="og:description" content="(.+?)" />', webpage, u'description')
-    vod_id =  self._html_search_regex(r'"filename" : "(.+?)"', webpage, u'playlist URL')
-    url = escape_url ("http://vod.streamcloud.be/vier_vod/mp4:_definst_/" + vod_id + ".mp4/playlist.m3u8")
+        webpage = self._download_webpage(url, display_id)
 
-    return {
-        'id': video_id,
-        'title': title,
-        'description': description,
-        'formats': self._extract_m3u8_formats(url, video_id, 'mp4'),
-    }
+        video_id = self._search_regex(
+            r'"nid"\s*:\s*"(\d+)"', webpage, 'video id')
+        application = self._search_regex(
+            r'"application"\s*:\s*"([^"]+)"', webpage, 'application', default='vier_vod')
+        filename = self._search_regex(
+            r'"filename"\s*:\s*"([^"]+)"', webpage, 'filename')
 
-class VierVideosIE (InfoExtractor):
-  _VALID_URL = r'http://www.vier.be/(?P<program>.*)/videos(\?page=(?P<page>\d*))?$'
-  _TESTS = [{
-    'url': 'http://www.vier.be/demoestuin/videos',
-    'info_dict': {
-      'id': 'demoestuin page(0)',
-    },
-    'playlist_mincount': 20,
-  },
-  {
-    'url': 'http://www.vier.be/demoestuin/videos?page=6',
-    'info_dict': {
-      'id': 'demoestuin page(6)',
-    },
-    'playlist_mincount': 20,
-  }]
+        playlist_url = 'http://vod.streamcloud.be/%s/mp4:_definst_/%s.mp4/playlist.m3u8' % (application, filename)
+        formats = self._extract_m3u8_formats(playlist_url, display_id, 'mp4')
 
-  def _real_extract (self, url):
-    mobj = re.match (self._VALID_URL, url)
+        title = self._og_search_title(webpage, default=display_id)
+        description = self._og_search_description(webpage, default=None)
+        thumbnail = self._og_search_thumbnail(webpage, default=None)
 
-    program = mobj.group ('program')
-    page = mobj.group ('page')
-    if page == None:
-      page = 0
-
-    videos_id = program + " page(" + str (page) + ")"
-    videos_page = self._download_webpage (url, videos_id, note='Retrieving videos page')
-
-    return {
-        '_type': 'playlist',
-        'id': videos_id,
-        'entries': [{
-            '_type': 'url',
-            'url': "http://www.vier.be/" + eurl[0],
-            'ie_key': 'Vier',
-          } for eurl in re.findall (r'<h3><a href="(.+?)">(.+?)</a></h3>', videos_page)]
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'formats': formats,
         }
+
+
+class VierVideosIE(InfoExtractor):
+    IE_NAME = 'vier:videos'
+    _VALID_URL = r'https?://(?:www\.)?vier\.be/(?P<program>[^/]+)/videos(?:\?page=(?P<page>\d+))?$'
+    _TESTS = [{
+        'url': 'http://www.vier.be/demoestuin/videos',
+        'info_dict': {
+            'id': 'demoestuin',
+        },
+        'playlist_mincount': 153,
+    }, {
+        'url': 'http://www.vier.be/demoestuin/videos?page=6',
+        'info_dict': {
+            'id': 'demoestuin-page6',
+        },
+        'playlist_mincount': 20,
+    }, {
+        'url': 'http://www.vier.be/demoestuin/videos?page=7',
+        'info_dict': {
+            'id': 'demoestuin-page7',
+        },
+        'playlist_mincount': 13,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        program = mobj.group('program')
+
+        webpage = self._download_webpage(url, program)
+
+        page_id = mobj.group('page')
+        if page_id:
+            page_id = int(page_id)
+            start_page = page_id
+            last_page = start_page + 1
+            playlist_id = '%s-page%d' % (program, page_id)
+        else:
+            start_page = 0
+            last_page = int(self._search_regex(
+                r'videos\?page=(\d+)">laatste</a>',
+                webpage, 'last page', default=0)) + 1
+            playlist_id = program
+
+        entries = []
+        for current_page_id in range(start_page, last_page):
+            current_page = self._download_webpage(
+                'http://www.vier.be/%s/videos?page=%d' % (program, current_page_id),
+                program,
+                'Downloading page %d' % (current_page_id + 1)) if current_page_id != page_id else webpage
+            page_entries = [
+                self.url_result('http://www.vier.be' + video_url, 'Vier')
+                for video_url in re.findall(
+                    r'<h3><a href="(/[^/]+/videos/[^/]+(?:/\d+)?)">', current_page)]
+            entries.extend(page_entries)
+
+        return self.playlist_result(entries, playlist_id)

From 6f790e582197b8f2366fc8fa94f76fe9d83954be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 20:16:43 +0600
Subject: [PATCH 0996/1937] Credit @lovebug356 for vier (#4617)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 29ce9e3e4..37306ac83 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -97,3 +97,4 @@ Petr Kutalek
 Will Glynn
 Max Reimann
 Cédric Luthi
+Thijs Vermeir

From fdbabca85f80e47961329972bcbf627f25638ce4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 20:21:41 +0600
Subject: [PATCH 0997/1937] [vier:videos] Tune _VALID_URL

---
 youtube_dl/extractor/vier.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index fe54ae475..0d9fb09a7 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -63,7 +63,7 @@ class VierIE(InfoExtractor):
 
 class VierVideosIE(InfoExtractor):
     IE_NAME = 'vier:videos'
-    _VALID_URL = r'https?://(?:www\.)?vier\.be/(?P<program>[^/]+)/videos(?:\?page=(?P<page>\d+))?$'
+    _VALID_URL = r'https?://(?:www\.)?vier\.be/(?P<program>[^/]+)/videos(?:\?.*\bpage=(?P<page>\d+))?'
     _TESTS = [{
         'url': 'http://www.vier.be/demoestuin/videos',
         'info_dict': {

From 317639758a0f1728e6e1d612800b6eab69261e89 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 20:37:54 +0600
Subject: [PATCH 0998/1937] [bbccouk] Improve _VALID_URL

---
 youtube_dl/extractor/bbccouk.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 53aea0ec9..73fe66b01 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -10,7 +10,7 @@ from ..compat import compat_HTTPError
 class BBCCoUkIE(SubtitlesInfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
-    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:programmes|iplayer/episode)/(?P<id>[\da-z]{8})'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:programmes|iplayer/(?:episode|playlist))/(?P<id>[\da-z]{8})'
 
     _TESTS = [
         {
@@ -84,6 +84,9 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
                 # rtmp download
                 'skip_download': True,
             }
+        }, {
+            'url': 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4',
+            'only_matching': True,
         }
     ]
 

From db546cf87fa4206e55f66ddcc041dcd12fe708c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 20:46:17 +0600
Subject: [PATCH 0999/1937] [generic] Add support for BBC iPlayer embeds
 (Closes #4619)

---
 youtube_dl/extractor/generic.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 40b2791c7..ce1e77daa 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -905,6 +905,11 @@ class GenericIE(InfoExtractor):
             return _playlist_from_matches(
                 matches, getter=unescapeHTML, ie='FunnyOrDie')
 
+        # Look for BBC iPlayer embed
+        matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
+        if matches:
+            return self.playlist_result([self.url_result(video_url, ie='BBCCoUk') for video_url in matches])
+
         # Look for embedded RUTV player
         rutv_url = RUTVIE._extract_url(webpage)
         if rutv_url:

From 8399267671938f072b9c6f6b2cbd2fa315b80a23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 20:54:30 +0600
Subject: [PATCH 1000/1937] [generic] Make getter None by default

---
 youtube_dl/extractor/generic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ce1e77daa..4bb7581ac 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -699,9 +699,9 @@ class GenericIE(InfoExtractor):
             r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
 
         # Helper method
-        def _playlist_from_matches(matches, getter, ie=None):
+        def _playlist_from_matches(matches, getter=None, ie=None):
             urlrs = orderedSet(
-                self.url_result(self._proto_relative_url(getter(m)), ie)
+                self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
                 for m in matches)
             return self.playlist_result(
                 urlrs, playlist_id=video_id, playlist_title=video_title)

From 476eae0c2a7cb4de1f00d050d733f1df1bd6cc0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 20:55:09 +0600
Subject: [PATCH 1001/1937] [generic] Generalize BBC iPlayer playlist
 extraction

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4bb7581ac..fd4ee4667 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -908,7 +908,7 @@ class GenericIE(InfoExtractor):
         # Look for BBC iPlayer embed
         matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
         if matches:
-            return self.playlist_result([self.url_result(video_url, ie='BBCCoUk') for video_url in matches])
+            return _playlist_from_matches(matches, ie='BBCCoUk')
 
         # Look for embedded RUTV player
         rutv_url = RUTVIE._extract_url(webpage)

From 29a9385ff083fde37bcd5fc1119fa7fa55cf6973 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 2 Jan 2015 15:56:26 +0100
Subject: [PATCH 1002/1937] release 2015.01.02

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 89707ceef..58b5021dc 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.01'
+__version__ = '2015.01.02'

From faa4ea68c08efc5bddb863455bab7c2fe64dcbc3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 20:56:42 +0600
Subject: [PATCH 1003/1937] [generic] Add BBC iPlayer playlist test

---
 youtube_dl/extractor/generic.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index fd4ee4667..493afb57d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -181,6 +181,14 @@ class GenericIE(InfoExtractor):
                 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
             },
         },
+        # BBC iPlayer embeds
+        {
+            'url': 'http://www.bbc.co.uk/blogs/adamcurtis/posts/BUGGER',
+            'info_dict': {
+                'title': 'BBC - Blogs -  Adam Curtis - BUGGER',
+            },
+            'playlist_mincount': 18,
+        },
         # RUTV embed
         {
             'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',

From d34f98289b302619b1213ef6e50e034021f811e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 21:12:25 +0600
Subject: [PATCH 1004/1937] [xhamster] Remove identical tests

---
 youtube_dl/extractor/xhamster.py | 22 +---------------------
 1 file changed, 1 insertion(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 519bfc191..4527567f8 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -42,28 +42,8 @@ class XHamsterIE(InfoExtractor):
         },
         {
             'url': 'https://xhamster.com/movies/2272726/amber_slayed_by_the_knight.html',
-            'info_dict': {
-                'id': '2272726',
-                'ext': 'mp4',
-                'title': 'Amber slayed by the Knight',
-                'upload_date': '20131009',
-                'uploader_id': 'amberblank',
-                'duration': 149,
-                'age_limit': 18,
-            }
+            'only_matching': True,
         },
-        {
-            "url": "https://xhamster.com/movies/1444747/hotkinkyjo_amp_asian_girl_elbow_deep_anal_fisting_and_footing.html",
-            'info_dict': {
-                "id": "1444747",
-                'ext': 'mp4',
-                "title": "HOTKINKYJO & ASIAN GIRL ELBOW DEEP ANAL FISTING AND FOOTING",
-                "upload_date": "20120922",
-                "uploader_id": "alex1981",
-                "duration": 80,
-                "age_limit": 18,
-            }
-        }
     ]
 
     def _real_extract(self, url):

From 324b2c78fad949545b6e0707bb5ee1e37ec6217d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 21:46:57 +0600
Subject: [PATCH 1005/1937] [xtube] Fix uploader regex

---
 youtube_dl/extractor/xtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index cf74d4fd5..95f1c8f3c 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -40,7 +40,7 @@ class XTubeIE(InfoExtractor):
             r'<p class="title">([^<]+)', webpage, 'title')
         video_uploader = self._html_search_regex(
             [r"var\s+contentOwnerId\s*=\s*'([^']+)",
-             r'By:\s*<a href="/community/profile\.php?user=([^"]+)'],
+             r'By:\s*<a href="/community/profile\.php\?user=([^"]+)'],
             webpage, 'uploader', fatal=False)
         video_description = self._html_search_regex(
             r'<p class="fieldsDesc">([^<]+)',

From 823a155293f744f147c83c2f8efc4aaf2c3760e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 22:09:00 +0600
Subject: [PATCH 1006/1937] [vier:videos] Tune _VALID_URL not to match single
 videos

---
 youtube_dl/extractor/vier.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vier.py b/youtube_dl/extractor/vier.py
index 0d9fb09a7..619039e51 100644
--- a/youtube_dl/extractor/vier.py
+++ b/youtube_dl/extractor/vier.py
@@ -63,7 +63,7 @@ class VierIE(InfoExtractor):
 
 class VierVideosIE(InfoExtractor):
     IE_NAME = 'vier:videos'
-    _VALID_URL = r'https?://(?:www\.)?vier\.be/(?P<program>[^/]+)/videos(?:\?.*\bpage=(?P<page>\d+))?'
+    _VALID_URL = r'https?://(?:www\.)?vier\.be/(?P<program>[^/]+)/videos(?:\?.*\bpage=(?P<page>\d+)|$)'
     _TESTS = [{
         'url': 'http://www.vier.be/demoestuin/videos',
         'info_dict': {

From c067545c175bb421fcfc255ae7df7a2fff1c5638 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Caletka?= <ondrej@caletka.cz>
Date: Fri, 2 Jan 2015 17:12:20 +0100
Subject: [PATCH 1007/1937] ceskatelevize: Closed captions support

---
 youtube_dl/extractor/ceskatelevize.py | 55 ++++++++++++++++++++++++++-
 1 file changed, 53 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index ba8376338..345ac5e85 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .subtitles import SubtitlesInfoExtractor
 from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
@@ -12,10 +12,11 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     float_or_none,
+    HEADRequest,
 )
 
 
-class CeskaTelevizeIE(InfoExtractor):
+class CeskaTelevizeIE(SubtitlesInfoExtractor):
     _VALID_URL = r'https?://www\.ceskatelevize\.cz/(porady|ivysilani)/(.+/)?(?P<id>[^?#]+)'
 
     _TESTS = [
@@ -104,6 +105,15 @@ class CeskaTelevizeIE(InfoExtractor):
         duration = float_or_none(item.get('duration'))
         thumbnail = item.get('previewImageUrl')
 
+        # subtitles
+        subtitles = self.extract_subtitles(video_id, webpage)
+
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, webpage)
+            return
+
+        subtitles = self._fix_subtitles(self.extract_subtitles(video_id, webpage))
+
         return {
             'id': episode_id,
             'title': title,
@@ -111,4 +121,45 @@ class CeskaTelevizeIE(InfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
+            'subtitles': subtitles,
         }
+
+    def _fix_subtitles(self, subtitles):
+        """ Convert milisecond-based subtitles to SRT """
+        if subtitles is None:
+            return subtitles  # subtitles not requested
+
+        def _msectotimecode(msec):
+            """ Helper utility to convert miliseconds to timecode """
+            components = []
+            for divider in [1000, 60, 60, 100]:
+                components.append(msec % divider)
+                msec //= divider
+            return "{3:02}:{2:02}:{1:02},{0:03}".format(*components)
+
+        def _fix_subtitle(subtitle):
+            for line in subtitle.splitlines():
+                m = re.match(r"^ *([0-9]+); *([0-9]+) +([0-9]+) *$", line)
+                if m:
+                    yield m.group(1)
+                    start, stop = (_msectotimecode(int(t)) for t in m.groups()[1:])
+                    yield "{} --> {}".format(start, stop)
+                else:
+                    yield line
+
+        fixed_subtitles = {}
+        for k, v in subtitles.items():
+            fixed_subtitles[k] = "\r\n".join(_fix_subtitle(v))
+        return fixed_subtitles
+
+    def _get_available_subtitles(self, video_id, webpage):
+        video_id = video_id.partition('-')[0]
+        url = 'http://imgct.ceskatelevize.cz/cache/data/ivysilani/' \
+              'subtitles/{}/{}/sub.txt'.format(video_id[:3], video_id)
+        req = HEADRequest(url)
+        sub = self._request_webpage(
+            req, video_id,
+            note="Checking subtitles",
+            errnote="No subtitles found",
+            fatal=False)
+        return {'cs': url} if sub else {}

From c491418526dc87d16188e30d6bf1fc01917ad62c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 22:13:26 +0600
Subject: [PATCH 1008/1937] [bbccouk] Update test

---
 youtube_dl/extractor/bbccouk.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 73fe66b01..5865a6e39 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -18,8 +18,8 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
             'info_dict': {
                 'id': 'b039d07m',
                 'ext': 'flv',
-                'title': 'Kaleidoscope: Leonard Cohen',
-                'description': 'md5:db4755d7a665ae72343779f7dacb402c',
+                'title': 'Kaleidoscope, Leonard Cohen',
+                'description': 'The Canadian poet and songwriter reflects on his musical career.',
                 'duration': 1740,
             },
             'params': {

From 46d32a12c9fd3e7639a51e27226a29c59ee0ca72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 22:23:00 +0600
Subject: [PATCH 1009/1937] [bet] Update test

---
 youtube_dl/extractor/bet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bet.py b/youtube_dl/extractor/bet.py
index 003e50002..4fbdd6f1c 100644
--- a/youtube_dl/extractor/bet.py
+++ b/youtube_dl/extractor/bet.py
@@ -16,7 +16,7 @@ class BetIE(InfoExtractor):
         {
             'url': 'http://www.bet.com/news/politics/2014/12/08/in-bet-exclusive-obama-talks-race-and-racism.html',
             'info_dict': {
-                'id': '417cd61c-c793-4e8e-b006-e445ecc45add',
+                'id': '406429c6-1b8a-463e-83fc-814adb81a9db',
                 'display_id': 'in-bet-exclusive-obama-talks-race-and-racism',
                 'ext': 'flv',
                 'title': 'BET News Presents: A Conversation With President Obama',

From 2522a0b7dab3acde63ed1384d53a6e5c2b7fa749 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 22:28:48 +0600
Subject: [PATCH 1010/1937] [kontrtube] Extract display_id

Trailing slash in URL is mandatory now
---
 youtube_dl/extractor/kontrtube.py | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/kontrtube.py b/youtube_dl/extractor/kontrtube.py
index 41fd62009..720bc939b 100644
--- a/youtube_dl/extractor/kontrtube.py
+++ b/youtube_dl/extractor/kontrtube.py
@@ -10,13 +10,14 @@ from ..utils import int_or_none
 class KontrTubeIE(InfoExtractor):
     IE_NAME = 'kontrtube'
     IE_DESC = 'KontrTube.ru - Труба зовёт'
-    _VALID_URL = r'http://(?:www\.)?kontrtube\.ru/videos/(?P<id>\d+)/.+'
+    _VALID_URL = r'http://(?:www\.)?kontrtube\.ru/videos/(?P<id>\d+)/(?P<display_id>[^/]+)/'
 
     _TEST = {
         'url': 'http://www.kontrtube.ru/videos/2678/nad-olimpiyskoy-derevney-v-sochi-podnyat-rossiyskiy-flag/',
         'md5': '975a991a4926c9a85f383a736a2e6b80',
         'info_dict': {
             'id': '2678',
+            'display_id': 'nad-olimpiyskoy-derevney-v-sochi-podnyat-rossiyskiy-flag',
             'ext': 'mp4',
             'title': 'Над олимпийской деревней в Сочи поднят российский флаг',
             'description': 'md5:80edc4c613d5887ae8ccf1d59432be41',
@@ -28,21 +29,28 @@ class KontrTubeIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        webpage = self._download_webpage(url, video_id, 'Downloading page')
+        webpage = self._download_webpage(
+            url, display_id, 'Downloading page')
 
-        video_url = self._html_search_regex(r"video_url: '(.+?)/?',", webpage, 'video URL')
-        thumbnail = self._html_search_regex(r"preview_url: '(.+?)/?',", webpage, 'video thumbnail', fatal=False)
+        video_url = self._html_search_regex(
+            r"video_url\s*:\s*'(.+?)/?',", webpage, 'video URL')
+        thumbnail = self._html_search_regex(
+            r"preview_url\s*:\s*'(.+?)/?',", webpage, 'video thumbnail', fatal=False)
         title = self._html_search_regex(
             r'<title>(.+?)</title>', webpage, 'video title')
-        description = self._html_search_meta('description', webpage, 'video description')
+        description = self._html_search_meta(
+            'description', webpage, 'video description')
 
         mobj = re.search(
-            r'<div class="col_2">Длительность: <span>(?P<minutes>\d+)м:(?P<seconds>\d+)с</span></div>', webpage)
+            r'<div class="col_2">Длительность: <span>(?P<minutes>\d+)м:(?P<seconds>\d+)с</span></div>',
+            webpage)
         duration = int(mobj.group('minutes')) * 60 + int(mobj.group('seconds')) if mobj else None
 
         view_count = self._html_search_regex(
-            r'<div class="col_2">Просмотров: <span>(\d+)</span></div>', webpage, 'view count', fatal=False)
+            r'<div class="col_2">Просмотров: <span>(\d+)</span></div>',
+            webpage, 'view count', fatal=False)
 
         comment_count = None
         comment_str = self._html_search_regex(
@@ -56,6 +64,7 @@ class KontrTubeIE(InfoExtractor):
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'url': video_url,
             'thumbnail': thumbnail,
             'title': title,

From 62420c73cb16472ead562339e22b038fac7aa950 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 2 Jan 2015 22:31:55 +0600
Subject: [PATCH 1011/1937] [played] Skip test

---
 youtube_dl/extractor/played.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/played.py b/youtube_dl/extractor/played.py
index 449d4836c..45716c75d 100644
--- a/youtube_dl/extractor/played.py
+++ b/youtube_dl/extractor/played.py
@@ -26,6 +26,7 @@ class PlayedIE(InfoExtractor):
             'ext': 'flv',
             'title': 'youtube-dl_test_video.mp4',
         },
+        'skip': 'Removed for copyright infringement.',  # oh wow
     }
 
     def _real_extract(self, url):

From 817f786fbb05070a7ad23aec32293cfcdd072bf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 2 Jan 2015 21:02:34 +0100
Subject: [PATCH 1012/1937] [canalplus] Raise an error if the video is
 georestricted (closes #4472)

---
 youtube_dl/extractor/canalplus.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 9873728df..11d18d74a 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -5,6 +5,8 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
+    HEADRequest,
     unified_strdate,
     url_basename,
     qualities,
@@ -76,6 +78,16 @@ class CanalplusIE(InfoExtractor):
 
         preference = qualities(['MOBILE', 'BAS_DEBIT', 'HAUT_DEBIT', 'HD', 'HLS', 'HDS'])
 
+        fmt_url = next(iter(media.find('VIDEOS'))).text
+        if '/geo' in fmt_url.lower():
+            response = self._request_webpage(
+                HEADRequest(fmt_url), video_id,
+                'Checking if the video is georestricted')
+            if '/blocage' in response.geturl():
+                raise ExtractorError(
+                    'The video is not available in your country',
+                    expected=True)
+
         formats = []
         for fmt in media.find('VIDEOS'):
             format_url = fmt.text

From a221f22969843866ad85fd5cdf98160c79d99e9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 2 Jan 2015 21:17:10 +0100
Subject: [PATCH 1013/1937] [crunchyroll] Fix format extraction

Reported in https://github.com/rg3/youtube-dl/issues/2782#issuecomment-68556780
---
 youtube_dl/extractor/crunchyroll.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 354046a9e..1680f532f 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -228,7 +228,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
         video_thumbnail = self._search_regex(r'<episode_image_url>([^<]+)', playerdata, 'thumbnail', fatal=False)
 
         formats = []
-        for fmt in re.findall(r'\?p([0-9]{3,4})=1', webpage):
+        for fmt in re.findall(r'showmedia\.([0-9]{3,4})p', webpage):
             stream_quality, stream_format = self._FORMAT_IDS[fmt]
             video_format = fmt + 'p'
             streamdata_req = compat_urllib_request.Request('http://www.crunchyroll.com/xml/')

From d26b1317edd31d85f8fa68d796f185f7ba65c282 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 3 Jan 2015 00:33:36 +0100
Subject: [PATCH 1014/1937] [downloader/mplayer] Use check_executable

---
 youtube_dl/downloader/mplayer.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/mplayer.py b/youtube_dl/downloader/mplayer.py
index c53195da0..34b23b5c2 100644
--- a/youtube_dl/downloader/mplayer.py
+++ b/youtube_dl/downloader/mplayer.py
@@ -6,6 +6,7 @@ import subprocess
 from .common import FileDownloader
 from ..compat import compat_subprocess_get_DEVNULL
 from ..utils import (
+    check_executable,
     encodeFilename,
 )
 
@@ -20,11 +21,7 @@ class MplayerFD(FileDownloader):
             'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
             '-dumpstream', '-dumpfile', tmpfilename, url]
         # Check for mplayer first
-        try:
-            subprocess.call(
-                ['mplayer', '-h'],
-                stdout=compat_subprocess_get_DEVNULL(), stderr=subprocess.STDOUT)
-        except (OSError, IOError):
+        if not check_executable('mplayer', ['-h']):
             self.report_error('MMS or RTSP download detected but "%s" could not be run' % args[0])
             return False
 

From c7e675940ca06a74f068164b25d3afb9b657ff24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 3 Jan 2015 20:43:40 +0600
Subject: [PATCH 1015/1937] [bbccouk] Add support for music clips (Closes
 #4143)

---
 youtube_dl/extractor/bbccouk.py | 33 ++++++++++++++++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 5865a6e39..1cf48fe0d 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -10,7 +10,7 @@ from ..compat import compat_HTTPError
 class BBCCoUkIE(SubtitlesInfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
-    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:programmes|iplayer/(?:episode|playlist))/(?P<id>[\da-z]{8})'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
     _TESTS = [
         {
@@ -84,9 +84,40 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
                 # rtmp download
                 'skip_download': True,
             }
+        }, {
+            'url': 'http://www.bbc.co.uk/music/clips/p02frcc3',
+            'note': 'Audio',
+            'info_dict': {
+                'id': 'p02frcch',
+                'ext': 'flv',
+                'title': 'Pete Tong, Past, Present and Future Special, Madeon - After Hours mix',
+                'description': 'French house superstar Madeon takes us out of the club and onto the after party.',
+                'duration': 3507,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            }
+        }, {
+            'url': 'http://www.bbc.co.uk/music/clips/p025c0zz',
+            'note': 'Video',
+            'info_dict': {
+                'id': 'p025c103',
+                'ext': 'flv',
+                'title': 'Reading and Leeds Festival, 2014, Rae Morris - Closer (Live on BBC Three)',
+                'description': 'Rae Morris performs Closer for BBC Three at Reading 2014',
+                'duration': 226,
+            },
+            'params': {
+                # rtmp download
+                'skip_download': True,
+            }
         }, {
             'url': 'http://www.bbc.co.uk/iplayer/playlist/p01dvks4',
             'only_matching': True,
+        }, {
+            'url': 'http://www.bbc.co.uk/music/clips#p02frcc3',
+            'only_matching': True,
         }
     ]
 

From 6e46c3f1fddc954596d429d15d7c035d12a625a9 Mon Sep 17 00:00:00 2001
From: Gaurav <gauravb7090@gmail.com>
Date: Sat, 3 Jan 2015 20:30:28 +0530
Subject: [PATCH 1016/1937] Added support for EllenTube along with EllenTV

---
 youtube_dl/extractor/ellentv.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 3e7923648..097317842 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -12,8 +12,8 @@ from ..utils import (
 
 
 class EllenTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ellentv\.com/videos/(?P<id>[a-z0-9_-]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?(ellentv|ellentube)\.com/videos/(?P<id>[a-z0-9_-]+)'
+    _TESTS = [{
         'url': 'http://www.ellentv.com/videos/0-7jqrsr18/',
         'md5': 'e4af06f3bf0d5f471921a18db5764642',
         'info_dict': {
@@ -23,7 +23,18 @@ class EllenTVIE(InfoExtractor):
             'timestamp': 1406876400,
             'upload_date': '20140801',
         }
-    }
+    },
+    {
+        'url': 'http://ellentube.com/videos/0-dvzmabd5/',
+        'md5': '98238118eaa2bbdf6ad7f708e3e4f4eb',
+        'info_dict': {
+            'id': '0-dvzmabd5',
+            'ext': 'mp4',
+            'title': '1 year old twin sister makes her brother laugh',
+            'timestamp': 1419542075,
+            'upload_date': '20141225',
+        }
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -33,6 +44,9 @@ class EllenTVIE(InfoExtractor):
         timestamp = parse_iso8601(self._search_regex(
             r'<span class="publish-date"><time datetime="([^"]+)">',
             webpage, 'timestamp'))
+        print "\n"
+        print timestamp
+        print "\n"
 
         return {
             'id': video_id,

From f4f339529cfe19d09a0c7103f64ef60b15032a2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 3 Jan 2015 21:44:47 +0600
Subject: [PATCH 1017/1937] [ellentv] Clean up and simplify

---
 youtube_dl/extractor/ellentv.py | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 097317842..89fa9f7e4 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -1,7 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
 import json
 
 from .common import InfoExtractor
@@ -12,7 +11,7 @@ from ..utils import (
 
 
 class EllenTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(ellentv|ellentube)\.com/videos/(?P<id>[a-z0-9_-]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:ellentv|ellentube)\.com/videos/(?P<id>[a-z0-9_-]+)'
     _TESTS = [{
         'url': 'http://www.ellentv.com/videos/0-7jqrsr18/',
         'md5': 'e4af06f3bf0d5f471921a18db5764642',
@@ -23,8 +22,7 @@ class EllenTVIE(InfoExtractor):
             'timestamp': 1406876400,
             'upload_date': '20140801',
         }
-    },
-    {
+    }, {
         'url': 'http://ellentube.com/videos/0-dvzmabd5/',
         'md5': '98238118eaa2bbdf6ad7f708e3e4f4eb',
         'info_dict': {
@@ -37,16 +35,12 @@ class EllenTVIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
         timestamp = parse_iso8601(self._search_regex(
             r'<span class="publish-date"><time datetime="([^"]+)">',
             webpage, 'timestamp'))
-        print "\n"
-        print timestamp
-        print "\n"
 
         return {
             'id': video_id,
@@ -69,8 +63,7 @@ class EllenTVClipsIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        playlist_id = mobj.group('id')
+        playlist_id = self._match_id(url)
 
         webpage = self._download_webpage(url, playlist_id)
         playlist = self._extract_playlist(webpage)

From baa3c3f0f64b1aba523f56c9af351b6fb3378fc6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 3 Jan 2015 21:54:18 +0600
Subject: [PATCH 1018/1937] [ellentv] Improve extraction

---
 youtube_dl/extractor/ellentv.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py
index 89fa9f7e4..fc92ff825 100644
--- a/youtube_dl/extractor/ellentv.py
+++ b/youtube_dl/extractor/ellentv.py
@@ -19,6 +19,7 @@ class EllenTVIE(InfoExtractor):
             'id': '0-7jqrsr18',
             'ext': 'mp4',
             'title': 'What\'s Wrong with These Photos? A Whole Lot',
+            'description': 'md5:35f152dc66b587cf13e6d2cf4fa467f6',
             'timestamp': 1406876400,
             'upload_date': '20140801',
         }
@@ -29,6 +30,7 @@ class EllenTVIE(InfoExtractor):
             'id': '0-dvzmabd5',
             'ext': 'mp4',
             'title': '1 year old twin sister makes her brother laugh',
+            'description': '1 year old twin sister makes her brother laugh',
             'timestamp': 1419542075,
             'upload_date': '20141225',
         }
@@ -38,14 +40,20 @@ class EllenTVIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
+        video_url = self._html_search_meta('VideoURL', webpage, 'url')
+        title = self._og_search_title(webpage, default=None) or self._search_regex(
+            r'pageName\s*=\s*"([^"]+)"', webpage, 'title')
+        description = self._html_search_meta(
+            'description', webpage, 'description') or self._og_search_description(webpage)
         timestamp = parse_iso8601(self._search_regex(
             r'<span class="publish-date"><time datetime="([^"]+)">',
             webpage, 'timestamp'))
 
         return {
             'id': video_id,
-            'title': self._og_search_title(webpage),
-            'url': self._html_search_meta('VideoURL', webpage, 'url'),
+            'url': video_url,
+            'title': title,
+            'description': description,
             'timestamp': timestamp,
         }
 

From e65566a9cc13d13bef3dc70eebfc9bda2e302b78 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 3 Jan 2015 18:33:38 +0100
Subject: [PATCH 1019/1937] [youtube] Correct handling when DASH manifest is
 not necessary to find all formats

---
 youtube_dl/YoutubeDL.py         |  4 +++-
 youtube_dl/extractor/common.py  |  2 ++
 youtube_dl/extractor/youtube.py | 11 +++++++++--
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e2b823f66..806e7b239 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1333,7 +1333,9 @@ class YoutubeDL(object):
         formats = info_dict.get('formats', [info_dict])
         idlen = max(len('format code'),
                     max(len(f['format_id']) for f in formats))
-        formats_s = [line(f, idlen) for f in formats]
+        formats_s = [
+            line(f, idlen) for f in formats
+            if f.get('preference') is None or f['preference'] >= -1000]
         if len(formats) > 1:
             formats_s[0] += (' ' if self._format_note(formats[0]) else '') + '(worst)'
             formats_s[-1] += (' ' if self._format_note(formats[-1]) else '') + '(best)'
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6e264f687..562e656e0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -92,6 +92,8 @@ class InfoExtractor(object):
                                  by this field, regardless of all other values.
                                  -1 for default (order by other properties),
                                  -2 or smaller for less than default.
+                                 < -1000 to hide the format (if there is
+                                    another one which is strictly better)
                     * language_preference  Is this in the correct requested
                                  language?
                                  10 if it's what the URL is about,
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8c7842ee8..e9bf39a00 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -256,7 +256,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
-        '138': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
+        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},  # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
         '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
@@ -736,6 +736,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'format_id': format_id,
                 'url': video_url,
                 'width': int_or_none(r.attrib.get('width')),
+                'height': int_or_none(r.attrib.get('height')),
                 'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
                 'asr': int_or_none(r.attrib.get('audioSamplingRate')),
                 'filesize': filesize,
@@ -746,7 +747,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                     fo for fo in formats
                     if fo['format_id'] == format_id)
             except StopIteration:
-                f.update(self._formats.get(format_id, {}))
+                f.update(self._formats.get(format_id, {}).items())
                 formats.append(f)
             else:
                 existing_format.update(f)
@@ -1040,6 +1041,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                     self.report_warning(
                         'Skipping DASH manifest: %r' % e, video_id)
                 else:
+                    # Hide the formats we found through non-DASH
+                    dash_keys = set(df['format_id'] for df in dash_formats)
+                    for f in formats:
+                        if f['format_id'] in dash_keys:
+                            f['format_id'] = 'nondash-%s' % f['format_id']
+                            f['preference'] -= 10000
                     formats.extend(dash_formats)
 
         self._sort_formats(formats)

From 19b05d886ef0bd23f4a4c6ee90f171c48cb55ace Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 3 Jan 2015 18:35:30 +0100
Subject: [PATCH 1020/1937] release 2015.01.03

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 58b5021dc..2a54b9bbe 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.02'
+__version__ = '2015.01.03'

From 382e05fa5686ea58606f684f3032b91801ece8f3 Mon Sep 17 00:00:00 2001
From: t0mm0 <dev@onairsoftware.co.uk>
Date: Sun, 4 Jan 2015 00:05:31 +0000
Subject: [PATCH 1021/1937] [tf1] add support for TFOU

---
 youtube_dl/extractor/tf1.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 6e61cc9e2..384b4c3f8 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -8,8 +8,8 @@ from .common import InfoExtractor
 
 class TF1IE(InfoExtractor):
     """TF1 uses the wat.tv player."""
-    _VALID_URL = r'http://videos\.tf1\.fr/.*-(?P<id>.*?)\.html'
-    _TEST = {
+    _VALID_URL = r'http://(?:videos\.tf1|www\.tfou)\.fr/.*?-(?P<id>\d+)(?:-\d+)?\.html'
+    _TESTS = {
         'url': 'http://videos.tf1.fr/auto-moto/citroen-grand-c4-picasso-2013-presentation-officielle-8062060.html',
         'info_dict': {
             'id': '10635995',
@@ -21,14 +21,25 @@ class TF1IE(InfoExtractor):
             # Sometimes wat serves the whole file with the --test option
             'skip_download': True,
         },
+    }, {
+        'url': 'http://www.tfou.fr/chuggington/videos/le-grand-mysterioso-chuggington-7085291-739.html',
+        'info_dict': {
+            'id': '7085291',
+            'ext': 'mp4',
+            'title': 'Le grand Mystérioso - Chuggington',
+            'description': 'Emery rêve qu\'un article lui soit consacré dans le journal.',
+        },
+        'params': {
+            # Sometimes wat serves the whole file with the --test option
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         embed_url = self._html_search_regex(
-            r'"(https://www.wat.tv/embedframe/.*?)"', webpage, 'embed url')
+            r'["\'](http(?:s)?://www.wat.tv/embedframe/.*?)["\']', webpage, 'embed url')
         embed_page = self._download_webpage(embed_url, video_id,
                                             'Downloading embed player page')
         wat_id = self._search_regex(r'UVID=(.*?)&', embed_page, 'wat id')

From 2bf098eda43f0c789e04f7f633149cc99873d182 Mon Sep 17 00:00:00 2001
From: t0mm0 <dev@onairsoftware.co.uk>
Date: Sun, 4 Jan 2015 00:43:55 +0000
Subject: [PATCH 1022/1937] [tf1] fix test

---
 youtube_dl/extractor/tf1.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 384b4c3f8..5c0fcef58 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -24,10 +24,11 @@ class TF1IE(InfoExtractor):
     }, {
         'url': 'http://www.tfou.fr/chuggington/videos/le-grand-mysterioso-chuggington-7085291-739.html',
         'info_dict': {
-            'id': '7085291',
+            'id': '12043945',
             'ext': 'mp4',
             'title': 'Le grand Mystérioso - Chuggington',
-            'description': 'Emery rêve qu\'un article lui soit consacré dans le journal.',
+            'description': 'Le grand Mystérioso - Emery rêve qu\'un article lui soit consacré dans le journal.',
+            'upload_date': '20150103',
         },
         'params': {
             # Sometimes wat serves the whole file with the --test option

From b158bb86934fb8863a697b6e65a1cbc5105d1741 Mon Sep 17 00:00:00 2001
From: t0mm0 <dev@onairsoftware.co.uk>
Date: Sun, 4 Jan 2015 00:45:23 +0000
Subject: [PATCH 1023/1937] [tf1] simplify regex

---
 youtube_dl/extractor/tf1.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 5c0fcef58..07cc81226 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -40,7 +40,7 @@ class TF1IE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         embed_url = self._html_search_regex(
-            r'["\'](http(?:s)?://www.wat.tv/embedframe/.*?)["\']', webpage, 'embed url')
+            r'["\'](https?://www.wat.tv/embedframe/.*?)["\']', webpage, 'embed url')
         embed_page = self._download_webpage(embed_url, video_id,
                                             'Downloading embed player page')
         wat_id = self._search_regex(r'UVID=(.*?)&', embed_page, 'wat id')

From b68ff259170711574a1fd779e86bdfea9aaafcf5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 4 Jan 2015 02:05:26 +0100
Subject: [PATCH 1024/1937] Add various anime sites (Closes #4554)

---
 AUTHORS                           |   1 +
 youtube_dl/extractor/__init__.py  |  19 ++++
 youtube_dl/extractor/gogoanime.py |  76 +++++++++++++++
 youtube_dl/extractor/play44.py    | 149 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/soulanime.py |  74 +++++++++++++++
 youtube_dl/extractor/videofun.py  |  36 ++++++++
 6 files changed, 355 insertions(+)
 create mode 100644 youtube_dl/extractor/gogoanime.py
 create mode 100644 youtube_dl/extractor/play44.py
 create mode 100644 youtube_dl/extractor/soulanime.py
 create mode 100644 youtube_dl/extractor/videofun.py

diff --git a/AUTHORS b/AUTHORS
index 37306ac83..9b548cf25 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -98,3 +98,4 @@ Will Glynn
 Max Reimann
 Cédric Luthi
 Thijs Vermeir
+Joel Leclerc
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9ccd1b32e..c3dc09f75 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -164,6 +164,10 @@ from .globo import GloboIE
 from .godtube import GodTubeIE
 from .goldenmoustache import GoldenMoustacheIE
 from .golem import GolemIE
+from .gogoanime import (
+    GoGoAnimeIE,
+    GoGoAnimeSearchIE
+)
 from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
 from .gorillavid import GorillaVidIE
@@ -313,6 +317,16 @@ from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
 from .planetaplay import PlanetaPlayIE
 from .played import PlayedIE
+from .play44 import (
+    Play44IE,
+    ByZooIE,
+    Video44IE,
+    VideoWingIE,
+    PlayPandaIE,
+    VideoZooIE,
+    PlayBBIE,
+    EasyVideoIE
+)
 from .playfm import PlayFMIE
 from .playvid import PlayvidIE
 from .podomatic import PodomaticIE
@@ -373,6 +387,10 @@ from .smotri import (
 from .snotr import SnotrIE
 from .sockshare import SockshareIE
 from .sohu import SohuIE
+from .soulanime import (
+    SoulAnimeWatchingIE,
+    SoulAnimeSeriesIE
+)
 from .soundcloud import (
     SoundcloudIE,
     SoundcloudSetIE,
@@ -467,6 +485,7 @@ from .viddler import ViddlerIE
 from .videobam import VideoBamIE
 from .videodetective import VideoDetectiveIE
 from .videolecturesnet import VideoLecturesNetIE
+from .videofun import VideoFunIE
 from .videofyme import VideofyMeIE
 from .videomega import VideoMegaIE
 from .videopremium import VideoPremiumIE
diff --git a/youtube_dl/extractor/gogoanime.py b/youtube_dl/extractor/gogoanime.py
new file mode 100644
index 000000000..d4f4ecc58
--- /dev/null
+++ b/youtube_dl/extractor/gogoanime.py
@@ -0,0 +1,76 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    compat_urllib_parse,
+    get_element_by_attribute,
+    unescapeHTML
+)
+
+
+class GoGoAnimeIE(InfoExtractor):
+    IE_NAME = 'gogoanime'
+    IE_DESC = 'GoGoAnime'
+    _VALID_URL = r'http://www.gogoanime.com/(?P<id>[A-Za-z0-9-]+)'
+
+    _TEST = {
+        'url': 'http://www.gogoanime.com/mahou-shoujo-madoka-magica-movie-1',
+        'info_dict': {
+            'id': 'mahou-shoujo-madoka-magica-movie-1'
+        },
+        'playlist_count': 3
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        page = self._download_webpage(url, video_id)
+
+        if 'Oops! Page Not Found</font>' in page:
+            raise ExtractorError('Video does not exist', expected=True)
+
+        content = get_element_by_attribute("class", "postcontent", page)
+        vids = re.findall(r'<iframe[^>]*?src=[\'"](h[^\'"]+)[\'"]', content)
+        vids = [
+            unescapeHTML(compat_urllib_parse.unquote(x))
+            for x in vids if not re.search(r".*videofun.*", x)]
+
+        if re.search(r'<div class="postcontent">[^<]*<p><iframe src=[\'"][^>]+></iframe><br />', page):
+            return self.playlist_result([self.url_result(vid) for vid in vids], video_id)
+
+        title = self._html_search_regex(
+            r'<div class="postdesc">[^<]*<h1>([^<]+)</h1>', page, 'title')
+
+        return {
+            '_type': 'url',
+            'id': video_id,
+            'url': vids[0],
+            'title': title,
+        }
+
+
+class GoGoAnimeSearchIE(InfoExtractor):
+    IE_NAME = 'gogoanime:search'
+    IE_DESC = 'GoGoAnime Search'
+
+    _VALID_URL = r'http://www\.gogoanime\.com/.*\?s=(?P<id>[^&]*)'
+    _TEST = {
+        'url': 'http://www.gogoanime.com/?s=bokusatsu',
+        'info_dict': {
+            'id': 'bokusatsu'
+        },
+        'playlist_count': 6
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        webpage = self._download_webpage(url, playlist_id)
+
+        posts = re.findall(
+            r'<div class="postlist">[^<]*<p[^>]*>[^<]*<a href="(?P<url>[^"]+)"',
+            webpage)
+
+        return self.playlist_result(
+            [self.url_result(p) for p in posts], playlist_id)
diff --git a/youtube_dl/extractor/play44.py b/youtube_dl/extractor/play44.py
new file mode 100644
index 000000000..b8696e516
--- /dev/null
+++ b/youtube_dl/extractor/play44.py
@@ -0,0 +1,149 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    compat_urllib_parse
+)
+
+
+class Play44IE(InfoExtractor):
+    _VALID_URL = r'http://[w.]*play44\.net/embed\.php[^/]*/(?P<id>.+)'
+
+    _TESTS = [{
+        'url': 'http://play44.net/embed.php?w=600&h=438&vid=M/mahou-shoujo-madoka-magica-07.flv',
+        'md5': 'e37e99d665f503dd2db952f7c4dba9e6',
+        'info_dict': {
+            'id': 'mahou-shoujo-madoka-magica-07',
+            'ext': 'flv',
+            'title': 'mahou-shoujo-madoka-magica-07',
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        page = self._download_webpage(url, video_id)
+
+        video_url = compat_urllib_parse.unquote(self._html_search_regex(
+            r'_url = "(https?://[^"]+?)";', page, 'url'))
+        title = self._search_regex(r'.*/(?P<title>[^.]*).', video_url, 'title')
+
+        return {
+            'id': title,
+            'url': video_url,
+            'title': title,
+        }
+
+
+class ByZooIE(Play44IE):
+    _VALID_URL = r'http://[w.]*byzoo\.org/embed\.php[^/]*/(?P<id>.+)'
+
+    _TESTS = [{
+        'url': 'http://byzoo.org/embed.php?w=600&h=438&vid=at/nw/mahou_shoujo_madoka_magica_movie_3_-_part1.mp4',
+        'md5': '455c83dabe2cd9fd74a87612b01fe017',
+        'info_dict': {
+            'id': 'mahou_shoujo_madoka_magica_movie_3_-_part1',
+            'ext': 'mp4',
+            'title': 'mahou_shoujo_madoka_magica_movie_3_-_part1',
+        }
+    }]
+
+
+class Video44IE(Play44IE):
+    _VALID_URL = r'http://[w.]*video44\.net/.*file=(?P<id>[^&].).*'
+
+    _TESTS = [{
+        'url': 'http://www.video44.net/gogo/?w=600&h=438&file=chaoshead-12.flv&sv=1',
+        'md5': '43eaec6d0beb10e8d42459b9f108aff3',
+        'info_dict': {
+            'id': 'chaoshead-12',
+            'ext': 'mp4',
+            'title': 'chaoshead-12',
+        }
+    }]
+
+
+class VideoWingIE(Play44IE):
+    _VALID_URL = r'''(?x)
+        http://[w.]*videowing\.[^/]*/
+        (?:
+            .*video=/*
+            |embed/
+        )
+        (?P<id>[^&?.]+)
+    '''
+
+    _TESTS = [{
+        'url': 'http://videowing.me/embed?w=718&h=438&video=ongoing/boku_wa_tomodachi_ga_sukunai_-_05.mp4',
+        'md5': '4ed320e353ed26c742c4f12a9c210b60',
+        'info_dict': {
+            'id': 'boku_wa_tomodachi_ga_sukunai_-_05',
+            'ext': 'mp4',
+            'title': 'boku_wa_tomodachi_ga_sukunai_-_05',
+        }
+    }, {
+        'url': 'http://videowing.me/embed/a8d6a39522df066bd734a69f2334497e?w=600&h=438',
+        'md5': '33fdd71581357018c226f95c5cedcfd7',
+        'info_dict': {
+            'id': 'mahoushoujomadokamagicamovie1part1',
+            'ext': 'flv',
+            'title': 'mahoushoujomadokamagicamovie1part1',
+        }
+    }]
+
+
+class PlayPandaIE(Play44IE):
+    _VALID_URL = r'http://[w.]*playpanda\.[^/]*/.*vid=/*(?P<id>[^&].).*'
+
+    _TESTS = [{
+        'url': 'http://playpanda.net/embed.php?w=718&h=438&vid=at/nw/boku_wa_tomodachi_ga_sukunai_-_05.mp4',
+        'md5': '4ed320e353ed26c742c4f12a9c210b60',
+        'info_dict': {
+            'id': 'boku_wa_tomodachi_ga_sukunai_-_05',
+            'ext': 'mp4',
+            'title': 'boku_wa_tomodachi_ga_sukunai_-_05',
+            'description': 'boku_wa_tomodachi_ga_sukunai_-_05'
+        }
+    }]
+
+
+class VideoZooIE(Play44IE):
+    _VALID_URL = r'http://[w.]*videozoo\.[^/]*/.*vid=/*(?P<id>[^&].).*'
+
+    _TESTS = [{
+        'url': 'http://videozoo.me/embed.php?w=718&h=438&vid=at/nw/boku_wa_tomodachi_ga_sukunai_-_05.mp4',
+        'md5': '4ed320e353ed26c742c4f12a9c210b60',
+        'info_dict': {
+            'id': 'boku_wa_tomodachi_ga_sukunai_-_05',
+            'ext': 'mp4',
+            'title': 'boku_wa_tomodachi_ga_sukunai_-_05',
+        }
+    }]
+
+
+class PlayBBIE(Play44IE):
+    _VALID_URL = r'http://[w.]*playbb\.[^/]*/.*vid=/*(?P<id>[^&].).*'
+
+    _TESTS = [{
+        'url': 'http://playbb.me/embed.php?w=718&h=438&vid=at/nw/boku_wa_tomodachi_ga_sukunai_-_05.mp4',
+        'md5': '4ed320e353ed26c742c4f12a9c210b60',
+        'info_dict': {
+            'id': 'boku_wa_tomodachi_ga_sukunai_-_05',
+            'ext': 'mp4',
+            'title': 'boku_wa_tomodachi_ga_sukunai_-_05',
+        }
+    }]
+
+
+class EasyVideoIE(Play44IE):
+    _VALID_URL = r'http://[w.]*easyvideo\.[^/]*/.*file=/*(?P<id>[^&.]+)'
+
+    _TESTS = [{
+        'url': 'http://easyvideo.me/gogo/?w=718&h=438&file=bokuwatomodachigasukunai-04.flv&sv=1',
+        'md5': '26178b57629b7650106d72b191137176',
+        'info_dict': {
+            'id': 'bokuwatomodachigasukunai-04',
+            'ext': 'mp4',
+            'title': 'bokuwatomodachigasukunai-04',
+        },
+        'skip': 'Blocked in Germany',
+    }]
diff --git a/youtube_dl/extractor/soulanime.py b/youtube_dl/extractor/soulanime.py
new file mode 100644
index 000000000..7adb10c03
--- /dev/null
+++ b/youtube_dl/extractor/soulanime.py
@@ -0,0 +1,74 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class SoulAnimeWatchingIE(InfoExtractor):
+    IE_NAME = "soulanime:watching"
+    IE_DESC = "SoulAnime video"
+    _TEST = {
+        'url': 'http://www.soul-anime.net/watching/seirei-tsukai-no-blade-dance-episode-9/',
+        'md5': '05fae04abf72298098b528e98abf4298',
+        'info_dict': {
+            'id': 'seirei-tsukai-no-blade-dance-episode-9',
+            'ext': 'mp4',
+            'title': 'seirei-tsukai-no-blade-dance-episode-9',
+            'description': 'seirei-tsukai-no-blade-dance-episode-9'
+        }
+    }
+    _VALID_URL = r'http://[w.]*soul-anime\.(?P<domain>[^/]+)/watch[^/]*/(?P<id>[^/]+)'
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        domain = mobj.group('domain')
+
+        page = self._download_webpage(url, video_id)
+
+        video_url_encoded = self._html_search_regex(
+            r'<div id="download">[^<]*<a href="(?P<url>[^"]+)"', page, 'url')
+        video_url = "http://www.soul-anime." + domain + video_url_encoded
+
+        vid = self._request_webpage(video_url, video_id)
+        ext = vid.info().gettype().split("/")[1]
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'ext': ext,
+            'title': video_id,
+            'description': video_id
+        }
+
+
+class SoulAnimeSeriesIE(InfoExtractor):
+    IE_NAME = "soulanime:series"
+    IE_DESC = "SoulAnime Series"
+
+    _VALID_URL = r'http://[w.]*soul-anime\.(?P<domain>[^/]+)/anime./(?P<id>[^/]+)'
+
+    _EPISODE_REGEX = r'<option value="(/watch[^/]*/[^"]+)">[^<]*</option>'
+
+    _TEST = {
+        'url': 'http://www.soul-anime.net/anime1/black-rock-shooter-tv/',
+        'info_dict': {
+            'id': 'black-rock-shooter-tv'
+        },
+        'playlist_count': 8
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        series_id = mobj.group('id')
+        domain = mobj.group('domain')
+
+        pattern = re.compile(self._EPISODE_REGEX)
+
+        page = self._download_webpage(url, series_id, "Downloading series page")
+        mobj = pattern.findall(page)
+
+        entries = [self.url_result("http://www.soul-anime." + domain + obj) for obj in mobj]
+
+        return self.playlist_result(entries, series_id)
diff --git a/youtube_dl/extractor/videofun.py b/youtube_dl/extractor/videofun.py
new file mode 100644
index 000000000..0364b9d32
--- /dev/null
+++ b/youtube_dl/extractor/videofun.py
@@ -0,0 +1,36 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    compat_urllib_parse
+)
+
+
+class VideoFunIE(InfoExtractor):
+    _VALID_URL = r'http://[w.]*videofun\.me/embed/(?P<id>[0-9a-f]+)'
+
+    _TEST = {
+        'url': 'http://videofun.me/embed/8267659be070860af600fee7deadbcdb?w=600&h=438',
+        'md5': 'e37e99d665f503dd2db952f7c4dba9e6',
+        'info_dict': {
+            'id': 'Mahou-Shoujo-Madoka-Magica-07',
+            'ext': 'flv',
+            'title': 'Mahou-Shoujo-Madoka-Magica-07',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, video_id, 'Downloading video page')
+
+        video_url_encoded = self._html_search_regex(
+            r'url: "(http://gateway\.videofun\.me[^"]+)"', webpage, 'video url')
+        video_url = compat_urllib_parse.unquote(video_url_encoded)
+        title = self._html_search_regex(r'.*/([^.]*)\.', video_url, 'title')
+
+        return {
+            'id': title,
+            'url': video_url,
+            'title': title,
+        }

From 95ceeec72200ed3b2c94a54650eb69dfe946e595 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 4 Jan 2015 02:05:35 +0100
Subject: [PATCH 1025/1937] Remove unused import

---
 youtube_dl/downloader/mplayer.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/downloader/mplayer.py b/youtube_dl/downloader/mplayer.py
index 34b23b5c2..72cef30ea 100644
--- a/youtube_dl/downloader/mplayer.py
+++ b/youtube_dl/downloader/mplayer.py
@@ -4,7 +4,6 @@ import os
 import subprocess
 
 from .common import FileDownloader
-from ..compat import compat_subprocess_get_DEVNULL
 from ..utils import (
     check_executable,
     encodeFilename,

From c11125f9ed952f9b7ebd06c15eacadcc6005dd8c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 4 Jan 2015 02:06:53 +0100
Subject: [PATCH 1026/1937] [tests] Remove format 138 from tests (#4559)

---
 test/test_YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index f8e4f930e..730f7ec26 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -218,7 +218,7 @@ class TestFormatSelection(unittest.TestCase):
             # 3D
             '85', '84', '102', '83', '101', '82', '100',
             # Dash video
-            '138', '137', '248', '136', '247', '135', '246',
+            '137', '248', '136', '247', '135', '246',
             '245', '244', '134', '243', '133', '242', '160',
             # Dash audio
             '141', '172', '140', '171', '139',

From 8848314c08284f6a4b8f3c3529bf2e3f1b72610c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 4 Jan 2015 02:08:18 +0100
Subject: [PATCH 1027/1937] [Makefile] Make offline tests actually work offline

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 71470eedb..e53a367ef 100644
--- a/Makefile
+++ b/Makefile
@@ -46,7 +46,7 @@ test:
 ot: offlinetest
 
 offlinetest: codetest
-	nosetests --verbose test --exclude test_download --exclude test_age_restriction --exclude test_subtitles --exclude test_write_annotations
+	nosetests --verbose test --exclude test_download --exclude test_age_restriction --exclude test_subtitles --exclude test_write_annotations --exclude test_youtube_lists
 
 tar: youtube-dl.tar.gz
 

From 2ccd1b10e58cc8e5173dc1aeedc2b3f0ef9b55bf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 4 Jan 2015 02:20:45 +0100
Subject: [PATCH 1028/1937] [soulanime] Fix under Python 3

---
 youtube_dl/extractor/soulanime.py | 10 ++++++++--
 youtube_dl/utils.py               | 11 +++++++++++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/soulanime.py b/youtube_dl/extractor/soulanime.py
index 7adb10c03..feef33e27 100644
--- a/youtube_dl/extractor/soulanime.py
+++ b/youtube_dl/extractor/soulanime.py
@@ -3,6 +3,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import (
+    HEADRequest,
+    urlhandle_detect_ext,
+)
 
 
 class SoulAnimeWatchingIE(InfoExtractor):
@@ -31,8 +35,10 @@ class SoulAnimeWatchingIE(InfoExtractor):
             r'<div id="download">[^<]*<a href="(?P<url>[^"]+)"', page, 'url')
         video_url = "http://www.soul-anime." + domain + video_url_encoded
 
-        vid = self._request_webpage(video_url, video_id)
-        ext = vid.info().gettype().split("/")[1]
+        ext_req = HEADRequest(video_url)
+        ext_handle = self._request_webpage(
+            ext_req, video_id, note='Determining extension')
+        ext = urlhandle_detect_ext(ext_handle)
 
         return {
             'id': video_id,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index efbe64fb3..bdfe053a7 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1550,3 +1550,14 @@ def ytdl_is_updateable():
 def args_to_str(args):
     # Get a short string representation for a subprocess command
     return ' '.join(shlex_quote(a) for a in args)
+
+
+def urlhandle_detect_ext(url_handle):
+    try:
+        url_handle.headers
+        getheader = lambda h: url_handle.headers[h]
+    except AttributeError:  # Python < 3
+        getheader = url_handle.info().getheader
+
+    return getheader('Content-Type').split("/")[1]
+

From 7a1818c99b1729796f62c341b1b3f809cd47dbf8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 4 Jan 2015 03:15:27 +0100
Subject: [PATCH 1029/1937] [vk] Add support for rutube embeds (Fixes #4514)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/rutube.py   | 31 +++++++++++++++++++++++++++++++
 youtube_dl/extractor/vk.py       |  9 +++++++++
 3 files changed, 41 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c3dc09f75..143cd5c49 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -359,6 +359,7 @@ from .ruhd import RUHDIE
 from .rutube import (
     RutubeIE,
     RutubeChannelIE,
+    RutubeEmbedIE,
     RutubeMovieIE,
     RutubePersonIE,
 )
diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py
index b72b5a586..5b1c3577a 100644
--- a/youtube_dl/extractor/rutube.py
+++ b/youtube_dl/extractor/rutube.py
@@ -70,6 +70,37 @@ class RutubeIE(InfoExtractor):
         }
 
 
+class RutubeEmbedIE(InfoExtractor):
+    IE_NAME = 'rutube:embed'
+    IE_DESC = 'Rutube embedded videos'
+    _VALID_URL = 'https?://rutube\.ru/video/embed/(?P<id>[0-9]+)'
+
+    _TEST = {
+        'url': 'http://rutube.ru/video/embed/6722881?vk_puid37=&vk_puid38=',
+        'info_dict': {
+            'id': 'a10e53b86e8f349080f718582ce4c661',
+            'ext': 'mp4',
+            'upload_date': '20131223',
+            'uploader_id': '297833',
+            'description': 'Видео группы ★http://vk.com/foxkidsreset★ музей Fox Kids и Jetix<br/><br/> восстановлено и сделано в шикоформате subziro89 http://vk.com/subziro89',
+            'uploader': 'subziro89 ILya',
+            'title': 'Мистический городок Эйри в Индиан 5 серия озвучка subziro89',
+        },
+        'params': {
+            'skip_download': 'Requires ffmpeg',
+        },
+    }
+
+    def _real_extract(self, url):
+        embed_id = self._match_id(url)
+        webpage = self._download_webpage(url, embed_id)
+
+        canonical_url = self._html_search_regex(
+            r'<link\s+rel="canonical"\s+href="([^"]+?)"', webpage,
+            'Canonical URL')
+        return self.url_result(canonical_url, 'Rutube')
+
+
 class RutubeChannelIE(InfoExtractor):
     IE_NAME = 'rutube:channel'
     IE_DESC = 'Rutube channels'
diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 542e9198a..129de6cf3 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -164,6 +164,15 @@ class VKIE(InfoExtractor):
             self.to_screen('Youtube video detected')
             return self.url_result(m_yt.group(1), 'Youtube')
 
+        m_rutube = re.search(
+            r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)
+        assert m_rutube
+        if m_rutube is not None:
+            self.to_screen('rutube video detected')
+            rutube_url = self._proto_relative_url(
+                m_rutube.group(1).replace('\\', ''))
+            return self.url_result(rutube_url)
+
         m_opts = re.search(r'(?s)var\s+opts\s*=\s*({.*?});', info_page)
         if m_opts:
             m_opts_url = re.search(r"url\s*:\s*'([^']+)", m_opts.group(1))

From 26886e6140a684058064c30237ef096332e1510f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 4 Jan 2015 03:15:48 +0100
Subject: [PATCH 1030/1937] release 2015.01.04

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2a54b9bbe..09813928a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.03'
+__version__ = '2015.01.04'

From f4858a71035549cf82b258d01dda5060aef707b7 Mon Sep 17 00:00:00 2001
From: Christopher Krooss <c.krooss@gmail.com>
Date: Sun, 4 Jan 2015 13:33:26 +0100
Subject: [PATCH 1031/1937] Add support for Radio Bremen

---
 youtube_dl/extractor/__init__.py    |  1 +
 youtube_dl/extractor/radiobremen.py | 55 +++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/radiobremen.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 143cd5c49..349f4fe71 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -339,6 +339,7 @@ from .prosiebensat1 import ProSiebenSat1IE
 from .pyvideo import PyvideoIE
 from .quickvid import QuickVidIE
 from .radiode import RadioDeIE
+from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
 from .rai import RaiIE
 from .rbmaradio import RBMARadioIE
diff --git a/youtube_dl/extractor/radiobremen.py b/youtube_dl/extractor/radiobremen.py
new file mode 100644
index 000000000..68c78c4f9
--- /dev/null
+++ b/youtube_dl/extractor/radiobremen.py
@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class RadioBremenIE(InfoExtractor):
+    _VALID_URL = r'http?://(?:www\.)?radiobremen\.de/mediathek/(index\.html)?\?id=(?P<video_id>[0-9]+)'
+    IE_NAME = 'radiobremen'
+
+    _TEST = {
+        'url': 'http://www.radiobremen.de/mediathek/index.html?id=114720',
+        'info_dict': {
+            'id': '114720',
+            'ext': 'mp4',
+            'height': 288,
+            'width': 512,
+            'title': 'buten un binnen vom 22. Dezember',
+            'description': 'Unter anderem mit diesen Themen: 45 Flüchtlinge sind in Worpswede angekommen +++ Freies Internet für alle: Bremer arbeiten an einem flächendeckenden W-Lan-Netzwerk +++ Aktivisten kämpfen für das Unibad +++ So war das Wetter 2014 +++',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('video_id')
+
+        meta_url = "http://www.radiobremen.de/apps/php/mediathek/metadaten.php?id=%s" % video_id
+        meta_doc = self._download_webpage(meta_url, video_id, 'Downloading metadata')
+        title = self._html_search_regex("<h1.*>(?P<title>.+)</h1>", meta_doc, "title")
+        description = self._html_search_regex("<p>(?P<description>.*)</p>", meta_doc, "description")
+        duration = self._html_search_regex("L&auml;nge:</td>\s+<td>(?P<duration>[0-9]+:[0-9]+)</td>", meta_doc, "duration")
+
+        page_doc = self._download_webpage(url, video_id, 'Downloading video information')
+        pattern = "ardformatplayerclassic\(\'playerbereich\',\'(?P<width>[0-9]+)\',\'.*\',\'(?P<video_id>[0-9]+)\',\'(?P<secret>[0-9]+)\',\'(?P<thumbnail>.+)\',\'\'\)"
+        mobj = re.search(pattern, page_doc)
+        width, video_id, secret, thumbnail = int(mobj.group("width")), mobj.group("video_id"), mobj.group("secret"), mobj.group("thumbnail")
+        video_url = "http://dl-ondemand.radiobremen.de/mediabase/{:}/{:}_{:}_{:}.mp4".format(video_id, video_id, secret, width)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'formats': [
+                {'url': video_url,
+                 'ext': 'mp4',
+                 'width': width,
+                 'protocol': 'http'
+                 }
+            ],
+            'thumbnail': thumbnail,
+        }

From 63948fc62c7f0bfcfe7b2ce102ab6e4e87de558c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 4 Jan 2015 13:40:30 +0100
Subject: [PATCH 1032/1937] [downloader/hls] Respect the 'prefer_ffmpeg' option

---
 youtube_dl/downloader/hls.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 5bb0f3cfd..aa58b52ab 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -11,7 +11,6 @@ from ..compat import (
     compat_urllib_request,
 )
 from ..utils import (
-    check_executable,
     encodeFilename,
 )
 
@@ -27,16 +26,13 @@ class HlsFD(FileDownloader):
             '-bsf:a', 'aac_adtstoasc',
             encodeFilename(tmpfilename, for_subprocess=True)]
 
-        for program in ['avconv', 'ffmpeg']:
-            if check_executable(program, ['-version']):
-                break
-        else:
+        ffpp = FFmpegPostProcessor(downloader=self)
+        program = ffpp._executable
+        if program is None:
             self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
             return False
-        cmd = [program] + args
-
-        ffpp = FFmpegPostProcessor(downloader=self)
         ffpp.check_version()
+        cmd = [program] + args
 
         retval = subprocess.call(cmd)
         if retval == 0:

From bc1fc5ddbcba784778cbdd98c051ff2493178515 Mon Sep 17 00:00:00 2001
From: Christopher Krooss <c.krooss@gmail.com>
Date: Sun, 4 Jan 2015 14:02:07 +0100
Subject: [PATCH 1033/1937] Don't check for height as it's not provided

---
 youtube_dl/extractor/radiobremen.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/radiobremen.py b/youtube_dl/extractor/radiobremen.py
index 68c78c4f9..6d130d3d9 100644
--- a/youtube_dl/extractor/radiobremen.py
+++ b/youtube_dl/extractor/radiobremen.py
@@ -16,7 +16,6 @@ class RadioBremenIE(InfoExtractor):
         'info_dict': {
             'id': '114720',
             'ext': 'mp4',
-            'height': 288,
             'width': 512,
             'title': 'buten un binnen vom 22. Dezember',
             'description': 'Unter anderem mit diesen Themen: 45 Flüchtlinge sind in Worpswede angekommen +++ Freies Internet für alle: Bremer arbeiten an einem flächendeckenden W-Lan-Netzwerk +++ Aktivisten kämpfen für das Unibad +++ So war das Wetter 2014 +++',

From bc3e582fe457f9239dc3a3386cbfd0e7db167404 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 4 Jan 2015 14:02:17 +0100
Subject: [PATCH 1034/1937] Don't use '-shortest' option for merging formats
 (closes #4220, closes #4580)

With avconv and older versions of ffmpeg the video is partially copied.
The duration difference between the audio and the video seem to be really small, so it's probably not noticeable.
---
 youtube_dl/postprocessor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 048525efc..473536dcc 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -520,7 +520,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
 class FFmpegMergerPP(FFmpegPostProcessor):
     def run(self, info):
         filename = info['filepath']
-        args = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0', '-shortest']
+        args = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0']
         self._downloader.to_screen('[ffmpeg] Merging formats into "%s"' % filename)
         self.run_ffmpeg_multiple_files(info['__files_to_merge'], filename, args)
         return True, info

From 9fda6ee39fa2da1949af5e9b95633e3df3c6f6b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 4 Jan 2015 14:06:23 +0100
Subject: [PATCH 1035/1937] [tf1] Remove unused import

---
 youtube_dl/extractor/tf1.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 07cc81226..025d0877c 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 

From 1d2d0e3ff2b4e55810039caf267bb9ad086f3610 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 4 Jan 2015 14:07:06 +0100
Subject: [PATCH 1036/1937] utils: Remove blank line at the end of file

---
 youtube_dl/utils.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index bdfe053a7..d4951c406 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1560,4 +1560,3 @@ def urlhandle_detect_ext(url_handle):
         getheader = url_handle.info().getheader
 
     return getheader('Content-Type').split("/")[1]
-

From 67c2bcdf4cf83f9ac32e5f1f50a8b4b38d2ac624 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 4 Jan 2015 19:19:15 +0100
Subject: [PATCH 1037/1937] Remove extractors which infringe copyright (#4554)

---
 youtube_dl/extractor/__init__.py  |  19 ----
 youtube_dl/extractor/gogoanime.py |  76 ---------------
 youtube_dl/extractor/play44.py    | 149 ------------------------------
 youtube_dl/extractor/videofun.py  |  36 --------
 4 files changed, 280 deletions(-)
 delete mode 100644 youtube_dl/extractor/gogoanime.py
 delete mode 100644 youtube_dl/extractor/play44.py
 delete mode 100644 youtube_dl/extractor/videofun.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 143cd5c49..613e8e05b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -164,10 +164,6 @@ from .globo import GloboIE
 from .godtube import GodTubeIE
 from .goldenmoustache import GoldenMoustacheIE
 from .golem import GolemIE
-from .gogoanime import (
-    GoGoAnimeIE,
-    GoGoAnimeSearchIE
-)
 from .googleplus import GooglePlusIE
 from .googlesearch import GoogleSearchIE
 from .gorillavid import GorillaVidIE
@@ -317,16 +313,6 @@ from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
 from .planetaplay import PlanetaPlayIE
 from .played import PlayedIE
-from .play44 import (
-    Play44IE,
-    ByZooIE,
-    Video44IE,
-    VideoWingIE,
-    PlayPandaIE,
-    VideoZooIE,
-    PlayBBIE,
-    EasyVideoIE
-)
 from .playfm import PlayFMIE
 from .playvid import PlayvidIE
 from .podomatic import PodomaticIE
@@ -388,10 +374,6 @@ from .smotri import (
 from .snotr import SnotrIE
 from .sockshare import SockshareIE
 from .sohu import SohuIE
-from .soulanime import (
-    SoulAnimeWatchingIE,
-    SoulAnimeSeriesIE
-)
 from .soundcloud import (
     SoundcloudIE,
     SoundcloudSetIE,
@@ -486,7 +468,6 @@ from .viddler import ViddlerIE
 from .videobam import VideoBamIE
 from .videodetective import VideoDetectiveIE
 from .videolecturesnet import VideoLecturesNetIE
-from .videofun import VideoFunIE
 from .videofyme import VideofyMeIE
 from .videomega import VideoMegaIE
 from .videopremium import VideoPremiumIE
diff --git a/youtube_dl/extractor/gogoanime.py b/youtube_dl/extractor/gogoanime.py
deleted file mode 100644
index d4f4ecc58..000000000
--- a/youtube_dl/extractor/gogoanime.py
+++ /dev/null
@@ -1,76 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    compat_urllib_parse,
-    get_element_by_attribute,
-    unescapeHTML
-)
-
-
-class GoGoAnimeIE(InfoExtractor):
-    IE_NAME = 'gogoanime'
-    IE_DESC = 'GoGoAnime'
-    _VALID_URL = r'http://www.gogoanime.com/(?P<id>[A-Za-z0-9-]+)'
-
-    _TEST = {
-        'url': 'http://www.gogoanime.com/mahou-shoujo-madoka-magica-movie-1',
-        'info_dict': {
-            'id': 'mahou-shoujo-madoka-magica-movie-1'
-        },
-        'playlist_count': 3
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        page = self._download_webpage(url, video_id)
-
-        if 'Oops! Page Not Found</font>' in page:
-            raise ExtractorError('Video does not exist', expected=True)
-
-        content = get_element_by_attribute("class", "postcontent", page)
-        vids = re.findall(r'<iframe[^>]*?src=[\'"](h[^\'"]+)[\'"]', content)
-        vids = [
-            unescapeHTML(compat_urllib_parse.unquote(x))
-            for x in vids if not re.search(r".*videofun.*", x)]
-
-        if re.search(r'<div class="postcontent">[^<]*<p><iframe src=[\'"][^>]+></iframe><br />', page):
-            return self.playlist_result([self.url_result(vid) for vid in vids], video_id)
-
-        title = self._html_search_regex(
-            r'<div class="postdesc">[^<]*<h1>([^<]+)</h1>', page, 'title')
-
-        return {
-            '_type': 'url',
-            'id': video_id,
-            'url': vids[0],
-            'title': title,
-        }
-
-
-class GoGoAnimeSearchIE(InfoExtractor):
-    IE_NAME = 'gogoanime:search'
-    IE_DESC = 'GoGoAnime Search'
-
-    _VALID_URL = r'http://www\.gogoanime\.com/.*\?s=(?P<id>[^&]*)'
-    _TEST = {
-        'url': 'http://www.gogoanime.com/?s=bokusatsu',
-        'info_dict': {
-            'id': 'bokusatsu'
-        },
-        'playlist_count': 6
-    }
-
-    def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-        webpage = self._download_webpage(url, playlist_id)
-
-        posts = re.findall(
-            r'<div class="postlist">[^<]*<p[^>]*>[^<]*<a href="(?P<url>[^"]+)"',
-            webpage)
-
-        return self.playlist_result(
-            [self.url_result(p) for p in posts], playlist_id)
diff --git a/youtube_dl/extractor/play44.py b/youtube_dl/extractor/play44.py
deleted file mode 100644
index b8696e516..000000000
--- a/youtube_dl/extractor/play44.py
+++ /dev/null
@@ -1,149 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    compat_urllib_parse
-)
-
-
-class Play44IE(InfoExtractor):
-    _VALID_URL = r'http://[w.]*play44\.net/embed\.php[^/]*/(?P<id>.+)'
-
-    _TESTS = [{
-        'url': 'http://play44.net/embed.php?w=600&h=438&vid=M/mahou-shoujo-madoka-magica-07.flv',
-        'md5': 'e37e99d665f503dd2db952f7c4dba9e6',
-        'info_dict': {
-            'id': 'mahou-shoujo-madoka-magica-07',
-            'ext': 'flv',
-            'title': 'mahou-shoujo-madoka-magica-07',
-        }
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        page = self._download_webpage(url, video_id)
-
-        video_url = compat_urllib_parse.unquote(self._html_search_regex(
-            r'_url = "(https?://[^"]+?)";', page, 'url'))
-        title = self._search_regex(r'.*/(?P<title>[^.]*).', video_url, 'title')
-
-        return {
-            'id': title,
-            'url': video_url,
-            'title': title,
-        }
-
-
-class ByZooIE(Play44IE):
-    _VALID_URL = r'http://[w.]*byzoo\.org/embed\.php[^/]*/(?P<id>.+)'
-
-    _TESTS = [{
-        'url': 'http://byzoo.org/embed.php?w=600&h=438&vid=at/nw/mahou_shoujo_madoka_magica_movie_3_-_part1.mp4',
-        'md5': '455c83dabe2cd9fd74a87612b01fe017',
-        'info_dict': {
-            'id': 'mahou_shoujo_madoka_magica_movie_3_-_part1',
-            'ext': 'mp4',
-            'title': 'mahou_shoujo_madoka_magica_movie_3_-_part1',
-        }
-    }]
-
-
-class Video44IE(Play44IE):
-    _VALID_URL = r'http://[w.]*video44\.net/.*file=(?P<id>[^&].).*'
-
-    _TESTS = [{
-        'url': 'http://www.video44.net/gogo/?w=600&h=438&file=chaoshead-12.flv&sv=1',
-        'md5': '43eaec6d0beb10e8d42459b9f108aff3',
-        'info_dict': {
-            'id': 'chaoshead-12',
-            'ext': 'mp4',
-            'title': 'chaoshead-12',
-        }
-    }]
-
-
-class VideoWingIE(Play44IE):
-    _VALID_URL = r'''(?x)
-        http://[w.]*videowing\.[^/]*/
-        (?:
-            .*video=/*
-            |embed/
-        )
-        (?P<id>[^&?.]+)
-    '''
-
-    _TESTS = [{
-        'url': 'http://videowing.me/embed?w=718&h=438&video=ongoing/boku_wa_tomodachi_ga_sukunai_-_05.mp4',
-        'md5': '4ed320e353ed26c742c4f12a9c210b60',
-        'info_dict': {
-            'id': 'boku_wa_tomodachi_ga_sukunai_-_05',
-            'ext': 'mp4',
-            'title': 'boku_wa_tomodachi_ga_sukunai_-_05',
-        }
-    }, {
-        'url': 'http://videowing.me/embed/a8d6a39522df066bd734a69f2334497e?w=600&h=438',
-        'md5': '33fdd71581357018c226f95c5cedcfd7',
-        'info_dict': {
-            'id': 'mahoushoujomadokamagicamovie1part1',
-            'ext': 'flv',
-            'title': 'mahoushoujomadokamagicamovie1part1',
-        }
-    }]
-
-
-class PlayPandaIE(Play44IE):
-    _VALID_URL = r'http://[w.]*playpanda\.[^/]*/.*vid=/*(?P<id>[^&].).*'
-
-    _TESTS = [{
-        'url': 'http://playpanda.net/embed.php?w=718&h=438&vid=at/nw/boku_wa_tomodachi_ga_sukunai_-_05.mp4',
-        'md5': '4ed320e353ed26c742c4f12a9c210b60',
-        'info_dict': {
-            'id': 'boku_wa_tomodachi_ga_sukunai_-_05',
-            'ext': 'mp4',
-            'title': 'boku_wa_tomodachi_ga_sukunai_-_05',
-            'description': 'boku_wa_tomodachi_ga_sukunai_-_05'
-        }
-    }]
-
-
-class VideoZooIE(Play44IE):
-    _VALID_URL = r'http://[w.]*videozoo\.[^/]*/.*vid=/*(?P<id>[^&].).*'
-
-    _TESTS = [{
-        'url': 'http://videozoo.me/embed.php?w=718&h=438&vid=at/nw/boku_wa_tomodachi_ga_sukunai_-_05.mp4',
-        'md5': '4ed320e353ed26c742c4f12a9c210b60',
-        'info_dict': {
-            'id': 'boku_wa_tomodachi_ga_sukunai_-_05',
-            'ext': 'mp4',
-            'title': 'boku_wa_tomodachi_ga_sukunai_-_05',
-        }
-    }]
-
-
-class PlayBBIE(Play44IE):
-    _VALID_URL = r'http://[w.]*playbb\.[^/]*/.*vid=/*(?P<id>[^&].).*'
-
-    _TESTS = [{
-        'url': 'http://playbb.me/embed.php?w=718&h=438&vid=at/nw/boku_wa_tomodachi_ga_sukunai_-_05.mp4',
-        'md5': '4ed320e353ed26c742c4f12a9c210b60',
-        'info_dict': {
-            'id': 'boku_wa_tomodachi_ga_sukunai_-_05',
-            'ext': 'mp4',
-            'title': 'boku_wa_tomodachi_ga_sukunai_-_05',
-        }
-    }]
-
-
-class EasyVideoIE(Play44IE):
-    _VALID_URL = r'http://[w.]*easyvideo\.[^/]*/.*file=/*(?P<id>[^&.]+)'
-
-    _TESTS = [{
-        'url': 'http://easyvideo.me/gogo/?w=718&h=438&file=bokuwatomodachigasukunai-04.flv&sv=1',
-        'md5': '26178b57629b7650106d72b191137176',
-        'info_dict': {
-            'id': 'bokuwatomodachigasukunai-04',
-            'ext': 'mp4',
-            'title': 'bokuwatomodachigasukunai-04',
-        },
-        'skip': 'Blocked in Germany',
-    }]
diff --git a/youtube_dl/extractor/videofun.py b/youtube_dl/extractor/videofun.py
deleted file mode 100644
index 0364b9d32..000000000
--- a/youtube_dl/extractor/videofun.py
+++ /dev/null
@@ -1,36 +0,0 @@
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    compat_urllib_parse
-)
-
-
-class VideoFunIE(InfoExtractor):
-    _VALID_URL = r'http://[w.]*videofun\.me/embed/(?P<id>[0-9a-f]+)'
-
-    _TEST = {
-        'url': 'http://videofun.me/embed/8267659be070860af600fee7deadbcdb?w=600&h=438',
-        'md5': 'e37e99d665f503dd2db952f7c4dba9e6',
-        'info_dict': {
-            'id': 'Mahou-Shoujo-Madoka-Magica-07',
-            'ext': 'flv',
-            'title': 'Mahou-Shoujo-Madoka-Magica-07',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(
-            url, video_id, 'Downloading video page')
-
-        video_url_encoded = self._html_search_regex(
-            r'url: "(http://gateway\.videofun\.me[^"]+)"', webpage, 'video url')
-        video_url = compat_urllib_parse.unquote(video_url_encoded)
-        title = self._html_search_regex(r'.*/([^.]*)\.', video_url, 'title')
-
-        return {
-            'id': title,
-            'url': video_url,
-            'title': title,
-        }

From 2f985f4bb4938ee13356bda0436fde18f8c0e434 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 5 Jan 2015 00:18:43 +0100
Subject: [PATCH 1038/1937] [youtube:toplist] Remove extractor

They use now normal playlists (their id is PL*).
---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/youtube.py  | 46 --------------------------------
 2 files changed, 47 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 613e8e05b..79e6bba45 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -546,7 +546,6 @@ from .youtube import (
     YoutubeSearchURLIE,
     YoutubeShowIE,
     YoutubeSubscriptionsIE,
-    YoutubeTopListIE,
     YoutubeTruncatedIDIE,
     YoutubeTruncatedURLIE,
     YoutubeUserIE,
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e9bf39a00..d1bbf0b01 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1206,9 +1206,6 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         if playlist_id.startswith('RD'):
             # Mixes require a custom extraction process
             return self._extract_mix(playlist_id)
-        if playlist_id.startswith('TL'):
-            raise ExtractorError('For downloading YouTube.com top lists, use '
-                                 'the "yttoplist" keyword, for example "youtube-dl \'yttoplist:music:Top Tracks\'"', expected=True)
 
         url = self._TEMPLATE_URL % playlist_id
         page = self._download_webpage(url, playlist_id)
@@ -1254,49 +1251,6 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         return self.playlist_result(url_results, playlist_id, playlist_title)
 
 
-class YoutubeTopListIE(YoutubePlaylistIE):
-    IE_NAME = 'youtube:toplist'
-    IE_DESC = ('YouTube.com top lists, "yttoplist:{channel}:{list title}"'
-               ' (Example: "yttoplist:music:Top Tracks")')
-    _VALID_URL = r'yttoplist:(?P<chann>.*?):(?P<title>.*?)$'
-    _TESTS = [{
-        'url': 'yttoplist:music:Trending',
-        'playlist_mincount': 5,
-        'skip': 'Only works for logged-in users',
-    }]
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        channel = mobj.group('chann')
-        title = mobj.group('title')
-        query = compat_urllib_parse.urlencode({'title': title})
-        channel_page = self._download_webpage(
-            'https://www.youtube.com/%s' % channel, title)
-        link = self._html_search_regex(
-            r'''(?x)
-                <a\s+href="([^"]+)".*?>\s*
-                <span\s+class="branded-page-module-title-text">\s*
-                <span[^>]*>.*?%s.*?</span>''' % re.escape(query),
-            channel_page, 'list')
-        url = compat_urlparse.urljoin('https://www.youtube.com/', link)
-
-        video_re = r'data-index="\d+".*?data-video-id="([0-9A-Za-z_-]{11})"'
-        ids = []
-        # sometimes the webpage doesn't contain the videos
-        # retry until we get them
-        for i in itertools.count(0):
-            msg = 'Downloading Youtube mix'
-            if i > 0:
-                msg += ', retry #%d' % i
-
-            webpage = self._download_webpage(url, title, msg)
-            ids = orderedSet(re.findall(video_re, webpage))
-            if ids:
-                break
-        url_results = self._ids_to_results(ids)
-        return self.playlist_result(url_results, playlist_title=title)
-
-
 class YoutubeChannelIE(InfoExtractor):
     IE_DESC = 'YouTube.com channels'
     _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/(?P<id>[0-9A-Za-z_-]+)'

From caf90bfaa5434d9ff7035d8575b842b076178ca3 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 5 Jan 2015 02:22:01 +0200
Subject: [PATCH 1039/1937] [webofstories] Add new extractor (Closes #4585)

---
 youtube_dl/extractor/__init__.py     |   1 +
 youtube_dl/extractor/webofstories.py | 102 +++++++++++++++++++++++++++
 2 files changed, 103 insertions(+)
 create mode 100644 youtube_dl/extractor/webofstories.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 79e6bba45..0c8729384 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -511,6 +511,7 @@ from .wdr import (
     WDRMobileIE,
     WDRMausIE,
 )
+from .webofstories import WebOfStoriesIE
 from .weibo import WeiboIE
 from .wimp import WimpIE
 from .wistia import WistiaIE
diff --git a/youtube_dl/extractor/webofstories.py b/youtube_dl/extractor/webofstories.py
new file mode 100644
index 000000000..396cf4e83
--- /dev/null
+++ b/youtube_dl/extractor/webofstories.py
@@ -0,0 +1,102 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class WebOfStoriesIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?webofstories\.com/play/(?:[^/]+/)?(?P<id>[0-9]+)'
+    _VIDEO_DOMAIN = 'http://eu-mobile.webofstories.com/'
+    _GREAT_LIFE_STREAMER = 'rtmp://eu-cdn1.webofstories.com/cfx/st/'
+    _USER_STREAMER = 'rtmp://eu-users.webofstories.com/cfx/st/'
+    _TESTS = [
+        {
+            'url': 'http://www.webofstories.com/play/hans.bethe/71',
+            'md5': '373e4dd915f60cfe3116322642ddf364',
+            'info_dict': {
+                'id': '4536',
+                'ext': 'mp4',
+                'title': 'The temperature of the sun',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'description': 'Hans Bethe talks about calculating the temperature of the sun',
+                'duration': 238,
+            }
+        },
+        {
+            'url': 'http://www.webofstories.com/play/55908',
+            'md5': '2985a698e1fe3211022422c4b5ed962c',
+            'info_dict': {
+                'id': '55908',
+                'ext': 'mp4',
+                'title': 'The story of Gemmata obscuriglobus',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'description': 'Planctomycete talks about The story of Gemmata obscuriglobus',
+                'duration': 169,
+            }
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        title = self._og_search_title(webpage)
+        description = self._html_search_meta('description', webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        story_filename = self._search_regex(
+            r'\.storyFileName\("([^"]+)"\)', webpage, 'story filename')
+        speaker_id = self._search_regex(
+            r'\.speakerId\("([^"]+)"\)', webpage, 'speaker ID')
+        story_id = self._search_regex(
+            r'\.storyId\((\d+)\)', webpage, 'story ID')
+        speaker_type = self._search_regex(
+            r'\.speakerType\("([^"]+)"\)', webpage, 'speaker type')
+        great_life = self._search_regex(
+            r'isGreatLifeStory\s*=\s*(true|false)', webpage, 'great life story')
+        is_great_life_series = great_life == 'true'
+        duration = int_or_none(self._search_regex(
+            r'\.duration\((\d+)\)', webpage, 'duration', fatal=False))
+
+        # URL building, see: http://www.webofstories.com/scripts/player.js
+        ms_prefix = ''
+        if speaker_type.lower() == 'ms':
+            ms_prefix = 'mini_sites/'
+
+        if is_great_life_series:
+            mp4_url = '{0:}lives/{1:}/{2:}.mp4'.format(
+                self._VIDEO_DOMAIN, speaker_id, story_filename)
+            rtmp_ext = 'flv'
+            streamer = self._GREAT_LIFE_STREAMER
+            play_path = 'stories/{0:}/{1:}'.format(
+                speaker_id, story_filename)
+        else:
+            mp4_url = '{0:}{1:}{2:}/{3:}.mp4'.format(
+                self._VIDEO_DOMAIN, ms_prefix, speaker_id, story_filename)
+            rtmp_ext = 'mp4'
+            streamer = self._USER_STREAMER
+            play_path = 'mp4:{0:}{1:}/{2}.mp4'.format(
+                ms_prefix, speaker_id, story_filename)
+
+        formats = [{
+            'format_id': 'mp4_sd',
+            'url': mp4_url,
+        }, {
+            'format_id': 'rtmp_sd',
+            'page_url': url,
+            'url': streamer,
+            'ext': rtmp_ext,
+            'play_path': play_path,
+        }]
+
+        self._sort_formats(formats)
+
+        return {
+            'id': story_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'description': description,
+            'duration': duration,
+        }

From adf3c58ad31e7376f085271a02fdfe56b1e75989 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 5 Jan 2015 02:55:12 +0200
Subject: [PATCH 1040/1937] [lrt] Fix missing provider key

Also, modernize a bit.
---
 youtube_dl/extractor/lrt.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/lrt.py b/youtube_dl/extractor/lrt.py
index d72d470aa..9c2fbdd96 100644
--- a/youtube_dl/extractor/lrt.py
+++ b/youtube_dl/extractor/lrt.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..utils import (
@@ -28,7 +27,6 @@ class LRTIE(InfoExtractor):
         'params': {
             'skip_download': True,  # HLS download
         },
-
     }
 
     def _real_extract(self, url):
@@ -44,7 +42,9 @@ class LRTIE(InfoExtractor):
 
         formats = []
         for js in re.findall(r'(?s)config:\s*(\{.*?\})', webpage):
-            data = json.loads(js_to_json(js))
+            data = self._parse_json(js, video_id, transform_source=js_to_json)
+            if 'provider' not in data:
+                continue
             if data['provider'] == 'rtmp':
                 formats.append({
                     'format_id': 'rtmp',

From bdf80aa542da15437545ae9c17cd5c80e17e171f Mon Sep 17 00:00:00 2001
From: Bart Kappenburg <bartkappenburg@gmail.com>
Date: Mon, 5 Jan 2015 11:51:24 +0100
Subject: [PATCH 1041/1937] Update rtlnl.py

Added support for the non-www version of rtlxl.nl by making "www." optional.
---
 youtube_dl/extractor/rtlnl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index d029b0ec5..a3ca79f2c 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -8,7 +8,7 @@ from ..utils import parse_duration
 
 class RtlXlIE(InfoExtractor):
     IE_NAME = 'rtlxl.nl'
-    _VALID_URL = r'https?://www\.rtlxl\.nl/#!/[^/]+/(?P<uuid>[^/?]+)'
+    _VALID_URL = r'https?://(www\.)?rtlxl\.nl/#!/[^/]+/(?P<uuid>[^/?]+)'
 
     _TEST = {
         'url': 'http://www.rtlxl.nl/#!/rtl-nieuws-132237/6e4203a6-0a5e-3596-8424-c599a59e0677',

From a4c3f486394ae8ead64e8e634433670639e3080f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 5 Jan 2015 11:46:40 +0100
Subject: [PATCH 1042/1937] [vimple] Replace tests

The first one seems to be no longer available and the second was an episode from a tv show.
---
 youtube_dl/extractor/vimple.py | 23 ++++++-----------------
 1 file changed, 6 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/vimple.py b/youtube_dl/extractor/vimple.py
index 33d370e1c..ee3d86117 100644
--- a/youtube_dl/extractor/vimple.py
+++ b/youtube_dl/extractor/vimple.py
@@ -14,28 +14,17 @@ class VimpleIE(InfoExtractor):
     IE_DESC = 'Vimple.ru'
     _VALID_URL = r'https?://(player.vimple.ru/iframe|vimple.ru)/(?P<id>[a-f0-9]{10,})'
     _TESTS = [
-        # Quality: Large, from iframe
         {
-            'url': 'http://player.vimple.ru/iframe/b132bdfd71b546d3972f9ab9a25f201c',
+            'url': 'http://vimple.ru/c0f6b1687dcd4000a97ebe70068039cf',
+            'md5': '2e750a330ed211d3fd41821c6ad9a279',
             'info_dict': {
-                'id': 'b132bdfd71b546d3972f9ab9a25f201c',
-                'title': 'great-escape-minecraft.flv',
+                'id': 'c0f6b1687dcd4000a97ebe70068039cf',
                 'ext': 'mp4',
-                'duration': 352,
-                'webpage_url': 'http://vimple.ru/b132bdfd71b546d3972f9ab9a25f201c',
+                'title': 'Sunset',
+                'duration': 20,
+                'thumbnail': 're:https?://.*?\.jpg',
             },
         },
-        # Quality: Medium, from mainpage
-        {
-            'url': 'http://vimple.ru/a15950562888453b8e6f9572dc8600cd',
-            'info_dict': {
-                'id': 'a15950562888453b8e6f9572dc8600cd',
-                'title': 'DB 01',
-                'ext': 'flv',
-                'duration': 1484,
-                'webpage_url': 'http://vimple.ru/a15950562888453b8e6f9572dc8600cd',
-            }
-        },
     ]
 
     def _real_extract(self, url):

From 628bc4d1e73ddef2b67eb6aba7b642c2e0ea894e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 5 Jan 2015 12:28:35 +0100
Subject: [PATCH 1043/1937] [khanacademy] Update test

---
 youtube_dl/extractor/khanacademy.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/khanacademy.py b/youtube_dl/extractor/khanacademy.py
index 408d00944..08a671fa8 100644
--- a/youtube_dl/extractor/khanacademy.py
+++ b/youtube_dl/extractor/khanacademy.py
@@ -22,8 +22,10 @@ class KhanAcademyIE(InfoExtractor):
             'description': 'The perfect cipher',
             'duration': 176,
             'uploader': 'Brit Cruise',
+            'uploader_id': 'khanacademy',
             'upload_date': '20120411',
-        }
+        },
+        'add_ie': ['Youtube'],
     }, {
         'url': 'https://www.khanacademy.org/math/applied-math/cryptography',
         'info_dict': {

From 75311a7e160912550e3c07642a5635f85f72cb0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 5 Jan 2015 12:29:32 +0100
Subject: [PATCH 1044/1937] .travis.yml: Remove my email from the list

---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index c6cc7a994..f14014414 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,7 +9,6 @@ notifications:
   email:
     - filippo.valsorda@gmail.com
     - phihag@phihag.de
-    - jaime.marquinez.ferrandiz+travis@gmail.com
     - yasoob.khld@gmail.com
 #  irc:
 #    channels:

From 87830900a95f95308dac565f9da12387edea65e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 5 Jan 2015 13:07:24 +0100
Subject: [PATCH 1045/1937] [generic] Update some tests

---
 youtube_dl/extractor/generic.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 493afb57d..5c41ff517 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -131,12 +131,13 @@ class GenericIE(InfoExtractor):
         # ooyala video
         {
             'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
-            'md5': '5644c6ca5d5782c1d0d350dad9bd840c',
+            'md5': '166dd577b433b4d4ebfee10b0824d8ff',
             'info_dict': {
                 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
                 'ext': 'mp4',
                 'title': '2cc213299525360.mov',  # that's what we get
             },
+            'add_ie': ['Ooyala'],
         },
         # google redirect
         {
@@ -146,7 +147,7 @@ class GenericIE(InfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20130224',
                 'uploader_id': 'TheVerge',
-                'description': 'Chris Ziegler takes a look at the Alcatel OneTouch Fire and the ZTE Open; two of the first Firefox OS handsets to be officially announced.',
+                'description': 're:^Chris Ziegler takes a look at the\.*',
                 'uploader': 'The Verge',
                 'title': 'First Firefox OS phones side-by-side',
             },

From cd791a5ea08b77dab37c15efa7e064c07144cb6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 5 Jan 2015 13:11:13 +0100
Subject: [PATCH 1046/1937] [ted] Add support for embed-ssl.ted.com embedded
 videos

---
 youtube_dl/extractor/generic.py | 2 +-
 youtube_dl/extractor/ted.py     | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5c41ff517..2d871f8b4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -926,7 +926,7 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded TED player
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>http://embed\.ted\.com/.+?)\1', webpage)
+                r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'TED')
 
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 944177426..10b3b706a 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -13,7 +13,7 @@ from ..compat import (
 class TEDIE(SubtitlesInfoExtractor):
     _VALID_URL = r'''(?x)
         (?P<proto>https?://)
-        (?P<type>www|embed)(?P<urlmain>\.ted\.com/
+        (?P<type>www|embed(?:-ssl)?)(?P<urlmain>\.ted\.com/
         (
             (?P<type_playlist>playlists(?:/\d+)?) # We have a playlist
             |
@@ -98,7 +98,7 @@ class TEDIE(SubtitlesInfoExtractor):
 
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url, re.VERBOSE)
-        if m.group('type') == 'embed':
+        if m.group('type').startswith('embed'):
             desktop_url = m.group('proto') + 'www' + m.group('urlmain')
             return self.url_result(desktop_url, 'TED')
         name = m.group('name')

From a285b6377b46518ca45d6a41481bf920b353a857 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 5 Jan 2015 13:59:49 +0100
Subject: [PATCH 1047/1937] [normalboots] Skip download in test, it uses rtmp

---
 youtube_dl/extractor/normalboots.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/normalboots.py b/youtube_dl/extractor/normalboots.py
index 3d35b11ac..c13ff0d65 100644
--- a/youtube_dl/extractor/normalboots.py
+++ b/youtube_dl/extractor/normalboots.py
@@ -22,7 +22,11 @@ class NormalbootsIE(InfoExtractor):
             'description': 'Jon is late for Christmas. Typical. Thanks to: Paul Ritchey for Co-Writing/Filming: http://www.youtube.com/user/ContinueShow Michael Azzi for Christmas Intro Animation: http://michafrar.tumblr.com/ Jerrod Waters for Christmas Intro Music: http://www.youtube.com/user/xXJerryTerryXx Casey Ormond for ‘Tense Battle Theme’:\xa0http://www.youtube.com/Kiamet/',
             'uploader': 'JonTron',
             'upload_date': '20140125',
-        }
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):

From 03ff2cc1c49c82daf2218b76e169c2d679447f03 Mon Sep 17 00:00:00 2001
From: oteng <otengkwaku@gmail.com>
Date: Mon, 5 Jan 2015 16:28:24 +0000
Subject: [PATCH 1048/1937] [Auengine] corrected extractions logic

The way the video download url was been extracted was
not working well so i change it for it to extract the
correct url
---
 .gitignore                       |  2 ++
 youtube_dl/extractor/auengine.py | 16 ++++++----------
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/.gitignore b/.gitignore
index 86312d4e4..0422adf44 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,3 +31,5 @@ updates_key.pem
 test/testdata
 .tox
 youtube-dl.zsh
+.idea
+.idea/*
\ No newline at end of file
diff --git a/youtube_dl/extractor/auengine.py b/youtube_dl/extractor/auengine.py
index 014a21952..17c3ad2ef 100644
--- a/youtube_dl/extractor/auengine.py
+++ b/youtube_dl/extractor/auengine.py
@@ -29,17 +29,12 @@ class AUEngineIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
         title = self._html_search_regex(r'<title>(?P<title>.+?)</title>', webpage, 'title')
         title = title.strip()
-        links = re.findall(r'\s(?:file|url):\s*["\']([^\'"]+)["\']', webpage)
-        links = map(compat_urllib_parse.unquote, links)
+        video_url = re.findall(r'http://\w+.auengine.com/vod/.*[^\W]', webpage)
+        video_url = map(compat_urllib_parse.unquote, video_url)[0]
+        thumbnail = re.findall(r'http://\w+.auengine.com/thumb/.*[^\W]', webpage)
+        thumbnail = map(compat_urllib_parse.unquote, thumbnail)[0]
 
-        thumbnail = None
-        video_url = None
-        for link in links:
-            if link.endswith('.png'):
-                thumbnail = link
-            elif '/videos/' in link:
-                video_url = link
-        if not video_url:
+        if video_url == "" and thumbnail =="":
             raise ExtractorError('Could not find video URL')
         ext = '.' + determine_ext(video_url)
         if ext == title[-len(ext):]:
@@ -52,3 +47,4 @@ class AUEngineIE(InfoExtractor):
             'thumbnail': thumbnail,
             'http_referer': 'http://www.auengine.com/flowplayer/flowplayer.commercial-3.2.14.swf',
         }
+

From 9d247bbd2d972953fbb9e8f9aee67472d3854883 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 5 Jan 2015 18:13:19 +0100
Subject: [PATCH 1049/1937] [radiobremen] Fix under Python 2.6 and fix duration

---
 youtube_dl/extractor/radiobremen.py | 30 +++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/radiobremen.py b/youtube_dl/extractor/radiobremen.py
index 6d130d3d9..9f7e6af15 100644
--- a/youtube_dl/extractor/radiobremen.py
+++ b/youtube_dl/extractor/radiobremen.py
@@ -5,10 +5,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import parse_duration
 
 
 class RadioBremenIE(InfoExtractor):
-    _VALID_URL = r'http?://(?:www\.)?radiobremen\.de/mediathek/(index\.html)?\?id=(?P<video_id>[0-9]+)'
+    _VALID_URL = r'http?://(?:www\.)?radiobremen\.de/mediathek/(?:index\.html)?\?id=(?P<id>[0-9]+)'
     IE_NAME = 'radiobremen'
 
     _TEST = {
@@ -16,6 +17,7 @@ class RadioBremenIE(InfoExtractor):
         'info_dict': {
             'id': '114720',
             'ext': 'mp4',
+            'duration': 1685,
             'width': 512,
             'title': 'buten un binnen vom 22. Dezember',
             'description': 'Unter anderem mit diesen Themen: 45 Flüchtlinge sind in Worpswede angekommen +++ Freies Internet für alle: Bremer arbeiten an einem flächendeckenden W-Lan-Netzwerk +++ Aktivisten kämpfen für das Unibad +++ So war das Wetter 2014 +++',
@@ -23,32 +25,32 @@ class RadioBremenIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
+        video_id = self._match_id(url)
 
         meta_url = "http://www.radiobremen.de/apps/php/mediathek/metadaten.php?id=%s" % video_id
         meta_doc = self._download_webpage(meta_url, video_id, 'Downloading metadata')
         title = self._html_search_regex("<h1.*>(?P<title>.+)</h1>", meta_doc, "title")
         description = self._html_search_regex("<p>(?P<description>.*)</p>", meta_doc, "description")
-        duration = self._html_search_regex("L&auml;nge:</td>\s+<td>(?P<duration>[0-9]+:[0-9]+)</td>", meta_doc, "duration")
+        duration = parse_duration(
+            self._html_search_regex("L&auml;nge:</td>\s+<td>(?P<duration>[0-9]+:[0-9]+)</td>", meta_doc, "duration"))
 
         page_doc = self._download_webpage(url, video_id, 'Downloading video information')
         pattern = "ardformatplayerclassic\(\'playerbereich\',\'(?P<width>[0-9]+)\',\'.*\',\'(?P<video_id>[0-9]+)\',\'(?P<secret>[0-9]+)\',\'(?P<thumbnail>.+)\',\'\'\)"
         mobj = re.search(pattern, page_doc)
-        width, video_id, secret, thumbnail = int(mobj.group("width")), mobj.group("video_id"), mobj.group("secret"), mobj.group("thumbnail")
-        video_url = "http://dl-ondemand.radiobremen.de/mediabase/{:}/{:}_{:}_{:}.mp4".format(video_id, video_id, secret, width)
+        video_url = (
+            "http://dl-ondemand.radiobremen.de/mediabase/%s/%s_%s_%s.mp4" %
+            (video_id, video_id, mobj.group("secret"), mobj.group('width')))
 
+        formats = [{
+            'url': video_url,
+            'ext': 'mp4',
+            'width': int(mobj.group("width")),
+        }]
         return {
             'id': video_id,
             'title': title,
             'description': description,
             'duration': duration,
-            'formats': [
-                {'url': video_url,
-                 'ext': 'mp4',
-                 'width': width,
-                 'protocol': 'http'
-                 }
-            ],
-            'thumbnail': thumbnail,
+            'formats': formats,
+            'thumbnail': mobj.group('thumbnail'),
         }

From aa80652f47b3df14664556913d4f14172c9ec4fb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 5 Jan 2015 18:14:09 +0100
Subject: [PATCH 1050/1937] [radiobremen] Add test for thumbnail

---
 youtube_dl/extractor/radiobremen.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/radiobremen.py b/youtube_dl/extractor/radiobremen.py
index 9f7e6af15..057dc15ab 100644
--- a/youtube_dl/extractor/radiobremen.py
+++ b/youtube_dl/extractor/radiobremen.py
@@ -20,6 +20,7 @@ class RadioBremenIE(InfoExtractor):
             'duration': 1685,
             'width': 512,
             'title': 'buten un binnen vom 22. Dezember',
+            'thumbnail': 're:https?://.*\.jpg$',
             'description': 'Unter anderem mit diesen Themen: 45 Flüchtlinge sind in Worpswede angekommen +++ Freies Internet für alle: Bremer arbeiten an einem flächendeckenden W-Lan-Netzwerk +++ Aktivisten kämpfen für das Unibad +++ So war das Wetter 2014 +++',
         },
     }

From 5e3e1c82d828bc54f6873d2c7bdab315713e9a02 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 5 Jan 2015 18:14:39 +0100
Subject: [PATCH 1051/1937] Credit @ckrooss for radiobremen (#4632)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 9b548cf25..a63c97ae0 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -99,3 +99,4 @@ Max Reimann
 Cédric Luthi
 Thijs Vermeir
 Joel Leclerc
+Christopher Krooss

From d7cc31b63e1efaf5762f38897d4c717901e127e3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 5 Jan 2015 18:16:47 +0100
Subject: [PATCH 1052/1937] [generic] PEP8

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2d871f8b4..7a5bf9392 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -926,7 +926,7 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded TED player
         mobj = re.search(
-                r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'TED')
 

From dda620e88c68e995afcc3cd35b9d360cb42527a0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 5 Jan 2015 18:17:03 +0100
Subject: [PATCH 1053/1937] [radiobremen] Make code more readable and more
 resilient to failures

---
 youtube_dl/extractor/radiobremen.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/radiobremen.py b/youtube_dl/extractor/radiobremen.py
index 057dc15ab..0d706312e 100644
--- a/youtube_dl/extractor/radiobremen.py
+++ b/youtube_dl/extractor/radiobremen.py
@@ -29,15 +29,21 @@ class RadioBremenIE(InfoExtractor):
         video_id = self._match_id(url)
 
         meta_url = "http://www.radiobremen.de/apps/php/mediathek/metadaten.php?id=%s" % video_id
-        meta_doc = self._download_webpage(meta_url, video_id, 'Downloading metadata')
-        title = self._html_search_regex("<h1.*>(?P<title>.+)</h1>", meta_doc, "title")
-        description = self._html_search_regex("<p>(?P<description>.*)</p>", meta_doc, "description")
-        duration = parse_duration(
-            self._html_search_regex("L&auml;nge:</td>\s+<td>(?P<duration>[0-9]+:[0-9]+)</td>", meta_doc, "duration"))
+        meta_doc = self._download_webpage(
+            meta_url, video_id, 'Downloading metadata')
+        title = self._html_search_regex(
+            r"<h1.*>(?P<title>.+)</h1>", meta_doc, "title")
+        description = self._html_search_regex(
+            r"<p>(?P<description>.*)</p>", meta_doc, "description", fatal=False)
+        duration = parse_duration(self._html_search_regex(
+            r"L&auml;nge:</td>\s+<td>(?P<duration>[0-9]+:[0-9]+)</td>",
+            meta_doc, "duration", fatal=False))
 
-        page_doc = self._download_webpage(url, video_id, 'Downloading video information')
-        pattern = "ardformatplayerclassic\(\'playerbereich\',\'(?P<width>[0-9]+)\',\'.*\',\'(?P<video_id>[0-9]+)\',\'(?P<secret>[0-9]+)\',\'(?P<thumbnail>.+)\',\'\'\)"
-        mobj = re.search(pattern, page_doc)
+        page_doc = self._download_webpage(
+            url, video_id, 'Downloading video information')
+        mobj = re.search(
+            r"ardformatplayerclassic\(\'playerbereich\',\'(?P<width>[0-9]+)\',\'.*\',\'(?P<video_id>[0-9]+)\',\'(?P<secret>[0-9]+)\',\'(?P<thumbnail>.+)\',\'\'\)",
+            page_doc)
         video_url = (
             "http://dl-ondemand.radiobremen.de/mediabase/%s/%s_%s_%s.mp4" %
             (video_id, video_id, mobj.group("secret"), mobj.group('width')))

From 6291438073e35adc94f573a43625fb54a64cf733 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 5 Jan 2015 18:21:32 +0100
Subject: [PATCH 1054/1937] [auengine] Simplify (#4643)

---
 youtube_dl/extractor/auengine.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/auengine.py b/youtube_dl/extractor/auengine.py
index 17c3ad2ef..a1b666be0 100644
--- a/youtube_dl/extractor/auengine.py
+++ b/youtube_dl/extractor/auengine.py
@@ -7,6 +7,7 @@ from ..compat import compat_urllib_parse
 from ..utils import (
     determine_ext,
     ExtractorError,
+    remove_end,
 )
 
 
@@ -27,18 +28,18 @@ class AUEngineIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
-        title = self._html_search_regex(r'<title>(?P<title>.+?)</title>', webpage, 'title')
-        title = title.strip()
-        video_url = re.findall(r'http://\w+.auengine.com/vod/.*[^\W]', webpage)
-        video_url = map(compat_urllib_parse.unquote, video_url)[0]
-        thumbnail = re.findall(r'http://\w+.auengine.com/thumb/.*[^\W]', webpage)
-        thumbnail = map(compat_urllib_parse.unquote, thumbnail)[0]
+        title = self._html_search_regex(
+            r'<title>\s*(?P<title>.+?)\s*</title>', webpage, 'title')
+        video_urls = re.findall(r'http://\w+.auengine.com/vod/.*[^\W]', webpage)
+        video_url = compat_urllib_parse.unquote(video_urls[0])
+        thumbnails = re.findall(r'http://\w+.auengine.com/thumb/.*[^\W]', webpage)
+        thumbnail = compat_urllib_parse.unquote(thumbnails[0])
 
-        if video_url == "" and thumbnail =="":
+        if not video_url:
             raise ExtractorError('Could not find video URL')
+
         ext = '.' + determine_ext(video_url)
-        if ext == title[-len(ext):]:
-            title = title[:-len(ext)]
+        title = remove_end(title, ext)
 
         return {
             'id': video_id,
@@ -47,4 +48,3 @@ class AUEngineIE(InfoExtractor):
             'thumbnail': thumbnail,
             'http_referer': 'http://www.auengine.com/flowplayer/flowplayer.commercial-3.2.14.swf',
         }
-

From f4bca0b348fe1f4f65c939b496973062180e0c4f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 5 Jan 2015 18:44:29 +0100
Subject: [PATCH 1055/1937] release 2015.01.05

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 09813928a..086f0ebf0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.04'
+__version__ = '2015.01.05'

From 8f9529cd0559bdbe6c568cfd765f9129666a77be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 5 Jan 2015 19:14:50 +0100
Subject: [PATCH 1056/1937] [motorsport] Fix extraction and make trailing '/'
 optional

They directly embed a youtube video now.
---
 youtube_dl/extractor/motorsport.py | 60 ++++++++++++------------------
 1 file changed, 23 insertions(+), 37 deletions(-)

diff --git a/youtube_dl/extractor/motorsport.py b/youtube_dl/extractor/motorsport.py
index f5ca74e97..c1a482dba 100644
--- a/youtube_dl/extractor/motorsport.py
+++ b/youtube_dl/extractor/motorsport.py
@@ -1,63 +1,49 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import hashlib
-import json
-import time
-
 from .common import InfoExtractor
 from ..compat import (
-    compat_parse_qs,
-    compat_str,
-)
-from ..utils import (
-    int_or_none,
+    compat_urlparse,
 )
 
 
 class MotorsportIE(InfoExtractor):
     IE_DESC = 'motorsport.com'
-    _VALID_URL = r'http://www\.motorsport\.com/[^/?#]+/video/(?:[^/?#]+/)(?P<id>[^/]+)/(?:$|[?#])'
+    _VALID_URL = r'http://www\.motorsport\.com/[^/?#]+/video/(?:[^/?#]+/)(?P<id>[^/]+)/?(?:$|[?#])'
     _TEST = {
         'url': 'http://www.motorsport.com/f1/video/main-gallery/red-bull-racing-2014-rules-explained/',
-        'md5': '5592cb7c5005d9b2c163df5ac3dc04e4',
         'info_dict': {
-            'id': '7063',
+            'id': '2-T3WuR-KMM',
             'ext': 'mp4',
             'title': 'Red Bull Racing: 2014 Rules Explained',
-            'duration': 207,
+            'duration': 208,
             'description': 'A new clip from Red Bull sees Daniel Ricciardo and Sebastian Vettel explain the 2014 Formula One regulations – which are arguably the most complex the sport has ever seen.',
-            'uploader': 'rainiere',
-            'thumbnail': r're:^http://.*motorsport\.com/.+\.jpg$'
-        }
+            'uploader': 'mcomstaff',
+            'uploader_id': 'UC334JIYKkVnyFoNCclfZtHQ',
+            'upload_date': '20140903',
+            'thumbnail': r're:^https?://.+\.jpg$'
+        },
+        'add_ie': ['Youtube'],
+        'params': {
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
 
-        flashvars_code = self._html_search_regex(
-            r'<embed id="player".*?flashvars="([^"]+)"', webpage, 'flashvars')
-        flashvars = compat_parse_qs(flashvars_code)
-        params = json.loads(flashvars['parameters'][0])
-
-        e = compat_str(int(time.time()) + 24 * 60 * 60)
-        base_video_url = params['location'] + '?e=' + e
-        s = 'h3hg713fh32'
-        h = hashlib.md5((s + base_video_url).encode('utf-8')).hexdigest()
-        video_url = base_video_url + '&h=' + h
-
-        uploader = self._html_search_regex(
-            r'(?s)<span class="label">Video by: </span>(.*?)</a>', webpage,
-            'uploader', fatal=False)
+        iframe_path = self._html_search_regex(
+            r'<iframe id="player_iframe"[^>]+src="([^"]+)"', webpage,
+            'iframe path')
+        iframe = self._download_webpage(
+            compat_urlparse.urljoin(url, iframe_path), display_id,
+            'Downloading iframe')
+        youtube_id = self._search_regex(
+            r'www.youtube.com/embed/(.{11})', iframe, 'youtube id')
 
         return {
-            'id': params['video_id'],
+            '_type': 'url_transparent',
             'display_id': display_id,
-            'title': params['title'],
-            'url': video_url,
-            'description': params.get('description'),
-            'thumbnail': params.get('main_thumb'),
-            'duration': int_or_none(params.get('duration')),
-            'uploader': uploader,
+            'url': 'https://youtube.com/watch?v=%s' % youtube_id,
         }

From 3b8f3a1504b46ff6085617ea22e027668d45a503 Mon Sep 17 00:00:00 2001
From: Antti Ajanki <antti.ajanki@iki.fi>
Date: Mon, 5 Jan 2015 20:12:29 +0200
Subject: [PATCH 1057/1937] [downloader/f4m] <metadata> is optional according
 to the F4M specs

---
 youtube_dl/downloader/f4m.py | 38 ++++++++++++++++++++++--------------
 1 file changed, 23 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index f9f6f3e73..87d3150c7 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -187,24 +187,27 @@ def build_fragments_list(boot_info):
     return res
 
 
-def write_flv_header(stream, metadata):
-    """Writes the FLV header and the metadata to stream"""
+def write_flv_header(stream):
+    """Writes the FLV header to stream"""
     # FLV header
     stream.write(b'FLV\x01')
     stream.write(b'\x05')
     stream.write(b'\x00\x00\x00\x09')
-    # FLV File body
     stream.write(b'\x00\x00\x00\x00')
-    # FLVTAG
-    # Script data
-    stream.write(b'\x12')
-    # Size of the metadata with 3 bytes
-    stream.write(struct_pack('!L', len(metadata))[1:])
-    stream.write(b'\x00\x00\x00\x00\x00\x00\x00')
-    stream.write(metadata)
-    # Magic numbers extracted from the output files produced by AdobeHDS.php
-    # (https://github.com/K-S-V/Scripts)
-    stream.write(b'\x00\x00\x01\x73')
+
+
+def write_metadata_tag(stream, metadata):
+    """Writes optional metadata tag to stream"""
+    if metadata:
+        # Script data
+        stream.write(b'\x12')
+        # Size of the metadata with 3 bytes
+        stream.write(struct_pack('!L', len(metadata))[1:])
+        stream.write(b'\x00\x00\x00\x00\x00\x00\x00')
+        stream.write(metadata)
+        # Magic numbers extracted from the output files produced by AdobeHDS.php
+        # (https://github.com/K-S-V/Scripts)
+        stream.write(b'\x00\x00\x01\x73')
 
 
 def _add_ns(prop):
@@ -256,7 +259,11 @@ class F4mFD(FileDownloader):
             bootstrap = self.ydl.urlopen(bootstrap_url).read()
         else:
             bootstrap = base64.b64decode(bootstrap_node.text)
-        metadata = base64.b64decode(media.find(_add_ns('metadata')).text)
+        metadata_node = media.find(_add_ns('metadata'))
+        if metadata_node is not None:
+            metadata = base64.b64decode(metadata_node.text)
+        else:
+            metadata = None
         boot_info = read_bootstrap_info(bootstrap)
 
         fragments_list = build_fragments_list(boot_info)
@@ -269,7 +276,8 @@ class F4mFD(FileDownloader):
 
         tmpfilename = self.temp_name(filename)
         (dest_stream, tmpfilename) = sanitize_open(tmpfilename, 'wb')
-        write_flv_header(dest_stream, metadata)
+        write_flv_header(dest_stream)
+        write_metadata_tag(dest_stream, metadata)
 
         # This dict stores the download progress, it's updated by the progress
         # hook

From 2c322cc5d65550de10d70d812ac3cd6742452252 Mon Sep 17 00:00:00 2001
From: Antti Ajanki <antti.ajanki@iki.fi>
Date: Mon, 5 Jan 2015 20:22:17 +0200
Subject: [PATCH 1058/1937] [downloader/f4m] The last value in a tag is the tag
 length

---
 youtube_dl/downloader/f4m.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 87d3150c7..744bf91cb 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -187,6 +187,10 @@ def build_fragments_list(boot_info):
     return res
 
 
+def write_unsigned_int(stream, val):
+    stream.write(struct_pack('!I', val))
+
+
 def write_flv_header(stream):
     """Writes the FLV header to stream"""
     # FLV header
@@ -198,6 +202,8 @@ def write_flv_header(stream):
 
 def write_metadata_tag(stream, metadata):
     """Writes optional metadata tag to stream"""
+    FLV_TAG_HEADER_LEN = 11
+
     if metadata:
         # Script data
         stream.write(b'\x12')
@@ -205,9 +211,7 @@ def write_metadata_tag(stream, metadata):
         stream.write(struct_pack('!L', len(metadata))[1:])
         stream.write(b'\x00\x00\x00\x00\x00\x00\x00')
         stream.write(metadata)
-        # Magic numbers extracted from the output files produced by AdobeHDS.php
-        # (https://github.com/K-S-V/Scripts)
-        stream.write(b'\x00\x00\x01\x73')
+        write_unsigned_int(stream, FLV_TAG_HEADER_LEN + len(metadata))
 
 
 def _add_ns(prop):

From f14f2a6d7909eae5cf0d6f447f8e8a3eb53752d4 Mon Sep 17 00:00:00 2001
From: Antti Ajanki <antti.ajanki@iki.fi>
Date: Mon, 5 Jan 2015 20:30:40 +0200
Subject: [PATCH 1059/1937] [downloader/f4m] Minor cleanup

---
 youtube_dl/downloader/f4m.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 744bf91cb..c460c167a 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -191,6 +191,10 @@ def write_unsigned_int(stream, val):
     stream.write(struct_pack('!I', val))
 
 
+def write_unsigned_int_24(stream, val):
+    stream.write(struct_pack('!I', val)[1:])
+
+
 def write_flv_header(stream):
     """Writes the FLV header to stream"""
     # FLV header
@@ -202,13 +206,12 @@ def write_flv_header(stream):
 
 def write_metadata_tag(stream, metadata):
     """Writes optional metadata tag to stream"""
+    SCRIPT_TAG = b'\x12'
     FLV_TAG_HEADER_LEN = 11
 
     if metadata:
-        # Script data
-        stream.write(b'\x12')
-        # Size of the metadata with 3 bytes
-        stream.write(struct_pack('!L', len(metadata))[1:])
+        stream.write(SCRIPT_TAG)
+        write_unsigned_int_24(stream, len(metadata))
         stream.write(b'\x00\x00\x00\x00\x00\x00\x00')
         stream.write(metadata)
         write_unsigned_int(stream, FLV_TAG_HEADER_LEN + len(metadata))

From ee61f6f3e2f4d03a3ad9c5ca8f4fc5cd9461bdbc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 5 Jan 2015 22:42:17 +0100
Subject: [PATCH 1060/1937] [youtube] Handle cases where format comes without a
 preference (Fixes #4648)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d1bbf0b01..4aa8615b4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1046,7 +1046,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                     for f in formats:
                         if f['format_id'] in dash_keys:
                             f['format_id'] = 'nondash-%s' % f['format_id']
-                            f['preference'] -= 10000
+                            f['preference'] = f.get('preference', 0) - 10000
                     formats.extend(dash_formats)
 
         self._sort_formats(formats)

From f8bb576c4f376c9514a57386b924e67484f52c36 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 5 Jan 2015 22:42:38 +0100
Subject: [PATCH 1061/1937] release 2015.01.05.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 086f0ebf0..2124e954f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.05'
+__version__ = '2015.01.05.1'

From 0df2dea73bc311e1b839de7d9b8fe84c8f550718 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Jan 2015 06:54:31 +0600
Subject: [PATCH 1062/1937] [giga] Add extractor (Closes #4090)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/giga.py     | 101 +++++++++++++++++++++++++++++++
 2 files changed, 102 insertions(+)
 create mode 100644 youtube_dl/extractor/giga.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8e47bd60d..b523e9644 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -159,6 +159,7 @@ from .gametrailers import GametrailersIE
 from .gdcvault import GDCVaultIE
 from .generic import GenericIE
 from .giantbomb import GiantBombIE
+from .giga import GigaIE
 from .glide import GlideIE
 from .globo import GloboIE
 from .godtube import GodTubeIE
diff --git a/youtube_dl/extractor/giga.py b/youtube_dl/extractor/giga.py
new file mode 100644
index 000000000..775890112
--- /dev/null
+++ b/youtube_dl/extractor/giga.py
@@ -0,0 +1,101 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import itertools
+
+from .common import InfoExtractor
+from ..utils import (
+    qualities,
+    compat_str,
+    parse_duration,
+    parse_iso8601,
+    str_to_int,
+)
+
+
+class GigaIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?giga\.de/(?:[^/]+/)*(?P<id>[^/]+)'
+    _TESTS = [{
+        'url': 'http://www.giga.de/filme/anime-awesome/trailer/anime-awesome-chihiros-reise-ins-zauberland-das-beste-kommt-zum-schluss/',
+        'md5': '6bc5535e945e724640664632055a584f',
+        'info_dict': {
+            'id': '2622086',
+            'display_id': 'anime-awesome-chihiros-reise-ins-zauberland-das-beste-kommt-zum-schluss',
+            'ext': 'mp4',
+            'title': 'Anime Awesome: Chihiros Reise ins Zauberland – Das Beste kommt zum Schluss',
+            'description': 'md5:afdf5862241aded4718a30dff6a57baf',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 578,
+            'timestamp': 1414749706,
+            'upload_date': '20141031',
+            'uploader': 'Robin Schweiger',
+            'view_count': int,
+        },
+    }, {
+        'url': 'http://www.giga.de/games/channel/giga-top-montag/giga-topmontag-die-besten-serien-2014/',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.giga.de/extra/netzkultur/videos/giga-games-tom-mats-robin-werden-eigene-wege-gehen-eine-ankuendigung/',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.giga.de/tv/jonas-liest-spieletitel-eingedeutscht-episode-2/',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._search_regex(
+            [r'data-video-id="(\d+)"', r'/api/video/jwplayer/#v=(\d+)'],
+            webpage, 'video id')
+
+        playlist = self._download_json(
+            'http://www.giga.de/api/syndication/video/video_id/%s/playlist.json?content=syndication/key/368b5f151da4ae05ced7fa296bdff65a/'
+            % video_id, video_id)[0]
+
+        quality = qualities(['normal', 'hd720'])
+
+        formats = []
+        for format_id in itertools.count(0):
+            fmt = playlist.get(compat_str(format_id))
+            if not fmt:
+                break
+            formats.append({
+                'url': fmt['src'],
+                'format_id': '%s-%s' % (fmt['quality'], fmt['type'].split('/')[-1]),
+                'quality': quality(fmt['quality']),
+            })
+        self._sort_formats(formats)
+
+        title = self._html_search_meta(
+            'title', webpage, 'title', fatal=True)
+        description = self._html_search_meta(
+            'description', webpage, 'description')
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        duration = parse_duration(self._search_regex(
+            r'(?s)(?:data-video-id="{0}"|data-video="[^"]*/api/video/jwplayer/#v={0}[^"]*")[^>]*>.+?<span class="duration">([^<]+)</span>'.format(video_id),
+            webpage, 'duration', fatal=False))
+
+        timestamp = parse_iso8601(self._search_regex(
+            r'datetime="([^"]+)"', webpage, 'upload date', fatal=False))
+        uploader = self._search_regex(
+            r'class="author">([^<]+)</a>', webpage, 'uploader', fatal=False)
+
+        view_count = str_to_int(self._search_regex(
+            r'<span class="views"><strong>([\d.]+)</strong>', webpage, 'view count', fatal=False))
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'view_count': view_count,
+            'formats': formats,
+        }

From 76b3c61012b5cd5a539e2a1a121d11427e6c4c6d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 6 Jan 2015 11:59:41 +0100
Subject: [PATCH 1063/1937] [youtube] Add formats 308 and 315 (closes #4650)

---
 youtube_dl/extractor/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4aa8615b4..f0efaf0d9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -287,7 +287,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
         '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
         '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
+        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
         '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'VP9'},
+        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'VP9'},
 
         # Dash webm audio
         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},

From 27a82a1b938f149b8c718e41ae57fcba01af299e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Jan 2015 05:03:14 +0600
Subject: [PATCH 1064/1937] [ceskatelevize] Simplify

---
 youtube_dl/extractor/ceskatelevize.py | 32 +++++++++------------------
 1 file changed, 11 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 345ac5e85..0a9bd01c9 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -12,7 +12,6 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     float_or_none,
-    HEADRequest,
 )
 
 
@@ -105,14 +104,16 @@ class CeskaTelevizeIE(SubtitlesInfoExtractor):
         duration = float_or_none(item.get('duration'))
         thumbnail = item.get('previewImageUrl')
 
-        # subtitles
-        subtitles = self.extract_subtitles(video_id, webpage)
+        subtitles = {}
+        subs = item.get('subtitles')
+        if subs:
+            subtitles['cs'] = subs[0]['url']
 
         if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, webpage)
+            self._list_available_subtitles(video_id, subtitles)
             return
 
-        subtitles = self._fix_subtitles(self.extract_subtitles(video_id, webpage))
+        subtitles = self._fix_subtitles(self.extract_subtitles(video_id, subtitles))
 
         return {
             'id': episode_id,
@@ -124,13 +125,14 @@ class CeskaTelevizeIE(SubtitlesInfoExtractor):
             'subtitles': subtitles,
         }
 
-    def _fix_subtitles(self, subtitles):
-        """ Convert milisecond-based subtitles to SRT """
+    @staticmethod
+    def _fix_subtitles(subtitles):
+        """ Convert millisecond-based subtitles to SRT """
         if subtitles is None:
             return subtitles  # subtitles not requested
 
         def _msectotimecode(msec):
-            """ Helper utility to convert miliseconds to timecode """
+            """ Helper utility to convert milliseconds to timecode """
             components = []
             for divider in [1000, 60, 60, 100]:
                 components.append(msec % divider)
@@ -139,7 +141,7 @@ class CeskaTelevizeIE(SubtitlesInfoExtractor):
 
         def _fix_subtitle(subtitle):
             for line in subtitle.splitlines():
-                m = re.match(r"^ *([0-9]+); *([0-9]+) +([0-9]+) *$", line)
+                m = re.match(r"^\s*([0-9]+);\s*([0-9]+)\s+([0-9]+)\s*$", line)
                 if m:
                     yield m.group(1)
                     start, stop = (_msectotimecode(int(t)) for t in m.groups()[1:])
@@ -151,15 +153,3 @@ class CeskaTelevizeIE(SubtitlesInfoExtractor):
         for k, v in subtitles.items():
             fixed_subtitles[k] = "\r\n".join(_fix_subtitle(v))
         return fixed_subtitles
-
-    def _get_available_subtitles(self, video_id, webpage):
-        video_id = video_id.partition('-')[0]
-        url = 'http://imgct.ceskatelevize.cz/cache/data/ivysilani/' \
-              'subtitles/{}/{}/sub.txt'.format(video_id[:3], video_id)
-        req = HEADRequest(url)
-        sub = self._request_webpage(
-            req, video_id,
-            note="Checking subtitles",
-            errnote="No subtitles found",
-            fatal=False)
-        return {'cs': url} if sub else {}

From 6309cb9b412a6e9914bd873201ad0fc38ac2659d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Jan 2015 05:03:34 +0600
Subject: [PATCH 1065/1937] [ceskatelevize] Fix python 2.6 format issue

---
 youtube_dl/extractor/ceskatelevize.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 0a9bd01c9..f70e090bb 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -145,7 +145,7 @@ class CeskaTelevizeIE(SubtitlesInfoExtractor):
                 if m:
                     yield m.group(1)
                     start, stop = (_msectotimecode(int(t)) for t in m.groups()[1:])
-                    yield "{} --> {}".format(start, stop)
+                    yield "{0} --> {1}".format(start, stop)
                 else:
                     yield line
 

From 0b54a5b10ab9da3257130b0061fb46c84b4d64ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Jan 2015 05:04:15 +0600
Subject: [PATCH 1066/1937] [ceskatelevize] Add subtitles tests

---
 test/test_subtitles.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index d34565191..6336dd317 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -17,6 +17,7 @@ from youtube_dl.extractor import (
     TEDIE,
     VimeoIE,
     WallaIE,
+    CeskaTelevizeIE,
 )
 
 
@@ -317,5 +318,32 @@ class TestWallaSubtitles(BaseTestSubtitles):
         self.assertEqual(len(subtitles), 0)
 
 
+class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
+    url = 'http://www.ceskatelevize.cz/ivysilani/10600540290-u6-uzasny-svet-techniky'
+    IE = CeskaTelevizeIE
+
+    def test_list_subtitles(self):
+        self.DL.expect_warning('Automatic Captions not supported by this server')
+        self.DL.params['listsubtitles'] = True
+        info_dict = self.getInfoDict()
+        self.assertEqual(info_dict, None)
+
+    def test_allsubtitles(self):
+        self.DL.expect_warning('Automatic Captions not supported by this server')
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['cs']))
+        self.assertEqual(md5(subtitles['cs']), '9bf52d9549533c32c427e264bf0847d4')
+
+    def test_nosubtitles(self):
+        self.DL.expect_warning('video doesn\'t have subtitles')
+        self.url = 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220'
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(len(subtitles), 0)
+
+
 if __name__ == '__main__':
     unittest.main()

From d6a31b17661b6d1a0d3fd987ef9570011e458e6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Jan 2015 05:05:18 +0600
Subject: [PATCH 1067/1937] Credit @oskar456 for ceskatelevize subtitles
 support (#4622)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index a63c97ae0..8f2010803 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -100,3 +100,4 @@ Cédric Luthi
 Thijs Vermeir
 Joel Leclerc
 Christopher Krooss
+Ondřej Caletka

From 799d88d3d8a9337fb0b7a858ec4ffb3aaacbc974 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Jan 2015 06:46:56 +0600
Subject: [PATCH 1068/1937] [nrktv] Add support for playlists (Closes #4656)

---
 youtube_dl/extractor/nrk.py | 128 ++++++++++++++++++++++++++++++------
 1 file changed, 107 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 43e8e619f..e950c76dd 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -72,7 +72,7 @@ class NRKIE(InfoExtractor):
 
 
 class NRKTVIE(InfoExtractor):
-    _VALID_URL = r'http://tv\.nrk(?:super)?\.no/(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})'
+    _VALID_URL = r'http://tv\.nrk(?:super)?\.no/(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
 
     _TESTS = [
         {
@@ -85,7 +85,7 @@ class NRKTVIE(InfoExtractor):
                 'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
                 'upload_date': '20140523',
                 'duration': 1741.52,
-            }
+            },
         },
         {
             'url': 'http://tv.nrk.no/program/mdfp15000514',
@@ -97,39 +97,125 @@ class NRKTVIE(InfoExtractor):
                 'description': 'md5:654c12511f035aed1e42bdf5db3b206a',
                 'upload_date': '20140524',
                 'duration': 4605.0,
-            }
+            },
         },
+        {
+            # single playlist video
+            'url': 'http://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015#del=2',
+            'md5': 'adbd1dbd813edaf532b0a253780719c2',
+            'info_dict': {
+                'id': 'MSPO40010515-part2',
+                'ext': 'flv',
+                'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 2:2)',
+                'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
+                'upload_date': '20150106',
+            },
+            'skip': 'Only works from Norway',
+            'params': {
+                'proxy': '127.0.0.1:8118',
+            },
+        },
+        {
+            'url': 'http://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
+            'playlist': [
+                {
+                    'md5': '9480285eff92d64f06e02a5367970a7a',
+                    'info_dict': {
+                        'id': 'MSPO40010515-part1',
+                        'ext': 'flv',
+                        'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 1:2)',
+                        'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
+                        'upload_date': '20150106',
+                    },
+                },
+                {
+                    'md5': 'adbd1dbd813edaf532b0a253780719c2',
+                    'info_dict': {
+                        'id': 'MSPO40010515-part2',
+                        'ext': 'flv',
+                        'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn 06.01.2015 (del 2:2)',
+                        'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
+                        'upload_date': '20150106',
+                    },
+                },
+            ],
+            'info_dict': {
+                'id': 'MSPO40010515',
+                'title': 'Tour de Ski: Sprint fri teknikk, kvinner og menn',
+                'description': 'md5:238b67b97a4ac7d7b4bf0edf8cc57d26',
+                'upload_date': '20150106',
+                'duration': 6947.5199999999995,
+            },
+            'skip': 'Only works from Norway',
+            'params': {
+                'proxy': '127.0.0.1:8118',
+            },
+        }
     ]
 
+    def _extract_f4m(self, manifest_url, video_id):
+        return self._extract_f4m_formats(manifest_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124', video_id)
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
+        part_id = mobj.group('part_id')
 
-        page = self._download_webpage(url, video_id)
+        webpage = self._download_webpage(url, video_id)
 
-        title = self._html_search_meta('title', page, 'title')
-        description = self._html_search_meta('description', page, 'description')
-        thumbnail = self._html_search_regex(r'data-posterimage="([^"]+)"', page, 'thumbnail', fatal=False)
-        upload_date = unified_strdate(self._html_search_meta('rightsfrom', page, 'upload date', fatal=False))
-        duration = float_or_none(
-            self._html_search_regex(r'data-duration="([^"]+)"', page, 'duration', fatal=False))
+        title = self._html_search_meta(
+            'title', webpage, 'title')
+        description = self._html_search_meta(
+            'description', webpage, 'description')
+
+        thumbnail = self._html_search_regex(
+            r'data-posterimage="([^"]+)"',
+            webpage, 'thumbnail', fatal=False)
+        upload_date = unified_strdate(self._html_search_meta(
+            'rightsfrom', webpage, 'upload date', fatal=False))
+        duration = float_or_none(self._html_search_regex(
+            r'data-duration="([^"]+)"',
+            webpage, 'duration', fatal=False))
+
+        # playlist
+        parts = re.findall(
+            r'<a href="#del=(\d+)"[^>]+data-argument="([^"]+)">([^<]+)</a>', webpage)
+        if parts:
+            entries = []
+            for current_part_id, stream_url, part_title in parts:
+                if part_id and current_part_id != part_id:
+                    continue
+                video_part_id = '%s-part%s' % (video_id, current_part_id)
+                formats = self._extract_f4m(stream_url, video_part_id)
+                entries.append({
+                    'id': video_part_id,
+                    'title': part_title,
+                    'description': description,
+                    'thumbnail': thumbnail,
+                    'upload_date': upload_date,
+                    'formats': formats,
+                })
+            if part_id:
+                if entries:
+                    return entries[0]
+            else:
+                playlist = self.playlist_result(entries, video_id, title, description)
+                playlist.update({
+                    'thumbnail': thumbnail,
+                    'upload_date': upload_date,
+                    'duration': duration,
+                })
+                return playlist
 
         formats = []
 
-        f4m_url = re.search(r'data-media="([^"]+)"', page)
+        f4m_url = re.search(r'data-media="([^"]+)"', webpage)
         if f4m_url:
-            formats.append({
-                'url': f4m_url.group(1) + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124',
-                'format_id': 'f4m',
-                'ext': 'flv',
-            })
+            formats.extend(self._extract_f4m(f4m_url.group(1), video_id))
 
-        m3u8_url = re.search(r'data-hls-media="([^"]+)"', page)
+        m3u8_url = re.search(r'data-hls-media="([^"]+)"', webpage)
         if m3u8_url:
-            formats.append({
-                'url': m3u8_url.group(1),
-                'format_id': 'm3u8',
-            })
+            formats.extend(self._extract_m3u8_formats(m3u8_url.group(1), video_id, 'mp4'))
 
         self._sort_formats(formats)
 

From 059006292523264f4e7c7e03df3729612af8099c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 07:20:20 +0100
Subject: [PATCH 1069/1937] Respect age_limit when listing extractors (Fixes
 #4653)

---
 test/helper.py                   | 14 ++------------
 test/test_utils.py               |  8 ++++++++
 youtube_dl/YoutubeDL.py          | 10 +++-------
 youtube_dl/__init__.py           |  8 +++-----
 youtube_dl/extractor/__init__.py | 13 +++++++++++++
 youtube_dl/extractor/common.py   | 30 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/xtube.py    |  2 ++
 youtube_dl/utils.py              | 10 ++++++++++
 8 files changed, 71 insertions(+), 24 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 96d58b7c1..77225e4f7 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -82,18 +82,8 @@ class FakeYDL(YoutubeDL):
 
 def gettestcases(include_onlymatching=False):
     for ie in youtube_dl.extractor.gen_extractors():
-        t = getattr(ie, '_TEST', None)
-        if t:
-            assert not hasattr(ie, '_TESTS'), \
-                '%s has _TEST and _TESTS' % type(ie).__name__
-            tests = [t]
-        else:
-            tests = getattr(ie, '_TESTS', [])
-        for t in tests:
-            if not include_onlymatching and t.get('only_matching', False):
-                continue
-            t['name'] = type(ie).__name__[:-len('IE')]
-            yield t
+        for tc in ie.get_testcases(include_onlymatching):
+            yield tc
 
 
 md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
diff --git a/test/test_utils.py b/test/test_utils.py
index dd49a6d17..16e1a1ddf 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -16,6 +16,7 @@ import json
 import xml.etree.ElementTree
 
 from youtube_dl.utils import (
+    age_restricted,
     args_to_str,
     clean_html,
     DateRange,
@@ -402,5 +403,12 @@ Trying to open render node...
 Success at /dev/dri/renderD128.
 ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
 
+    def test_age_restricted(self):
+        self.assertFalse(age_restricted(None, 10))  # unrestricted content
+        self.assertFalse(age_restricted(1, None))  # unrestricted policy
+        self.assertFalse(age_restricted(8, 10))
+        self.assertTrue(age_restricted(18, 14))
+        self.assertFalse(age_restricted(18, 18))
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 806e7b239..36c71f947 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -63,6 +63,7 @@ from .utils import (
     YoutubeDLHandler,
     prepend_extension,
     args_to_str,
+    age_restricted,
 )
 from .cache import Cache
 from .extractor import get_info_extractor, gen_extractors
@@ -550,13 +551,8 @@ class YoutubeDL(object):
             max_views = self.params.get('max_views')
             if max_views is not None and view_count > max_views:
                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
-        age_limit = self.params.get('age_limit')
-        if age_limit is not None:
-            actual_age_limit = info_dict.get('age_limit')
-            if actual_age_limit is None:
-                actual_age_limit = 0
-            if age_limit < actual_age_limit:
-                return 'Skipping "' + title + '" because it is age restricted'
+        if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
+            return 'Skipping "%s" because it is age restricted' % title
         if self.in_download_archive(info_dict):
             return '%s has already been recorded in archive' % video_title
         return None
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 56f560d26..4c21188a9 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -38,7 +38,7 @@ from .update import update_self
 from .downloader import (
     FileDownloader,
 )
-from .extractor import gen_extractors
+from .extractor import list_extractors
 from .YoutubeDL import YoutubeDL
 
 
@@ -95,17 +95,15 @@ def _real_main(argv=None):
     _enc = preferredencoding()
     all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
 
-    extractors = gen_extractors()
-
     if opts.list_extractors:
-        for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
+        for ie in list_extractors(opts.age_limit):
             compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
             matchedUrls = [url for url in all_urls if ie.suitable(url)]
             for mu in matchedUrls:
                 compat_print('  ' + mu)
         sys.exit(0)
     if opts.list_extractor_descriptions:
-        for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
+        for ie in list_extractors(opts.age_limit):
             if not ie._WORKING:
                 continue
             desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b523e9644..0145e350d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -560,6 +560,8 @@ from .zingmp3 import (
     ZingMp3AlbumIE,
 )
 
+from ..utils import age_restricted
+
 _ALL_CLASSES = [
     klass
     for name, klass in globals().items()
@@ -575,6 +577,17 @@ def gen_extractors():
     return [klass() for klass in _ALL_CLASSES]
 
 
+def list_extractors(age_limit):
+    """
+    Return a list of extractors that are suitable for the given age,
+    sorted by extractor ID.
+    """
+
+    return sorted(
+        filter(lambda ie: ie.is_suitable(age_limit), gen_extractors()),
+        key=lambda ie: ie.IE_NAME.lower())
+
+
 def get_info_extractor(ie_name):
     """Returns the info extractor class with the given ie_name"""
     return globals()[ie_name + 'IE']
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 562e656e0..df32b5ca0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -21,6 +21,7 @@ from ..compat import (
     compat_str,
 )
 from ..utils import (
+    age_restricted,
     clean_html,
     compiled_regex_type,
     ExtractorError,
@@ -877,6 +878,35 @@ class InfoExtractor(object):
             None, '/', True, False, expire_time, '', None, None, None)
         self._downloader.cookiejar.set_cookie(cookie)
 
+    def get_testcases(self, include_onlymatching=False):
+        t = getattr(self, '_TEST', None)
+        if t:
+            assert not hasattr(self, '_TESTS'), \
+                '%s has _TEST and _TESTS' % type(self).__name__
+            tests = [t]
+        else:
+            tests = getattr(self, '_TESTS', [])
+        for t in tests:
+            if not include_onlymatching and t.get('only_matching', False):
+                continue
+            t['name'] = type(self).__name__[:-len('IE')]
+            yield t
+
+    def is_suitable(self, age_limit):
+        """ Test whether the extractor is generally suitable for the given
+        age limit (i.e. pornographic sites are not, all others usually are) """
+
+        any_restricted = False
+        for tc in self.get_testcases(include_onlymatching=False):
+            if 'playlist' in tc:
+                tc = tc['playlist'][0]
+            is_restricted = age_restricted(
+                tc.get('info_dict', {}).get('age_limit'), age_limit)
+            if not is_restricted:
+                return True
+            any_restricted = any_restricted or is_restricted
+        return not any_restricted
+
 
 class SearchInfoExtractor(InfoExtractor):
     """
diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 95f1c8f3c..e8490b028 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -95,6 +95,7 @@ class XTubeUserIE(InfoExtractor):
         'url': 'http://www.xtube.com/community/profile.php?user=greenshowers',
         'info_dict': {
             'id': 'greenshowers',
+            'age_limit': 18,
         },
         'playlist_mincount': 155,
     }
@@ -124,6 +125,7 @@ class XTubeUserIE(InfoExtractor):
         return {
             '_type': 'playlist',
             'id': username,
+            'age_limit': 18,
             'entries': [{
                 '_type': 'url',
                 'url': eurl,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d4951c406..29739a483 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1560,3 +1560,13 @@ def urlhandle_detect_ext(url_handle):
         getheader = url_handle.info().getheader
 
     return getheader('Content-Type').split("/")[1]
+
+
+def age_restricted(content_limit, age_limit):
+    """ Returns True iff the content should be blocked """
+
+    if age_limit is None:  # No limit set
+        return False
+    if content_limit is None:
+        return False  # Content available for everyone
+    return age_limit < content_limit

From 8ee341500dea885c79316fb8f12adde2028c55a5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 07:21:24 +0100
Subject: [PATCH 1070/1937] [viki] Modernize

---
 youtube_dl/extractor/viki.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 15f315298..944901e14 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -17,7 +17,6 @@ class VikiIE(SubtitlesInfoExtractor):
     _VALID_URL = r'^https?://(?:www\.)?viki\.com/videos/(?P<id>[0-9]+v)'
     _TEST = {
         'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
-        'md5': 'a21454021c2646f5433514177e2caa5f',
         'info_dict': {
             'id': '1023585v',
             'ext': 'mp4',
@@ -31,8 +30,7 @@ class VikiIE(SubtitlesInfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
         title = self._og_search_title(webpage)

From 75e51819d010b38d9a3464f8749688c8d53fc123 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 07:22:28 +0100
Subject: [PATCH 1071/1937] release 2015.01.07

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2124e954f..f9c934351 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.05.1'
+__version__ = '2015.01.07'

From fd86c2026de4ca071d1f6bd432da68c93fcf184e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 07:31:38 +0100
Subject: [PATCH 1072/1937] release 2015.01.07.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f9c934351..e20525b58 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.07'
+__version__ = '2015.01.07.1'

From ccc5842bc9d96abaf6e54b91101f7e281b05663c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 07:37:21 +0100
Subject: [PATCH 1073/1937] [gameone] Modernize

---
 youtube_dl/extractor/gameone.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gameone.py b/youtube_dl/extractor/gameone.py
index 75f180928..a07d69841 100644
--- a/youtube_dl/extractor/gameone.py
+++ b/youtube_dl/extractor/gameone.py
@@ -57,8 +57,7 @@ class GameOneIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
         og_video = self._og_search_video_url(webpage, secure=False)

From 2bad0e5d202aa041ab9ddb9e7dcffe69848a5349 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 07:41:05 +0100
Subject: [PATCH 1074/1937] [/__init__] Define public API

---
 devscripts/gh-pages/update-sites.py | 2 +-
 youtube_dl/__init__.py              | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/devscripts/gh-pages/update-sites.py b/devscripts/gh-pages/update-sites.py
index f0f0481c7..d3ef5f0b5 100755
--- a/devscripts/gh-pages/update-sites.py
+++ b/devscripts/gh-pages/update-sites.py
@@ -16,7 +16,7 @@ def main():
         template = tmplf.read()
 
     ie_htmls = []
-    for ie in sorted(youtube_dl.gen_extractors(), key=lambda i: i.IE_NAME.lower()):
+    for ie in youtube_dl.list_extractors(age_limit=None):
         ie_html = '<b>{}</b>'.format(ie.IE_NAME)
         ie_desc = getattr(ie, 'IE_DESC', None)
         if ie_desc is False:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 4c21188a9..4aa7fba6a 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -38,7 +38,7 @@ from .update import update_self
 from .downloader import (
     FileDownloader,
 )
-from .extractor import list_extractors
+from .extractor import gen_extractors, list_extractors
 from .YoutubeDL import YoutubeDL
 
 
@@ -363,3 +363,5 @@ def main(argv=None):
         sys.exit('ERROR: fixed output name but more than one file to download')
     except KeyboardInterrupt:
         sys.exit('\nERROR: Interrupted by user')
+
+__all__ = ['main', 'YoutubeDL', 'gen_extractors', 'list_extractors']

From 900813a328813ae9095ebfd2b2b846a9d98e2821 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 07:41:48 +0100
Subject: [PATCH 1075/1937] release 2015.01.07.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e20525b58..164ce0268 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.07.1'
+__version__ = '2015.01.07.2'

From c758bf9fd7625b8db3c64a06e100ddc5a4ae07d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 7 Jan 2015 11:56:22 +0100
Subject: [PATCH 1076/1937] [nrktv] Remove 'proxy' parameter from tests

---
 youtube_dl/extractor/nrk.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index e950c76dd..321ce5ce7 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -111,9 +111,6 @@ class NRKTVIE(InfoExtractor):
                 'upload_date': '20150106',
             },
             'skip': 'Only works from Norway',
-            'params': {
-                'proxy': '127.0.0.1:8118',
-            },
         },
         {
             'url': 'http://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
@@ -147,9 +144,6 @@ class NRKTVIE(InfoExtractor):
                 'duration': 6947.5199999999995,
             },
             'skip': 'Only works from Norway',
-            'params': {
-                'proxy': '127.0.0.1:8118',
-            },
         }
     ]
 

From 5d8993b06adb24b9c275a42a67dc6bbeae590407 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 7 Jan 2015 11:59:15 +0100
Subject: [PATCH 1077/1937] [extractor/__init__] Remove unused import

---
 youtube_dl/extractor/__init__.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0145e350d..8dacc2c54 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -560,8 +560,6 @@ from .zingmp3 import (
     ZingMp3AlbumIE,
 )
 
-from ..utils import age_restricted
-
 _ALL_CLASSES = [
     klass
     for name, klass in globals().items()

From c5d666d3741d076d668c16f0dd3601589484b7c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 7 Jan 2015 16:09:43 +0100
Subject: [PATCH 1078/1937] Fix build with python 2.6

* Packages cannot be executed
* '.format' needs the index of the argument

(Reported in https://github.com/Homebrew/homebrew/issues/35616)
---
 Makefile                          | 2 +-
 devscripts/make_supportedsites.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index e53a367ef..578079879 100644
--- a/Makefile
+++ b/Makefile
@@ -63,7 +63,7 @@ youtube-dl: youtube_dl/*.py youtube_dl/*/*.py
 	chmod a+x youtube-dl
 
 README.md: youtube_dl/*.py youtube_dl/*/*.py
-	COLUMNS=80 python -m youtube_dl --help | python devscripts/make_readme.py
+	COLUMNS=80 python youtube_dl/__main__.py --help | python devscripts/make_readme.py
 
 CONTRIBUTING.md: README.md
 	python devscripts/make_contributing.py README.md CONTRIBUTING.md
diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py
index 140010644..3df4385a6 100644
--- a/devscripts/make_supportedsites.py
+++ b/devscripts/make_supportedsites.py
@@ -23,12 +23,12 @@ def main():
 
     def gen_ies_md(ies):
         for ie in ies:
-            ie_md = '**{}**'.format(ie.IE_NAME)
+            ie_md = '**{0}**'.format(ie.IE_NAME)
             ie_desc = getattr(ie, 'IE_DESC', None)
             if ie_desc is False:
                 continue
             if ie_desc is not None:
-                ie_md += ': {}'.format(ie.IE_DESC)
+                ie_md += ': {0}'.format(ie.IE_DESC)
             if not ie.working():
                 ie_md += ' (Currently broken)'
             yield ie_md

From 58a84b8cb69e967dd95fa309c0160838853c7d79 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Jan 2015 01:33:22 +0600
Subject: [PATCH 1079/1937] [bilibili] Fix extraction (Closes #4660)

---
 youtube_dl/extractor/bilibili.py | 67 ++++++++++++++------------------
 1 file changed, 29 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 241b904a9..75d744852 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -4,9 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_parse_qs
 from ..utils import (
-    ExtractorError,
     int_or_none,
     unified_strdate,
 )
@@ -54,45 +52,38 @@ class BiliBiliIE(InfoExtractor):
         thumbnail = self._html_search_meta(
             'thumbnailUrl', video_code, 'thumbnail', fatal=False)
 
-        player_params = compat_parse_qs(self._html_search_regex(
-            r'<iframe .*?class="player" src="https://secure\.bilibili\.(?:tv|com)/secure,([^"]+)"',
-            webpage, 'player params'))
+        cid = self._search_regex(r'cid=(\d+)', webpage, 'cid')
 
-        if 'cid' in player_params:
-            cid = player_params['cid'][0]
+        lq_doc = self._download_xml(
+            'http://interface.bilibili.com/v_cdn_play?appkey=1&cid=%s' % cid,
+            video_id,
+            note='Downloading LQ video info'
+        )
+        lq_durl = lq_doc.find('./durl')
+        formats = [{
+            'format_id': 'lq',
+            'quality': 1,
+            'url': lq_durl.find('./url').text,
+            'filesize': int_or_none(
+                lq_durl.find('./size'), get_attr='text'),
+        }]
 
-            lq_doc = self._download_xml(
-                'http://interface.bilibili.cn/v_cdn_play?cid=%s' % cid,
-                video_id,
-                note='Downloading LQ video info'
-            )
-            lq_durl = lq_doc.find('.//durl')
-            formats = [{
-                'format_id': 'lq',
-                'quality': 1,
-                'url': lq_durl.find('./url').text,
+        hq_doc = self._download_xml(
+            'http://interface.bilibili.com/playurl?appkey=1&cid=%s' % cid,
+            video_id,
+            note='Downloading HQ video info',
+            fatal=False,
+        )
+        if hq_doc is not False:
+            hq_durl = hq_doc.find('./durl')
+            formats.append({
+                'format_id': 'hq',
+                'quality': 2,
+                'ext': 'flv',
+                'url': hq_durl.find('./url').text,
                 'filesize': int_or_none(
-                    lq_durl.find('./size'), get_attr='text'),
-            }]
-
-            hq_doc = self._download_xml(
-                'http://interface.bilibili.cn/playurl?cid=%s' % cid,
-                video_id,
-                note='Downloading HQ video info',
-                fatal=False,
-            )
-            if hq_doc is not False:
-                hq_durl = hq_doc.find('.//durl')
-                formats.append({
-                    'format_id': 'hq',
-                    'quality': 2,
-                    'ext': 'flv',
-                    'url': hq_durl.find('./url').text,
-                    'filesize': int_or_none(
-                        hq_durl.find('./size'), get_attr='text'),
-                })
-        else:
-            raise ExtractorError('Unsupported player parameters: %r' % (player_params,))
+                    hq_durl.find('./size'), get_attr='text'),
+            })
 
         self._sort_formats(formats)
         return {

From fdd6e18b75ef580ff9097f8589e38dcce3a7fa3a Mon Sep 17 00:00:00 2001
From: Kieran O'Reilly <kieranoreilly@college.harvard.edu>
Date: Wed, 7 Jan 2015 20:23:36 -0800
Subject: [PATCH 1080/1937] Spelling

Corrected the spelling of incapacitated
---
 CONTRIBUTING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 0ff7b395a..7917abfc6 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -44,7 +44,7 @@ In particular, every site support request issue should only pertain to services
 
 ###  Is anyone going to need the feature?
 
-Only post features that you (or an incapicated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them.
+Only post features that you (or an incapacitated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them.
 
 ###  Is your question about youtube-dl?
 

From 52fc3ba405cf9d87a4ada50acded5dbe10eca492 Mon Sep 17 00:00:00 2001
From: Li4ick <danielciugurean@yahoo.com>
Date: Thu, 8 Jan 2015 14:17:26 +0200
Subject: [PATCH 1081/1937] Change path name to MSDN standard.

<Yourname> changed to <user name>, which is more general.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 915bcd0cd..1df4fe2f1 100644
--- a/README.md
+++ b/README.md
@@ -326,7 +326,7 @@ which means you can modify it, redistribute it or use it however you like.
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl/config`. On Windows, the configuration file locations are `%APPDATA%\youtube-dl\config.txt` and `C:\Users\<Yourname>\youtube-dl.conf`.
+You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl/config`. On Windows, the configuration file locations are `%APPDATA%\youtube-dl\config.txt` and `C:\Users\<user name>\youtube-dl.conf`.
 
 # OUTPUT TEMPLATE
 

From 398133cf554895859cf587654e67049d2e21b570 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Jan 2015 19:07:28 +0600
Subject: [PATCH 1082/1937] [huffpost] Make extraction more robust (Closes
 #4663)

---
 youtube_dl/extractor/huffpost.py | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/huffpost.py b/youtube_dl/extractor/huffpost.py
index 4ccf6b9b8..a38eae421 100644
--- a/youtube_dl/extractor/huffpost.py
+++ b/youtube_dl/extractor/huffpost.py
@@ -39,8 +39,9 @@ class HuffPostIE(InfoExtractor):
         data = self._download_json(api_url, video_id)['data']
 
         video_title = data['title']
-        duration = parse_duration(data['running_time'])
-        upload_date = unified_strdate(data['schedule']['starts_at'])
+        duration = parse_duration(data.get('running_time'))
+        upload_date = unified_strdate(
+            data.get('schedule', {}).get('starts_at') or data.get('segment_start_date_time'))
         description = data.get('description')
 
         thumbnails = []
@@ -59,16 +60,11 @@ class HuffPostIE(InfoExtractor):
             'ext': 'mp4',
             'url': url,
             'vcodec': 'none' if key.startswith('audio/') else None,
-        } for key, url in data['sources']['live'].items()]
-        if data.get('fivemin_id'):
-            fid = data['fivemin_id']
-            fcat = str(int(fid) // 100 + 1)
-            furl = 'http://avideos.5min.com/2/' + fcat[-3:] + '/' + fcat + '/' + fid + '.mp4'
-            formats.append({
-                'format': 'fivemin',
-                'url': furl,
-                'preference': 1,
-            })
+        } for key, url in data.get('sources', {}).get('live', {}).items()]
+
+        if not formats and data.get('fivemin_id'):
+            return self.url_result('5min:%s' % data['fivemin_id'])
+
         self._sort_formats(formats)
 
         return {

From 99673f04bc5b179241d121cf2aa50f25b6b1c7d0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 11:21:40 +0100
Subject: [PATCH 1083/1937] [washingtonpost] Modernize and correct test case

---
 youtube_dl/extractor/washingtonpost.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/washingtonpost.py b/youtube_dl/extractor/washingtonpost.py
index 88bbbb219..c17bebd6e 100644
--- a/youtube_dl/extractor/washingtonpost.py
+++ b/youtube_dl/extractor/washingtonpost.py
@@ -10,14 +10,14 @@ from ..utils import (
 
 
 class WashingtonPostIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:www\.)?washingtonpost\.com/.*?/(?P<id>[^/]+)/(?:$|[?#])'
+    _VALID_URL = r'https?://(?:www\.)?washingtonpost\.com/.*?/(?P<id>[^/]+)/(?:$|[?#])'
     _TEST = {
         'url': 'http://www.washingtonpost.com/sf/national/2014/03/22/sinkhole-of-bureaucracy/',
         'info_dict': {
             'title': 'Sinkhole of bureaucracy',
         },
         'playlist': [{
-            'md5': 'c3f4b4922ffa259243f68e928db2db8c',
+            'md5': '79132cc09ec5309fa590ae46e4cc31bc',
             'info_dict': {
                 'id': 'fc433c38-b146-11e3-b8b3-44b1d1cd4c1f',
                 'ext': 'mp4',
@@ -29,7 +29,7 @@ class WashingtonPostIE(InfoExtractor):
                 'upload_date': '20140322',
             },
         }, {
-            'md5': 'f645a07652c2950cd9134bb852c5f5eb',
+            'md5': 'e1d5734c06865cc504ad99dc2de0d443',
             'info_dict': {
                 'id': '41255e28-b14a-11e3-b8b3-44b1d1cd4c1f',
                 'ext': 'mp4',
@@ -44,10 +44,9 @@ class WashingtonPostIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        page_id = mobj.group('id')
-
+        page_id = self._match_id(url)
         webpage = self._download_webpage(url, page_id)
+
         title = self._og_search_title(webpage)
         uuids = re.findall(r'data-video-uuid="([^"]+)"', webpage)
         entries = []

From adb1307b9aeae4f011ba454a5d64d343e23503c8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 11:34:27 +0100
Subject: [PATCH 1084/1937] [imdb] Remove test md5

They seem to reencode quite frequently, so simply remove the md5 sum.
---
 youtube_dl/extractor/imdb.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py
index 13a53a0cb..f29df36b5 100644
--- a/youtube_dl/extractor/imdb.py
+++ b/youtube_dl/extractor/imdb.py
@@ -16,7 +16,6 @@ class ImdbIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.imdb.com/video/imdb/vi2524815897',
-        'md5': '9f34fa777ade3a6e57a054fdbcb3a068',
         'info_dict': {
             'id': '2524815897',
             'ext': 'mp4',

From 211503c39f5ce6e875d18545007060c2764f6f7e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 11:43:20 +0100
Subject: [PATCH 1085/1937] [teachertube] Modernize

---
 youtube_dl/extractor/teachertube.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/teachertube.py b/youtube_dl/extractor/teachertube.py
index 6c3445d79..82675431f 100644
--- a/youtube_dl/extractor/teachertube.py
+++ b/youtube_dl/extractor/teachertube.py
@@ -57,9 +57,7 @@ class TeacherTubeIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_meta('title', webpage, 'title', fatal=True)

From bec2248141c9cc3e44794d7ee48a6bcfc4904eac Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 11:43:36 +0100
Subject: [PATCH 1086/1937] [InfoExtractor/common] Correct and test meta tag
 matching

---
 test/test_InfoExtractor.py     | 18 ++++++++++++++++++
 youtube_dl/extractor/common.py |  2 +-
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 13c18ed95..be8d12997 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -40,5 +40,23 @@ class TestInfoExtractor(unittest.TestCase):
         self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
         self.assertEqual(ie._og_search_thumbnail(html), 'http://domain.com/pic.jpg?key1=val1&key2=val2')
 
+    def test_html_search_meta(self):
+        ie = self.ie
+        html = '''
+            <meta name="a" content="1" />
+            <meta name='b' content='2'>
+            <meta name="c" content='3'>
+            <meta name=d content='4'>
+            <meta property="e" content='5' >
+            <meta content="6" name="f">
+        '''
+
+        self.assertEqual(ie._html_search_meta('a', html), '1')
+        self.assertEqual(ie._html_search_meta('b', html), '2')
+        self.assertEqual(ie._html_search_meta('c', html), '3')
+        self.assertEqual(ie._html_search_meta('d', html), '4')
+        self.assertEqual(ie._html_search_meta('e', html), '5')
+        self.assertEqual(ie._html_search_meta('f', html), '6')
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index df32b5ca0..d703893dc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -594,7 +594,7 @@ class InfoExtractor(object):
         return self._html_search_regex(
             r'''(?isx)<meta
                     (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
-                    [^>]+content=(["\'])(?P<content>.*?)\1''' % re.escape(name),
+                    [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(name),
             html, display_name, fatal=fatal, group='content', **kwargs)
 
     def _dc_search_uploader(self, html):

From 5b78caca94b8913b224222fb94a81481cf8637e9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 11:45:19 +0100
Subject: [PATCH 1087/1937] [mit] Amend test definitions

---
 youtube_dl/extractor/mit.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index 78787e8f1..3c61a850f 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -105,6 +105,9 @@ class OCWMITIE(InfoExtractor):
                 'ext': 'mp4',
                 'title': 'Lecture 7: Multiple Discrete Random Variables: Expectations, Conditioning, Independence',
                 'description': 'In this lecture, the professor discussed multiple random variables, expectations, and binomial distribution.',
+                'upload_date': '20121109',
+                'uploader_id': 'MIT',
+                'uploader': 'MIT OpenCourseWare',
                 # 'subtitles': 'http://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-041-probabilistic-systems-analysis-and-applied-probability-fall-2010/video-lectures/lecture-7-multiple-variables-expectations-independence/MIT6_041F11_lec07_300k.mp4.srt'
             }
         },
@@ -114,6 +117,9 @@ class OCWMITIE(InfoExtractor):
                 'id': '7K1sB05pE0A',
                 'ext': 'mp4',
                 'title': 'Session 1: Introduction to Derivatives',
+                'upload_date': '20090818',
+                'uploader_id': 'MIT',
+                'uploader': 'MIT OpenCourseWare',
                 'description': 'This section contains lecture video excerpts, lecture notes, an interactive mathlet with supporting documents, and problem solving videos.',
                 # 'subtitles': 'http://ocw.mit.edu//courses/mathematics/18-01sc-single-variable-calculus-fall-2010/ocw-18.01-f07-lec01_300k.SRT'
             }

From 796858a53f3bb987e47ebe28fb0c1963703541cb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 7 Jan 2015 11:47:03 +0100
Subject: [PATCH 1088/1937] [sexykarma] Add age_limit designation

---
 youtube_dl/extractor/sexykarma.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sexykarma.py b/youtube_dl/extractor/sexykarma.py
index c833fc8ee..6446d26dc 100644
--- a/youtube_dl/extractor/sexykarma.py
+++ b/youtube_dl/extractor/sexykarma.py
@@ -24,7 +24,7 @@ class SexyKarmaIE(InfoExtractor):
             'title': 'Taking a quick pee.',
             'thumbnail': 're:^https?://.*\.jpg$',
             'uploader': 'wildginger7',
-            'upload_date': '20141007',
+            'upload_date': '20141008',
             'duration': 22,
             'view_count': int,
             'comment_count': int,
@@ -45,6 +45,7 @@ class SexyKarmaIE(InfoExtractor):
             'view_count': int,
             'comment_count': int,
             'categories': list,
+            'age_limit': 18,
         }
     }, {
         'url': 'http://www.watchindianporn.net/video/desi-dancer-namrata-stripping-completely-nude-and-dancing-on-a-hot-number-dW2mtctxJfs.html',
@@ -61,6 +62,7 @@ class SexyKarmaIE(InfoExtractor):
             'view_count': int,
             'comment_count': int,
             'categories': list,
+            'age_limit': 18,
         }
     }]
 
@@ -114,4 +116,5 @@ class SexyKarmaIE(InfoExtractor):
             'view_count': view_count,
             'comment_count': comment_count,
             'categories': categories,
+            'age_limit': 18,
         }

From ec3a6a31378925c72facbed1005499780b2d78da Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 14:09:43 +0100
Subject: [PATCH 1089/1937] [tunein] Ignore reliability if it's >90% (#4097)

---
 youtube_dl/extractor/tunein.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/tunein.py b/youtube_dl/extractor/tunein.py
index 4ce5aeeba..b6b1f2568 100644
--- a/youtube_dl/extractor/tunein.py
+++ b/youtube_dl/extractor/tunein.py
@@ -24,7 +24,7 @@ class TuneInIE(InfoExtractor):
     _INFO_DICT = {
         'id': '34682',
         'title': 'Jazz 24 on 88.5 Jazz24 - KPLU-HD2',
-        'ext': 'AAC',
+        'ext': 'aac',
         'thumbnail': 're:^https?://.*\.png$',
         'location': 'Tacoma, WA',
     }
@@ -78,14 +78,21 @@ class TuneInIE(InfoExtractor):
         for stream in streams:
             if stream.get('Type') == 'Live':
                 is_live = True
+            reliability = stream.get('Reliability')
+            format_note = (
+                'Reliability: %d%%' % reliability
+                if reliability is not None else None)
             formats.append({
+                'preference': (
+                    0 if reliability is None or reliability > 90
+                    else 1),
                 'abr': stream.get('Bandwidth'),
-                'ext': stream.get('MediaType'),
+                'ext': stream.get('MediaType').lower(),
                 'acodec': stream.get('MediaType'),
                 'vcodec': 'none',
                 'url': stream.get('Url'),
-                # Sometimes streams with the highest quality do not exist
-                'preference': stream.get('Reliability'),
+                'source_preference': reliability,
+                'format_note': format_note,
             })
         self._sort_formats(formats)
 

From 9ed99402f5732f84d402c2b098ab20e9ebbea549 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 14:27:01 +0100
Subject: [PATCH 1090/1937] [youtube] Fix test case

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f0efaf0d9..e71956071 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -414,7 +414,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'id': 'HtVdAasjOgU',
                 'ext': 'mp4',
                 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
-                'description': 'md5:eca57043abae25130f58f655ad9a7771',
+                'description': 're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
                 'uploader': 'The Witcher',
                 'uploader_id': 'WitcherGame',
                 'upload_date': '20140605',

From d51a853d5cc4aa17a1c6d60a986387bf242449af Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 14:29:12 +0100
Subject: [PATCH 1091/1937] [zdf] Fix test case

---
 youtube_dl/extractor/zdf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 74c76a9a0..98f15177b 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -119,7 +119,7 @@ class ZDFChannelIE(InfoExtractor):
         'info_dict': {
             'id': '1586442',
         },
-        'playlist_count': 4,
+        'playlist_count': 3,
     }
     _PAGE_SIZE = 50
 

From bd6b25ce0e5fabd3cb20651a1a3ed8a9cf8748e4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 16:03:13 +0100
Subject: [PATCH 1092/1937] [fktv] Fix download URL

---
 youtube_dl/extractor/fktv.py | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index d09d1c13a..190d9f9ad 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -13,7 +13,7 @@ from ..utils import (
 
 class FKTVIE(InfoExtractor):
     IE_NAME = 'fernsehkritik.tv'
-    _VALID_URL = r'http://(?:www\.)?fernsehkritik\.tv/folge-(?P<ep>[0-9]+)(?:/.*)?'
+    _VALID_URL = r'http://(?:www\.)?fernsehkritik\.tv/folge-(?P<id>[0-9]+)(?:/.*)?'
 
     _TEST = {
         'url': 'http://fernsehkritik.tv/folge-1',
@@ -26,29 +26,32 @@ class FKTVIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        episode = int(mobj.group('ep'))
+        episode = int(self._match_id(url))
 
-        server = random.randint(2, 4)
-        video_thumbnail = 'http://fernsehkritik.tv/images/magazin/folge%d.jpg' % episode
-        start_webpage = self._download_webpage('http://fernsehkritik.tv/folge-%d/Start' % episode,
+        video_thumbnail = 'http://fernsehkritik.tv/images/magazin/folge%s.jpg' % episode
+        start_webpage = self._download_webpage('http://fernsehkritik.tv/folge-%s/Start' % episode,
                                                episode)
         playlist = self._search_regex(r'playlist = (\[.*?\]);', start_webpage,
                                       'playlist', flags=re.DOTALL)
         files = json.loads(re.sub('{[^{}]*?}', '{}', playlist))
-        # TODO: return a single multipart video
+
         videos = []
         for i, _ in enumerate(files, 1):
             video_id = '%04d%d' % (episode, i)
-            video_url = 'http://dl%d.fernsehkritik.tv/fernsehkritik%d%s.flv' % (server, episode, '' if i == 1 else '-%d' % i)
+            video_url = 'http://fernsehkritik.tv/js/directme.php?file=%s%s.flv' % (episode, '' if i == 1 else '-%d' % i)
             videos.append({
+                'ext': 'flv',
                 'id': video_id,
                 'url': video_url,
                 'title': clean_html(get_element_by_id('eptitle', start_webpage)),
                 'description': clean_html(get_element_by_id('contentlist', start_webpage)),
                 'thumbnail': video_thumbnail
             })
-        return videos
+        return {
+            '_type': 'multi_video',
+            'entries': videos,
+            'id': 'folge-%s' % episode,
+        }
 
 
 class FKTVPosteckeIE(InfoExtractor):

From 0217c783771a6c8543046869e934f4ced56febc7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 16:14:16 +0100
Subject: [PATCH 1093/1937] [YoutubeDL] Allow selection by more extensions

---
 test/test_YoutubeDL.py  | 33 +++++++++++++++++++++++++++++++++
 youtube_dl/YoutubeDL.py |  2 +-
 youtube_dl/options.py   | 10 ++++++----
 3 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 730f7ec26..85d87f2c3 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -8,6 +8,8 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+import copy
+
 from test.helper import FakeYDL, assertRegexpMatches
 from youtube_dl import YoutubeDL
 from youtube_dl.extractor import YoutubeIE
@@ -192,6 +194,37 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], 'vid-high')
 
+    def test_format_selection_audio_exts(self):
+        formats = [
+            {'format_id': 'mp3-64', 'ext': 'mp3', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
+            {'format_id': 'ogg-64', 'ext': 'ogg', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
+            {'format_id': 'aac-64', 'ext': 'aac', 'abr': 64, 'url': 'http://_', 'vcodec': 'none'},
+            {'format_id': 'mp3-32', 'ext': 'mp3', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
+            {'format_id': 'aac-32', 'ext': 'aac', 'abr': 32, 'url': 'http://_', 'vcodec': 'none'},
+        ]
+
+        info_dict = _make_result(formats)
+        ydl = YDL({'format': 'best'})
+        ie = YoutubeIE(ydl)
+        ie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(copy.deepcopy(info_dict))
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'aac-64')
+
+        ydl = YDL({'format': 'mp3'})
+        ie = YoutubeIE(ydl)
+        ie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(copy.deepcopy(info_dict))
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'mp3-64')
+
+        ydl = YDL({'prefer_free_formats': True})
+        ie = YoutubeIE(ydl)
+        ie._sort_formats(info_dict['formats'])
+        ydl.process_ie_result(copy.deepcopy(info_dict))
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'ogg-64')
+
     def test_format_selection_video(self):
         formats = [
             {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': '_'},
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 36c71f947..24d6c2de7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -786,7 +786,7 @@ class YoutubeDL(object):
             if video_formats:
                 return video_formats[0]
         else:
-            extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a']
+            extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
             if format_spec in extensions:
                 filter_f = lambda f: f['ext'] == format_spec
             else:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index a018c1d71..058342dec 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -267,10 +267,12 @@ def parseOpts(overrideArguments=None):
         action='store', dest='format', metavar='FORMAT', default=None,
         help=(
             'video format code, specify the order of preference using'
-            ' slashes: -f 22/17/18 .  -f mp4 , -f m4a and  -f flv  are also'
-            ' supported. You can also use the special names "best",'
-            ' "bestvideo", "bestaudio", "worst", "worstvideo" and'
-            ' "worstaudio". By default, youtube-dl will pick the best quality.'
+            ' slashes, as in -f 22/17/18 . '
+            ' Instead of format codes, you can select by extension for the '
+            'extensions aac, m4a, mp3, mp4, ogg, wav, webm. '
+            'You can also use the special names "best",'
+            ' "bestvideo", "bestaudio", "worst". '
+            ' By default, youtube-dl will pick the best quality.'
             ' Use commas to download multiple audio formats, such as'
             ' -f  136/137/mp4/bestvideo,140/m4a/bestaudio.'
             ' You can merge the video and audio of two formats into a single'

From 11497d5bba4a8a99d5a1907d2a34fe7be7e04c37 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 16:15:08 +0100
Subject: [PATCH 1094/1937] release 2015.01.08

---
 README.md             | 17 +++++++++--------
 youtube_dl/version.py |  2 +-
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 1df4fe2f1..24b4d7970 100644
--- a/README.md
+++ b/README.md
@@ -248,14 +248,15 @@ which means you can modify it, redistribute it or use it however you like.
 
 ## Video Format Options:
     -f, --format FORMAT              video format code, specify the order of
-                                     preference using slashes: -f 22/17/18 .  -f
-                                     mp4 , -f m4a and  -f flv  are also
-                                     supported. You can also use the special
-                                     names "best", "bestvideo", "bestaudio",
-                                     "worst", "worstvideo" and "worstaudio". By
-                                     default, youtube-dl will pick the best
-                                     quality. Use commas to download multiple
-                                     audio formats, such as -f
+                                     preference using slashes, as in -f 22/17/18
+                                     .  Instead of format codes, you can select
+                                     by extension for the extensions aac, m4a,
+                                     mp3, mp4, ogg, wav, webm. You can also use
+                                     the special names "best", "bestvideo",
+                                     "bestaudio", "worst".  By default, youtube-
+                                     dl will pick the best quality. Use commas
+                                     to download multiple audio formats, such as
+                                     -f
                                      136/137/mp4/bestvideo,140/m4a/bestaudio.
                                      You can merge the video and audio of two
                                      formats into a single file using -f <video-
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 164ce0268..32019e362 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.07.2'
+__version__ = '2015.01.08'

From 811cacdc2c62bea63c6fbd4a3e570744257c37a6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 17:02:37 +0100
Subject: [PATCH 1095/1937] [bet] Correct test IDs

---
 youtube_dl/extractor/bet.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bet.py b/youtube_dl/extractor/bet.py
index 4fbdd6f1c..d2abd4d77 100644
--- a/youtube_dl/extractor/bet.py
+++ b/youtube_dl/extractor/bet.py
@@ -16,7 +16,7 @@ class BetIE(InfoExtractor):
         {
             'url': 'http://www.bet.com/news/politics/2014/12/08/in-bet-exclusive-obama-talks-race-and-racism.html',
             'info_dict': {
-                'id': '406429c6-1b8a-463e-83fc-814adb81a9db',
+                'id': '740ab250-bb94-4a8a-8787-fe0de7c74471',
                 'display_id': 'in-bet-exclusive-obama-talks-race-and-racism',
                 'ext': 'flv',
                 'title': 'BET News Presents: A Conversation With President Obama',
@@ -35,7 +35,7 @@ class BetIE(InfoExtractor):
         {
             'url': 'http://www.bet.com/video/news/national/2014/justice-for-ferguson-a-community-reacts.html',
             'info_dict': {
-                'id': '4160e53b-ad41-43b1-980f-8d85f63121f4',
+                'id': 'bcd1b1df-673a-42cf-8d01-b282db608f2d',
                 'display_id': 'justice-for-ferguson-a-community-reacts',
                 'ext': 'flv',
                 'title': 'Justice for Ferguson: A Community Reacts',
@@ -55,7 +55,6 @@ class BetIE(InfoExtractor):
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
-
         webpage = self._download_webpage(url, display_id)
 
         media_url = compat_urllib_parse.unquote(self._search_regex(

From db6e6250050d011b801f01a665e644bb41be40cf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 17:05:46 +0100
Subject: [PATCH 1096/1937] [buzzfeed] Fix test

---
 youtube_dl/extractor/buzzfeed.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/buzzfeed.py b/youtube_dl/extractor/buzzfeed.py
index a40a1bbc4..a5d2af174 100644
--- a/youtube_dl/extractor/buzzfeed.py
+++ b/youtube_dl/extractor/buzzfeed.py
@@ -33,7 +33,7 @@ class BuzzFeedIE(InfoExtractor):
             'skip_download': True,  # Got enough YouTube download tests
         },
         'info_dict': {
-            'description': 'Munchkin the Teddy Bear is back !',
+            'description': 're:Munchkin the Teddy Bear is back ?!',
             'title': 'You Need To Stop What You\'re Doing And Watching This Dog Walk On A Treadmill',
         },
         'playlist': [{
@@ -42,9 +42,9 @@ class BuzzFeedIE(InfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20141124',
                 'uploader_id': 'CindysMunchkin',
-                'description': '© 2014 Munchkin the Shih Tzu\nAll rights reserved\nFacebook: http://facebook.com/MunchkintheShihTzu',
+                'description': 're:© 2014 Munchkin the Shih Tzu',
                 'uploader': 'Munchkin the Shih Tzu',
-                'title': 'Munchkin the Teddy Bear gets her exercise',
+                'title': 're:Munchkin the Teddy Bear gets her exercise',
             },
         }]
     }]

From 5fbf25a681c2f7f6f3470e783fcac65b7c7c339d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 17:09:28 +0100
Subject: [PATCH 1097/1937] [test_age_restriction] remove misbehaving test

We now test for the age_limit being set right in test_download, so we don't need more than two tests for the actual age limit handling.
---
 test/test_age_restriction.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/test/test_age_restriction.py b/test/test_age_restriction.py
index 5be065c43..6f5513faa 100644
--- a/test/test_age_restriction.py
+++ b/test/test_age_restriction.py
@@ -45,11 +45,6 @@ class TestAgeRestriction(unittest.TestCase):
             'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
             '505835.mp4', 2, old_age=25)
 
-    def test_pornotube(self):
-        self._assert_restricted(
-            'http://pornotube.com/c/173/m/1689755/Marilyn-Monroe-Bathing',
-            '1689755.flv', 13)
-
 
 if __name__ == '__main__':
     unittest.main()

From 740a7fcbc8673a8edcfc9ebdab967ccb99fb8c08 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 17:11:17 +0100
Subject: [PATCH 1098/1937] [gdcvault] Skip test that is now restricted

---
 youtube_dl/extractor/gdcvault.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index d453ec010..fed968f51 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -39,7 +39,8 @@ class GDCVaultIE(InfoExtractor):
                 'id': '1015301',
                 'ext': 'flv',
                 'title': 'Thexder Meets Windows 95, or Writing Great Games in the Windows 95 Environment',
-            }
+            },
+            'skip': 'Requires login',
         }
     ]
 

From 9298d4e3df4b6237f4f9e4f37b9e55775c5e3c0c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 17:36:56 +0100
Subject: [PATCH 1099/1937] [discovery] Fix extractor

---
 youtube_dl/extractor/discovery.py | 46 +++++++++++++++----------------
 1 file changed, 22 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/discovery.py b/youtube_dl/extractor/discovery.py
index 52c2d7ddf..d3e667528 100644
--- a/youtube_dl/extractor/discovery.py
+++ b/youtube_dl/extractor/discovery.py
@@ -1,47 +1,45 @@
 from __future__ import unicode_literals
 
-import re
-import json
-
 from .common import InfoExtractor
+from ..utils import (
+    parse_iso8601,
+    int_or_none,
+)
 
 
 class DiscoveryIE(InfoExtractor):
-    _VALID_URL = r'http://www\.discovery\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9\-]*)(.htm)?'
+    _VALID_URL = r'http://www\.discovery\.com\/[a-zA-Z0-9\-]*/[a-zA-Z0-9\-]*/videos/(?P<id>[a-zA-Z0-9_\-]*)(?:\.htm)?'
     _TEST = {
         'url': 'http://www.discovery.com/tv-shows/mythbusters/videos/mission-impossible-outtakes.htm',
-        'md5': 'e12614f9ee303a6ccef415cb0793eba2',
+        'md5': '3c69d77d9b0d82bfd5e5932a60f26504',
         'info_dict': {
-            'id': '614784',
-            'ext': 'mp4',
-            'title': 'MythBusters: Mission Impossible Outtakes',
+            'id': 'mission-impossible-outtakes',
+            'ext': 'flv',
+            'title': 'Mission Impossible Outtakes',
             'description': ('Watch Jamie Hyneman and Adam Savage practice being'
                             ' each other -- to the point of confusing Jamie\'s dog -- and '
                             'don\'t miss Adam moon-walking as Jamie ... behind Jamie\'s'
                             ' back.'),
             'duration': 156,
+            'timestamp': 1303099200,
+            'upload_date': '20110418',
         },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        video_list_json = self._search_regex(r'var videoListJSON = ({.*?});',
-                                             webpage, 'video list', flags=re.DOTALL)
-        video_list = json.loads(video_list_json)
-        info = video_list['clips'][0]
-        formats = []
-        for f in info['mp4']:
-            formats.append(
-                {'url': f['src'], 'ext': 'mp4', 'tbr': int(f['bitrate'][:-1])})
+        info = self._parse_json(self._search_regex(
+            r'(?s)<script type="application/ld\+json">(.*?)</script>',
+            webpage, 'video info'), video_id)
 
         return {
-            'id': info['contentId'],
-            'title': video_list['name'],
-            'formats': formats,
-            'description': info['videoCaption'],
-            'thumbnail': info.get('videoStillURL') or info.get('thumbnailURL'),
-            'duration': info['duration'],
+            'id': video_id,
+            'title': info['name'],
+            'url': info['contentURL'],
+            'description': info.get('description'),
+            'thumbnail': info.get('thumbnailUrl'),
+            'timestamp': parse_iso8601(info.get('uploadDate')),
+            'duration': int_or_none(info.get('duration')),
         }

From 92b065dc539649d29804ebe329c89de9503b7098 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 17:50:46 +0100
Subject: [PATCH 1100/1937] [tudou] Fix extraction

---
 youtube_dl/extractor/tudou.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/tudou.py b/youtube_dl/extractor/tudou.py
index 161e47624..c89de5ba4 100644
--- a/youtube_dl/extractor/tudou.py
+++ b/youtube_dl/extractor/tudou.py
@@ -9,7 +9,7 @@ from .common import InfoExtractor
 
 
 class TudouIE(InfoExtractor):
-    _VALID_URL = r'(?:http://)?(?:www\.)?tudou\.com/(?:listplay|programs|albumplay)/(?:view|(.+?))/(?:([^/]+)|([^/]+))(?:\.html)?'
+    _VALID_URL = r'https?://(?:www\.)?tudou\.com/(?:listplay|programs(?:/view)?|albumplay)/.*?/(?P<id>[^/?#]+?)(?:\.html)?/?(?:$|[?#])'
     _TESTS = [{
         'url': 'http://www.tudou.com/listplay/zzdE77v6Mmo/2xN2duXMxmw.html',
         'md5': '140a49ed444bd22f93330985d8475fcb',
@@ -27,13 +27,6 @@ class TudouIE(InfoExtractor):
             'title': 'La Sylphide-Bolshoi-Ekaterina Krysanova & Vyacheslav Lopatin 2012',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
-    }, {
-        'url': 'http://www.tudou.com/albumplay/TenTw_JgiPM/PzsAs5usU9A.html',
-        'info_dict': {
-            'title': 'todo.mp4',
-        },
-        'add_ie': ['Youku'],
-        'skip': 'Only works from China'
     }]
 
     def _url_for_id(self, id, quality=None):
@@ -45,8 +38,7 @@ class TudouIE(InfoExtractor):
         return final_url
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(2)
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         m = re.search(r'vcode:\s*[\'"](.+?)[\'"]', webpage)
@@ -87,4 +79,9 @@ class TudouIE(InfoExtractor):
             }
             result.append(part_info)
 
-        return result
+        return {
+            '_type': 'multi_video',
+            'entries': result,
+            'id': video_id,
+            'title': title,
+        }

From c0bdf32a3cb0f6a08ba92cb5261d020ef2f61892 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 8 Jan 2015 18:01:41 +0100
Subject: [PATCH 1101/1937] Add --print-json (Closes #2845)

---
 youtube_dl/__init__.py | 9 +++++----
 youtube_dl/options.py  | 5 +++++
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 4aa7fba6a..844992d8e 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -197,7 +197,8 @@ def _real_main(argv=None):
                      ' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
                      ' template'.format(outtmpl))
 
-    any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
+    any_getting = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
+    any_printing = opts.print_json
     download_archive_fn = compat_expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
 
     # PostProcessors
@@ -243,7 +244,7 @@ def _real_main(argv=None):
         'password': opts.password,
         'twofactor': opts.twofactor,
         'videopassword': opts.videopassword,
-        'quiet': (opts.quiet or any_printing),
+        'quiet': (opts.quiet or any_getting or any_printing),
         'no_warnings': opts.no_warnings,
         'forceurl': opts.geturl,
         'forcetitle': opts.gettitle,
@@ -253,9 +254,9 @@ def _real_main(argv=None):
         'forceduration': opts.getduration,
         'forcefilename': opts.getfilename,
         'forceformat': opts.getformat,
-        'forcejson': opts.dumpjson,
+        'forcejson': opts.dumpjson or opts.print_json,
         'dump_single_json': opts.dump_single_json,
-        'simulate': opts.simulate or any_printing,
+        'simulate': opts.simulate or any_getting,
         'skip_download': opts.skip_download,
         'format': opts.format,
         'format_limit': opts.format_limit,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 058342dec..feaca03a3 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -445,6 +445,11 @@ def parseOpts(overrideArguments=None):
         '-J', '--dump-single-json',
         action='store_true', dest='dump_single_json', default=False,
         help='simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist information in a single line.')
+    verbosity.add_option(
+        '--print-json',
+        action='store_true', dest='print_json', default=False,
+        help='Be quiet and print the video information as JSON (video is still being downloaded).',
+    )
     verbosity.add_option(
         '--newline',
         action='store_true', dest='progress_with_newline', default=False,

From ff0813313ac33b6abd03e9322b706dc83c5aeb14 Mon Sep 17 00:00:00 2001
From: Your Name <you@example.com>
Date: Thu, 8 Jan 2015 18:35:33 -0600
Subject: [PATCH 1102/1937] Minor style changes

---
 youtube_dl/extractor/audiomack.py | 54 +++++++++++++++----------------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index b6f3039c7..73cde78d3 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -9,21 +9,21 @@ import time
 
 
 class AudiomackIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?audiomack\.com/(song)/(?P<id>[\w/-]+)'
+    _VALID_URL = r'https?://(?:www\.)?audiomack\.com/song/(?P<id>[\w/-]+)'
     IE_NAME = 'audiomack'
     _TESTS = [
-        # audiomack
+        # hosted on audiomack
         {
             'url': 'http://www.audiomack.com/song/roosh-williams/extraordinary',
             'info_dict':
             {
                 'id': '310086',
-                "ext": "mp3",
-                "artist": "Roosh Williams",
+                'ext': 'mp3',
+                'artist': 'Roosh Williams',
                 'title': 'Extraordinary'
             }
         },
-        # audiomack through soundcloud
+        # audiomack wrapper around soundcloud song
         {
             'add_ie': ['Soundcloud'],
             'url': 'http://www.audiomack.com/song/xclusiveszone/take-kare',
@@ -41,12 +41,12 @@ class AudiomackIE(InfoExtractor):
     @staticmethod
     def create_song_dictionary(api_response, album_url_tag, track_no=0):
         # All keys are the same in audiomack api and InfoExtractor format
-        entry = {key: api_response[key] for key in ["title", "artist", "id", "url"] if key in api_response}
+        entry = {key: api_response[key] for key in ['title', 'artist', 'id', 'url'] if key in api_response}
         # Fudge values in the face of missing metadata
-        if "id" not in entry:
-            entry["id"] = track_no
-        if "title" not in entry:
-            entry["title"] = album_url_tag
+        if 'id' not in entry:
+            entry['id'] = track_no
+        if 'title' not in entry:
+            entry['title'] = album_url_tag
         return entry
 
     def _real_extract(self, url):
@@ -57,18 +57,18 @@ class AudiomackIE(InfoExtractor):
 
         # Request the extended version of the api for extra fields like artist and title
         api_response = self._download_json(
-            "http://www.audiomack.com/api/music/url/song/%s?extended=1&_=%d" % (
+            'http://www.audiomack.com/api/music/url/song/%s?extended=1&_=%d' % (
                 album_url_tag, time.time()),
             album_url_tag)
 
         # API is inconsistent with errors
-        if "url" not in api_response or not api_response["url"] or "error" in api_response:
-            raise ExtractorError("Invalid url %s", url)
+        if 'url' not in api_response or not api_response['url'] or 'error' in api_response:
+            raise ExtractorError('Invalid url %s', url)
 
         # Audiomack wraps a lot of soundcloud tracks in their branded wrapper
         # if so, pass the work off to the soundcloud extractor
-        if SoundcloudIE.suitable(api_response["url"]):
-            return {'_type': 'url', 'url': api_response["url"], 'ie_key': 'Soundcloud'}
+        if SoundcloudIE.suitable(api_response['url']):
+            return {'_type': 'url', 'url': api_response['url'], 'ie_key': 'Soundcloud'}
 
         return self.create_song_dictionary(api_response, album_url_tag)
 
@@ -80,17 +80,17 @@ class AudiomackAlbumIE(InfoExtractor):
         # Standard album playlist
         {
             'url': 'http://www.audiomack.com/album/flytunezcom/tha-tour-part-2-mixtape',
-            "playlist_count": 15,
+            'playlist_count': 15,
             'info_dict':
             {
-                'id': "812251",
-                'title': "Tha Tour: Part 2 (Official Mixtape)"
+                'id': '812251',
+                'title': 'Tha Tour: Part 2 (Official Mixtape)'
             }
         },
         # Album playlist ripped from fakeshoredrive with no metadata
         {
-            "url": "http://www.audiomack.com/album/fakeshoredrive/ppp-pistol-p-project",
-            "playlist_count": 10
+            'url': 'http://www.audiomack.com/album/fakeshoredrive/ppp-pistol-p-project',
+            'playlist_count': 10
         }
     ]
 
@@ -99,27 +99,27 @@ class AudiomackAlbumIE(InfoExtractor):
         # this title is whatever the user types in, and is rarely
         # the proper song title.  Real metadata is in the api response
         album_url_tag = self._match_id(url)
-        result = {"_type": "playlist", "entries": []}
+        result = {'_type': 'playlist', 'entries': []}
         # There is no one endpoint for album metadata - instead it is included/repeated in each song's metadata
         # Therefore we don't know how many songs the album has and must infi-loop until failure
         track_no = 0
         while True:
             # Get song's metadata
-            api_response = self._download_json("http://www.audiomack.com/api/music/url/album/%s/%d?extended=1&_=%d"
+            api_response = self._download_json('http://www.audiomack.com/api/music/url/album/%s/%d?extended=1&_=%d'
                                                % (album_url_tag, track_no, time.time()), album_url_tag)
 
             # Total failure, only occurs when url is totally wrong
             # Won't happen in middle of valid playlist (next case)
-            if "url" not in api_response or "error" in api_response:
-                raise ExtractorError("Invalid url for track %d of album url %s" % (track_no, url))
+            if 'url' not in api_response or 'error' in api_response:
+                raise ExtractorError('Invalid url for track %d of album url %s' % (track_no, url))
             # URL is good but song id doesn't exist - usually means end of playlist
-            elif not api_response["url"]:
+            elif not api_response['url']:
                 break
             else:
                 # Pull out the album metadata and add to result (if it exists)
-                for resultkey, apikey in [("id", "album_id"), ("title", "album_title")]:
+                for resultkey, apikey in [('id', 'album_id'), ('title', 'album_title')]:
                     if apikey in api_response and resultkey not in result:
                         result[resultkey] = api_response[apikey]
-                result["entries"].append(AudiomackIE.create_song_dictionary(api_response, album_url_tag, track_no))
+                result['entries'].append(AudiomackIE.create_song_dictionary(api_response, album_url_tag, track_no))
             track_no += 1
         return result

From b0e87c311078cf8510c720d50d4960a87f43296b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 19:02:06 +0100
Subject: [PATCH 1103/1937] [ffmpeg] Correctly encode paths on Windows

On Python 2.x on Windows, if there are any unicode arguments in the command argument list, the whole list is converted to unicode internally.
Therefore, we need to call encodeArgument on every argument.

Fixes #4337 and #4668.
---
 youtube_dl/postprocessor/ffmpeg.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 473536dcc..d1b342c7a 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -80,8 +80,9 @@ class FFmpegPostProcessor(PostProcessor):
 
         files_cmd = []
         for path in input_paths:
-            files_cmd.extend(['-i', encodeFilename(path, True)])
-        cmd = ([self._executable, '-y'] + files_cmd
+            files_cmd.extend([encodeArgument('-i'), encodeFilename(path, True)])
+        cmd = ([encodeFilename(self._executable, True), encodeArgument('-y')] +
+               files_cmd
                + [encodeArgument(o) for o in opts] +
                [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])
 
@@ -122,8 +123,8 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
             raise PostProcessingError('ffprobe or avprobe not found. Please install one.')
         try:
             cmd = [
-                self._probe_executable,
-                '-show_streams',
+                encodeFilename(self._probe_executable, True),
+                encodeArgument('-show_streams'),
                 encodeFilename(self._ffmpeg_filename_argument(path), True)]
             handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE)
             output = handle.communicate()[0]

From 5e4166478de0a8a7ddafd013919acd08b4211910 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 19:17:15 +0100
Subject: [PATCH 1104/1937] [README] Add an FAQ entry for how to install on
 Windows

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index 24b4d7970..fe380619f 100644
--- a/README.md
+++ b/README.md
@@ -450,6 +450,14 @@ Since June 2012 (#342) youtube-dl is packed as an executable zipfile, simply unz
 
 To run the exe you need to install first the [Microsoft Visual C++ 2008 Redistributable Package](http://www.microsoft.com/en-us/download/details.aspx?id=29).
 
+### On Windows, how should I set up ffmpeg and youtube-dl? Where should I put the exe files?
+
+If you put youtube-dl and ffmpeg in the same directory that you're running the command from, it will work, but that's rather cumbersome.
+
+To make a different directory work - either for ffmpeg, or for youtube-dl, or for both - simply create the directory (say, `C:\bin`, or `C:\Users\<User name>\bin`), put all the executables directly in there, and then [set your PATH environment variable](https://www.java.com/en/download/help/path.xml) to include that directory.
+
+From then on, after restarting your shell, you will be able to access both youtube-dl and ffmpeg (and youtube-dl will be able to find ffmpeg) by simply typing `youtube-dl` or `ffmpeg`, no matter what directory you're in.
+
 ### How can I detect whether a given URL is supported by youtube-dl?
 
 For one, have a look at the [list of supported sites](docs/supportedsites). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.

From 9932a65370564e08be11b0611e52b63ff451742d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 9 Jan 2015 20:13:53 +0100
Subject: [PATCH 1105/1937] [vk] Remove debug assert statement (fixes #4672,
 fixes #4514)

---
 youtube_dl/extractor/vk.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 129de6cf3..81e02a624 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -166,7 +166,6 @@ class VKIE(InfoExtractor):
 
         m_rutube = re.search(
             r'\ssrc="((?:https?:)?//rutube\.ru\\?/video\\?/embed(?:.*?))\\?"', info_page)
-        assert m_rutube
         if m_rutube is not None:
             self.to_screen('rutube video detected')
             rutube_url = self._proto_relative_url(

From aa2fd59857a3cc16e542dd93b762a607c98e9202 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 20:16:55 +0100
Subject: [PATCH 1106/1937] [update] Use utils HTTPS handler (Fixes #4666)

On FreeBSD, the default HTTPS handler is missing certificates, so use our own.
---
 youtube_dl/update.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 2d2703368..3f9c5249d 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -13,6 +13,7 @@ from .compat import (
     compat_str,
     compat_urllib_request,
 )
+from .utils import make_HTTPS_handler
 from .version import __version__
 
 
@@ -58,9 +59,12 @@ def update_self(to_screen, verbose):
         to_screen('It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
         return
 
+    https_handler = make_HTTPS_handler(False)
+    opener = compat_urllib_request.build_opener(https_handler)
+
     # Check if there is a new version
     try:
-        newversion = compat_urllib_request.urlopen(VERSION_URL).read().decode('utf-8').strip()
+        newversion = opener.open(VERSION_URL).read().decode('utf-8').strip()
     except:
         if verbose:
             to_screen(compat_str(traceback.format_exc()))
@@ -72,7 +76,7 @@ def update_self(to_screen, verbose):
 
     # Download and check versions info
     try:
-        versions_info = compat_urllib_request.urlopen(JSON_URL).read().decode('utf-8')
+        versions_info = opener.open(JSON_URL).read().decode('utf-8')
         versions_info = json.loads(versions_info)
     except:
         if verbose:
@@ -120,7 +124,7 @@ def update_self(to_screen, verbose):
             return
 
         try:
-            urlh = compat_urllib_request.urlopen(version['exe'][0])
+            urlh = opener.open(version['exe'][0])
             newcontent = urlh.read()
             urlh.close()
         except (IOError, OSError):
@@ -166,7 +170,7 @@ start /b "" cmd /c del "%%~f0"&exit /b"
     # Zip unix package
     elif isinstance(globals().get('__loader__'), zipimporter):
         try:
-            urlh = compat_urllib_request.urlopen(version['bin'][0])
+            urlh = opener.open(version['bin'][0])
             newcontent = urlh.read()
             urlh.close()
         except (IOError, OSError):

From 754c838903778911174e2777aebf4d1348780198 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 20:20:55 +0100
Subject: [PATCH 1107/1937] release 2015.01.09

---
 README.md             | 2 ++
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index fe380619f..3d632a553 100644
--- a/README.md
+++ b/README.md
@@ -219,6 +219,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      for each command-line argument. If the URL
                                      refers to a playlist, dump the whole
                                      playlist information in a single line.
+    --print-json                     Be quiet and print the video information as
+                                     JSON (video is still being downloaded).
     --newline                        output progress bar as new lines
     --no-progress                    do not print progress bar
     --console-title                  display progress in console titlebar
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 32019e362..2d942852f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.08'
+__version__ = '2015.01.09'

From 230b2287dd75430d0dcc3c22b407d916cb608b77 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 20:44:20 +0100
Subject: [PATCH 1108/1937] [youtube] Add acodec information

The codec seems to be consistently aac, so state that in our metadata.
---
 youtube_dl/extractor/youtube.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e71956071..bc18276d6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -264,9 +264,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'h264'},
 
         # Dash mp4 audio
-        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 48, 'preference': -50},
-        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 128, 'preference': -50},
-        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 256, 'preference': -50},
+        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 48, 'preference': -50},
+        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 128, 'preference': -50},
+        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 256, 'preference': -50},
 
         # Dash webm
         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},

From f90ad27375206433cdcb6609816893cb3c29c70d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 20:50:23 +0100
Subject: [PATCH 1109/1937] [YoutubeDL] Copy over format metadata when merging
 (Fixes #4671)

---
 youtube_dl/YoutubeDL.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 24d6c2de7..e34b36016 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -913,6 +913,14 @@ class YoutubeDL(object):
                                 'requested_formats': formats_info,
                                 'format': rf,
                                 'ext': formats_info[0]['ext'],
+                                'width': formats_info[0].get('width'),
+                                'height': formats_info[0].get('height'),
+                                'resolution': formats_info[0].get('resolution'),
+                                'fps': formats_info[0].get('fps'),
+                                'vcodec': formats_info[0].get('vcodec'),
+                                'vbr': formats_info[0].get('vbr'),
+                                'acodec': formats_info[1].get('acodec'),
+                                'abr': formats_info[1].get('abr'),
                             }
                         else:
                             selected_format = None

From 7b6faddfc808ba798ab654312e98d01005cda9ee Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 20:52:49 +0100
Subject: [PATCH 1110/1937] [wdr] Modernize

---
 youtube_dl/extractor/wdr.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 8e25ecf28..d1c46ccb3 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -172,8 +172,7 @@ class WDRMausIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
         param_code = self._html_search_regex(
@@ -224,5 +223,3 @@ class WDRMausIE(InfoExtractor):
             'thumbnail': thumbnail,
             'upload_date': upload_date,
         }
-
-# TODO test _1

From 176cf9e0c3ef9efd4864b940ac1c2a1c0236593a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 21:33:07 +0100
Subject: [PATCH 1111/1937] [wdr] Support overviews (Fixes #4651)

---
 youtube_dl/extractor/wdr.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index d1c46ccb3..45466e31b 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import itertools
 import re
 
 from .common import InfoExtractor
@@ -67,6 +68,10 @@ class WDRIE(InfoExtractor):
                 'upload_date': '20140717',
             },
         },
+        {
+            'url': 'http://www1.wdr.de/mediathek/video/sendungen/quarks_und_co/filterseite-quarks-und-co100.html',
+            'playlist_mincount': 146,
+        }
     ]
 
     def _real_extract(self, url):
@@ -81,6 +86,27 @@ class WDRIE(InfoExtractor):
                 self.url_result(page_url + href, 'WDR')
                 for href in re.findall(r'<a href="/?(.+?%s\.html)" rel="nofollow"' % self._PLAYER_REGEX, webpage)
             ]
+
+            if entries:  # Playlist page
+                return self.playlist_result(entries, page_id)
+
+            # Overview page
+            entries = []
+            for page_num in itertools.count(2):
+                hrefs = re.findall(
+                    r'<li class="mediathekvideo"\s*>\s*<img[^>]*>\s*<a href="(/mediathek/video/[^"]+)"',
+                    webpage)
+                entries.extend(
+                    self.url_result(page_url + href, 'WDR')
+                    for href in hrefs)
+                next_url_m = re.search(
+                    r'<li class="nextToLast">\s*<a href="([^"]+)"', webpage)
+                if not next_url_m:
+                    break
+                next_url = page_url + next_url_m.group(1)
+                webpage = self._download_webpage(
+                    next_url, page_id,
+                    note='Downloading playlist page %d' % page_num)
             return self.playlist_result(entries, page_id)
 
         flashvars = compat_parse_qs(

From 4baea47c42c291564dcbbbfc93955afbd4a80460 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 21:33:16 +0100
Subject: [PATCH 1112/1937] release 2015.01.09.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2d942852f..8270f7638 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.09'
+__version__ = '2015.01.09.1'

From b8da6b9fc6b05dff5dfbe98e20546b7e5e0ab170 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 22:43:49 +0100
Subject: [PATCH 1113/1937] [elpais] Modernize

---
 youtube_dl/extractor/elpais.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/elpais.py b/youtube_dl/extractor/elpais.py
index 4277202a2..00a69e631 100644
--- a/youtube_dl/extractor/elpais.py
+++ b/youtube_dl/extractor/elpais.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import unified_strdate
 
@@ -24,9 +22,7 @@ class ElPaisIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         prefix = self._html_search_regex(

From d120e9013f8dbab7e82fe1878e9b5e5e9bb94e88 Mon Sep 17 00:00:00 2001
From: Andrei Troie <andreitroie90@gmail.com>
Date: Fri, 9 Jan 2015 22:03:56 +0000
Subject: [PATCH 1114/1937] Added an option to specify an output format for
 merges when downloading separate video & audio

---
 youtube_dl/options.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index feaca03a3..0ed5e3db4 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -302,6 +302,12 @@ def parseOpts(overrideArguments=None):
         '--youtube-skip-dash-manifest',
         action='store_false', dest='youtube_include_dash_manifest',
         help='Do not download the DASH manifest on YouTube videos')
+    video_format.add_option(
+    	'--merge-output-format',
+        action='store', dest='merge_output_format', metavar='FORMAT' ,default=None,
+        help=(
+            'If a merge is required (e.g. bestvideo+bestaudio), output to given container format (e.g. mkv, mp4, ogg, webm, flv) '
+            'Ignored if no merge is required'))
 
     subtitles = optparse.OptionGroup(parser, 'Subtitle Options')
     subtitles.add_option(

From dd622d7c4ec15aec6ea5d66ed88a236147373e95 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 23:59:18 +0100
Subject: [PATCH 1115/1937] [netzkino] Add new extractor (Fixes #4669)

---
 test/helper.py                   | 14 ++++++
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/common.py   | 11 ++++
 youtube_dl/extractor/netzkino.py | 86 ++++++++++++++++++++++++++++++++
 youtube_dl/utils.py              |  4 ++
 5 files changed, 116 insertions(+)
 create mode 100644 youtube_dl/extractor/netzkino.py

diff --git a/test/helper.py b/test/helper.py
index 77225e4f7..c416f388c 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -110,6 +110,20 @@ def expect_info_dict(self, got_dict, expected_dict):
         else:
             if isinstance(expected, compat_str) and expected.startswith('md5:'):
                 got = 'md5:' + md5(got_dict.get(info_field))
+            elif isinstance(expected, compat_str) and expected.startswith('mincount:'):
+                got = got_dict.get(info_field)
+                self.assertTrue(
+                    isinstance(got, list),
+                    'Expected field %s to be a list, but it is of type %s' % (
+                        info_field, type(got).__name__))
+                expected_num = int(expected.partition(':')[2])
+                assertGreaterEqual(
+                    self, len(got), expected_num,
+                    'Expected %d items in field %s, but only got %d' % (
+                        expected_num, info_field, len(got)
+                    )
+                )
+                continue
             else:
                 got = got_dict.get(info_field)
             self.assertEqual(expected, got,
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 8dacc2c54..5da7568ca 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -274,6 +274,7 @@ from .nbc import (
 )
 from .ndr import NDRIE
 from .ndtv import NDTVIE
+from .netzkino import NetzkinoIE
 from .nerdcubed import NerdCubedFeedIE
 from .newgrounds import NewgroundsIE
 from .newstube import NewstubeIE
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d703893dc..b4cd59e43 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -147,6 +147,17 @@ class InfoExtractor(object):
     like_count:     Number of positive ratings of the video
     dislike_count:  Number of negative ratings of the video
     comment_count:  Number of comments on the video
+    comments:       A list of comments, each with one or more of the following
+                    properties (all but one of text or html optional):
+                        * "author" - human-readable name of the comment author
+                        * "author_id" - user ID of the comment author
+                        * "id" - Comment ID
+                        * "html" - Comment as HTML
+                        * "text" - Plain text of the comment
+                        * "timestamp" - UNIX timestamp of comment
+                        * "parent" - ID of the comment this one is replying to.
+                                     Set to "root" to indicate that this is a
+                                     comment to the original video.
     age_limit:      Age restriction for the video, as an integer (years)
     webpage_url:    The url to the video webpage, if given to youtube-dl it
                     should allow to get the same result again. (It will be set
diff --git a/youtube_dl/extractor/netzkino.py b/youtube_dl/extractor/netzkino.py
new file mode 100644
index 000000000..93567d1e3
--- /dev/null
+++ b/youtube_dl/extractor/netzkino.py
@@ -0,0 +1,86 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    int_or_none,
+    js_to_json,
+    parse_iso8601,
+)
+
+
+class NetzkinoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?netzkino\.de/\#!/(?P<category>[^/]+)/(?P<id>[^/]+)'
+
+    _TEST = {
+        'url': 'http://www.netzkino.de/#!/scifikino/rakete-zum-mond',
+        'md5': '92a3f8b76f8d7220acce5377ea5d4873',
+        'info_dict': {
+            'id': 'rakete-zum-mond',
+            'ext': 'mp4',
+            'title': 'Rakete zum Mond (Endstation Mond, Destination Moon)',
+            'comments': 'mincount:3',
+            'description': 'md5:1eddeacc7e62d5a25a2d1a7290c64a28',
+            'upload_date': '20120813',
+            'thumbnail': 're:https?://.*\.jpg$',
+            'timestamp': 1344858571,
+            'age_limit': 12,
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        category_id = mobj.group('category')
+        video_id = mobj.group('id')
+
+        api_url = 'http://api.netzkino.de.simplecache.net/capi-2.0a/categories/%s.json?d=www' % category_id
+        api_info = self._download_json(api_url, video_id)
+        info = next(
+            p for p in api_info['posts'] if p['slug'] == video_id)
+        custom_fields = info['custom_fields']
+
+        production_js = self._download_webpage(
+            'http://www.netzkino.de/beta/dist/production.min.js', video_id,
+            note='Downloading player code')
+        avo_js = self._search_regex(
+            r'window\.avoCore\s*=.*?urlTemplate:\s*(\{.*?"\})',
+            production_js, 'URL templates')
+        templates = self._parse_json(
+            avo_js, video_id, transform_source=js_to_json)
+
+        suffix = {
+            'hds': '.mp4/manifest.f4m',
+            'hls': '.mp4/master.m3u8',
+            'pmd': '.mp4',
+        }
+        film_fn = custom_fields['Streaming'][0]
+        formats = [{
+            'format_id': key,
+            'ext': 'mp4',
+            'url': tpl.replace('{}', film_fn) + suffix[key],
+        } for key, tpl in templates.items()]
+        self._sort_formats(formats)
+
+        comments = [{
+            'timestamp': parse_iso8601(c.get('date'), delimiter=' '),
+            'id': c['id'],
+            'author': c['name'],
+            'html': c['content'],
+            'parent': 'root' if c.get('parent', 0) == 0 else c['parent'],
+        } for c in info.get('comments', [])]
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'comments': comments,
+            'title': info['title'],
+            'age_limit': int_or_none(custom_fields.get('FSK')[0]),
+            'timestamp': parse_iso8601(info.get('date'), delimiter=' '),
+            'description': clean_html(info.get('content')),
+            'thumbnail': info.get('thumbnail'),
+            'playlist_title': api_info.get('title'),
+            'playlist_id': category_id,
+        }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 29739a483..079e8d2c3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -205,6 +205,10 @@ def get_element_by_attribute(attribute, value, html):
 
 def clean_html(html):
     """Clean an HTML snippet into a readable string"""
+
+    if html is None:  # Convenience for sanitizing descriptions etc.
+        return html
+
     # Newline vs <br />
     html = html.replace('\n', ' ')
     html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)

From 130239460370870c33f259ed1549878388996d3f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 9 Jan 2015 23:59:29 +0100
Subject: [PATCH 1116/1937] release 2015.01.09.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8270f7638..8c57c7413 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.09.1'
+__version__ = '2015.01.09.2'

From 34c781a24d1cc9b424ddc3ab3c4c616f49642c6a Mon Sep 17 00:00:00 2001
From: Andrei Troie <andreitroie90@gmail.com>
Date: Sat, 10 Jan 2015 00:03:11 +0000
Subject: [PATCH 1117/1937] Passing the option into the main program's
 arguments

---
 youtube_dl/__init__.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 844992d8e..b4ae35a2b 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -166,6 +166,9 @@ def _real_main(argv=None):
     if opts.recodevideo is not None:
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
             parser.error('invalid video recode format specified')
+    if opts.merge_output_format is not None and not '+' in opts.format: #if merge format output is used on videos that don't require merging, ignore
+        opts.merge_output_format = None
+        
     if opts.date is not None:
         date = DateRange.day(opts.date)
     else:
@@ -323,6 +326,7 @@ def _real_main(argv=None):
         'encoding': opts.encoding,
         'exec_cmd': opts.exec_cmd,
         'extract_flat': opts.extract_flat,
+        'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
     }
 

From d02115f83727ae0d4f394073235dc97086c64266 Mon Sep 17 00:00:00 2001
From: Andrei Troie <andreitroie90@gmail.com>
Date: Sat, 10 Jan 2015 00:29:06 +0000
Subject: [PATCH 1118/1937] Use the option in preparing the merge output
 filename

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 24d6c2de7..6d5c401df 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -912,7 +912,7 @@ class YoutubeDL(object):
                             selected_format = {
                                 'requested_formats': formats_info,
                                 'format': rf,
-                                'ext': formats_info[0]['ext'],
+                                'ext': self.params['merge_output_format'] if self.params['merge_output_format'] is not None else formats_info[0]['ext'],
                             }
                         else:
                             selected_format = None

From bd1a281ede1548ae0f0c74360eb518751c2a4526 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 02:03:00 +0100
Subject: [PATCH 1119/1937] [options] PEP8 and simpler --merge-output-format
 handling (#4673)

---
 youtube_dl/__init__.py | 4 +---
 youtube_dl/options.py  | 6 +++---
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index b4ae35a2b..8e7b74466 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -166,9 +166,7 @@ def _real_main(argv=None):
     if opts.recodevideo is not None:
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
             parser.error('invalid video recode format specified')
-    if opts.merge_output_format is not None and not '+' in opts.format: #if merge format output is used on videos that don't require merging, ignore
-        opts.merge_output_format = None
-        
+
     if opts.date is not None:
         date = DateRange.day(opts.date)
     else:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 0ed5e3db4..14006178d 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -303,10 +303,10 @@ def parseOpts(overrideArguments=None):
         action='store_false', dest='youtube_include_dash_manifest',
         help='Do not download the DASH manifest on YouTube videos')
     video_format.add_option(
-    	'--merge-output-format',
-        action='store', dest='merge_output_format', metavar='FORMAT' ,default=None,
+        '--merge-output-format',
+        action='store', dest='merge_output_format', metavar='FORMAT', default=None,
         help=(
-            'If a merge is required (e.g. bestvideo+bestaudio), output to given container format (e.g. mkv, mp4, ogg, webm, flv) '
+            'If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv.'
             'Ignored if no merge is required'))
 
     subtitles = optparse.OptionGroup(parser, 'Subtitle Options')

From 904fffffebcf26b8004d6298001f17236857a5d8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 02:24:46 +0100
Subject: [PATCH 1120/1937] [audiomack] Better titles, simplify code

---
 youtube_dl/extractor/audiomack.py | 60 +++++++++++++++++++------------
 1 file changed, 37 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 73cde78d3..8bfe50214 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -1,11 +1,15 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import itertools
+import time
+
 from .common import InfoExtractor
 from .soundcloud import SoundcloudIE
-from ..utils import ExtractorError
-
-import time
+from ..utils import (
+    ExtractorError,
+    url_basename,
+)
 
 
 class AudiomackIE(InfoExtractor):
@@ -19,7 +23,7 @@ class AudiomackIE(InfoExtractor):
             {
                 'id': '310086',
                 'ext': 'mp3',
-                'artist': 'Roosh Williams',
+                'uploader': 'Roosh Williams',
                 'title': 'Extraordinary'
             }
         },
@@ -38,17 +42,6 @@ class AudiomackIE(InfoExtractor):
         },
     ]
 
-    @staticmethod
-    def create_song_dictionary(api_response, album_url_tag, track_no=0):
-        # All keys are the same in audiomack api and InfoExtractor format
-        entry = {key: api_response[key] for key in ['title', 'artist', 'id', 'url'] if key in api_response}
-        # Fudge values in the face of missing metadata
-        if 'id' not in entry:
-            entry['id'] = track_no
-        if 'title' not in entry:
-            entry['title'] = album_url_tag
-        return entry
-
     def _real_extract(self, url):
         # URLs end with [uploader name]/[uploader title]
         # this title is whatever the user types in, and is rarely
@@ -70,7 +63,12 @@ class AudiomackIE(InfoExtractor):
         if SoundcloudIE.suitable(api_response['url']):
             return {'_type': 'url', 'url': api_response['url'], 'ie_key': 'Soundcloud'}
 
-        return self.create_song_dictionary(api_response, album_url_tag)
+        return {
+            'id': api_response.get('id', album_url_tag),
+            'uploader': api_response.get('artist'),
+            'title': api_response.get('title'),
+            'url': api_response['url'],
+        }
 
 
 class AudiomackAlbumIE(InfoExtractor):
@@ -90,7 +88,17 @@ class AudiomackAlbumIE(InfoExtractor):
         # Album playlist ripped from fakeshoredrive with no metadata
         {
             'url': 'http://www.audiomack.com/album/fakeshoredrive/ppp-pistol-p-project',
-            'playlist_count': 10
+            'playlist': [{
+                'info_dict': {
+                    'title': '9.-heaven-or-hell-chimaca-ft-zuse-prod-by-dj-fu',
+                    'id': '9.-heaven-or-hell-chimaca-ft-zuse-prod-by-dj-fu',
+                    'ext': 'mp3',
+                }
+            }],
+            'params': {
+                'playliststart': 8,
+                'playlistend': 8,
+            }
         }
     ]
 
@@ -102,11 +110,12 @@ class AudiomackAlbumIE(InfoExtractor):
         result = {'_type': 'playlist', 'entries': []}
         # There is no one endpoint for album metadata - instead it is included/repeated in each song's metadata
         # Therefore we don't know how many songs the album has and must infi-loop until failure
-        track_no = 0
-        while True:
+        for track_no in itertools.count():
             # Get song's metadata
-            api_response = self._download_json('http://www.audiomack.com/api/music/url/album/%s/%d?extended=1&_=%d'
-                                               % (album_url_tag, track_no, time.time()), album_url_tag)
+            api_response = self._download_json(
+                'http://www.audiomack.com/api/music/url/album/%s/%d?extended=1&_=%d'
+                % (album_url_tag, track_no, time.time()), album_url_tag,
+                note='Querying song information (%d)' % (track_no + 1))
 
             # Total failure, only occurs when url is totally wrong
             # Won't happen in middle of valid playlist (next case)
@@ -120,6 +129,11 @@ class AudiomackAlbumIE(InfoExtractor):
                 for resultkey, apikey in [('id', 'album_id'), ('title', 'album_title')]:
                     if apikey in api_response and resultkey not in result:
                         result[resultkey] = api_response[apikey]
-                result['entries'].append(AudiomackIE.create_song_dictionary(api_response, album_url_tag, track_no))
-            track_no += 1
+                song_id = url_basename(api_response['url']).rpartition('.')[0]
+                result['entries'].append({
+                    'id': api_response.get('id', song_id),
+                    'uploader': api_response.get('artist'),
+                    'title': api_response.get('title', song_id),
+                    'url': api_response['url'],
+                })
         return result

From fb4b030aafee72801b47d4715391bd5822f80e13 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 02:38:27 +0100
Subject: [PATCH 1121/1937] [tvp] Update tests and improve output

---
 youtube_dl/extractor/tvp.py | 164 +++++++++++++++++-------------------
 1 file changed, 78 insertions(+), 86 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index cf21d1202..cc26f417a 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -8,79 +8,62 @@ from .common import InfoExtractor
 
 class TvpIE(InfoExtractor):
     IE_NAME = 'tvp.pl'
-    _VALID_URL = r'https?://(?P<type>vod|www)\.tvp\.pl/.*/(?P<id>\d+)$'
+    _VALID_URL = r'https?://(?:vod|www)\.tvp\.pl/.*/(?P<id>\d+)$'
 
-    _TESTS = [
-        {
-            'url': 'http://www.tvp.pl/warszawa/magazyny/campusnews/wideo/31102013/12878238',
-            'info_dict': {
-                'id': '12878238',
-                'ext': 'wmv',
-                'title': 'CAMPUSnews, 31.10.2013 - Odcinek 2',
-                'description': '',
-            },
-            'skip': 'Download has to use same server IP as extraction. Therefore, a good (load-balancing) DNS resolver will make the download fail.',
-        }, {
-            'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem/wideo/odc-2/4278035',
-            'info_dict': {
-                'id': '4278035',
-                'ext': 'wmv',
-                'title': 'Ogniem i mieczem, odc. 2',
-                'description': 'Bohun dowiaduje się o złamaniu przez kniahinię danego mu słowa i wyrusza do Rozłogów. Helenie w ostatniej chwili udaje się uciec dzięki pomocy Zagłoby.',
-            },
-            'skip': 'As above',
-        }, {
-            'url': 'http://vod.tvp.pl/seriale/obyczajowe/czas-honoru/sezon-1-1-13/i-seria-odc-13/194536',
-            'info_dict': {
-                'id': '194536',
-                'ext': 'mp4',
-                'title': 'Czas honoru, I seria – odc. 13',
-                'description': 'WŁADEK\nCzesław prosi Marię o dostarczenie Władkowi zarazki tyfusu. Jeśli zachoruje zostanie przewieziony do szpitala skąd łatwiej będzie go odbić. Czy matka zdecyduje się zarazić syna? Karol odwiedza Wandę przyznaje się, że ją oszukiwał, ale ostrzega też, że grozi jej aresztowanie i nalega, żeby wyjechała z Warszawy. Czy dziewczyna zdecyduje się znów oddalić od ukochanego? Rozpoczyna się akcja odbicia Władka.',
-            },
-        }, {
-            'url': 'http://www.tvp.pl/there-can-be-anything-so-i-shortened-it/17916176',
-            'info_dict': {
-                'id': '17916176',
-                'ext': 'mp4',
-                'title': 'rozmaitosci, TVP Gorzów pokaże filmy studentów z podroży dookoła świata',
-                'description': '',
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': 'true',
-            },
-        }, {
-            'url': 'http://vod.tvp.pl/seriale/obyczajowe/na-sygnale/sezon-2-27-/odc-39/17834272',
-            'info_dict': {
-                'id': '17834272',
-                'ext': 'mp4',
-                'title': 'Na sygnale, odc. 39',
-                'description': 'Ekipa Wiktora ratuje młodą matkę, która spadła ze schodów trzymając na rękach noworodka. Okazuje się, że dziewczyna jest surogatką, a biologiczni rodzice dziecka próbują zmusić ją do oddania synka…',
-            },
-            'params': {
-                # m3u8 download
-                'skip_download': 'true',
-            },
+    _TESTS = [{
+        'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem/wideo/odc-2/4278035',
+        'info_dict': {
+            'id': '4278035',
+            'ext': 'wmv',
+            'title': 'Ogniem i mieczem, odc. 2',
+            'description': 'Bohun dowiaduje się o złamaniu przez kniahinię danego mu słowa i wyrusza do Rozłogów. Helenie w ostatniej chwili udaje się uciec dzięki pomocy Zagłoby.',
         },
-    ]
+    }, {
+        'url': 'http://vod.tvp.pl/seriale/obyczajowe/czas-honoru/sezon-1-1-13/i-seria-odc-13/194536',
+        'info_dict': {
+            'id': '194536',
+            'ext': 'mp4',
+            'title': 'Czas honoru, I seria – odc. 13',
+            #  'description': 'WŁADEK\nCzesław prosi Marię o dostarczenie Władkowi zarazki tyfusu. Jeśli zachoruje zostanie przewieziony do szpitala skąd łatwiej będzie go odbić. Czy matka zdecyduje się zarazić syna? Karol odwiedza Wandę przyznaje się, że ją oszukiwał, ale ostrzega też, że grozi jej aresztowanie i nalega, żeby wyjechała z Warszawy. Czy dziewczyna zdecyduje się znów oddalić od ukochanego? Rozpoczyna się akcja odbicia Władka.',
+        },
+    }, {
+        'url': 'http://www.tvp.pl/there-can-be-anything-so-i-shortened-it/17916176',
+        'info_dict': {
+            'id': '17916176',
+            'ext': 'mp4',
+            'title': 'TVP Gorzów pokaże filmy studentów z podroży dookoła świata',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': 'true',
+        },
+    }, {
+        'url': 'http://vod.tvp.pl/seriale/obyczajowe/na-sygnale/sezon-2-27-/odc-39/17834272',
+        'info_dict': {
+            'id': '17834272',
+            'ext': 'mp4',
+            'title': 'Na sygnale, odc. 39',
+            'description': 'Ekipa Wiktora ratuje młodą matkę, która spadła ze schodów trzymając na rękach noworodka. Okazuje się, że dziewczyna jest surogatką, a biologiczni rodzice dziecka próbują zmusić ją do oddania synka…',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': 'true',
+        },
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(
             'http://www.tvp.pl/sess/tvplayer.php?object_id=%s' % video_id, video_id)
+
         title = self._og_search_title(webpage)
         series = self._search_regex(
             r'{name:\s*([\'"])SeriesTitle\1,\s*value:\s*\1(?P<series>.*?)\1},',
             webpage, 'series', group='series', default=None)
         if series is not None and series not in title:
             title = '%s, %s' % (series, title)
-        info_dict = {
-            'id': video_id,
-            'title': title,
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'description': self._og_search_description(webpage),
-        }
+        description = self._og_search_description(webpage, default=None)
+
         video_url = self._search_regex(
             r'0:{src:([\'"])(?P<url>.*?)\1', webpage, 'formats', group='url', default=None)
         if video_url is None:
@@ -92,55 +75,64 @@ class TvpIE(InfoExtractor):
         if ext != 'ism/manifest':
             if '/' in ext:
                 ext = 'mp4'
-            info_dict.update({
-                'ext': ext,
+            formats = [{
+                'format_id': 'direct',
                 'url': video_url,
-            })
+                'ext': ext,
+            }]
         else:
             m3u8_url = re.sub('([^/]*)\.ism/manifest', r'\1.ism/\1.m3u8', video_url)
             formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4')
-            info_dict.update({
-                'formats': formats,
-            })
-        return info_dict
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'description': description,
+            'formats': formats,
+        }
 
 
 class TvpSeriesIE(InfoExtractor):
     IE_NAME = 'tvp.pl:Series'
     _VALID_URL = r'https?://vod\.tvp\.pl/(?:[^/]+/){2}(?P<id>[^/]+)/?$'
 
-    _TESTS = [
-        {
-            'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem',
-            'info_dict': {
-                'title': 'Ogniem i mieczem',
-                'id': '4278026',
-            },
-            'playlist_count': 4,
-        }, {
-            'url': 'http://vod.tvp.pl/audycje/podroze/boso-przez-swiat',
-            'info_dict': {
-                'title': 'Boso przez świat',
-                'id': '9329207',
-            },
-            'playlist_count': 86,
-        }
-    ]
+    _TESTS = [{
+        'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem',
+        'info_dict': {
+            'title': 'Ogniem i mieczem',
+            'id': '4278026',
+        },
+        'playlist_count': 4,
+    }, {
+        'url': 'http://vod.tvp.pl/audycje/podroze/boso-przez-swiat',
+        'info_dict': {
+            'title': 'Boso przez świat',
+            'id': '9329207',
+        },
+        'playlist_count': 86,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id, tries=5)
+
         title = self._html_search_regex(
             r'(?s) id=[\'"]path[\'"]>(?:.*? / ){2}(.*?)</span>', webpage, 'series')
         playlist_id = self._search_regex(r'nodeId:\s*(\d+)', webpage, 'playlist id')
         playlist = self._download_webpage(
             'http://vod.tvp.pl/vod/seriesAjax?type=series&nodeId=%s&recommend'
-            'edId=0&sort=&page=0&pageSize=10000' % playlist_id, display_id, tries=5)
+            'edId=0&sort=&page=0&pageSize=10000' % playlist_id, display_id, tries=5,
+            note='Downloading playlist')
+
         videos_paths = re.findall(
             '(?s)class="shortTitle">.*?href="(/[^"]+)', playlist)
         entries = [
             self.url_result('http://vod.tvp.pl%s' % v_path, ie=TvpIE.ie_key())
             for v_path in videos_paths]
+
         return {
             '_type': 'playlist',
             'id': playlist_id,

From 6271f1cad9331cbdb274906d8330736fb888bda2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 05:45:51 +0100
Subject: [PATCH 1122/1937] [youtube|ffmpeg] Automatically correct video with
 non-square pixels (Fixes #4674)

---
 youtube_dl/YoutubeDL.py              | 29 ++++++++++++++++++++++++++++
 youtube_dl/__init__.py               |  1 +
 youtube_dl/extractor/common.py       |  3 +++
 youtube_dl/extractor/youtube.py      | 24 +++++++++++++++++++++++
 youtube_dl/options.py                |  7 +++++++
 youtube_dl/postprocessor/__init__.py |  2 ++
 youtube_dl/postprocessor/ffmpeg.py   | 23 ++++++++++++++++++++++
 7 files changed, 89 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 61675d8ec..4cc3ec2fb 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -70,6 +70,7 @@ from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader
 from .downloader.rtmp import rtmpdump_version
 from .postprocessor import (
+    FFmpegFixupStretchedPP,
     FFmpegMergerPP,
     FFmpegPostProcessor,
     get_postprocessor,
@@ -204,6 +205,12 @@ class YoutubeDL(object):
                        Progress hooks are guaranteed to be called at least once
                        (with status "finished") if the download is successful.
     merge_output_format: Extension to use when merging formats.
+    fixup:             Automatically correct known faults of the file.
+                       One of:
+                       - "never": do nothing
+                       - "warn": only emit a warning
+                       - "detect_or_warn": check whether we can do anything
+                                           about it, warn otherwise
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
@@ -924,6 +931,7 @@ class YoutubeDL(object):
                                 'fps': formats_info[0].get('fps'),
                                 'vcodec': formats_info[0].get('vcodec'),
                                 'vbr': formats_info[0].get('vbr'),
+                                'stretched_ratio': formats_info[0].get('stretched_ratio'),
                                 'acodec': formats_info[1].get('acodec'),
                                 'abr': formats_info[1].get('abr'),
                                 'ext': output_ext,
@@ -1154,6 +1162,27 @@ class YoutubeDL(object):
                     return
 
             if success:
+                # Fixup content
+                stretched_ratio = info_dict.get('stretched_ratio')
+                if stretched_ratio is not None and stretched_ratio != 1:
+                    fixup_policy = self.params.get('fixup')
+                    if fixup_policy is None:
+                        fixup_policy = 'detect_or_warn'
+                    if fixup_policy == 'warn':
+                        self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
+                            info_dict['id'], stretched_ratio))
+                    elif fixup_policy == 'detect_or_warn':
+                        stretched_pp = FFmpegFixupStretchedPP(self)
+                        if stretched_pp.available:
+                            info_dict.setdefault('__postprocessors', [])
+                            info_dict['__postprocessors'].append(stretched_pp)
+                        else:
+                            self.report_warning(
+                                '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
+                                    info_dict['id'], stretched_ratio))
+                    else:
+                        assert fixup_policy == 'ignore'
+
                 try:
                     self.post_process(filename, info_dict)
                 except (PostProcessingError) as err:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 8e7b74466..659a92a3b 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -326,6 +326,7 @@ def _real_main(argv=None):
         'extract_flat': opts.extract_flat,
         'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
+        'fixup': opts.fixup,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index cd155a090..363e2000c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -114,6 +114,9 @@ class InfoExtractor(object):
                                  to add to the request.
                     * http_post_data  Additional data to send with a POST
                                  request.
+                    * stretched_ratio  If given and not 1, indicates that the
+                                       video's pixels are not square.
+                                       width : height ratio as float.
     url:            Final video URL.
     ext:            Video filename extension.
     format:         The video format, defaults to ext (used for --get-format)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index bc18276d6..c7611a3a0 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -465,6 +465,20 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'skip_download': 'requires avconv',
             }
         },
+        # Non-square pixels
+        {
+            'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
+            'info_dict': {
+                'id': '_b-2C3KPAM0',
+                'ext': 'mp4',
+                'stretched_ratio': 16 / 9.,
+                'upload_date': '20110310',
+                'uploader_id': 'AllenMeow',
+                'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
+                'uploader': '孫艾倫',
+                'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
+            },
+        }
     ]
 
     def __init__(self, *args, **kwargs):
@@ -1051,6 +1065,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                             f['preference'] = f.get('preference', 0) - 10000
                     formats.extend(dash_formats)
 
+        # Check for malformed aspect ratio
+        stretched_m = re.search(
+            r'<meta\s+property="og:video:tag".*?content="yt:stretch=(?P<w>[0-9]+):(?P<h>[0-9]+)">',
+            video_webpage)
+        if stretched_m:
+            ratio = float(stretched_m.group('w')) / float(stretched_m.group('h'))
+            for f in formats:
+                if f.get('vcodec') != 'none':
+                    f['stretched_ratio'] = ratio
+
         self._sort_formats(formats)
 
         return {
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 14006178d..e5602bb3a 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -631,6 +631,13 @@ def parseOpts(overrideArguments=None):
         '--xattrs',
         action='store_true', dest='xattrs', default=False,
         help='write metadata to the video file\'s xattrs (using dublin core and xdg standards)')
+    postproc.add_option(
+        '--fixup',
+        metavar='POLICY', dest='fixup', default='detect_or_warn',
+        help='(experimental) Automatically correct known faults of the file. '
+             'One of never (do nothing), warn (only emit a warning), '
+             'detect_or_warn(check whether we can do anything about it, warn '
+             'otherwise')
     postproc.add_option(
         '--prefer-avconv',
         action='store_false', dest='prefer_ffmpeg',
diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index 7f505b58e..f8507951c 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -6,6 +6,7 @@ from .ffmpeg import (
     FFmpegAudioFixPP,
     FFmpegEmbedSubtitlePP,
     FFmpegExtractAudioPP,
+    FFmpegFixupStretchedPP,
     FFmpegMergerPP,
     FFmpegMetadataPP,
     FFmpegVideoConvertorPP,
@@ -24,6 +25,7 @@ __all__ = [
     'FFmpegAudioFixPP',
     'FFmpegEmbedSubtitlePP',
     'FFmpegExtractAudioPP',
+    'FFmpegFixupStretchedPP',
     'FFmpegMergerPP',
     'FFmpegMetadataPP',
     'FFmpegPostProcessor',
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index d1b342c7a..6e9194fa6 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -50,6 +50,10 @@ class FFmpegPostProcessor(PostProcessor):
         programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
         return dict((p, get_exe_version(p, args=['-version'])) for p in programs)
 
+    @property
+    def available(self):
+        return self._executable is not None
+
     @property
     def _executable(self):
         if self._downloader.params.get('prefer_ffmpeg', False):
@@ -540,3 +544,22 @@ class FFmpegAudioFixPP(FFmpegPostProcessor):
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
         return True, info
+
+
+class FFmpegFixupStretchedPP(FFmpegPostProcessor):
+    def run(self, info):
+        stretched_ratio = info.get('stretched_ratio')
+        if stretched_ratio is None or stretched_ratio == 1:
+            return
+
+        filename = info['filepath']
+        temp_filename = prepend_extension(filename, 'temp')
+
+        options = ['-c', 'copy', '-aspect', '%f' % stretched_ratio]
+        self._downloader.to_screen('[ffmpeg] Fixing aspect ratio in "%s"' % filename)
+        self.run_ffmpeg(filename, temp_filename, options)
+
+        os.remove(encodeFilename(filename))
+        os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+
+        return True, info

From f71fdb0acc0e168d2a5e4f60be96217647cc872e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 05:51:07 +0100
Subject: [PATCH 1123/1937] [eighttracks] Improve waiting (#3954)

---
 youtube_dl/extractor/eighttracks.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index 9df2141c3..a9b18195b 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -4,11 +4,12 @@ from __future__ import unicode_literals
 import json
 import random
 import re
-import time
 
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
+)
+from ..utils import (
     ExtractorError,
 )
 
@@ -115,7 +116,7 @@ class EightTracksIE(InfoExtractor):
         mix_id = data['id']
         track_count = data['tracks_count']
         duration = data['duration']
-        avg_song_duration = duration / track_count
+        avg_song_duration = float(duration) / track_count
         first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id)
         next_url = first_url
         entries = []
@@ -136,7 +137,7 @@ class EightTracksIE(InfoExtractor):
                         raise
                     else:
                         download_tries += 1
-                        time.sleep(avg_song_duration)
+                        self._sleep(avg_song_duration, playlist_id)
 
             api_data = json.loads(api_json)
             track_data = api_data['set']['track']

From 6ef9f88299e1595bbe467fb527ada9b003ae5779 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 05:51:22 +0100
Subject: [PATCH 1124/1937] release 2015.01.10

---
 README.md             | 9 +++++++++
 youtube_dl/version.py | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 3d632a553..87aaec8a3 100644
--- a/README.md
+++ b/README.md
@@ -272,6 +272,10 @@ which means you can modify it, redistribute it or use it however you like.
     -F, --list-formats               list all available formats
     --youtube-skip-dash-manifest     Do not download the DASH manifest on
                                      YouTube videos
+    --merge-output-format FORMAT     If a merge is required (e.g.
+                                     bestvideo+bestaudio), output to given
+                                     container format. One of mkv, mp4, ogg,
+                                     webm, flv.Ignored if no merge is required
 
 ## Subtitle Options:
     --write-sub                      write subtitle file
@@ -318,6 +322,11 @@ which means you can modify it, redistribute it or use it however you like.
     --add-metadata                   write metadata to the video file
     --xattrs                         write metadata to the video file's xattrs
                                      (using dublin core and xdg standards)
+    --fixup POLICY                   (experimental) Automatically correct known
+                                     faults of the file. One of never (do
+                                     nothing), warn (only emit a warning),
+                                     detect_or_warn(check whether we can do
+                                     anything about it, warn otherwise
     --prefer-avconv                  Prefer avconv over ffmpeg for running the
                                      postprocessors (default)
     --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8c57c7413..60258d36a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.09.2'
+__version__ = '2015.01.10'

From 52afb2ac1b6551350cf4d28ad0d758a136186297 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 06:13:18 +0100
Subject: [PATCH 1125/1937] [ffmpeg] Call encodeFilename on filenames

---
 youtube_dl/postprocessor/ffmpeg.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 8bf5bebc3..26b99e43c 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -82,7 +82,8 @@ class FFmpegPostProcessor(PostProcessor):
     def run_ffmpeg_multiple_files(self, input_paths, out_path, opts):
         self.check_version()
 
-        oldest_mtime = min(os.stat(path).st_mtime for path in input_paths)
+        oldest_mtime = min(
+            os.stat(encodeFilename(path)).st_mtime for path in input_paths)
 
         files_cmd = []
         for path in input_paths:
@@ -100,7 +101,7 @@ class FFmpegPostProcessor(PostProcessor):
             stderr = stderr.decode('utf-8', 'replace')
             msg = stderr.strip().split('\n')[-1]
             raise FFmpegPostProcessorError(msg)
-        os.utime(out_path, (oldest_mtime, oldest_mtime))
+        os.utime(encodeFilename(out_path), (oldest_mtime, oldest_mtime))
         if self._deletetempfiles:
             for ipath in input_paths:
                 os.remove(ipath)

From b9f030cc26891fa8ee680742f5db17f819e5e81a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 14:23:54 +0100
Subject: [PATCH 1126/1937] [orf] Fix typo

---
 youtube_dl/extractor/orf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 4fed83bd6..3d2e5d2c9 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -160,7 +160,7 @@ class ORFOE1IE(InfoExtractor):
 
 
 class ORFFM4IE(InfoExtractor):
-    IE_DESC = 'orf:fm4'
+    IE_NAME = 'orf:fm4'
     IE_DESC = 'radio FM4'
     _VALID_URL = r'http://fm4\.orf\.at/7tage/?#(?P<date>[0-9]+)/(?P<show>\w+)'
 

From 385f8ae4684e7c7fd04d66b6066d980c93e2d79f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 14:25:11 +0100
Subject: [PATCH 1127/1937] [eighttracks] PEP8

---
 youtube_dl/extractor/eighttracks.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index a9b18195b..fb5dbbe2b 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -122,7 +122,6 @@ class EightTracksIE(InfoExtractor):
         entries = []
 
         for i in range(track_count):
-            
             api_json = None
             download_tries = 0
 

From 27ca82ebc66c91359173e45b25d992118511e330 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 14:27:27 +0100
Subject: [PATCH 1128/1937] [orf:oe1] Add konsole URL schema (Fixes #4675)

---
 youtube_dl/extractor/orf.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 3d2e5d2c9..4e293392b 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -128,13 +128,16 @@ class ORFTVthekIE(InfoExtractor):
         }
 
 
-# Audios on ORF radio are only available for 7 days, so we can't add tests.
-
-
 class ORFOE1IE(InfoExtractor):
     IE_NAME = 'orf:oe1'
     IE_DESC = 'Radio Österreich 1'
-    _VALID_URL = r'http://oe1\.orf\.at/programm/(?P<id>[0-9]+)'
+    _VALID_URL = r'http://oe1\.orf\.at/(?:programm/|konsole.*?#\?track_id=)(?P<id>[0-9]+)'
+
+    # Audios on ORF radio are only available for 7 days, so we can't add tests.
+    _TEST = {
+        'url': 'http://oe1.orf.at/konsole?show=on_demand#?track_id=394211',
+        'only_matching': True,
+    }
 
     def _real_extract(self, url):
         show_id = self._match_id(url)

From 776dc3992a8995d22c172f7616d4ffd9cabc884e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Jan 2015 19:43:52 +0600
Subject: [PATCH 1129/1937] [utils] Clarify more day-month-first ambiguous
 formats

---
 youtube_dl/utils.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 079e8d2c3..a12b0a7de 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -679,9 +679,6 @@ def unified_strdate(date_str, day_first=True):
         '%b %dth %Y %I:%M%p',
         '%Y-%m-%d',
         '%Y/%m/%d',
-        '%d.%m.%Y',
-        '%d/%m/%Y',
-        '%d/%m/%y',
         '%Y/%m/%d %H:%M:%S',
         '%Y-%m-%d %H:%M:%S',
         '%Y-%m-%d %H:%M:%S.%f',
@@ -696,10 +693,16 @@ def unified_strdate(date_str, day_first=True):
     ]
     if day_first:
         format_expressions.extend([
+            '%d.%m.%Y',
+            '%d/%m/%Y',
+            '%d/%m/%y',
             '%d/%m/%Y %H:%M:%S',
         ])
     else:
         format_expressions.extend([
+            '%m.%d.%Y',
+            '%m/%d/%Y',
+            '%m/%d/%y',
             '%m/%d/%Y %H:%M:%S',
         ])
     for expression in format_expressions:

From d89c6e336a3a20085b4ea35f2afcfcecedab5c53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 10 Jan 2015 19:44:29 +0600
Subject: [PATCH 1130/1937] [atttechchannel] Add extractor (Closes #3938)

---
 youtube_dl/extractor/__init__.py       |  1 +
 youtube_dl/extractor/atttechchannel.py | 55 ++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/atttechchannel.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a8579d083..b908eb38c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -26,6 +26,7 @@ from .arte import (
     ArteTVEmbedIE,
 )
 from .atresplayer import AtresPlayerIE
+from .atttechchannel import ATTTechChannelIE
 from .audiomack import AudiomackIE, AudiomackAlbumIE
 from .auengine import AUEngineIE
 from .azubu import AzubuIE
diff --git a/youtube_dl/extractor/atttechchannel.py b/youtube_dl/extractor/atttechchannel.py
new file mode 100644
index 000000000..b01d35bb2
--- /dev/null
+++ b/youtube_dl/extractor/atttechchannel.py
@@ -0,0 +1,55 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import unified_strdate
+
+
+class ATTTechChannelIE(InfoExtractor):
+    _VALID_URL = r'https?://techchannel\.att\.com/play-video\.cfm/([^/]+/)*(?P<id>.+)'
+    _TEST = {
+        'url': 'http://techchannel.att.com/play-video.cfm/2014/1/27/ATT-Archives-The-UNIX-System-Making-Computers-Easier-to-Use',
+        'info_dict': {
+            'id': '11316',
+            'display_id': 'ATT-Archives-The-UNIX-System-Making-Computers-Easier-to-Use',
+            'ext': 'flv',
+            'title': 'AT&T Archives : The UNIX System: Making Computers Easier to Use',
+            'description': 'A 1982 film about UNIX is the foundation for software in use around Bell Labs and AT&T.',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20140127',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_url = self._search_regex(
+            r"url\s*:\s*'(rtmp://[^']+)'",
+            webpage, 'video URL')
+
+        video_id = self._search_regex(
+            r'mediaid\s*=\s*(\d+)',
+            webpage, 'video id', fatal=False)
+
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+        upload_date = unified_strdate(self._search_regex(
+            r'[Rr]elease\s+date:\s*(\d{1,2}/\d{1,2}/\d{4})',
+            webpage, 'upload date', fatal=False), False)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'url': video_url,
+            'ext': 'flv',
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+        }

From 76bfaf6daf73ac74f7a9068c51d14ab0a5ec5e52 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 17:46:01 +0100
Subject: [PATCH 1131/1937] [nrk] Improve subtitle support (#3092)

---
 youtube_dl/extractor/nrk.py | 42 ++++++++++++++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 321ce5ce7..f6de26022 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -7,8 +7,10 @@ from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     float_or_none,
+    parse_duration,
     unified_strdate,
 )
+from .subtitles import SubtitlesInfoExtractor
 
 
 class NRKIE(InfoExtractor):
@@ -71,8 +73,8 @@ class NRKIE(InfoExtractor):
         }
 
 
-class NRKTVIE(InfoExtractor):
-    _VALID_URL = r'http://tv\.nrk(?:super)?\.no/(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
+class NRKTVIE(SubtitlesInfoExtractor):
+    _VALID_URL = r'(?P<baseurl>http://tv\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
 
     _TESTS = [
         {
@@ -147,6 +149,29 @@ class NRKTVIE(InfoExtractor):
         }
     ]
 
+    def _seconds2str(self, s):
+        return '%02d:%02d:%02d.%03d' % (s / 3600, (s % 3600) / 60, s % 60, (s % 1) * 1000)
+
+    def _debug_print(self, txt):
+        if self._downloader.params.get('verbose', False):
+            self.to_screen('[debug] %s' % txt)
+
+    def _extract_captions(self, subtitlesurl, video_id, baseurl):
+        url = "%s%s" % (baseurl, subtitlesurl)
+        self._debug_print('%s: Subtitle url: %s' % (video_id, url))
+        captions = self._download_xml(url, video_id, 'Downloading subtitles')
+        lang = captions.get('lang', 'no')
+        ps = captions.findall('./{0}body/{0}div/{0}p'.format('{http://www.w3.org/ns/ttml}'))
+        srt = ''
+        for pos, p in enumerate(ps):
+            begin = parse_duration(p.get('begin'))
+            duration = parse_duration(p.get('dur'))
+            starttime = self._seconds2str(begin)
+            endtime = self._seconds2str(begin + duration)
+            text = '\n'.join(p.itertext())
+            srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (str(pos), starttime, endtime, text)
+        return {lang: srt}
+
     def _extract_f4m(self, manifest_url, video_id):
         return self._extract_f4m_formats(manifest_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124', video_id)
 
@@ -154,6 +179,7 @@ class NRKTVIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         part_id = mobj.group('part_id')
+        baseurl = mobj.group('baseurl')
 
         webpage = self._download_webpage(url, video_id)
 
@@ -210,9 +236,18 @@ class NRKTVIE(InfoExtractor):
         m3u8_url = re.search(r'data-hls-media="([^"]+)"', webpage)
         if m3u8_url:
             formats.extend(self._extract_m3u8_formats(m3u8_url.group(1), video_id, 'mp4'))
-
         self._sort_formats(formats)
 
+        subtitles_url = self._html_search_regex(
+            r'data-subtitlesurl[ ]*=[ ]*"([^"]+)"',
+            webpage, 'subtitle URL', default=None)
+        subtitles = None
+        if subtitles_url:
+            subtitles = self._extract_captions(subtitles_url, video_id, baseurl)
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, subtitles)
+            return
+
         return {
             'id': video_id,
             'title': title,
@@ -221,4 +256,5 @@ class NRKTVIE(InfoExtractor):
             'upload_date': upload_date,
             'duration': duration,
             'formats': formats,
+            'subtitles': subtitles,
         }

From ea1d5bdcddc4b9bdac0ef5555f080484c33467a9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 17:57:21 +0100
Subject: [PATCH 1132/1937] [rte] Make more robust and add a new testcase
 (#4015)

---
 youtube_dl/extractor/rte.py | 38 ++++++++++++++++++++-----------------
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index 8d4e94e55..b5a67373b 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -9,34 +9,38 @@ from ..utils import (
 
 
 class RteIE(InfoExtractor):
-    _VALID_URL = r'http?://(?:www\.)?rte\.ie/player/in/show/(?P<id>[0-9]+)/'
+    _VALID_URL = r'http?://(?:www\.)?rte\.ie/player/[^/]{2,3}/show/(?P<id>[0-9]+)/'
     _TEST = {
-        'url': 'http://www.rte.ie/player/in/show/10336191/',
+        'url': 'http://www.rte.ie/player/de/show/10363114/',
         'info_dict': {
-            'id': '10336191',
+            'id': '10363114',
             'ext': 'mp4',
-            'title': 'Nine News',
+            'title': 'One News',
             'thumbnail': 're:^https?://.*\.jpg$',
             'description': 'The One O\'Clock News followed by Weather.',
-            'duration': 1622.963,
+            'duration': 436.844,
+        },
+        'params': {
+            'skip_download': 'f4m fails with --test atm'
         }
     }
     
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         webpage = self._download_webpage(url, video_id)
+
         title = self._og_search_title(webpage)
-        
-        description = self._search_regex(r'<meta name="description" content="(.*?)" />', webpage, 'description')
-        duration = float_or_none(self._html_search_meta('duration', webpage, 'duration'), 1000)
-        
-        thumbnail_id = self._search_regex(r'<meta name="thumbnail" content="uri:irus:(.*?)" />', webpage, 'thumbnail')
-        thumbnail = 'http://img.rasset.ie/' + thumbnail_id + '.jpg' 
-        
+        description = self._html_search_meta('description', webpage, 'description')
+        duration = float_or_none(self._html_search_meta(
+            'duration', webpage, 'duration', fatal=False), 1000)
+
+        thumbnail_id = self._search_regex(
+            r'<meta name="thumbnail" content="uri:irus:(.*?)" />', webpage, 'thumbnail')
+        thumbnail = 'http://img.rasset.ie/' + thumbnail_id + '.jpg'
+
         feeds_url = self._html_search_meta("feeds-prefix", webpage, 'feeds url') + video_id
         json_string = self._download_json(feeds_url, video_id)
-        
+
         # f4m_url = server + relative_url
         f4m_url = json_string['shows'][0]['media:group'][0]['rte:server'] + json_string['shows'][0]['media:group'][0]['url']
         f4m_formats = self._extract_f4m_formats(f4m_url, video_id)
@@ -46,8 +50,8 @@ class RteIE(InfoExtractor):
             'ext': 'mp4',
             'width': f['width'],
             'height': f['height'],
-        } for f in f4m_formats ]
-        
+        } for f in f4m_formats]
+
         return {
             'id': video_id,
             'title': title,
@@ -55,4 +59,4 @@ class RteIE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
             'duration': duration,
-        }
\ No newline at end of file
+        }

From 3462af03e6c8c64b916fe90ecea0ef7719438ad2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 17:59:07 +0100
Subject: [PATCH 1133/1937] [rte] PEP8

---
 youtube_dl/extractor/rte.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rte.py b/youtube_dl/extractor/rte.py
index b5a67373b..04158b993 100644
--- a/youtube_dl/extractor/rte.py
+++ b/youtube_dl/extractor/rte.py
@@ -24,7 +24,7 @@ class RteIE(InfoExtractor):
             'skip_download': 'f4m fails with --test atm'
         }
     }
-    
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)

From c80ede5f1380ce49cf8ad26d84e4eb1aa2d7c004 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 18:03:36 +0100
Subject: [PATCH 1134/1937] [karaoketv] Simplify (#3853)

---
 youtube_dl/extractor/karaoketv.py | 32 +++++++++++++------------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/karaoketv.py b/youtube_dl/extractor/karaoketv.py
index 4d50308cc..4856200d8 100644
--- a/youtube_dl/extractor/karaoketv.py
+++ b/youtube_dl/extractor/karaoketv.py
@@ -1,12 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-import json
-import sys
-
 from .common import InfoExtractor
-from ..utils import compat_urllib_parse, ExtractorError
+from ..compat import compat_urllib_parse
+from ..utils import (
+    ExtractorError,
+    js_to_json,
+)
 
 
 class KaraoketvIE(InfoExtractor):
@@ -21,22 +21,16 @@ class KaraoketvIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-
-        # BUG: SSL23_GET_SERVER_HELLO:unknown protocol 
-        if sys.hexversion < 0x03000000:
-            raise ExtractorError("Only python 3 supported.\n")
-
-        mobj = re.match(self._VALID_URL, url)
-        
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        settings_json = compat_urllib_parse.unquote_plus(self._search_regex(r'config=(.*)', self._og_search_video_url(webpage ,video_id), ''))
-        
-        urls_info_webpage = self._download_webpage(settings_json, 'Downloading settings json')
+        page_video_url = self._og_search_video_url(webpage, video_id)
+        config_json = compat_urllib_parse.unquote_plus(self._search_regex(
+            r'config=(.*)', page_video_url, 'configuration'))
 
-        urls_info_json = json.loads(urls_info_webpage.replace('\'', '"'))
+        urls_info_json = self._download_json(
+            config_json, video_id, 'Downloading configuration',
+            transform_source=js_to_json)
 
         url = urls_info_json['playlist'][0]['url']
 
@@ -44,4 +38,4 @@ class KaraoketvIE(InfoExtractor):
             'id': video_id,
             'title': self._og_search_title(webpage),
             'url': url,
-        }
\ No newline at end of file
+        }

From 6ce08764a1faba3eeecbd2a04643f3266c8c5111 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 18:58:03 +0100
Subject: [PATCH 1135/1937] Credit @dinesh for rte.ie (#4015)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 8f2010803..1e4cadae5 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -101,3 +101,4 @@ Thijs Vermeir
 Joel Leclerc
 Christopher Krooss
 Ondřej Caletka
+Dinesh S

From ed3958d714d8e68b3b48697333c72ff6438002f4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jan 2015 00:40:46 +0600
Subject: [PATCH 1136/1937] [collegerama] Add extractor (#4540)

---
 youtube_dl/extractor/__init__.py    |  1 +
 youtube_dl/extractor/collegerama.py | 92 +++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+)
 create mode 100644 youtube_dl/extractor/collegerama.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 647e0a8c2..2f3a322a7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -70,6 +70,7 @@ from .cnn import (
     CNNArticleIE,
 )
 from .collegehumor import CollegeHumorIE
+from .collegerama import CollegeRamaIE
 from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
 from .comcarcoff import ComCarCoffIE
 from .commonmistakes import CommonMistakesIE
diff --git a/youtube_dl/extractor/collegerama.py b/youtube_dl/extractor/collegerama.py
new file mode 100644
index 000000000..fedd48490
--- /dev/null
+++ b/youtube_dl/extractor/collegerama.py
@@ -0,0 +1,92 @@
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_request
+from ..utils import (
+    float_or_none,
+    int_or_none,
+)
+
+
+class CollegeRamaIE(InfoExtractor):
+    _VALID_URL = r'https?://collegerama\.tudelft\.nl/Mediasite/Play/(?P<id>[\da-f]+)'
+    _TESTS = [
+        {
+            'url': 'https://collegerama.tudelft.nl/Mediasite/Play/585a43626e544bdd97aeb71a0ec907a01d',
+            'md5': '481fda1c11f67588c0d9d8fbdced4e39',
+            'info_dict': {
+                'id': '585a43626e544bdd97aeb71a0ec907a01d',
+                'ext': 'mp4',
+                'title': 'Een nieuwe wereld: waarden, bewustzijn en techniek van de mensheid 2.0.',
+                'description': '',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 7713.088,
+                'timestamp': 1413309600,
+                'upload_date': '20141014',
+            },
+        },
+        {
+            'url': 'https://collegerama.tudelft.nl/Mediasite/Play/86a9ea9f53e149079fbdb4202b521ed21d?catalog=fd32fd35-6c99-466c-89d4-cd3c431bc8a4',
+            'md5': 'ef1fdded95bdf19b12c5999949419c92',
+            'info_dict': {
+                'id': '86a9ea9f53e149079fbdb4202b521ed21d',
+                'ext': 'wmv',
+                'title': '64ste Vakantiecursus: Afvalwater',
+                'description': 'md5:7fd774865cc69d972f542b157c328305',
+                'duration': 10853,
+                'timestamp': 1326446400,
+                'upload_date': '20120113',
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        player_options_request = {
+            "getPlayerOptionsRequest": {
+                "ResourceId": video_id,
+                "QueryString": "",
+            }
+        }
+
+        request = compat_urllib_request.Request(
+            'http://collegerama.tudelft.nl/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions',
+            json.dumps(player_options_request))
+        request.add_header('Content-Type', 'application/json')
+
+        player_options = self._download_json(request, video_id)
+
+        presentation = player_options['d']['Presentation']
+        title = presentation['Title']
+        description = presentation.get('Description')
+        thumbnail = None
+        duration = float_or_none(presentation.get('Duration'), 1000)
+        timestamp = int_or_none(presentation.get('UnixTime'), 1000)
+
+        formats = []
+        for stream in presentation['Streams']:
+            for video in stream['VideoUrls']:
+                thumbnail_url = stream.get('ThumbnailUrl')
+                if thumbnail_url:
+                    thumbnail = 'http://collegerama.tudelft.nl' + thumbnail_url
+                format_id = video['MediaType']
+                if format_id == 'SS':
+                    continue
+                formats.append({
+                    'url': video['Location'],
+                    'format_id': format_id,
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'timestamp': timestamp,
+            'formats': formats,
+        }

From be4a824d74add1a3b78b8244dff12f4f078f168a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 19:55:36 +0100
Subject: [PATCH 1137/1937] Add new option --source-address

Closes #3618, fixes #721, fixes #2481, fixes #4551, closes #1020.
---
 youtube_dl/YoutubeDL.py |  6 ++---
 youtube_dl/__init__.py  |  1 +
 youtube_dl/compat.py    | 28 +++++++++++++++++++++
 youtube_dl/options.py   | 24 ++++++++++++------
 youtube_dl/update.py    |  2 +-
 youtube_dl/utils.py     | 56 +++++++++++++++++++++++++++++++++++------
 6 files changed, 98 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4cc3ec2fb..fd2c0e044 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -211,6 +211,7 @@ class YoutubeDL(object):
                        - "warn": only emit a warning
                        - "detect_or_warn": check whether we can do anything
                                            about it, warn otherwise
+    source_address:    (Experimental) Client-side IP address to bind to.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
@@ -1493,9 +1494,8 @@ class YoutubeDL(object):
         proxy_handler = compat_urllib_request.ProxyHandler(proxies)
 
         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
-        https_handler = make_HTTPS_handler(
-            self.params.get('nocheckcertificate', False), debuglevel=debuglevel)
-        ydlh = YoutubeDLHandler(debuglevel=debuglevel)
+        https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
+        ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
         opener = compat_urllib_request.build_opener(
             https_handler, proxy_handler, cookie_processor, ydlh)
         # Delete the default user-agent header, which would otherwise apply in
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 659a92a3b..d74a304b7 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -327,6 +327,7 @@ def _real_main(argv=None):
         'merge_output_format': opts.merge_output_format,
         'postprocessors': postprocessors,
         'fixup': opts.fixup,
+        'source_address': opts.source_address,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 46d438846..44a902573 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -4,6 +4,7 @@ import getpass
 import optparse
 import os
 import re
+import socket
 import subprocess
 import sys
 
@@ -307,6 +308,32 @@ else:
     compat_kwargs = lambda kwargs: kwargs
 
 
+if sys.version_info < (2, 7):
+    def compat_socket_create_connection(address, timeout, source_address=None):
+        host, port = address
+        err = None
+        for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
+            af, socktype, proto, canonname, sa = res
+            sock = None
+            try:
+                sock = socket.socket(af, socktype, proto)
+                sock.settimeout(timeout)
+                if source_address:
+                    sock.bind(source_address)
+                sock.connect(sa)
+                return sock
+            except socket.error as _:
+                err = _
+                if sock is not None:
+                    sock.close()
+        if err is not None:
+            raise err
+        else:
+            raise error("getaddrinfo returns an empty list")
+else:
+    compat_socket_create_connection = socket.create_connection
+
+
 # Fix https://github.com/rg3/youtube-dl/issues/4223
 # See http://bugs.python.org/issue9161 for what is broken
 def workaround_optparse_bug9161():
@@ -343,6 +370,7 @@ __all__ = [
     'compat_parse_qs',
     'compat_print',
     'compat_str',
+    'compat_socket_create_connection',
     'compat_subprocess_get_DEVNULL',
     'compat_urllib_error',
     'compat_urllib_parse',
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index e5602bb3a..31351d43d 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -148,14 +148,6 @@ def parseOpts(overrideArguments=None):
         '--extractor-descriptions',
         action='store_true', dest='list_extractor_descriptions', default=False,
         help='Output descriptions of all supported extractors')
-    general.add_option(
-        '--proxy', dest='proxy',
-        default=None, metavar='URL',
-        help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
-    general.add_option(
-        '--socket-timeout',
-        dest='socket_timeout', type=float, default=None,
-        help='Time to wait before giving up, in seconds')
     general.add_option(
         '--default-search',
         dest='default_search', metavar='PREFIX',
@@ -173,6 +165,21 @@ def parseOpts(overrideArguments=None):
         default=False,
         help='Do not extract the videos of a playlist, only list them.')
 
+    network = optparse.OptionGroup(parser, 'Network Options')
+    network.add_option(
+        '--proxy', dest='proxy',
+        default=None, metavar='URL',
+        help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection')
+    network.add_option(
+        '--socket-timeout',
+        dest='socket_timeout', type=float, default=None, metavar='SECONDS',
+        help='Time to wait before giving up, in seconds')
+    network.add_option(
+        '--source-address',
+        metavar='IP', dest='source_address', default=None,
+        help='Client-side IP address to bind to (experimental)',
+    )
+
     selection = optparse.OptionGroup(parser, 'Video Selection')
     selection.add_option(
         '--playlist-start',
@@ -652,6 +659,7 @@ def parseOpts(overrideArguments=None):
         help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'')
 
     parser.add_option_group(general)
+    parser.add_option_group(network)
     parser.add_option_group(selection)
     parser.add_option_group(downloader)
     parser.add_option_group(filesystem)
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 3f9c5249d..d8be4049f 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -59,7 +59,7 @@ def update_self(to_screen, verbose):
         to_screen('It looks like you installed youtube-dl with a package manager, pip, setup.py or a tarball. Please use that to update.')
         return
 
-    https_handler = make_HTTPS_handler(False)
+    https_handler = make_HTTPS_handler({})
     opener = compat_urllib_request.build_opener(https_handler)
 
     # Check if there is a new version
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a12b0a7de..cc5f510f4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -10,6 +10,7 @@ import ctypes
 import datetime
 import email.utils
 import errno
+import functools
 import gzip
 import itertools
 import io
@@ -34,7 +35,9 @@ from .compat import (
     compat_chr,
     compat_getenv,
     compat_html_entities,
+    compat_http_client,
     compat_parse_qs,
+    compat_socket_create_connection,
     compat_str,
     compat_urllib_error,
     compat_urllib_parse,
@@ -391,13 +394,14 @@ def formatSeconds(secs):
         return '%d' % secs
 
 
-def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
+def make_HTTPS_handler(params, **kwargs):
+    opts_no_check_certificate = params.get('nocheckcertificate', False)
     if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
         context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
         if opts_no_check_certificate:
             context.verify_mode = ssl.CERT_NONE
         try:
-            return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
+            return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
         except TypeError:
             # Python 2.7.8
             # (create_default_context present but HTTPSHandler has no context=)
@@ -420,17 +424,14 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs):
                 except ssl.SSLError:
                     self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23)
 
-        class HTTPSHandlerV3(compat_urllib_request.HTTPSHandler):
-            def https_open(self, req):
-                return self.do_open(HTTPSConnectionV3, req)
-        return HTTPSHandlerV3(**kwargs)
+        return YoutubeDLHTTPSHandler(params, https_conn_class=HTTPSConnectionV3, **kwargs)
     else:  # Python < 3.4
         context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
         context.verify_mode = (ssl.CERT_NONE
                                if opts_no_check_certificate
                                else ssl.CERT_REQUIRED)
         context.set_default_verify_paths()
-        return compat_urllib_request.HTTPSHandler(context=context, **kwargs)
+        return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
 
 
 class ExtractorError(Exception):
@@ -544,6 +545,26 @@ class ContentTooShortError(Exception):
         self.expected = expected
 
 
+def _create_http_connection(ydl_handler, http_class, is_https=False, *args, **kwargs):
+    hc = http_class(*args, **kwargs)
+    source_address = ydl_handler._params.get('source_address')
+    if source_address is not None:
+        sa = (source_address, 0)
+        if hasattr(hc, 'source_address'):  # Python 2.7+
+            hc.source_address = sa
+        else:  # Python 2.6
+            def _hc_connect(self, *args, **kwargs):
+                sock = compat_socket_create_connection(
+                    (self.host, self.port), self.timeout, sa)
+                if is_https:
+                    self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+                else:
+                    self.sock = sock
+            hc.connect = functools.partial(_hc_connect, hc)
+
+    return hc
+
+
 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     """Handler for HTTP requests and responses.
 
@@ -562,6 +583,15 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     public domain.
     """
 
+    def __init__(self, params, *args, **kwargs):
+        compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
+        self._params = params
+
+    def http_open(self, req):
+        return self.do_open(functools.partial(
+            _create_http_connection, self, compat_http_client.HTTPConnection),
+            req)
+
     @staticmethod
     def deflate(data):
         try:
@@ -631,6 +661,18 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     https_response = http_response
 
 
+class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
+    def __init__(self, params, https_conn_class=None, *args, **kwargs):
+        compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
+        self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
+        self._params = params
+
+    def https_open(self, req):
+        return self.do_open(functools.partial(
+            _create_http_connection, self, self._https_conn_class, True),
+            req)
+
+
 def parse_iso8601(date_str, delimiter='T'):
     """ Return a UNIX timestamp from the given date """
 

From 500b8b41c1a1c27da64def50514c247e05f6fbbe Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 20:02:02 +0100
Subject: [PATCH 1138/1937] [options] Add -4 and -6 options

Fixes #520, fixes #3626.
---
 youtube_dl/options.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 31351d43d..e25466c39 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -179,6 +179,16 @@ def parseOpts(overrideArguments=None):
         metavar='IP', dest='source_address', default=None,
         help='Client-side IP address to bind to (experimental)',
     )
+    network.add_option(
+        '-4', '--force-ipv4',
+        action='store_const', const='0.0.0.0', dest='source_address',
+        help='Make all connections via IPv4 (experimental)',
+    )
+    network.add_option(
+        '-6', '--force-ipv6',
+        action='store_const', const='::', dest='source_address',
+        help='Make all connections via IPv6 (experimental)',
+    )
 
     selection = optparse.OptionGroup(parser, 'Video Selection')
     selection.add_option(

From c5a59d9391d23fc2f13cd87ba27982c98caa29ff Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 20:05:28 +0100
Subject: [PATCH 1139/1937] [utils] Fix call to _create_http_connection

Avoid confusion over args/kwargs.
---
 youtube_dl/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index cc5f510f4..42f0b07ce 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -545,7 +545,7 @@ class ContentTooShortError(Exception):
         self.expected = expected
 
 
-def _create_http_connection(ydl_handler, http_class, is_https=False, *args, **kwargs):
+def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
     hc = http_class(*args, **kwargs)
     source_address = ydl_handler._params.get('source_address')
     if source_address is not None:
@@ -589,7 +589,7 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
 
     def http_open(self, req):
         return self.do_open(functools.partial(
-            _create_http_connection, self, compat_http_client.HTTPConnection),
+            _create_http_connection, self, compat_http_client.HTTPConnection, False),
             req)
 
     @staticmethod

From 149f05c7b6adcb55b96c4ca60abc4e42d98623a7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 20:06:13 +0100
Subject: [PATCH 1140/1937] release 2015.01.10.1

---
 README.md             | 16 ++++++++++++----
 youtube_dl/version.py |  2 +-
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 87aaec8a3..c56d906b5 100644
--- a/README.md
+++ b/README.md
@@ -60,10 +60,6 @@ which means you can modify it, redistribute it or use it however you like.
                                      they would handle
     --extractor-descriptions         Output descriptions of all supported
                                      extractors
-    --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in
-                                     an empty string (--proxy "") for direct
-                                     connection
-    --socket-timeout None            Time to wait before giving up, in seconds
     --default-search PREFIX          Use this prefix for unqualified URLs. For
                                      example "gvsearch2:" downloads two videos
                                      from google videos for  youtube-dl "large
@@ -82,6 +78,18 @@ which means you can modify it, redistribute it or use it however you like.
     --flat-playlist                  Do not extract the videos of a playlist,
                                      only list them.
 
+## Network Options:
+    --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in
+                                     an empty string (--proxy "") for direct
+                                     connection
+    --socket-timeout SECONDS         Time to wait before giving up, in seconds
+    --source-address IP              Client-side IP address to bind to
+                                     (experimental)
+    -4, --force-ipv4                 Make all connections via IPv4
+                                     (experimental)
+    -6, --force-ipv6                 Make all connections via IPv6
+                                     (experimental)
+
 ## Video Selection:
     --playlist-start NUMBER          playlist video to start at (default is 1)
     --playlist-end NUMBER            playlist video to end at (default is last)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 60258d36a..267ffeeb4 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.10'
+__version__ = '2015.01.10.1'

From 58b1f00d19d608470540b111f3825eefafbdae97 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 21:02:27 +0100
Subject: [PATCH 1141/1937] [YoutubeDL] Add new --call-home option for
 debugging

---
 youtube_dl/YoutubeDL.py | 14 ++++++++++++++
 youtube_dl/__init__.py  |  1 +
 youtube_dl/options.py   |  4 ++++
 3 files changed, 19 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index fd2c0e044..84cea472c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -58,6 +58,7 @@ from .utils import (
     takewhile_inclusive,
     UnavailableVideoError,
     url_basename,
+    version_tuple,
     write_json_file,
     write_string,
     YoutubeDLHandler,
@@ -212,6 +213,8 @@ class YoutubeDL(object):
                        - "detect_or_warn": check whether we can do anything
                                            about it, warn otherwise
     source_address:    (Experimental) Client-side IP address to bind to.
+    call_home:         (Experimental) Boolean, true iff we are allowed to
+                       contact the youtube-dl servers for debugging.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
@@ -1464,6 +1467,17 @@ class YoutubeDL(object):
                 proxy_map.update(handler.proxies)
         self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
 
+        if self.params.get('call_home', False):
+            ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
+            self._write_string('[debug] Public IP address: %s\n' % ipaddr)
+            latest_version = self.urlopen(
+                'https://yt-dl.org/latest/version').read().decode('utf-8')
+            if version_tuple(latest_version) > version_tuple(__version__):
+                self.report_warning(
+                    'You are using an outdated version (newest version: %s)! '
+                    'See https://yt-dl.org/update if you need help updating.' %
+                    latest_version)
+
     def _setup_opener(self):
         timeout_val = self.params.get('socket_timeout')
         self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index d74a304b7..ddf6260d1 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -328,6 +328,7 @@ def _real_main(argv=None):
         'postprocessors': postprocessors,
         'fixup': opts.fixup,
         'source_address': opts.source_address,
+        'call_home': opts.call_home,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index e25466c39..0176bae75 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -505,6 +505,10 @@ def parseOpts(overrideArguments=None):
         '--print-traffic',
         dest='debug_printtraffic', action='store_true', default=False,
         help='Display sent and read HTTP traffic')
+    verbosity.add_option(
+        '-C', '--call-home',
+        dest='call_home', action='store_true', default=False,
+        help='Contact the youtube-dl server for debugging. (Experimental)')
 
     filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
     filesystem.add_option(

From cd3063f3fa81f1ca30ff9994b7346ad7ebadd0fe Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 21:03:00 +0100
Subject: [PATCH 1142/1937] release 2015.01.10.2

---
 README.md             | 2 ++
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c56d906b5..3113f70ee 100644
--- a/README.md
+++ b/README.md
@@ -239,6 +239,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      files in the current directory to debug
                                      problems
     --print-traffic                  Display sent and read HTTP traffic
+    -C, --call-home                  Contact the youtube-dl server for
+                                     debugging. (Experimental)
 
 ## Workarounds:
     --encoding ENCODING              Force the specified encoding (experimental)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 267ffeeb4..e5e320c76 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.10.1'
+__version__ = '2015.01.10.2'

From c071733fd441109633bb9bb57007fae754b7610d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 21:07:44 +0100
Subject: [PATCH 1143/1937] [README] Highlight that bug reports should include
 the -v output

---
 README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 3113f70ee..006e8d709 100644
--- a/README.md
+++ b/README.md
@@ -627,7 +627,9 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
 
 Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues> . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the irc channel #youtube-dl on freenode.
 
-Please include the full output of the command when run with `--verbose`. The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
+**Please include the full output of youtube-dl when run with `-v`**.
+
+The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
 
 Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist):
 

From 8bfa75451b6506db324e766ef5d815d10eee5619 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 10 Jan 2015 21:09:15 +0100
Subject: [PATCH 1144/1937] [options] Add --no-call-home

While we're at it, also drop "experimental" moniker for --call-home - should work fine.
---
 youtube_dl/YoutubeDL.py | 4 ++--
 youtube_dl/options.py   | 6 +++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 84cea472c..47e143d89 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -213,8 +213,8 @@ class YoutubeDL(object):
                        - "detect_or_warn": check whether we can do anything
                                            about it, warn otherwise
     source_address:    (Experimental) Client-side IP address to bind to.
-    call_home:         (Experimental) Boolean, true iff we are allowed to
-                       contact the youtube-dl servers for debugging.
+    call_home:         Boolean, true iff we are allowed to contact the
+                       youtube-dl servers for debugging.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 0176bae75..a30974efd 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -508,7 +508,11 @@ def parseOpts(overrideArguments=None):
     verbosity.add_option(
         '-C', '--call-home',
         dest='call_home', action='store_true', default=False,
-        help='Contact the youtube-dl server for debugging. (Experimental)')
+        help='Contact the youtube-dl server for debugging.')
+    verbosity.add_option(
+        '--no-call-home',
+        dest='call_home', action='store_false', default=False,
+        help='Do NOT contact the youtube-dl server for debugging.')
 
     filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
     filesystem.add_option(

From 987493aef37cce273a10ddfbe540a41a4c6a67cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jan 2015 15:13:03 +0600
Subject: [PATCH 1145/1937] [test_compat] Fix alphabetic order to make
 test_all_present pass

---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 44a902573..07ceaa170 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -369,8 +369,8 @@ __all__ = [
     'compat_ord',
     'compat_parse_qs',
     'compat_print',
-    'compat_str',
     'compat_socket_create_connection',
+    'compat_str',
     'compat_subprocess_get_DEVNULL',
     'compat_urllib_error',
     'compat_urllib_parse',

From d156a1d981b2773e1c35c9eb7d9b9a3bbcea7c96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jan 2015 15:25:29 +0600
Subject: [PATCH 1146/1937] [xboxclips] Fix extraction

---
 youtube_dl/extractor/xboxclips.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xboxclips.py b/youtube_dl/extractor/xboxclips.py
index 9cf867807..236ff403b 100644
--- a/youtube_dl/extractor/xboxclips.py
+++ b/youtube_dl/extractor/xboxclips.py
@@ -30,7 +30,7 @@ class XboxClipsIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         video_url = self._html_search_regex(
-            r'>(?:Link|Download): <a href="([^"]+)">', webpage, 'video URL')
+            r'>(?:Link|Download): <a[^>]+href="([^"]+)"', webpage, 'video URL')
         title = self._html_search_regex(
             r'<title>XboxClips \| ([^<]+)</title>', webpage, 'title')
         upload_date = unified_strdate(self._html_search_regex(

From d5bb814d348558a9700f9d8d2a063a2de2de4274 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jan 2015 15:42:53 +0600
Subject: [PATCH 1147/1937] [veehd] Capture removed video message

---
 youtube_dl/extractor/veehd.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/veehd.py b/youtube_dl/extractor/veehd.py
index 815f58468..4f76ca403 100644
--- a/youtube_dl/extractor/veehd.py
+++ b/youtube_dl/extractor/veehd.py
@@ -8,6 +8,7 @@ from ..compat import (
     compat_urlparse,
 )
 from ..utils import (
+    ExtractorError,
     clean_html,
     get_element_by_id,
 )
@@ -34,6 +35,10 @@ class VeeHDIE(InfoExtractor):
         # See https://github.com/rg3/youtube-dl/issues/2102
         self._download_webpage(url, video_id, 'Requesting webpage')
         webpage = self._download_webpage(url, video_id)
+
+        if 'This video has been removed<' in webpage:
+            raise ExtractorError('Video %s has been removed' % video_id, expected=True)
+
         player_path = self._search_regex(
             r'\$\("#playeriframe"\).attr\({src : "(.+?)"',
             webpage, 'player path')

From 8ad6b5ed9f90e07fb2847b38818d1352c71fcb18 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 11 Jan 2015 10:47:39 +0100
Subject: [PATCH 1148/1937] [compat] Correct socket error class reference

---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 07ceaa170..4453b34fc 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -329,7 +329,7 @@ if sys.version_info < (2, 7):
         if err is not None:
             raise err
         else:
-            raise error("getaddrinfo returns an empty list")
+            raise socket.error("getaddrinfo returns an empty list")
 else:
     compat_socket_create_connection = socket.create_connection
 

From 6a5fa754903370c8e522cb3c85fd1952c1d9ff9c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 11 Jan 2015 10:48:20 +0100
Subject: [PATCH 1149/1937] [karaoketv] Remove unused import

---
 youtube_dl/extractor/karaoketv.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/karaoketv.py b/youtube_dl/extractor/karaoketv.py
index 4856200d8..e3b43ff8d 100644
--- a/youtube_dl/extractor/karaoketv.py
+++ b/youtube_dl/extractor/karaoketv.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import compat_urllib_parse
 from ..utils import (
-    ExtractorError,
     js_to_json,
 )
 

From a798e64c1549f4bdf240dec4492203b304a39783 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jan 2015 16:20:16 +0600
Subject: [PATCH 1150/1937] [veehd] Improve extraction

---
 youtube_dl/extractor/veehd.py | 37 +++++++++++++++++++++++++----------
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/veehd.py b/youtube_dl/extractor/veehd.py
index 4f76ca403..5dc38dd94 100644
--- a/youtube_dl/extractor/veehd.py
+++ b/youtube_dl/extractor/veehd.py
@@ -47,18 +47,35 @@ class VeeHDIE(InfoExtractor):
         self._download_webpage(player_url, video_id, 'Requesting player page')
         player_page = self._download_webpage(
             player_url, video_id, 'Downloading player page')
-        config_json = self._search_regex(
-            r'value=\'config=({.+?})\'', player_page, 'config json')
-        config = json.loads(config_json)
 
-        video_url = compat_urlparse.unquote(config['clip']['url'])
+        config_json = self._search_regex(
+            r'value=\'config=({.+?})\'', player_page, 'config json', default=None)
+
+        if config_json:
+            config = json.loads(config_json)
+            video_url = compat_urlparse.unquote(config['clip']['url'])
+        else:
+            iframe_src = self._search_regex(
+                r'<iframe[^>]+src="/?([^"]+)"', player_page, 'iframe url')
+            iframe_url = 'http://veehd.com/%s' % iframe_src
+
+            self._download_webpage(iframe_url, video_id, 'Requesting iframe page')
+            iframe_page = self._download_webpage(
+                iframe_url, video_id, 'Downloading iframe page')
+
+            video_url = self._search_regex(
+                r"file\s*:\s*'([^']+)'", iframe_page, 'video url')
+
         title = clean_html(get_element_by_id('videoName', webpage).rpartition('|')[0])
-        uploader_id = self._html_search_regex(r'<a href="/profile/\d+">(.+?)</a>',
-                                              webpage, 'uploader')
-        thumbnail = self._search_regex(r'<img id="veehdpreview" src="(.+?)"',
-                                       webpage, 'thumbnail')
-        description = self._html_search_regex(r'<td class="infodropdown".*?<div>(.*?)<ul',
-                                              webpage, 'description', flags=re.DOTALL)
+        uploader_id = self._html_search_regex(
+            r'<a href="/profile/\d+">(.+?)</a>',
+            webpage, 'uploader')
+        thumbnail = self._search_regex(
+            r'<img id="veehdpreview" src="(.+?)"',
+            webpage, 'thumbnail')
+        description = self._html_search_regex(
+            r'<td class="infodropdown".*?<div>(.*?)<ul',
+            webpage, 'description', flags=re.DOTALL)
 
         return {
             '_type': 'video',

From aa24de39aae6b9a6ad59720258b9018f544a888f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jan 2015 16:20:39 +0600
Subject: [PATCH 1151/1937] [veehd] Update test

---
 youtube_dl/extractor/veehd.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/veehd.py b/youtube_dl/extractor/veehd.py
index 5dc38dd94..96353f525 100644
--- a/youtube_dl/extractor/veehd.py
+++ b/youtube_dl/extractor/veehd.py
@@ -18,13 +18,13 @@ class VeeHDIE(InfoExtractor):
     _VALID_URL = r'https?://veehd\.com/video/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://veehd.com/video/4686958',
+        'url': 'http://veehd.com/video/4639434_Solar-Sinter',
         'info_dict': {
-            'id': '4686958',
+            'id': '4639434',
             'ext': 'mp4',
-            'title': 'Time Lapse View from Space ( ISS)',
-            'uploader_id': 'spotted',
-            'description': 'md5:f0094c4cf3a72e22bc4e4239ef767ad7',
+            'title': 'Solar Sinter',
+            'uploader_id': 'VideoEyes',
+            'description': 'md5:46a840e8692ddbaffb5f81d9885cb457',
         },
     }
 

From f3ff1a3696c4080468e2cc5810c34273b148bd3e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 23 Feb 2014 11:29:42 +0100
Subject: [PATCH 1152/1937] YoutubeDL: Make the decision about removing the
 original file after each postprocessor is run (fixes #2261)

If one of the processors said the file should be kept, it wouldn't pay
attention to the response from the following processors. This was wrong if the
'keep_video' option was False, if the first extractor modifies the original file
and then we extract its audio we don't want to keep the original video file.
---
 youtube_dl/YoutubeDL.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 47e143d89..65581ec63 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1235,14 +1235,15 @@ class YoutubeDL(object):
         """Run all the postprocessors on the given file."""
         info = dict(ie_info)
         info['filepath'] = filename
-        keep_video = None
         pps_chain = []
         if ie_info.get('__postprocessors') is not None:
             pps_chain.extend(ie_info['__postprocessors'])
         pps_chain.extend(self._pps)
         for pp in pps_chain:
+            keep_video = None
+            old_filename = info['filepath']
             try:
-                keep_video_wish, new_info = pp.run(info)
+                keep_video_wish, info = pp.run(info)
                 if keep_video_wish is not None:
                     if keep_video_wish:
                         keep_video = keep_video_wish
@@ -1251,12 +1252,12 @@ class YoutubeDL(object):
                         keep_video = keep_video_wish
             except PostProcessingError as e:
                 self.report_error(e.msg)
-        if keep_video is False and not self.params.get('keepvideo', False):
-            try:
-                self.to_screen('Deleting original file %s (pass -k to keep)' % filename)
-                os.remove(encodeFilename(filename))
-            except (IOError, OSError):
-                self.report_warning('Unable to remove downloaded video file')
+            if keep_video is False and not self.params.get('keepvideo', False):
+                try:
+                    self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
+                    os.remove(encodeFilename(old_filename))
+                except (IOError, OSError):
+                    self.report_warning('Unable to remove downloaded video file')
 
     def _make_archive_id(self, info_dict):
         # Future-proof against any change in case

From 4340decad25e264863e27538847e17918b4c1ae2 Mon Sep 17 00:00:00 2001
From: Pierre <p.mdawar@hotmail.com>
Date: Thu, 25 Sep 2014 19:37:20 +0300
Subject: [PATCH 1153/1937] check for overwriting files in the downloader
 (fixes #3916, closes #3829)

---
 youtube_dl/YoutubeDL.py         | 81 ++++++++++++++++-----------------
 youtube_dl/downloader/common.py | 13 +++++-
 2 files changed, 51 insertions(+), 43 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 65581ec63..772fddd45 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1121,49 +1121,46 @@ class YoutubeDL(object):
                                             (info_dict['thumbnail'], compat_str(err)))
 
         if not self.params.get('skip_download', False):
-            if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
-                success = True
-            else:
-                try:
-                    def dl(name, info):
-                        fd = get_suitable_downloader(info)(self, self.params)
-                        for ph in self._progress_hooks:
-                            fd.add_progress_hook(ph)
-                        if self.params.get('verbose'):
-                            self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
-                        return fd.download(name, info)
-                    if info_dict.get('requested_formats') is not None:
-                        downloaded = []
-                        success = True
-                        merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
-                        if not merger._executable:
-                            postprocessors = []
-                            self.report_warning('You have requested multiple '
-                                                'formats but ffmpeg or avconv are not installed.'
-                                                ' The formats won\'t be merged')
-                        else:
-                            postprocessors = [merger]
-                        for f in info_dict['requested_formats']:
-                            new_info = dict(info_dict)
-                            new_info.update(f)
-                            fname = self.prepare_filename(new_info)
-                            fname = prepend_extension(fname, 'f%s' % f['format_id'])
-                            downloaded.append(fname)
-                            partial_success = dl(fname, new_info)
-                            success = success and partial_success
-                        info_dict['__postprocessors'] = postprocessors
-                        info_dict['__files_to_merge'] = downloaded
+            try:
+                def dl(name, info):
+                    fd = get_suitable_downloader(info)(self, self.params)
+                    for ph in self._progress_hooks:
+                        fd.add_progress_hook(ph)
+                    if self.params.get('verbose'):
+                        self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
+                    return fd.download(name, info)
+                if info_dict.get('requested_formats') is not None:
+                    downloaded = []
+                    success = True
+                    merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
+                    if not merger._executable:
+                        postprocessors = []
+                        self.report_warning('You have requested multiple '
+                                            'formats but ffmpeg or avconv are not installed.'
+                                            ' The formats won\'t be merged')
                     else:
-                        # Just a single file
-                        success = dl(filename, info_dict)
-                except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-                    self.report_error('unable to download video data: %s' % str(err))
-                    return
-                except (OSError, IOError) as err:
-                    raise UnavailableVideoError(err)
-                except (ContentTooShortError, ) as err:
-                    self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
-                    return
+                        postprocessors = [merger]
+                    for f in info_dict['requested_formats']:
+                        new_info = dict(info_dict)
+                        new_info.update(f)
+                        fname = self.prepare_filename(new_info)
+                        fname = prepend_extension(fname, 'f%s' % f['format_id'])
+                        downloaded.append(fname)
+                        partial_success = dl(fname, new_info)
+                        success = success and partial_success
+                    info_dict['__postprocessors'] = postprocessors
+                    info_dict['__files_to_merge'] = downloaded
+                else:
+                    # Just a single file
+                    success = dl(filename, info_dict)
+            except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+                self.report_error('unable to download video data: %s' % str(err))
+                return
+            except (OSError, IOError) as err:
+                raise UnavailableVideoError(err)
+            except (ContentTooShortError, ) as err:
+                self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
+                return
 
             if success:
                 # Fixup content
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 584bde732..de6b9311d 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -284,8 +284,19 @@ class FileDownloader(object):
         """Download to a filename using the info from info_dict
         Return True on success and False otherwise
         """
+        nooverwrites_and_exists = (
+            self.params.get('nooverwrites', False)
+            and os.path.exists(encodeFilename(filename))
+        )
+
+        continuedl_and_exists = (
+            self.params.get('continuedl', False)
+            and os.path.isfile(encodeFilename(filename))
+            and not self.params.get('nopart', False)
+        )
+
         # Check file already present
-        if filename != '-' and self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False):
+        if filename != '-' and nooverwrites_and_exists or continuedl_and_exists:
             self.report_file_already_downloaded(filename)
             self._hook_progress({
                 'filename': filename,

From 6ccbb335d2af17a0882a728ced155e3b00271044 Mon Sep 17 00:00:00 2001
From: Johan <Josso@users.noreply.github.com>
Date: Sun, 11 Jan 2015 17:39:16 +0100
Subject: [PATCH 1154/1937] [drtv] Updated with support for https

---
 youtube_dl/extractor/drtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index 93b3c9f36..c44adb109 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -6,7 +6,7 @@ from ..utils import parse_iso8601
 
 
 class DRTVIE(SubtitlesInfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?dr\.dk/tv/se/(?:[^/]+/)+(?P<id>[\da-z-]+)(?:[/#?]|$)'
+    _VALID_URL = r'https?://(?:www\.)?dr\.dk/tv/se/(?:[^/]+/)+(?P<id>[\da-z-]+)(?:[/#?]|$)'
 
     _TEST = {
         'url': 'http://www.dr.dk/tv/se/partiets-mand/partiets-mand-7-8',

From 2aeb06d6dc8c615cf98b7d21ad152bb393726b8a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 11 Jan 2015 17:40:45 +0100
Subject: [PATCH 1155/1937] [utils] Improve colon handling (Fixes #4683)

---
 test/test_utils.py  | 4 ++++
 youtube_dl/utils.py | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 16e1a1ddf..5e16f30a2 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -79,6 +79,10 @@ class TestUtil(unittest.TestCase):
         tests = '\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0446\u0430'
         self.assertEqual(sanitize_filename(tests), tests)
 
+        self.assertEqual(
+            sanitize_filename('New World record at 0:12:34'),
+            'New World record at 0_12_34')
+
         forbidden = '"\0\\/'
         for fc in forbidden:
             for fbc in forbidden:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 42f0b07ce..deaf71803 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -287,6 +287,8 @@ def sanitize_filename(s, restricted=False, is_id=False):
             return '_'
         return char
 
+    # Handle timestamps
+    s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
     result = ''.join(map(replace_insane, s))
     if not is_id:
         while '__' in result:

From 15aecd871147bcec28b341b334b2e37eaf6cef04 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 11 Jan 2015 17:47:04 +0100
Subject: [PATCH 1156/1937] release 2015.01.11

---
 README.md             | 4 +++-
 youtube_dl/version.py | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 006e8d709..c26546f2a 100644
--- a/README.md
+++ b/README.md
@@ -240,7 +240,9 @@ which means you can modify it, redistribute it or use it however you like.
                                      problems
     --print-traffic                  Display sent and read HTTP traffic
     -C, --call-home                  Contact the youtube-dl server for
-                                     debugging. (Experimental)
+                                     debugging.
+    --no-call-home                   Do NOT contact the youtube-dl server for
+                                     debugging.
 
 ## Workarounds:
     --encoding ENCODING              Force the specified encoding (experimental)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e5e320c76..0a184ad08 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.10.2'
+__version__ = '2015.01.11'

From a0977064ce9e79fd55f9da904ae3ae0a502b46c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jan 2015 23:18:45 +0600
Subject: [PATCH 1157/1937] [npo] Fix non asf streams (Closes #4680)

---
 youtube_dl/extractor/npo.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 8da76ae45..47802518d 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -10,6 +10,7 @@ from ..utils import (
     strip_jsonp,
     url_basename,
     fix_xml_ampersands,
+    determine_ext,
 )
 
 
@@ -125,6 +126,12 @@ class NPOIE(InfoExtractor):
                 stream_url = stream.get('url')
                 if not stream_url:
                     continue
+                if determine_ext(stream_url).lower() != 'asf':
+                    formats.append({
+                        'url': stream_url,
+                        'quality': stream.get('kwaliteit'),
+                    })
+                    continue
                 asx = self._download_xml(
                     stream_url, video_id,
                     'Downloading stream %d ASX playlist' % i,

From c85f368370e3e72f65fa1a40f6b6f785b4b2faf6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 11 Jan 2015 23:45:52 +0600
Subject: [PATCH 1158/1937] [npo] Make extension check less strict and add test
 (#4680)

---
 youtube_dl/extractor/npo.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 47802518d..fcd4fea94 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -68,6 +68,16 @@ class NPOIE(InfoExtractor):
                 'skip_download': True,
             }
         },
+        # non asf in streams
+        {
+            'url': 'http://www.npo.nl/hoe-gaat-europa-verder-na-parijs/10-01-2015/WO_NOS_762771',
+            'md5': 'b3da13de374cbe2d5332a7e910bef97f',
+            'info_dict': {
+                'id': 'WO_NOS_762771',
+                'ext': 'mp4',
+                'title': 'Hoe gaat Europa verder na Parijs?',
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -126,7 +136,7 @@ class NPOIE(InfoExtractor):
                 stream_url = stream.get('url')
                 if not stream_url:
                     continue
-                if determine_ext(stream_url).lower() != 'asf':
+                if not '.asf' in stream_url:
                     formats.append({
                         'url': stream_url,
                         'quality': stream.get('kwaliteit'),

From 8e7a9016d5e79679f4b948f00c794b8ff31a41e4 Mon Sep 17 00:00:00 2001
From: "Johan K. Jensen" <jj@johanjensen.dk>
Date: Sun, 11 Jan 2015 23:23:10 +0100
Subject: [PATCH 1159/1937] [DRBonanza] Add new extractor (fixing #4581)

---
 youtube_dl/extractor/__init__.py  |   1 +
 youtube_dl/extractor/drbonanza.py | 120 ++++++++++++++++++++++++++++++
 2 files changed, 121 insertions(+)
 create mode 100644 youtube_dl/extractor/drbonanza.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2f3a322a7..5a1c2f69b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -93,6 +93,7 @@ from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
 from .dotsub import DotsubIE
 from .dreisat import DreiSatIE
+from .drbonanza import DRBonanzaIE
 from .drtuber import DrTuberIE
 from .drtv import DRTVIE
 from .dvtv import DVTVIE
diff --git a/youtube_dl/extractor/drbonanza.py b/youtube_dl/extractor/drbonanza.py
new file mode 100644
index 000000000..8cbdb57b3
--- /dev/null
+++ b/youtube_dl/extractor/drbonanza.py
@@ -0,0 +1,120 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from .common import ExtractorError
+from ..utils import parse_iso8601
+import json
+import re
+
+class DRBonanzaIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?dr\.dk/bonanza/(?:[^/]+/)+(?:[^/])+?(?:assetId=(?P<id>\d+))?(?:[#&]|$)'
+
+    _TESTS = [{
+        'url': 'http://www.dr.dk/bonanza/serie/portraetter/Talkshowet.htm?assetId=65517',
+        'md5': 'fe330252ddea607635cf2eb2c99a0af3',
+        'info_dict': {
+            'id': '65517',
+            'ext': 'mp4',
+            'title': 'Talkshowet - Leonard Cohen',
+            'description': 'md5:8f34194fb30cd8c8a30ad8b27b70c0ca',
+            'timestamp': 1295537932,
+            'upload_date': '20110120',
+            'duration': 3664000,
+        },
+    },{
+        'url': 'http://www.dr.dk/bonanza/radio/serie/sport/fodbold.htm?assetId=59410',
+        'md5': '6dfe039417e76795fb783c52da3de11d',
+        'info_dict': {
+            'id': '59410',
+            'ext': 'mp3',
+            'title': 'EM fodbold 1992 Danmark - Tyskland finale Transmission',
+            'description': 'md5:501e5a195749480552e214fbbed16c4e',
+            'timestamp': 1223274900,
+            'upload_date': '20081006',
+            'duration': 7369000,
+        },
+    }]
+
+    def _real_extract(self, url):
+        url_id = self._match_id(url)
+        
+        webpage = self._download_webpage(url, url_id if url_id else "")
+        
+        if url_id:
+            info = json.loads(self._html_search_regex(r'({.*?' + url_id + '.*})', webpage, 'json'))
+        else:
+            # Just fetch the first video on that page
+            info = json.loads(self._html_search_regex(r'bonanzaFunctions.newPlaylist\(({.*})\)', webpage, 'json'))
+        
+        asset_id = str(info['AssetId'])
+        title = info['Title'].rstrip(' \'\"-,.:;!?')
+        duration = info['Duration']
+        timestamp = parse_iso8601(re.sub(r'\.\d+$', '', info['Created'])) # First published online. "FirstPublished" contains the date for original airing.
+        
+        def parse_filename_info(url):
+            match = re.search(r'/\d+_(?P<width>\d+)x(?P<height>\d+)x(?P<bitrate>\d+)K\.(?P<ext>\w+)$', url)
+            if match:
+                return {'width': int(match.group(1)), 'height': int(match.group(2)), 'bitrate': int(match.group(3)), 'ext': match.group(4)}
+            match = re.search(r'/\d+_(?P<bitrate>\d+)K\.(?P<ext>\w+)$', url)
+            if match:
+                return {'bitrate': int(match.group(1)), 'ext': match.group(2)}
+            return {'width': None, 'height': None, 'bitrate': None, 'ext': None}
+        
+        video_types = ['VideoHigh', 'VideoMid', 'VideoLow']
+        preferencemap = {
+            'VideoHigh': -1,
+            'VideoMid': -2,
+            'VideoLow': -3,
+            'Audio': -4,
+        }
+        
+        formats = []
+        for file in info['Files']:
+            if info['Type'] == "Video":
+                if file['Type'] in video_types:
+                    fileinfo = parse_filename_info(file['Location'])
+                    formats.append({
+                        'url': file['Location'],
+                        'format_id': file['Type'].replace('Video', ''),
+                        'preference': preferencemap.get(file['Type'], -10),
+                        'width': fileinfo['width'],
+                        'height': fileinfo['height'],
+                        'vbr': fileinfo['bitrate'],
+                        'ext': fileinfo['ext'],
+                    })
+                elif file['Type'] == "Thumb":
+                    thumbnail = file['Location']
+            elif info['Type'] == "Audio":
+                if file['Type'] == "Audio":
+                    fileinfo = parse_filename_info(file['Location'])
+                    formats.append({
+                        'url': file['Location'],
+                        'format_id': file['Type'],
+                        'abr': fileinfo['bitrate'],
+                        'ext': fileinfo['ext'],
+                        'vcodec': 'none',
+                    })
+                elif file['Type'] == "Thumb":
+                    thumbnail = file['Location']
+        
+        description = "{}\n{}\n{}\n".format(info['Description'], info['Actors'], info['Colophon'])
+
+        for f in formats:
+            f['url'] = f['url'].replace('rtmp://vod-bonanza.gss.dr.dk/bonanza/', 'http://vodfiles.dr.dk/')
+            f['url'] = f['url'].replace('mp4:bonanza', 'bonanza')
+        
+        self._sort_formats(formats)
+        
+        display_id = re.sub(r'[^\w\d-]', '', re.sub(r' ', '-', title.lower())) + '-' + asset_id
+        display_id = re.sub(r'-+', '-', display_id)
+        
+        return {
+            'id': asset_id,
+            'display_id': display_id,
+            'title': title,
+            'formats': formats,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'duration': duration,
+        }

From 8e2ec95575fc8157da1ec3ec408737f200aa44d7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 11 Jan 2015 23:40:27 +0100
Subject: [PATCH 1160/1937] [drbonanza] Simplify and fix duration (#4687)

---
 youtube_dl/extractor/drbonanza.py | 79 ++++++++++++++++++-------------
 1 file changed, 45 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/drbonanza.py b/youtube_dl/extractor/drbonanza.py
index 8cbdb57b3..7626219ba 100644
--- a/youtube_dl/extractor/drbonanza.py
+++ b/youtube_dl/extractor/drbonanza.py
@@ -1,11 +1,15 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-from .common import ExtractorError
-from ..utils import parse_iso8601
 import json
 import re
 
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
+
+
 class DRBonanzaIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dr\.dk/bonanza/(?:[^/]+/)+(?:[^/])+?(?:assetId=(?P<id>\d+))?(?:[#&]|$)'
 
@@ -17,11 +21,12 @@ class DRBonanzaIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Talkshowet - Leonard Cohen',
             'description': 'md5:8f34194fb30cd8c8a30ad8b27b70c0ca',
+            'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
             'timestamp': 1295537932,
             'upload_date': '20110120',
-            'duration': 3664000,
+            'duration': 3664,
         },
-    },{
+    }, {
         'url': 'http://www.dr.dk/bonanza/radio/serie/sport/fodbold.htm?assetId=59410',
         'md5': '6dfe039417e76795fb783c52da3de11d',
         'info_dict': {
@@ -29,37 +34,47 @@ class DRBonanzaIE(InfoExtractor):
             'ext': 'mp3',
             'title': 'EM fodbold 1992 Danmark - Tyskland finale Transmission',
             'description': 'md5:501e5a195749480552e214fbbed16c4e',
+            'thumbnail': 're:^https?://.*\.(?:gif|jpg)$',
             'timestamp': 1223274900,
             'upload_date': '20081006',
-            'duration': 7369000,
+            'duration': 7369,
         },
     }]
 
     def _real_extract(self, url):
         url_id = self._match_id(url)
-        
-        webpage = self._download_webpage(url, url_id if url_id else "")
-        
+        webpage = self._download_webpage(url, url_id)
+
         if url_id:
-            info = json.loads(self._html_search_regex(r'({.*?' + url_id + '.*})', webpage, 'json'))
+            info = json.loads(self._html_search_regex(r'({.*?%s.*})' % url_id, webpage, 'json'))
         else:
             # Just fetch the first video on that page
             info = json.loads(self._html_search_regex(r'bonanzaFunctions.newPlaylist\(({.*})\)', webpage, 'json'))
-        
+
         asset_id = str(info['AssetId'])
         title = info['Title'].rstrip(' \'\"-,.:;!?')
-        duration = info['Duration']
-        timestamp = parse_iso8601(re.sub(r'\.\d+$', '', info['Created'])) # First published online. "FirstPublished" contains the date for original airing.
-        
+        duration = int_or_none(info.get('Duration'), scale=1000)
+        # First published online. "FirstPublished" contains the date for original airing.
+        timestamp = parse_iso8601(
+            re.sub(r'\.\d+$', '', info['Created']))
+
         def parse_filename_info(url):
             match = re.search(r'/\d+_(?P<width>\d+)x(?P<height>\d+)x(?P<bitrate>\d+)K\.(?P<ext>\w+)$', url)
             if match:
-                return {'width': int(match.group(1)), 'height': int(match.group(2)), 'bitrate': int(match.group(3)), 'ext': match.group(4)}
+                return {
+                    'width': int(match.group('width')),
+                    'height': int(match.group('height')),
+                    'vbr': int(match.group('bitrate')),
+                    'ext': match.group('ext')
+                }
             match = re.search(r'/\d+_(?P<bitrate>\d+)K\.(?P<ext>\w+)$', url)
             if match:
-                return {'bitrate': int(match.group(1)), 'ext': match.group(2)}
-            return {'width': None, 'height': None, 'bitrate': None, 'ext': None}
-        
+                return {
+                    'vbr': int(match.group('bitrate')),
+                    'ext': match.group(2)
+                }
+            return {}
+
         video_types = ['VideoHigh', 'VideoMid', 'VideoLow']
         preferencemap = {
             'VideoHigh': -1,
@@ -67,47 +82,43 @@ class DRBonanzaIE(InfoExtractor):
             'VideoLow': -3,
             'Audio': -4,
         }
-        
+
         formats = []
         for file in info['Files']:
             if info['Type'] == "Video":
                 if file['Type'] in video_types:
-                    fileinfo = parse_filename_info(file['Location'])
-                    formats.append({
+                    format = parse_filename_info(file['Location'])
+                    format.update({
                         'url': file['Location'],
                         'format_id': file['Type'].replace('Video', ''),
                         'preference': preferencemap.get(file['Type'], -10),
-                        'width': fileinfo['width'],
-                        'height': fileinfo['height'],
-                        'vbr': fileinfo['bitrate'],
-                        'ext': fileinfo['ext'],
                     })
+                    formats.append(format)
                 elif file['Type'] == "Thumb":
                     thumbnail = file['Location']
             elif info['Type'] == "Audio":
                 if file['Type'] == "Audio":
-                    fileinfo = parse_filename_info(file['Location'])
-                    formats.append({
+                    format = parse_filename_info(file['Location'])
+                    format.update({
                         'url': file['Location'],
                         'format_id': file['Type'],
-                        'abr': fileinfo['bitrate'],
-                        'ext': fileinfo['ext'],
                         'vcodec': 'none',
                     })
+                    formats.append(format)
                 elif file['Type'] == "Thumb":
                     thumbnail = file['Location']
-        
-        description = "{}\n{}\n{}\n".format(info['Description'], info['Actors'], info['Colophon'])
+
+        description = '%s\n%s\n%s\n' % (
+            info['Description'], info['Actors'], info['Colophon'])
 
         for f in formats:
             f['url'] = f['url'].replace('rtmp://vod-bonanza.gss.dr.dk/bonanza/', 'http://vodfiles.dr.dk/')
             f['url'] = f['url'].replace('mp4:bonanza', 'bonanza')
-        
         self._sort_formats(formats)
-        
+
         display_id = re.sub(r'[^\w\d-]', '', re.sub(r' ', '-', title.lower())) + '-' + asset_id
         display_id = re.sub(r'-+', '-', display_id)
-        
+
         return {
             'id': asset_id,
             'display_id': display_id,

From 263255eb8d45377e54e23841b55f0609f0e7e8a4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 11 Jan 2015 23:42:24 +0100
Subject: [PATCH 1161/1937] Credit @Josso for drbonanza (#4581)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 1e4cadae5..35e256d49 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -102,3 +102,4 @@ Joel Leclerc
 Christopher Krooss
 Ondřej Caletka
 Dinesh S
+Johan K. Jensen

From 5c4a81d9345f0520a9d40acbb8212fb94bcc9e24 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 11 Jan 2015 23:43:09 +0100
Subject: [PATCH 1162/1937] [npo] Remove unused import

---
 youtube_dl/extractor/npo.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index fcd4fea94..6d20f755a 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -4,13 +4,12 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    unified_strdate,
+    fix_xml_ampersands,
     parse_duration,
     qualities,
     strip_jsonp,
+    unified_strdate,
     url_basename,
-    fix_xml_ampersands,
-    determine_ext,
 )
 
 
@@ -136,7 +135,7 @@ class NPOIE(InfoExtractor):
                 stream_url = stream.get('url')
                 if not stream_url:
                     continue
-                if not '.asf' in stream_url:
+                if '.asf' not in stream_url:
                     formats.append({
                         'url': stream_url,
                         'quality': stream.get('kwaliteit'),

From 227d4822fff13ea449f11531ce54c56b2b01432b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jan 2015 21:06:26 +0600
Subject: [PATCH 1163/1937] [utils] Disallow non string types in parse_duration
 (Closes #4679)

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index deaf71803..98732e8e9 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1269,7 +1269,7 @@ def float_or_none(v, scale=1, invscale=1, default=None):
 
 
 def parse_duration(s):
-    if s is None:
+    if not isinstance(s, basestring if sys.version_info < (3, 0) else compat_str):
         return None
 
     s = s.strip()

From a5fb718c50d394aeee827f6e9381eceb839128de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 12 Jan 2015 21:39:58 +0600
Subject: [PATCH 1164/1937] [test_utils] Add more tests for parse_duration

---
 test/test_utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 5e16f30a2..399cdfdbd 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -212,6 +212,8 @@ class TestUtil(unittest.TestCase):
 
     def test_parse_duration(self):
         self.assertEqual(parse_duration(None), None)
+        self.assertEqual(parse_duration(False), None)
+        self.assertEqual(parse_duration('invalid'), None)
         self.assertEqual(parse_duration('1'), 1)
         self.assertEqual(parse_duration('1337:12'), 80232)
         self.assertEqual(parse_duration('9:12:43'), 33163)

From 3d5f7a3947a8d304bc7ad46217f171996e95c475 Mon Sep 17 00:00:00 2001
From: Johannes Knoedtel <johannes@dev-baron.de>
Date: Mon, 12 Jan 2015 22:26:20 +0100
Subject: [PATCH 1165/1937] [utils] Prevent override of custom headers.

The dict of headers of request objects in urllib has its keys always
capitalized.

This causes the lookup to fail and overwrite the header. If for example
a Extractor tries to add a "User-Agent" header the internal
representation in the request object is "User-agent". The header is
therefore clobbered by the "User-Agent" in std_headers, because the
strings are not equal.
---
 youtube_dl/utils.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 98732e8e9..daf94abd1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -611,7 +611,9 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
 
     def http_request(self, req):
         for h, v in std_headers.items():
-            if h not in req.headers:
+            # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
+            # The dict keys are capitalized because of this bug by urllib
+            if h.capitalize() not in req.headers:
                 req.add_header(h, v)
         if 'Youtubedl-no-compression' in req.headers:
             if 'Accept-encoding' in req.headers:

From fbef83f399b6a54380cecfce6d00964cc1e063d0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 13 Jan 2015 08:10:08 +0100
Subject: [PATCH 1166/1937] [README] Add FAQ for playing from another machine
 (Fixes #4693)

---
 README.md | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index c26546f2a..078e9df82 100644
--- a/README.md
+++ b/README.md
@@ -444,9 +444,15 @@ Apparently YouTube requires you to pass a CAPTCHA test if you download too much.
 
 Once the video is fully downloaded, use any video player, such as [vlc](http://www.videolan.org) or [mplayer](http://www.mplayerhq.hu/).
 
-### The links provided by youtube-dl -g are not working anymore
+### I extracted a video URL with -g, but it does not play on another machine / in my webbrowser.
 
-The URLs youtube-dl outputs require the downloader to have the correct cookies. Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by youtube-dl.
+It depends a lot on the service. In many cases, requests for the video (to download/play it) must come from the same IP address and with the same cookies.  Use the `--cookies` option to write the required cookies into a file, and advise your downloader to read cookies from that file. Some sites also require a common user agent to be used, use `--dump-user-agent` to see the one in use by youtube-dl.
+
+It may be beneficial to use IPv6; in some cases, the restrictions are only applied to IPv4. Some services (sometimes only for a subset of videos) do not restrict the video URL by IP address, cookie, or user-agent, but these are the exception rather than the rule.
+
+Please bear in mind that some URL protocols are **not** supported by browsers out of the box, including RTMP. If you are using -g, your own downloader must support these as well.
+
+If you want to play the video on a machine that is not running youtube-dl, you can relay the video content from the machine that runs youtube-dl. You can use `-o -` to let youtube-dl stream a video to stdout, or simply allow the player to download the files written by youtube-dl in turn.
 
 ### ERROR: no fmt_url_map or conn information found in video info
 

From 04e0bac233fd4f094983d612d6c76b5590393230 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 13 Jan 2015 20:54:03 +0600
Subject: [PATCH 1167/1937] [npo:live] Add extractor (Closes #4691)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/npo.py      | 99 +++++++++++++++++++++++++++++---
 2 files changed, 91 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5a1c2f69b..2d31c298f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -295,6 +295,7 @@ from .nowness import NownessIE
 from .nowvideo import NowVideoIE
 from .npo import (
     NPOIE,
+    NPOLiveIE,
     TegenlichtVproIE,
 )
 from .nrk import (
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 6d20f755a..8d496488b 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -13,7 +13,16 @@ from ..utils import (
 )
 
 
-class NPOIE(InfoExtractor):
+class NPOBaseIE(InfoExtractor):
+    def _get_token(self, video_id):
+        token_page = self._download_webpage(
+            'http://ida.omroep.nl/npoplayer/i.js',
+            video_id, note='Downloading token')
+        return self._search_regex(
+            r'npoplayer\.token = "(.+?)"', token_page, 'token')
+
+
+class NPOIE(NPOBaseIE):
     IE_NAME = 'npo.nl'
     _VALID_URL = r'https?://www\.npo\.nl/[^/]+/[^/]+/(?P<id>[^/?]+)'
 
@@ -80,8 +89,7 @@ class NPOIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         return self._get_info(video_id)
 
     def _get_info(self, video_id):
@@ -91,12 +99,8 @@ class NPOIE(InfoExtractor):
             # We have to remove the javascript callback
             transform_source=strip_jsonp,
         )
-        token_page = self._download_webpage(
-            'http://ida.omroep.nl/npoplayer/i.js',
-            video_id,
-            note='Downloading token'
-        )
-        token = self._search_regex(r'npoplayer\.token = "(.+?)"', token_page, 'token')
+
+        token = self._get_token(video_id)
 
         formats = []
 
@@ -170,6 +174,83 @@ class NPOIE(InfoExtractor):
         }
 
 
+class NPOLiveIE(NPOBaseIE):
+    IE_NAME = 'npo.nl:live'
+    _VALID_URL = r'https?://www\.npo\.nl/live/(?P<id>.+)'
+
+    _TEST = {
+        'url': 'http://www.npo.nl/live/npo-1',
+        'info_dict': {
+            'id': 'LI_NEDERLAND1_136692',
+            'display_id': 'npo-1',
+            'ext': 'mp4',
+            'title': 're:^Nederland 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'Livestream',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        live_id = self._search_regex(
+            r'data-prid="([^"]+)"', webpage, 'live id')
+
+        metadata = self._download_json(
+            'http://e.omroep.nl/metadata/%s' % live_id,
+            display_id, transform_source=strip_jsonp)
+
+        token = self._get_token(display_id)
+
+        formats = []
+
+        streams = metadata.get('streams')
+        if streams:
+            for stream in streams:
+                stream_type = stream.get('type').lower()
+                if stream_type == 'ss':
+                    continue
+                stream_info = self._download_json(
+                    'http://ida.omroep.nl/aapi/?stream=%s&token=%s&type=jsonp'
+                    % (stream.get('url'), token),
+                    display_id, 'Downloading %s JSON' % stream_type)
+                if stream_info.get('error_code', 0) or stream_info.get('errorcode', 0):
+                    continue
+                stream_url = self._download_json(
+                    stream_info['stream'], display_id,
+                    'Downloading %s URL' % stream_type,
+                    transform_source=strip_jsonp)
+                if stream_type == 'hds':
+                    f4m_formats = self._extract_f4m_formats(stream_url, display_id)
+                    # f4m downloader downloads only piece of live stream
+                    for f4m_format in f4m_formats:
+                        f4m_format['preference'] = -1
+                    formats.extend(f4m_formats)
+                elif stream_type == 'hls':
+                    formats.extend(self._extract_m3u8_formats(stream_url, display_id, 'mp4'))
+                else:
+                    formats.append({
+                        'url': stream_url,
+                    })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': live_id,
+            'display_id': display_id,
+            'title': self._live_title(metadata['titel']),
+            'description': metadata['info'],
+            'thumbnail': metadata.get('images', [{'url': None}])[-1]['url'],
+            'formats': formats,
+            'is_live': True,
+        }
+
+
 class TegenlichtVproIE(NPOIE):
     IE_NAME = 'tegenlicht.vpro.nl'
     _VALID_URL = r'https?://tegenlicht\.vpro\.nl/afleveringen/.*?'

From 034206cec1903dc8c5829012e7130a63dc014265 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 14 Jan 2015 00:14:59 +0200
Subject: [PATCH 1168/1937] [lnkgo] Add new extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/lnkgo.py    | 122 +++++++++++++++++++++++++++++++
 2 files changed, 123 insertions(+)
 create mode 100644 youtube_dl/extractor/lnkgo.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2d31c298f..0902eb437 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -226,6 +226,7 @@ from .livestream import (
     LivestreamOriginalIE,
     LivestreamShortenerIE,
 )
+from .lnkgo import LnkGoIE
 from .lrt import LRTIE
 from .lynda import (
     LyndaIE,
diff --git a/youtube_dl/extractor/lnkgo.py b/youtube_dl/extractor/lnkgo.py
new file mode 100644
index 000000000..cfae60154
--- /dev/null
+++ b/youtube_dl/extractor/lnkgo.py
@@ -0,0 +1,122 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    unified_strdate,
+)
+
+
+class LnkGoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?lnkgo\.alfa\.lt/visi\-video/(?P<show>[^/]+)/ziurek\-(?P<display_id>[A-Za-z0-9\-]+)'
+    _TESTS = [{
+        'url': 'http://lnkgo.alfa.lt/visi-video/yra-kaip-yra/ziurek-yra-kaip-yra-162',
+        'info_dict': {
+            'id': '46712',
+            'ext': 'mp4',
+            'title': 'Yra kaip yra',
+            'upload_date': '20150107',
+            'description': 'md5:d82a5e36b775b7048617f263a0e3475e',
+            'age_limit': 7,
+            'duration': 3019,
+            'thumbnail': 're:^https?://.*\.jpg$'
+        },
+        'params': {
+            'skip_download': True,  # HLS download
+        },
+    }, {
+        'url': 'http://lnkgo.alfa.lt/visi-video/aktualai-pratesimas/ziurek-nerdas-taiso-kompiuteri-2',
+        'info_dict': {
+            'id': '47289',
+            'ext': 'mp4',
+            'title': 'Nėrdas: Kompiuterio Valymas',
+            'upload_date': '20150113',
+            'description': 'md5:7352d113a242a808676ff17e69db6a69',
+            'age_limit': 18,
+            'duration': 346,
+            'thumbnail': 're:^https?://.*\.jpg$'
+        },
+        'params': {
+            'skip_download': True,  # HLS download
+        },
+    }]
+    _AGE_LIMITS = {
+        'N-7': 7,
+        'N-14': 14,
+        'S': 18,
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        display_id = mobj.group('display_id')
+
+        webpage = self._download_webpage(
+            url, display_id, 'Downloading player webpage')
+
+        video_id = self._search_regex(
+            r'data-ep="([^"]+)"', webpage, 'video ID')
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+
+        thumbnail = self._og_search_thumbnail(webpage)
+        thumbnail_w = int_or_none(
+            self._og_search_property('image:width', webpage, 'thumbnail width', fatal=False))
+        thumbnail_h = int_or_none(
+            self._og_search_property('image:height', webpage, 'thumbnail height', fatal=False))
+        thumbnails = [{
+            'url': thumbnail,
+            'width': thumbnail_w,
+            'height': thumbnail_h,
+        }]
+
+        upload_date = unified_strdate(self._search_regex(
+            r'class="meta-item\sair-time">.*?<strong>([^<]+)</strong>', webpage, 'upload date', fatal=False))
+        duration = int_or_none(self._search_regex(
+            r'VideoDuration = "([^"]+)"', webpage, 'duration', fatal=False))
+
+        pg_rating = self._search_regex(
+            r'pgrating="([^"]+)"', webpage, 'PG rating', fatal=False, default='')
+        age_limit = self._AGE_LIMITS.get(pg_rating, 0)
+
+        sources_js = self._search_regex(
+            r'(?s)sources:\s(\[.*?\]),', webpage, 'sources')
+        sources = self._parse_json(
+            sources_js, video_id, transform_source=js_to_json)
+
+        formats = []
+        for source in sources:
+            if source.get('provider') == 'rtmp':
+                m = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<play_path>.+)$', source['file'])
+                if not m:
+                    continue
+                formats.append({
+                    'format_id': 'rtmp',
+                    'ext': 'flv',
+                    'url': m.group('url'),
+                    'play_path': m.group('play_path'),
+                    'page_url': url,
+                })
+            elif source.get('file').endswith('.m3u8'):
+                formats.append({
+                    'format_id': 'hls',
+                    'ext': source.get('type', 'mp4'),
+                    'url': source['file'],
+                })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'duration': duration,
+            'description': description,
+            'age_limit': age_limit,
+            'upload_date': upload_date,
+        }

From a69801e2c6f53985acf475f391f7f7205775d631 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 14 Jan 2015 00:16:34 +0200
Subject: [PATCH 1169/1937] [utils] Add additional format to unified_strdate

---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 399cdfdbd..206760d99 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -148,6 +148,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_strdate('8/7/2009'), '20090708')
         self.assertEqual(unified_strdate('Dec 14, 2012'), '20121214')
         self.assertEqual(unified_strdate('2012/10/11 01:56:38 +0000'), '20121011')
+        self.assertEqual(unified_strdate('1968 12 10'), '19681210')
         self.assertEqual(unified_strdate('1968-12-10'), '19681210')
         self.assertEqual(unified_strdate('28/01/2014 21:00:00 +0100'), '20140128')
         self.assertEqual(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 98732e8e9..38c18c127 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -721,6 +721,7 @@ def unified_strdate(date_str, day_first=True):
         '%b %dst %Y %I:%M%p',
         '%b %dnd %Y %I:%M%p',
         '%b %dth %Y %I:%M%p',
+        '%Y %m %d',
         '%Y-%m-%d',
         '%Y/%m/%d',
         '%Y/%m/%d %H:%M:%S',

From f2cbc96c3ea3f0c3b447b50b445de736e535d63f Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 14 Jan 2015 00:51:48 +0200
Subject: [PATCH 1170/1937] [lnkgo] Make more robust

---
 youtube_dl/extractor/lnkgo.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/lnkgo.py b/youtube_dl/extractor/lnkgo.py
index cfae60154..a8e357859 100644
--- a/youtube_dl/extractor/lnkgo.py
+++ b/youtube_dl/extractor/lnkgo.py
@@ -62,16 +62,18 @@ class LnkGoIE(InfoExtractor):
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage)
 
-        thumbnail = self._og_search_thumbnail(webpage)
         thumbnail_w = int_or_none(
             self._og_search_property('image:width', webpage, 'thumbnail width', fatal=False))
         thumbnail_h = int_or_none(
             self._og_search_property('image:height', webpage, 'thumbnail height', fatal=False))
-        thumbnails = [{
-            'url': thumbnail,
-            'width': thumbnail_w,
-            'height': thumbnail_h,
-        }]
+        thumbnail = {
+            'url': self._og_search_thumbnail(webpage),
+        }
+        if thumbnail_w and thumbnail_h:
+            thumbnail.update({
+                'width': thumbnail_w,
+                'height': thumbnail_h,
+            })
 
         upload_date = unified_strdate(self._search_regex(
             r'class="meta-item\sair-time">.*?<strong>([^<]+)</strong>', webpage, 'upload date', fatal=False))
@@ -80,7 +82,7 @@ class LnkGoIE(InfoExtractor):
 
         pg_rating = self._search_regex(
             r'pgrating="([^"]+)"', webpage, 'PG rating', fatal=False, default='')
-        age_limit = self._AGE_LIMITS.get(pg_rating, 0)
+        age_limit = self._AGE_LIMITS.get(pg_rating.upper(), 0)
 
         sources_js = self._search_regex(
             r'(?s)sources:\s(\[.*?\]),', webpage, 'sources')
@@ -114,7 +116,7 @@ class LnkGoIE(InfoExtractor):
             'display_id': display_id,
             'title': title,
             'formats': formats,
-            'thumbnails': thumbnails,
+            'thumbnails': [thumbnail],
             'duration': duration,
             'description': description,
             'age_limit': age_limit,

From e92d4a11f570abb47215b13f88020f52479766e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Jan 2015 20:27:14 +0600
Subject: [PATCH 1171/1937] [spiegel] Test format video URLs for 404 (Closes
 #4579)

---
 youtube_dl/extractor/spiegel.py | 49 ++++++++++++++++++++++-----------
 1 file changed, 33 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index 1e55a9ffb..f345883c7 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -4,7 +4,14 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import (
+    compat_urlparse,
+    compat_HTTPError,
+)
+from ..utils import (
+    HEADRequest,
+    ExtractorError,
+)
 from .spiegeltv import SpiegeltvIE
 
 
@@ -60,21 +67,31 @@ class SpiegelIE(InfoExtractor):
         xml_url = base_url + video_id + '.xml'
         idoc = self._download_xml(xml_url, video_id)
 
-        formats = [
-            {
-                'format_id': n.tag.rpartition('type')[2],
-                'url': base_url + n.find('./filename').text,
-                'width': int(n.find('./width').text),
-                'height': int(n.find('./height').text),
-                'abr': int(n.find('./audiobitrate').text),
-                'vbr': int(n.find('./videobitrate').text),
-                'vcodec': n.find('./codec').text,
-                'acodec': 'MP4A',
-            }
-            for n in list(idoc)
-            # Blacklist type 6, it's extremely LQ and not available on the same server
-            if n.tag.startswith('type') and n.tag != 'type6'
-        ]
+        formats = []
+        for n in list(idoc):
+            if n.tag.startswith('type') and n.tag != 'type6':
+                format_id = n.tag.rpartition('type')[2]
+                video_url = base_url + n.find('./filename').text
+                # Test video URLs beforehand as some of them are invalid
+                try:
+                    self._request_webpage(
+                        HEADRequest(video_url), video_id,
+                        'Checking %s video URL' % format_id)
+                except ExtractorError as e:
+                    if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+                        self.report_warning(
+                            '%s video URL is invalid, skipping' % format_id, video_id)
+                        continue
+                formats.append({
+                    'format_id': format_id,
+                    'url': video_url,
+                    'width': int(n.find('./width').text),
+                    'height': int(n.find('./height').text),
+                    'abr': int(n.find('./audiobitrate').text),
+                    'vbr': int(n.find('./videobitrate').text),
+                    'vcodec': n.find('./codec').text,
+                    'acodec': 'MP4A',
+                })
         duration = float(idoc[0].findall('./duration')[0].text)
 
         self._sort_formats(formats)

From c9ef44ce296ded2d5ba4b36ad60b0a7add7944bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 14 Jan 2015 21:50:36 +0600
Subject: [PATCH 1172/1937] [smotri] Improve extraction (Closes #4698)

---
 youtube_dl/extractor/smotri.py | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index baef3daa0..26f361c93 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -90,6 +90,20 @@ class SmotriIE(InfoExtractor):
             },
             'skip': 'Video is not approved by moderator',
         },
+        # not approved by moderator, but available
+        {
+            'url': 'http://smotri.com/video/view/?id=v28888533b73',
+            'md5': 'f44bc7adac90af518ef1ecf04893bb34',
+            'info_dict': {
+                'id': 'v28888533b73',
+                'ext': 'mp4',
+                'title': 'Russian Spies Killed By ISIL Child Soldier',
+                'uploader': 'Mopeder',
+                'uploader_id': 'mopeder',
+                'duration': 71,
+                'thumbnail': 'http://frame9.loadup.ru/d7/32/2888853.2.3.jpg',
+            },
+        },
         # swf player
         {
             'url': 'http://pics.smotri.com/scrubber_custom8.swf?file=v9188090500',
@@ -146,13 +160,16 @@ class SmotriIE(InfoExtractor):
 
         video = self._download_json(request, video_id, 'Downloading video JSON')
 
-        if video.get('_moderate_no') or not video.get('moderated'):
-            raise ExtractorError('Video %s has not been approved by moderator' % video_id, expected=True)
-
-        if video.get('error'):
-            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
-
         video_url = video.get('_vidURL') or video.get('_vidURL_mp4')
+
+        if not video_url:
+            if video.get('_moderate_no') or not video.get('moderated'):
+                raise ExtractorError(
+                    'Video %s has not been approved by moderator' % video_id, expected=True)
+
+            if video.get('error'):
+                raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
         title = video['title']
         thumbnail = video['_imgURL']
         upload_date = unified_strdate(video['added'])

From be5f2c192cde200b6fcc8d754fde8b5e479a300c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 15 Jan 2015 02:06:48 +0100
Subject: [PATCH 1173/1937] [ssl] Correct connect creation

We want to authenticate the server, see https://docs.python.org/dev/library/ssl.html#ssl.Purpose.SERVER_AUTH .
---
 youtube_dl/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 38c18c127..2546fa45d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -399,8 +399,9 @@ def formatSeconds(secs):
 def make_HTTPS_handler(params, **kwargs):
     opts_no_check_certificate = params.get('nocheckcertificate', False)
     if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
-        context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+        context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
         if opts_no_check_certificate:
+            context.check_hostname = False
             context.verify_mode = ssl.CERT_NONE
         try:
             return YoutubeDLHTTPSHandler(params, context=context, **kwargs)

From 4cfaf85c65d1b00ee5b198acb44f1ac65366e0d8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 15 Jan 2015 12:42:11 +0100
Subject: [PATCH 1174/1937] release 2015.01.15

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0a184ad08..6eccf2bf5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.11'
+__version__ = '2015.01.15'

From c8dfe360eb642b2957116814bfdef67686ab000d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 15 Jan 2015 21:43:35 +0600
Subject: [PATCH 1175/1937] [atresplayer] Add authentication support (Closes
 #4700)

---
 youtube_dl/extractor/atresplayer.py | 32 ++++++++++++++++++++++++++++-
 1 file changed, 31 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 72e83bfc2..5db1941b3 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -4,9 +4,12 @@ import time
 import hmac
 
 from .common import InfoExtractor
-from ..utils import (
+from ..compat import (
     compat_str,
+    compat_urllib_parse,
     compat_urllib_request,
+)
+from ..utils import (
     int_or_none,
     float_or_none,
     xpath_text,
@@ -44,6 +47,33 @@ class AtresPlayerIE(InfoExtractor):
     _PLAYER_URL_TEMPLATE = 'https://servicios.atresplayer.com/episode/getplayer.json?episodePk=%s'
     _EPISODE_URL_TEMPLATE = 'http://www.atresplayer.com/episodexml/%s'
 
+    _LOGIN_URL = 'https://servicios.atresplayer.com/j_spring_security_check'
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_form = {
+            'j_username': username,
+            'j_password': password,
+        }
+
+        request = compat_urllib_request.Request(
+            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+        request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+        response = self._download_webpage(
+            request, None, 'Logging in as %s' % username)
+
+        error = self._html_search_regex(
+            r'(?s)<ul class="list_error">(.+?)</ul>', response, 'error', default=None)
+        if error:
+            raise ExtractorError(
+                'Unable to login: %s' % error, expected=True)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
 

From a45c0a5d67b87e9ea16e2812f44753c9cb946636 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 15 Jan 2015 19:57:36 +0200
Subject: [PATCH 1176/1937] [videomega] Fix extraction (Closes #4703)

---
 youtube_dl/extractor/videomega.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 7a78f0d26..9fc64d172 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urllib_request,
 )
 from ..utils import (
     remove_start,
@@ -16,20 +17,23 @@ class VideoMegaIE(InfoExtractor):
         (?:iframe\.php)?\?ref=(?P<id>[A-Za-z0-9]+)
         '''
     _TEST = {
-        'url': 'http://videomega.tv/?ref=GKeGPVedBe',
-        'md5': '240fb5bcf9199961f48eb17839b084d6',
+        'url': 'http://videomega.tv/?ref=QR0HCUHI1661IHUCH0RQ',
+        'md5': 'bf5c2f95c4c917536e80936af7bc51e1',
         'info_dict': {
-            'id': 'GKeGPVedBe',
+            'id': 'QR0HCUHI1661IHUCH0RQ',
             'ext': 'mp4',
-            'title': 'XXL - All Sports United',
+            'title': 'Big Buck Bunny',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        url = 'http://videomega.tv/iframe.php?ref={0:}'.format(video_id)
-        webpage = self._download_webpage(url, video_id)
+
+        iframe_url = 'http://videomega.tv/iframe.php?ref={0:}'.format(video_id)
+        req = compat_urllib_request.Request(iframe_url)
+        req.add_header('Referer', url)
+        webpage = self._download_webpage(req, video_id)
 
         escaped_data = self._search_regex(
             r'unescape\("([^"]+)"\)', webpage, 'escaped data')
@@ -37,13 +41,13 @@ class VideoMegaIE(InfoExtractor):
 
         thumbnail = self._search_regex(
             r'image:\s*"([^"]+)"', playlist, 'thumbnail', fatal=False)
-        url = self._search_regex(r'file:\s*"([^"]+)"', playlist, 'URL')
+        video_url = self._search_regex(r'file:\s*"([^"]+)"', playlist, 'URL')
         title = remove_start(self._html_search_regex(
             r'<title>(.*?)</title>', webpage, 'title'), 'VideoMega.tv - ')
 
         formats = [{
             'format_id': 'sd',
-            'url': url,
+            'url': video_url,
         }]
         self._sort_formats(formats)
 
@@ -52,4 +56,5 @@ class VideoMegaIE(InfoExtractor):
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
+            'http_referer': iframe_url,
         }

From a9b6b5cd15cff004b4109ab64b181fff23c10770 Mon Sep 17 00:00:00 2001
From: Phil Kulak <pkulak@gmail.com>
Date: Thu, 15 Jan 2015 10:23:05 -0800
Subject: [PATCH 1177/1937] Looks like Google switched to a new JS compiler
 that includes dollar signs in function names.

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c7611a3a0..073b78fb5 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -588,7 +588,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
     def _parse_sig_js(self, jscode):
         funcname = self._search_regex(
-            r'\.sig\|\|([a-zA-Z0-9]+)\(', jscode,
+            r'\.sig\|\|([a-zA-Z0-9\$]+)\(', jscode,
             'Initial JS player signature function name')
 
         jsi = JSInterpreter(jscode)

From aaeb86f68289496595930a7d6a04457917826d13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 16 Jan 2015 01:25:03 +0600
Subject: [PATCH 1178/1937] [youtube] Add test for #4706

---
 youtube_dl/extractor/youtube.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 073b78fb5..2a1f8be0a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -394,6 +394,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'format': '141',
             },
         },
+        # JS player signature function name containing $
+        {
+            'url': 'https://www.youtube.com/watch?v=nfWlot6h_JM',
+            'info_dict': {
+                'id': 'nfWlot6h_JM',
+                'ext': 'm4a',
+                'title': 'Taylor Swift - Shake It Off',
+                'description': 'md5:2acfda1b285bdd478ccec22f9918199d',
+                'uploader': 'TaylorSwiftVEVO',
+                'uploader_id': 'TaylorSwiftVEVO',
+                'upload_date': '20140818',
+            },
+            'params': {
+                'youtube_include_dash_manifest': True,
+                'format': '141',
+            },
+        },
         # Controversy video
         {
             'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
@@ -588,7 +605,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
     def _parse_sig_js(self, jscode):
         funcname = self._search_regex(
-            r'\.sig\|\|([a-zA-Z0-9\$]+)\(', jscode,
+            r'\.sig\|\|([a-zA-Z0-9$]+)\(', jscode,
             'Initial JS player signature function name')
 
         jsi = JSInterpreter(jscode)

From 31d4a6e2121ab5954aec5cfb48d6b1f1cbffefbd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 15 Jan 2015 22:38:11 +0100
Subject: [PATCH 1179/1937] release 2015.01.15.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6eccf2bf5..458813b2d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.15'
+__version__ = '2015.01.15.1'

From e205db3bcd7f7a9b0beaa13a4774bdcb26091e2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 16 Jan 2015 13:29:01 +0100
Subject: [PATCH 1180/1937] FFmpegEmbedSubtitlePP: don't fail if the video
 doesn't have an audio stream (fixes #4718)

Instead of specifying which streams ffmpeg must copy, we tell it to copy all.
---
 youtube_dl/postprocessor/ffmpeg.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 26b99e43c..a75277778 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -475,7 +475,13 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
         filename = information['filepath']
         input_files = [filename] + [subtitles_filename(filename, lang, self._subformat) for lang in sub_langs]
 
-        opts = ['-map', '0:0', '-map', '0:1', '-c:v', 'copy', '-c:a', 'copy']
+        opts = [
+            '-map', '0',
+            '-c', 'copy',
+            # Don't copy the existing subtitles, we may be running the
+            # postprocessor a second time
+            '-map', '-0:s',
+        ]
         for (i, lang) in enumerate(sub_langs):
             opts.extend(['-map', '%d:0' % (i + 1), '-c:s:%d' % i, 'mov_text'])
             lang_code = self._conver_lang_code(lang)

From 2875cf01bb2229cb6cceeac72fb6dd770f7a96d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 16 Jan 2015 13:37:37 +0100
Subject: [PATCH 1181/1937] FFmpegEmbedSubtitlePP: simplify command

---
 youtube_dl/postprocessor/ffmpeg.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index a75277778..5b0ff32b1 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -481,15 +481,15 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
             # Don't copy the existing subtitles, we may be running the
             # postprocessor a second time
             '-map', '-0:s',
+            '-c:s', 'mov_text',
         ]
         for (i, lang) in enumerate(sub_langs):
-            opts.extend(['-map', '%d:0' % (i + 1), '-c:s:%d' % i, 'mov_text'])
+            opts.extend(['-map', '%d:0' % (i + 1)])
             lang_code = self._conver_lang_code(lang)
             if lang_code is not None:
                 opts.extend(['-metadata:s:s:%d' % i, 'language=%s' % lang_code])
-        opts.extend(['-f', 'mp4'])
 
-        temp_filename = filename + '.temp'
+        temp_filename = prepend_extension(filename, 'temp')
         self._downloader.to_screen('[ffmpeg] Embedding subtitles in \'%s\'' % filename)
         self.run_ffmpeg_multiple_files(input_files, temp_filename, opts)
         os.remove(encodeFilename(filename))

From 4f4f6428224b641eb97ffd4b90be9d67034fc64a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 16 Jan 2015 13:44:36 +0100
Subject: [PATCH 1182/1937] [npo] Remove unused import

---
 youtube_dl/extractor/npo.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 8d496488b..175b14583 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     fix_xml_ampersands,

From 9d22a7dfb07ad6ff23680e11238744b2bb1735aa Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 16 Jan 2015 13:44:44 +0100
Subject: [PATCH 1183/1937] [fourtube] Fix extraction

---
 youtube_dl/extractor/fourtube.py | 76 ++++++++++++++++++--------------
 youtube_dl/utils.py              |  2 +-
 2 files changed, 45 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/fourtube.py b/youtube_dl/extractor/fourtube.py
index 7187e0752..b2284ab01 100644
--- a/youtube_dl/extractor/fourtube.py
+++ b/youtube_dl/extractor/fourtube.py
@@ -7,10 +7,9 @@ from ..compat import (
     compat_urllib_request,
 )
 from ..utils import (
-    clean_html,
     parse_duration,
+    parse_iso8601,
     str_to_int,
-    unified_strdate,
 )
 
 
@@ -28,68 +27,81 @@ class FourTubeIE(InfoExtractor):
             'uploader': 'WCP Club',
             'uploader_id': 'wcp-club',
             'upload_date': '20131031',
+            'timestamp': 1383263892,
             'duration': 583,
+            'view_count': int,
+            'like_count': int,
+            'categories': list,
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage_url = 'http://www.4tube.com/videos/' + video_id
-        webpage = self._download_webpage(webpage_url, video_id)
+        webpage = self._download_webpage(url, video_id)
 
-        self.report_extraction(video_id)
+        title = self._html_search_meta('name', webpage)
+        timestamp = parse_iso8601(self._html_search_meta(
+            'uploadDate', webpage))
+        thumbnail = self._html_search_meta('thumbnailUrl', webpage)
+        uploader_id = self._html_search_regex(
+            r'<a class="img-avatar" href="[^"]+/channels/([^/"]+)" title="Go to [^"]+ page">',
+            webpage, 'uploader id')
+        uploader = self._html_search_regex(
+            r'<a class="img-avatar" href="[^"]+/channels/[^/"]+" title="Go to ([^"]+) page">',
+            webpage, 'uploader')
 
-        playlist_json = self._html_search_regex(r'var playerConfigPlaylist\s+=\s+([^;]+)', webpage, 'Playlist')
-        media_id = self._search_regex(r'idMedia:\s*(\d+)', playlist_json, 'Media Id')
-        sources = self._search_regex(r'sources:\s*\[([^\]]*)\]', playlist_json, 'Sources').split(',')
-        title = self._search_regex(r'title:\s*"([^"]*)', playlist_json, 'Title')
-        thumbnail_url = self._search_regex(r'image:\s*"([^"]*)', playlist_json, 'Thumbnail', fatal=False)
+        categories_html = self._search_regex(
+            r'(?s)><i class="icon icon-tag"></i>\s*Categories / Tags\s*.*?<ul class="list">(.*?)</ul>',
+            webpage, 'categories', fatal=False)
+        categories = None
+        if categories_html:
+            categories = [
+                c.strip() for c in re.findall(
+                    r'(?s)<li><a.*?>(.*?)</a>', categories_html)]
 
-        uploader_str = self._search_regex(r'<span>Uploaded by</span>(.*?)<span>', webpage, 'uploader', fatal=False)
-        mobj = re.search(r'<a href="/sites/(?P<id>[^"]+)"><strong>(?P<name>[^<]+)</strong></a>', uploader_str)
-        (uploader, uploader_id) = (mobj.group('name'), mobj.group('id')) if mobj else (clean_html(uploader_str), None)
+        view_count = str_to_int(self._search_regex(
+            r'<meta itemprop="interactionCount" content="UserPlays:([0-9,]+)">',
+            webpage, 'view count', fatal=False))
+        like_count = str_to_int(self._search_regex(
+            r'<meta itemprop="interactionCount" content="UserLikes:([0-9,]+)">',
+            webpage, 'like count', fatal=False))
+        duration = parse_duration(self._html_search_meta('duration', webpage))
 
-        upload_date = None
-        view_count = None
-        duration = None
-        description = self._html_search_meta('description', webpage, 'description')
-        if description:
-            upload_date = self._search_regex(r'Published Date: (\d{2} [a-zA-Z]{3} \d{4})', description, 'upload date',
-                                             fatal=False)
-            if upload_date:
-                upload_date = unified_strdate(upload_date)
-            view_count = self._search_regex(r'Views: ([\d,\.]+)', description, 'view count', fatal=False)
-            if view_count:
-                view_count = str_to_int(view_count)
-            duration = parse_duration(self._search_regex(r'Length: (\d+m\d+s)', description, 'duration', fatal=False))
+        params_js = self._search_regex(
+            r'\$\.ajax\(url,\ opts\);\s*\}\s*\}\)\(([0-9,\[\] ]+)\)',
+            webpage, 'initialization parameters'
+        )
+        params = self._parse_json('[%s]' % params_js, video_id)
+        media_id = params[0]
+        sources = ['%s' % p for p in params[2]]
 
-        token_url = "http://tkn.4tube.com/{0}/desktop/{1}".format(media_id, "+".join(sources))
+        token_url = 'http://tkn.4tube.com/{0}/desktop/{1}'.format(
+            media_id, '+'.join(sources))
         headers = {
             b'Content-Type': b'application/x-www-form-urlencoded',
             b'Origin': b'http://www.4tube.com',
         }
         token_req = compat_urllib_request.Request(token_url, b'{}', headers)
         tokens = self._download_json(token_req, video_id)
-
         formats = [{
             'url': tokens[format]['token'],
             'format_id': format + 'p',
             'resolution': format + 'p',
             'quality': int(format),
         } for format in sources]
-
         self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
             'formats': formats,
-            'thumbnail': thumbnail_url,
+            'categories': categories,
+            'thumbnail': thumbnail,
             'uploader': uploader,
             'uploader_id': uploader_id,
-            'upload_date': upload_date,
+            'timestamp': timestamp,
+            'like_count': like_count,
             'view_count': view_count,
             'duration': duration,
             'age_limit': 18,
-            'webpage_url': webpage_url,
         }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2546fa45d..7832ed87f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1277,7 +1277,7 @@ def parse_duration(s):
     s = s.strip()
 
     m = re.match(
-        r'''(?ix)T?
+        r'''(?ix)(?:P?T)?
         (?:
             (?P<only_mins>[0-9.]+)\s*(?:mins?|minutes?)\s*|
             (?P<only_hours>[0-9.]+)\s*(?:hours?)|

From cce81f192c15f3766adb7340315a877bc4fc592b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 16 Jan 2015 14:20:25 +0100
Subject: [PATCH 1184/1937] [bandcamp:album] Fix title extraction (Fixes #4721)

---
 youtube_dl/extractor/bandcamp.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index b45d68a61..aea0263d6 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -161,7 +161,8 @@ class BandcampAlbumIE(InfoExtractor):
         entries = [
             self.url_result(compat_urlparse.urljoin(url, t_path), ie=BandcampIE.ie_key())
             for t_path in tracks_paths]
-        title = self._search_regex(r'album_title : "(.*?)"', webpage, 'title')
+        title = self._search_regex(
+            r'album_title\s*:\s*"(.*?)"', webpage, 'title', fatal=False)
         return {
             '_type': 'playlist',
             'id': playlist_id,

From 910c55205293097e79b2a134b69b572984e37846 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 16 Jan 2015 14:20:38 +0100
Subject: [PATCH 1185/1937] release 2015.01.16

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 458813b2d..63a79a7ee 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.15.1'
+__version__ = '2015.01.16'

From ba319696a99f34342a321d23bb41d3d442a773cc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 17 Jan 2015 23:56:34 +0100
Subject: [PATCH 1186/1937] [options] Clarify that --password can be left out
 (#4723)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index a30974efd..f25c12e52 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -264,7 +264,7 @@ def parseOpts(overrideArguments=None):
     authentication.add_option(
         '-p', '--password',
         dest='password', metavar='PASSWORD',
-        help='account password')
+        help='account password. If this option is left out, youtube-dl will ask interactively.')
     authentication.add_option(
         '-2', '--twofactor',
         dest='twofactor', metavar='TWOFACTOR',

From fdb2ed7455d509a5f0b17ad5c1d721d5484bde8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 18 Jan 2015 08:09:18 +0600
Subject: [PATCH 1187/1937] [abc7news] Add extractor (Closes #4734)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/abc7news.py | 68 ++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+)
 create mode 100644 youtube_dl/extractor/abc7news.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0902eb437..2b9d4455d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from .abc import ABCIE
+from .abc7news import Abc7NewsIE
 from .academicearth import AcademicEarthCourseIE
 from .addanime import AddAnimeIE
 from .adobetv import AdobeTVIE
diff --git a/youtube_dl/extractor/abc7news.py b/youtube_dl/extractor/abc7news.py
new file mode 100644
index 000000000..c04949c21
--- /dev/null
+++ b/youtube_dl/extractor/abc7news.py
@@ -0,0 +1,68 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import parse_iso8601
+
+
+class Abc7NewsIE(InfoExtractor):
+    _VALID_URL = r'https?://abc7news\.com(?:/[^/]+/(?P<display_id>[^/]+))?/(?P<id>\d+)'
+    _TESTS = [
+        {
+            'url': 'http://abc7news.com/entertainment/east-bay-museum-celebrates-vintage-synthesizers/472581/',
+            'info_dict': {
+                'id': '472581',
+                'display_id': 'east-bay-museum-celebrates-vintage-synthesizers',
+                'ext': 'mp4',
+                'title': 'East Bay museum celebrates history of synthesized music',
+                'description': 'md5:a4f10fb2f2a02565c1749d4adbab4b10',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'timestamp': 1421123075,
+                'upload_date': '20150113',
+                'uploader': 'Jonathan Bloom',
+            },
+            'params': {
+                # m3u8 download
+                'skip_download': True,
+            },
+        },
+        {
+            'url': 'http://abc7news.com/472581',
+            'only_matching': True,
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id') or video_id
+
+        webpage = self._download_webpage(url, display_id)
+
+        m3u8 = self._html_search_meta(
+            'contentURL', webpage, 'm3u8 url', fatal=True)
+
+        formats = self._extract_m3u8_formats(m3u8, display_id, 'mp4')
+        self._sort_formats(formats)
+
+        title = self._og_search_title(webpage).strip()
+        description = self._og_search_description(webpage).strip()
+        thumbnail = self._og_search_thumbnail(webpage)
+        timestamp = parse_iso8601(self._search_regex(
+            r'<div class="meta">\s*<time class="timeago" datetime="([^"]+)">',
+            webpage, 'upload date', fatal=False))
+        uploader = self._search_regex(
+            r'rel="author">([^<]+)</a>',
+            webpage, 'uploader', default=None)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'uploader': uploader,
+            'formats': formats,
+        }

From bd3cbe0716c3cfc4c6ce7a5a338e881a06b19ef8 Mon Sep 17 00:00:00 2001
From: David Powell <bitlooter@users.noreply.github.com>
Date: Sun, 18 Jan 2015 15:01:05 -0800
Subject: [PATCH 1188/1937] Embed description and URL as MP4 tags

---
 youtube_dl/postprocessor/ffmpeg.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 5b0ff32b1..faccdc43d 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -509,6 +509,10 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
             metadata['artist'] = info['uploader']
         elif info.get('uploader_id') is not None:
             metadata['artist'] = info['uploader_id']
+        if info.get('description') is not None:
+            metadata['description'] = info['description']
+        if info.get('webpage_url') is not None:
+            metadata['comment'] = info['webpage_url']
 
         if not metadata:
             self._downloader.to_screen('[ffmpeg] There isn\'t any metadata to add')

From c511f13f223d843dd4affc30c76bf333a6863227 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 19 Jan 2015 10:10:05 +0100
Subject: [PATCH 1189/1937] [ndtv] Modernize

---
 youtube_dl/extractor/ndtv.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ndtv.py b/youtube_dl/extractor/ndtv.py
index 95e7d63aa..2a1ca80df 100644
--- a/youtube_dl/extractor/ndtv.py
+++ b/youtube_dl/extractor/ndtv.py
@@ -27,9 +27,7 @@ class NDTVIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         filename = self._search_regex(

From 030aa5d9e7f482ab7baba18a9ff45e41021d60e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 19 Jan 2015 23:00:22 +0600
Subject: [PATCH 1190/1937] [tvp] Fix extraction

---
 youtube_dl/extractor/tvp.py | 37 +++++++++++++++++--------------------
 1 file changed, 17 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index cc26f417a..f57d609d4 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -12,61 +12,59 @@ class TvpIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://vod.tvp.pl/filmy-fabularne/filmy-za-darmo/ogniem-i-mieczem/wideo/odc-2/4278035',
+        'md5': 'cdd98303338b8a7f7abab5cd14092bf2',
         'info_dict': {
             'id': '4278035',
             'ext': 'wmv',
             'title': 'Ogniem i mieczem, odc. 2',
-            'description': 'Bohun dowiaduje się o złamaniu przez kniahinię danego mu słowa i wyrusza do Rozłogów. Helenie w ostatniej chwili udaje się uciec dzięki pomocy Zagłoby.',
         },
     }, {
         'url': 'http://vod.tvp.pl/seriale/obyczajowe/czas-honoru/sezon-1-1-13/i-seria-odc-13/194536',
+        'md5': '8aa518c15e5cc32dfe8db400dc921fbb',
         'info_dict': {
             'id': '194536',
             'ext': 'mp4',
             'title': 'Czas honoru, I seria – odc. 13',
-            #  'description': 'WŁADEK\nCzesław prosi Marię o dostarczenie Władkowi zarazki tyfusu. Jeśli zachoruje zostanie przewieziony do szpitala skąd łatwiej będzie go odbić. Czy matka zdecyduje się zarazić syna? Karol odwiedza Wandę przyznaje się, że ją oszukiwał, ale ostrzega też, że grozi jej aresztowanie i nalega, żeby wyjechała z Warszawy. Czy dziewczyna zdecyduje się znów oddalić od ukochanego? Rozpoczyna się akcja odbicia Władka.',
         },
     }, {
         'url': 'http://www.tvp.pl/there-can-be-anything-so-i-shortened-it/17916176',
+        'md5': 'c3b15ed1af288131115ff17a17c19dda',
         'info_dict': {
             'id': '17916176',
             'ext': 'mp4',
             'title': 'TVP Gorzów pokaże filmy studentów z podroży dookoła świata',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': 'true',
-        },
     }, {
         'url': 'http://vod.tvp.pl/seriale/obyczajowe/na-sygnale/sezon-2-27-/odc-39/17834272',
+        'md5': 'c3b15ed1af288131115ff17a17c19dda',
         'info_dict': {
             'id': '17834272',
             'ext': 'mp4',
             'title': 'Na sygnale, odc. 39',
-            'description': 'Ekipa Wiktora ratuje młodą matkę, która spadła ze schodów trzymając na rękach noworodka. Okazuje się, że dziewczyna jest surogatką, a biologiczni rodzice dziecka próbują zmusić ją do oddania synka…',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': 'true',
         },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(
             'http://www.tvp.pl/sess/tvplayer.php?object_id=%s' % video_id, video_id)
 
-        title = self._og_search_title(webpage)
-        series = self._search_regex(
-            r'{name:\s*([\'"])SeriesTitle\1,\s*value:\s*\1(?P<series>.*?)\1},',
+        title = self._search_regex(
+            r'name\s*:\s*([\'"])Title\1\s*,\s*value\s*:\s*\1(?P<title>.+?)\1',
+            webpage, 'title', group='title')
+        series_title = self._search_regex(
+            r'name\s*:\s*([\'"])SeriesTitle\1\s*,\s*value\s*:\s*\1(?P<series>.+?)\1',
             webpage, 'series', group='series', default=None)
-        if series is not None and series not in title:
-            title = '%s, %s' % (series, title)
-        description = self._og_search_description(webpage, default=None)
+        if series_title:
+            title = '%s, %s' % (series_title, title)
+
+        thumbnail = self._search_regex(
+            r"poster\s*:\s*'([^']+)'", webpage, 'thumbnail', default=None)
 
         video_url = self._search_regex(
             r'0:{src:([\'"])(?P<url>.*?)\1', webpage, 'formats', group='url', default=None)
-        if video_url is None:
+        if not video_url:
             video_url = self._download_json(
                 'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s' % video_id,
                 video_id)['video_url']
@@ -89,8 +87,7 @@ class TvpIE(InfoExtractor):
         return {
             'id': video_id,
             'title': title,
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'description': description,
+            'thumbnail': thumbnail,
             'formats': formats,
         }
 

From 2df54b4ba81a2615d8f5a9016061eed389ba5b6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 20 Jan 2015 21:23:51 +0600
Subject: [PATCH 1191/1937] [nbcnews] Ignore HTTP errors while coping with
 playlists (Closes #4749)

---
 youtube_dl/extractor/nbc.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 690c46b6a..7ae66616a 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -6,6 +6,7 @@ import json
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
+    compat_HTTPError,
 )
 from ..utils import (
     ExtractorError,
@@ -78,6 +79,16 @@ class NBCNewsIE(InfoExtractor):
             },
             'add_ie': ['ThePlatform'],
         },
+        {
+            'url': 'http://www.nbcnews.com/feature/dateline-full-episodes/full-episode-family-business-n285156',
+            'md5': 'fdbf39ab73a72df5896b6234ff98518a',
+            'info_dict': {
+                'id': 'Wjf9EDR3A_60',
+                'ext': 'mp4',
+                'title': 'FULL EPISODE: Family Business',
+                'description': 'md5:757988edbaae9d7be1d585eb5d55cc04',
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -115,10 +126,19 @@ class NBCNewsIE(InfoExtractor):
                 if not base_url:
                     continue
                 playlist_url = base_url + '?form=MPXNBCNewsAPI'
-                all_videos = self._download_json(playlist_url, title)['videos']
 
                 try:
-                    info = next(v for v in all_videos if v['mpxId'] == mpxid)
+                    all_videos = self._download_json(playlist_url, title)
+                except ExtractorError as ee:
+                    if isinstance(ee.cause, compat_HTTPError):
+                        continue
+                    raise
+
+                if not all_videos or not 'videos' in all_videos:
+                    continue
+
+                try:
+                    info = next(v for v in all_videos['videos'] if v['mpxId'] == mpxid)
                     break
                 except StopIteration:
                     continue

From 564bb5e964fc263012784baa579a10eee822a436 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Jan 2015 02:15:28 +0600
Subject: [PATCH 1192/1937] [tinypic] Tweak VALID_URL regex (Closes #4754)

---
 youtube_dl/extractor/tinypic.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/tinypic.py b/youtube_dl/extractor/tinypic.py
index 4fe89dbe5..e036b8cdf 100644
--- a/youtube_dl/extractor/tinypic.py
+++ b/youtube_dl/extractor/tinypic.py
@@ -9,17 +9,23 @@ from ..utils import ExtractorError
 class TinyPicIE(InfoExtractor):
     IE_NAME = 'tinypic'
     IE_DESC = 'tinypic.com videos'
-    _VALID_URL = r'http://tinypic\.com/player\.php\?v=(?P<id>[^&]+)&s=\d+'
+    _VALID_URL = r'http://(?:.+?\.)?tinypic\.com/player\.php\?v=(?P<id>[^&]+)&s=\d+'
 
-    _TEST = {
-        'url': 'http://tinypic.com/player.php?v=6xw7tc%3E&s=5#.UtqZmbRFCM8',
-        'md5': '609b74432465364e72727ebc6203f044',
-        'info_dict': {
-            'id': '6xw7tc',
-            'ext': 'flv',
-            'title': 'shadow phenomenon weird',
+    _TESTS = [
+        {
+            'url': 'http://tinypic.com/player.php?v=6xw7tc%3E&s=5#.UtqZmbRFCM8',
+            'md5': '609b74432465364e72727ebc6203f044',
+            'info_dict': {
+                'id': '6xw7tc',
+                'ext': 'flv',
+                'title': 'shadow phenomenon weird',
+            },
+        },
+        {
+            'url': 'http://de.tinypic.com/player.php?v=dy90yh&s=8',
+            'only_matching': True,
         }
-    }
+    ]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From 5fe511258998666e7617c22a5db27506c16eb955 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 21 Jan 2015 10:27:18 +0100
Subject: [PATCH 1193/1937] [CNNArticle] Update test

---
 youtube_dl/extractor/cnn.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 93e8d0de3..4bb58f25e 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -143,13 +143,13 @@ class CNNArticleIE(InfoExtractor):
     _VALID_URL = r'https?://(?:(?:edition|www)\.)?cnn\.com/(?!video/)'
     _TEST = {
         'url': 'http://www.cnn.com/2014/12/21/politics/obama-north-koreas-hack-not-war-but-cyber-vandalism/',
-        'md5': '275b326f85d80dff7592a9820f5dc887',
+        'md5': '689034c2a3d9c6dc4aa72d65a81efd01',
         'info_dict': {
-            'id': 'bestoftv/2014/12/21/sotu-crowley-president-obama-north-korea-not-going-to-be-intimidated.cnn',
+            'id': 'bestoftv/2014/12/21/ip-north-korea-obama.cnn',
             'ext': 'mp4',
-            'title': 'Obama: We\'re not going to be intimidated',
-            'description': 'md5:e735586f3dc936075fa654a4d91b21f9',
-            'upload_date': '20141220',
+            'title': 'Obama: Cyberattack not an act of war',
+            'description': 'md5:51ce6750450603795cad0cdfbd7d05c5',
+            'upload_date': '20141221',
         },
         'add_ie': ['CNN'],
     }

From efcddaebe9302eaaea1cbb5570c5c2bd25abf1fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 21 Jan 2015 10:31:57 +0100
Subject: [PATCH 1194/1937] [cnn] Use edition.cnn.com for getting the
 information (fixes #4757)

Some videos (like http://edition.cnn.com/videos/us/2015/01/20/orig-yellowstone-oil-spill.cnn) will fail if we use cnn.com.
---
 youtube_dl/extractor/cnn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index 4bb58f25e..90ea07438 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -51,7 +51,7 @@ class CNNIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         path = mobj.group('path')
         page_title = mobj.group('title')
-        info_url = 'http://cnn.com/video/data/3.0/%s/index.xml' % path
+        info_url = 'http://edition.cnn.com/video/data/3.0/%s/index.xml' % path
         info = self._download_xml(info_url, page_title)
 
         formats = []

From 47e0e1e0e2306dd7119844bbbc3ef39ccb175ed1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 21 Jan 2015 10:36:15 +0100
Subject: [PATCH 1195/1937] [nbc] Fix pep8 issue

---
 youtube_dl/extractor/nbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 7ae66616a..f840f6532 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -134,7 +134,7 @@ class NBCNewsIE(InfoExtractor):
                         continue
                     raise
 
-                if not all_videos or not 'videos' in all_videos:
+                if not all_videos or 'videos' not in all_videos:
                     continue
 
                 try:

From c8dc41a6e7eea2a1235238cfeadb4caf5619d0ae Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 21 Jan 2015 23:05:47 +0800
Subject: [PATCH 1196/1937] [StreetVoice] Add new extractor

---
 youtube_dl/extractor/__init__.py    |  1 +
 youtube_dl/extractor/streetvoice.py | 36 +++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 youtube_dl/extractor/streetvoice.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2b9d4455d..9e942e6f9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -409,6 +409,7 @@ from .stanfordoc import StanfordOpenClassroomIE
 from .steam import SteamIE
 from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
+from .streetvoice import StreetVoiceIE
 from .sunporno import SunPornoIE
 from .swrmediathek import SWRMediathekIE
 from .syfy import SyfyIE
diff --git a/youtube_dl/extractor/streetvoice.py b/youtube_dl/extractor/streetvoice.py
new file mode 100644
index 000000000..55ceea75b
--- /dev/null
+++ b/youtube_dl/extractor/streetvoice.py
@@ -0,0 +1,36 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class StreetVoiceIE(InfoExtractor):
+    _VALID_URL = r'http://tw.streetvoice.com/[^/]+/songs/(?P<id>[0-9]+)/'
+    _TESTS = [
+        {
+            'url': 'http://tw.streetvoice.com/skippylu/songs/94440/',
+            'md5': '15974627fc01a29e492c98593c2fd472',
+            'info_dict': {
+                'id': '94440',
+                'ext': 'mp3',
+                'title': '輸',
+                'description': '輸 - Crispy脆樂團'
+            }
+        }
+    ]
+
+    def _real_extract(self, url):
+        song_id = self._match_id(url)
+
+        api_url = 'http://tw.streetvoice.com/music/api/song/%s/' % song_id
+        info_dict = self._download_json(api_url, song_id)
+
+        author = info_dict['musician']['name']
+        title = info_dict['name']
+        return {
+            'id': song_id,
+            'ext': 'mp3',
+            'title': title,
+            'url': info_dict['file'],
+            'description': '%s - %s' % (title, author)
+        }

From c5db6bb32ba708bc5c92a68487b07be2374f416d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Jan 2015 22:27:21 +0600
Subject: [PATCH 1197/1937] [twitch] Refactor and add support for past
 broadcasts

---
 youtube_dl/extractor/__init__.py |   8 +-
 youtube_dl/extractor/twitch.py   | 345 ++++++++++++++++---------------
 2 files changed, 184 insertions(+), 169 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2b9d4455d..0b0f3be19 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -458,7 +458,13 @@ from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
 from .twentyfourvideo import TwentyFourVideoIE
-from .twitch import TwitchIE
+from .twitch import (
+    TwitchVideoIE,
+    TwitchChapterIE,
+    TwitchVodIE,
+    TwitchProfileIE,
+    TwitchPastBroadcastsIE,
+)
 from .ubu import UbuIE
 from .udemy import (
     UdemyIE,
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index b11a1d561..f70002f68 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -15,44 +15,11 @@ from ..utils import (
 )
 
 
-class TwitchIE(InfoExtractor):
-    # TODO: One broadcast may be split into multiple videos. The key
-    # 'broadcast_id' is the same for all parts, and 'broadcast_part'
-    # starts at 1 and increases. Can we treat all parts as one video?
-    _VALID_URL = r"""(?x)^(?:http://)?(?:www\.)?twitch\.tv/
-        (?:
-            (?P<channelid>[^/]+)|
-            (?:(?:[^/]+)/v/(?P<vodid>[^/]+))|
-            (?:(?:[^/]+)/b/(?P<videoid>[^/]+))|
-            (?:(?:[^/]+)/c/(?P<chapterid>[^/]+))
-        )
-        /?(?:\#.*)?$
-        """
-    _PAGE_LIMIT = 100
+class TwitchBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'http://(?:www\.)?twitch\.tv'
+
     _API_BASE = 'https://api.twitch.tv'
     _LOGIN_URL = 'https://secure.twitch.tv/user/login'
-    _TESTS = [{
-        'url': 'http://www.twitch.tv/riotgames/b/577357806',
-        'info_dict': {
-            'id': 'a577357806',
-            'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
-        },
-        'playlist_mincount': 12,
-    }, {
-        'url': 'http://www.twitch.tv/acracingleague/c/5285812',
-        'info_dict': {
-            'id': 'c5285812',
-            'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
-        },
-        'playlist_mincount': 3,
-    }, {
-        'url': 'http://www.twitch.tv/vanillatv',
-        'info_dict': {
-            'id': 'vanillatv',
-            'title': 'VanillaTV',
-        },
-        'playlist_mincount': 412,
-    }]
 
     def _handle_error(self, response):
         if not isinstance(response, dict):
@@ -64,71 +31,10 @@ class TwitchIE(InfoExtractor):
                 expected=True)
 
     def _download_json(self, url, video_id, note='Downloading JSON metadata'):
-        response = super(TwitchIE, self)._download_json(url, video_id, note)
+        response = super(TwitchBaseIE, self)._download_json(url, video_id, note)
         self._handle_error(response)
         return response
 
-    def _extract_media(self, item, item_id):
-        ITEMS = {
-            'a': 'video',
-            'v': 'vod',
-            'c': 'chapter',
-        }
-        info = self._extract_info(self._download_json(
-            '%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
-            'Downloading %s info JSON' % ITEMS[item]))
-
-        if item == 'v':
-            access_token = self._download_json(
-                '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
-                'Downloading %s access token' % ITEMS[item])
-            formats = self._extract_m3u8_formats(
-                'http://usher.twitch.tv/vod/%s?nauth=%s&nauthsig=%s'
-                % (item_id, access_token['token'], access_token['sig']),
-                item_id, 'mp4')
-            info['formats'] = formats
-            return info
-
-        response = self._download_json(
-            '%s/api/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
-            'Downloading %s playlist JSON' % ITEMS[item])
-        entries = []
-        chunks = response['chunks']
-        qualities = list(chunks.keys())
-        for num, fragment in enumerate(zip(*chunks.values()), start=1):
-            formats = []
-            for fmt_num, fragment_fmt in enumerate(fragment):
-                format_id = qualities[fmt_num]
-                fmt = {
-                    'url': fragment_fmt['url'],
-                    'format_id': format_id,
-                    'quality': 1 if format_id == 'live' else 0,
-                }
-                m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
-                if m:
-                    fmt['height'] = int(m.group('height'))
-                formats.append(fmt)
-            self._sort_formats(formats)
-            entry = dict(info)
-            entry['id'] = '%s_%d' % (entry['id'], num)
-            entry['title'] = '%s part %d' % (entry['title'], num)
-            entry['formats'] = formats
-            entries.append(entry)
-        return self.playlist_result(entries, info['id'], info['title'])
-
-    def _extract_info(self, info):
-        return {
-            'id': info['_id'],
-            'title': info['title'],
-            'description': info['description'],
-            'duration': info['length'],
-            'thumbnail': info['preview'],
-            'uploader': info['channel']['display_name'],
-            'uploader_id': info['channel']['name'],
-            'timestamp': parse_iso8601(info['recorded_at']),
-            'view_count': info['views'],
-        }
-
     def _real_initialize(self):
         self._login()
 
@@ -167,81 +73,184 @@ class TwitchIE(InfoExtractor):
             raise ExtractorError(
                 'Unable to login: %s' % m.group('msg').strip(), expected=True)
 
+
+class TwitchItemBaseIE(TwitchBaseIE):
+    def _download_info(self, item, item_id):
+        return self._extract_info(self._download_json(
+            '%s/kraken/videos/%s%s' % (self._API_BASE, item, item_id), item_id,
+            'Downloading %s info JSON' % self._ITEM_TYPE))
+
+    def _extract_media(self, item_id):
+        info = self._download_info(self._ITEM_SHORTCUT, item_id)
+        response = self._download_json(
+            '%s/api/videos/%s%s' % (self._API_BASE, self._ITEM_SHORTCUT, item_id), item_id,
+            'Downloading %s playlist JSON' % self._ITEM_TYPE)
+        entries = []
+        chunks = response['chunks']
+        qualities = list(chunks.keys())
+        for num, fragment in enumerate(zip(*chunks.values()), start=1):
+            formats = []
+            for fmt_num, fragment_fmt in enumerate(fragment):
+                format_id = qualities[fmt_num]
+                fmt = {
+                    'url': fragment_fmt['url'],
+                    'format_id': format_id,
+                    'quality': 1 if format_id == 'live' else 0,
+                }
+                m = re.search(r'^(?P<height>\d+)[Pp]', format_id)
+                if m:
+                    fmt['height'] = int(m.group('height'))
+                formats.append(fmt)
+            self._sort_formats(formats)
+            entry = dict(info)
+            entry['id'] = '%s_%d' % (entry['id'], num)
+            entry['title'] = '%s part %d' % (entry['title'], num)
+            entry['formats'] = formats
+            entries.append(entry)
+        return self.playlist_result(entries, info['id'], info['title'])
+
+    def _extract_info(self, info):
+        return {
+            'id': info['_id'],
+            'title': info['title'],
+            'description': info['description'],
+            'duration': info['length'],
+            'thumbnail': info['preview'],
+            'uploader': info['channel']['display_name'],
+            'uploader_id': info['channel']['name'],
+            'timestamp': parse_iso8601(info['recorded_at']),
+            'view_count': info['views'],
+        }
+
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        if mobj.group('chapterid'):
-            return self._extract_media('c', mobj.group('chapterid'))
+        return self._extract_media(self._match_id(url))
 
-            """
-            webpage = self._download_webpage(url, chapter_id)
-            m = re.search(r'PP\.archive_id = "([0-9]+)";', webpage)
-            if not m:
-                raise ExtractorError('Cannot find archive of a chapter')
-            archive_id = m.group(1)
 
-            api = api_base + '/broadcast/by_chapter/%s.xml' % chapter_id
-            doc = self._download_xml(
-                api, chapter_id,
-                note='Downloading chapter information',
-                errnote='Chapter information download failed')
-            for a in doc.findall('.//archive'):
-                if archive_id == a.find('./id').text:
-                    break
-            else:
-                raise ExtractorError('Could not find chapter in chapter information')
+class TwitchVideoIE(TwitchItemBaseIE):
+    IE_NAME = 'twitch:video'
+    _VALID_URL = r'%s/[^/]+/b/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _ITEM_TYPE = 'video'
+    _ITEM_SHORTCUT = 'a'
 
-            video_url = a.find('./video_file_url').text
-            video_ext = video_url.rpartition('.')[2] or 'flv'
+    _TEST = {
+        'url': 'http://www.twitch.tv/riotgames/b/577357806',
+        'info_dict': {
+            'id': 'a577357806',
+            'title': 'Worlds Semifinals - Star Horn Royal Club vs. OMG',
+        },
+        'playlist_mincount': 12,
+    }
 
-            chapter_api_url = 'https://api.twitch.tv/kraken/videos/c' + chapter_id
-            chapter_info = self._download_json(
-                chapter_api_url, 'c' + chapter_id,
-                note='Downloading chapter metadata',
-                errnote='Download of chapter metadata failed')
 
-            bracket_start = int(doc.find('.//bracket_start').text)
-            bracket_end = int(doc.find('.//bracket_end').text)
+class TwitchChapterIE(TwitchItemBaseIE):
+    IE_NAME = 'twitch:chapter'
+    _VALID_URL = r'%s/[^/]+/c/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _ITEM_TYPE = 'chapter'
+    _ITEM_SHORTCUT = 'c'
 
-            # TODO determine start (and probably fix up file)
-            #  youtube-dl -v http://www.twitch.tv/firmbelief/c/1757457
-            #video_url += '?start=' + TODO:start_timestamp
-            # bracket_start is 13290, but we want 51670615
-            self._downloader.report_warning('Chapter detected, but we can just download the whole file. '
-                                            'Chapter starts at %s and ends at %s' % (formatSeconds(bracket_start), formatSeconds(bracket_end)))
+    _TEST = {
+        'url': 'http://www.twitch.tv/acracingleague/c/5285812',
+        'info_dict': {
+            'id': 'c5285812',
+            'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
+        },
+        'playlist_mincount': 3,
+    }
 
-            info = {
-                'id': 'c' + chapter_id,
-                'url': video_url,
-                'ext': video_ext,
-                'title': chapter_info['title'],
-                'thumbnail': chapter_info['preview'],
-                'description': chapter_info['description'],
-                'uploader': chapter_info['channel']['display_name'],
-                'uploader_id': chapter_info['channel']['name'],
-            }
-            return info
-            """
-        elif mobj.group('videoid'):
-            return self._extract_media('a', mobj.group('videoid'))
-        elif mobj.group('vodid'):
-            return self._extract_media('v', mobj.group('vodid'))
-        elif mobj.group('channelid'):
-            channel_id = mobj.group('channelid')
-            info = self._download_json(
-                '%s/kraken/channels/%s' % (self._API_BASE, channel_id),
-                channel_id, 'Downloading channel info JSON')
-            channel_name = info.get('display_name') or info.get('name')
-            entries = []
-            offset = 0
-            limit = self._PAGE_LIMIT
-            for counter in itertools.count(1):
-                response = self._download_json(
-                    '%s/kraken/channels/%s/videos/?offset=%d&limit=%d'
-                    % (self._API_BASE, channel_id, offset, limit),
-                    channel_id, 'Downloading channel videos JSON page %d' % counter)
-                videos = response['videos']
-                if not videos:
-                    break
-                entries.extend([self.url_result(video['url'], 'Twitch') for video in videos])
-                offset += limit
-            return self.playlist_result(entries, channel_id, channel_name)
+
+class TwitchVodIE(TwitchItemBaseIE):
+    IE_NAME = 'twitch:vod'
+    _VALID_URL = r'%s/[^/]+/v/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _ITEM_TYPE = 'vod'
+    _ITEM_SHORTCUT = 'v'
+
+    _TEST = {
+        'url': 'http://www.twitch.tv/ksptv/v/3622000',
+        'info_dict': {
+            'id': 'v3622000',
+            'ext': 'mp4',
+            'title': '''KSPTV: Squadcast: "Everyone's on vacation so here's Dahud" Edition!''',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 6951,
+            'timestamp': 1419028564,
+            'upload_date': '20141219',
+            'uploader': 'KSPTV',
+            'uploader_id': 'ksptv',
+            'view_count': int,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        item_id = self._match_id(url)
+        info = self._download_info(self._ITEM_SHORTCUT, item_id)
+        access_token = self._download_json(
+            '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
+            'Downloading %s access token' % self._ITEM_TYPE)
+        formats = self._extract_m3u8_formats(
+            'http://usher.twitch.tv/vod/%s?nauth=%s&nauthsig=%s'
+            % (item_id, access_token['token'], access_token['sig']),
+            item_id, 'mp4')
+        info['formats'] = formats
+        return info
+
+
+class TwitchPlaylistBaseIE(TwitchBaseIE):
+    _PLAYLIST_URL = '%s/kraken/channels/%%s/videos/?offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
+    _PAGE_LIMIT = 100
+
+    def _extract_playlist(self, channel_id):
+        info = self._download_json(
+            '%s/kraken/channels/%s' % (self._API_BASE, channel_id),
+            channel_id, 'Downloading channel info JSON')
+        channel_name = info.get('display_name') or info.get('name')
+        entries = []
+        offset = 0
+        limit = self._PAGE_LIMIT
+        for counter in itertools.count(1):
+            response = self._download_json(
+                self._PLAYLIST_URL % (channel_id, offset, limit),
+                channel_id, 'Downloading %s videos JSON page %d' % (self._PLAYLIST_TYPE, counter))
+            videos = response['videos']
+            if not videos:
+                break
+            entries.extend([self.url_result(video['url']) for video in videos])
+            offset += limit
+        return self.playlist_result(entries, channel_id, channel_name)
+
+    def _real_extract(self, url):
+        return self._extract_playlist(self._match_id(url))
+
+
+class TwitchProfileIE(TwitchPlaylistBaseIE):
+    IE_NAME = 'twitch:profile'
+    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+    _PLAYLIST_TYPE = 'profile'
+
+    _TEST = {
+        'url': 'http://www.twitch.tv/vanillatv/profile',
+        'info_dict': {
+            'id': 'vanillatv',
+            'title': 'VanillaTV',
+        },
+        'playlist_mincount': 412,
+    }
+
+
+class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
+    IE_NAME = 'twitch:profile'
+    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+    _PLAYLIST_URL = TwitchPlaylistBaseIE._PLAYLIST_URL + '&broadcasts=true'
+    _PLAYLIST_TYPE = 'past broadcasts'
+
+    _TEST = {
+        'url': 'http://www.twitch.tv/spamfish/profile/past_broadcasts',
+        'info_dict': {
+            'id': 'spamfish',
+            'title': 'Spamfish',
+        },
+        'playlist_mincount': 54,
+    }

From bb0aa4cb3ca1b21084e6e6229e4a69014e61ccff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Jan 2015 22:53:51 +0600
Subject: [PATCH 1198/1937] [streetvoice] Improve

---
 youtube_dl/extractor/streetvoice.py | 53 ++++++++++++++++++-----------
 1 file changed, 34 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/streetvoice.py b/youtube_dl/extractor/streetvoice.py
index 55ceea75b..6a57fa60a 100644
--- a/youtube_dl/extractor/streetvoice.py
+++ b/youtube_dl/extractor/streetvoice.py
@@ -2,35 +2,50 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import unified_strdate
 
 
 class StreetVoiceIE(InfoExtractor):
-    _VALID_URL = r'http://tw.streetvoice.com/[^/]+/songs/(?P<id>[0-9]+)/'
-    _TESTS = [
-        {
-            'url': 'http://tw.streetvoice.com/skippylu/songs/94440/',
-            'md5': '15974627fc01a29e492c98593c2fd472',
-            'info_dict': {
-                'id': '94440',
-                'ext': 'mp3',
-                'title': '輸',
-                'description': '輸 - Crispy脆樂團'
-            }
+    _VALID_URL = r'https?://(?:.+?\.)?streetvoice\.com/[^/]+/songs/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://streetvoice.com/skippylu/songs/94440/',
+        'md5': '15974627fc01a29e492c98593c2fd472',
+        'info_dict': {
+            'id': '94440',
+            'ext': 'mp3',
+            'filesize': 4167053,
+            'title': '輸',
+            'description': 'Crispy脆樂團 - 輸',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 260,
+            'upload_date': '20091018',
+            'uploader': 'Crispy脆樂團',
+            'uploader_id': '627810',
         }
-    ]
+    }, {
+        'url': 'http://tw.streetvoice.com/skippylu/songs/94440/',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         song_id = self._match_id(url)
 
-        api_url = 'http://tw.streetvoice.com/music/api/song/%s/' % song_id
-        info_dict = self._download_json(api_url, song_id)
+        song = self._download_json(
+            'http://streetvoice.com/music/api/song/%s' % song_id, song_id)
+
+        title = song['name']
+        author = song['musician']['name']
 
-        author = info_dict['musician']['name']
-        title = info_dict['name']
         return {
             'id': song_id,
-            'ext': 'mp3',
+            'url': song['file'],
+            'filesize': song.get('size'),
             'title': title,
-            'url': info_dict['file'],
-            'description': '%s - %s' % (title, author)
+            'description': '%s - %s' % (author, title),
+            'thumbnail': self._proto_relative_url(song.get('image'), 'http:'),
+            'duration': song.get('length'),
+            'upload_date': unified_strdate(song.get('created_at')),
+            'uploader': author,
+            'uploader_id': compat_str(song['musician']['id']),
         }

From c62b44976545401af60fd35f0188fd09a93e449b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Jan 2015 22:56:28 +0600
Subject: [PATCH 1199/1937] Credit @yan12125 for streetvoice (#4758)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 35e256d49..b8bf3cb6f 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -103,3 +103,4 @@ Christopher Krooss
 Ondřej Caletka
 Dinesh S
 Johan K. Jensen
+Yen Chi Hsuan

From 8bb1bdfae91246e1306ee31674c06a2ef7e42a08 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Jan 2015 23:06:16 +0600
Subject: [PATCH 1200/1937] [twitch:past_broadcasts] Fix IE_NAME

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index f70002f68..8866e5182 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -241,7 +241,7 @@ class TwitchProfileIE(TwitchPlaylistBaseIE):
 
 
 class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
-    IE_NAME = 'twitch:profile'
+    IE_NAME = 'twitch:past_broadcasts'
     _VALID_URL = r'%s/(?P<id>[^/]+)/profile/past_broadcasts/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
     _PLAYLIST_URL = TwitchPlaylistBaseIE._PLAYLIST_URL + '&broadcasts=true'
     _PLAYLIST_TYPE = 'past broadcasts'

From e5763a7a7e630cfd94b8993a9592cb3243890e0c Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 21 Jan 2015 21:47:55 +0200
Subject: [PATCH 1201/1937] [hearthisat] Add new extractor (Closes #4743)

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/hearthisat.py | 92 ++++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+)
 create mode 100644 youtube_dl/extractor/hearthisat.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c41f2de0f..be3fecd0d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -176,6 +176,7 @@ from .goshgay import GoshgayIE
 from .grooveshark import GroovesharkIE
 from .groupon import GrouponIE
 from .hark import HarkIE
+from .hearthisat import HearThisAtIE
 from .heise import HeiseIE
 from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
diff --git a/youtube_dl/extractor/hearthisat.py b/youtube_dl/extractor/hearthisat.py
new file mode 100644
index 000000000..c1177a9c5
--- /dev/null
+++ b/youtube_dl/extractor/hearthisat.py
@@ -0,0 +1,92 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_request
+from ..utils import (
+    str_to_int,
+    urlencode_postdata,
+)
+
+
+class HearThisAtIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?hearthis\.at/(?P<artist>[^/]+)/(?P<title>[A-Za-z0-9\-]+)/?$'
+    _PLAYLIST_URL = 'https://hearthis.at/playlist.php'
+    _TEST = {
+        'url': 'https://hearthis.at/moofi/dr-kreep',
+        'md5': 'd594c573227a89f4256f0b03e68c80cc',
+        'info_dict': {
+            'id': '150939',
+            'ext': 'mp3',
+            'title': 'Moofi - Dr. Kreep',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1421564134,
+            'description': 'Creepy Patch. Mutable Instruments Braids Vowel + Formant Mode.',
+            'upload_date': '20150118',
+            'comment_count': int,
+            'view_count': int,
+            'like_count': int,
+            'duration': 71,
+            'categories': ['Experimental'],
+        }
+    }
+
+    def _real_extract(self, url):
+        m = re.match(self._VALID_URL, url)
+        display_id = '{artist:s} - {title:s}'.format(**m.groupdict())
+
+        webpage = self._download_webpage(url, display_id)
+        track_id = self._search_regex(
+            r'intTrackId\s*=\s*(\d+)', webpage, 'track ID')
+
+        payload = urlencode_postdata({'tracks[]': track_id})
+        req = compat_urllib_request.Request(self._PLAYLIST_URL, payload)
+        req.add_header('Content-type', 'application/x-www-form-urlencoded')
+
+        track = self._download_json(req, track_id, 'Downloading playlist')[0]
+        title = '{artist:s} - {title:s}'.format(**track)
+
+        categories = None
+        if track.get('category'):
+            categories = [track['category']]
+
+        description = self._og_search_description(webpage)
+        thumbnail = self._og_search_thumbnail(webpage)
+
+        meta_span = r'<span[^>]+class="%s".*?</i>([^<]+)</span>'
+        view_count = str_to_int(self._search_regex(
+            meta_span % 'plays_count', webpage, 'view count', fatal=False))
+        like_count = str_to_int(self._search_regex(
+            meta_span % 'likes_count', webpage, 'like count', fatal=False))
+        comment_count = str_to_int(self._search_regex(
+            meta_span % 'comment_count', webpage, 'comment count', fatal=False))
+        duration = str_to_int(self._search_regex(
+            r'data-length="(\d+)', webpage, 'duration', fatal=False))
+        timestamp = str_to_int(self._search_regex(
+            r'<span[^>]+class="calctime"[^>]+data-time="(\d+)', webpage, 'timestamp', fatal=False))
+
+        track_url = self._search_regex(
+            r'<a[^>]+data-mp3="([^"]+)"', webpage, 'track URL')
+
+        formats = [{
+            'format_id': 'mp3',
+            'url': track_url,
+            'vcodec': 'none',
+        }]
+
+        return {
+            'id': track_id,
+            'display-id': display_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'description': description,
+            'duration': duration,
+            'timestamp': timestamp,
+            'view_count': view_count,
+            'comment_count': comment_count,
+            'like_count': like_count,
+            'categories': categories,
+        }

From b55ee18ff3a9642fe25a977e1152472877294493 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 12:04:07 +0100
Subject: [PATCH 1202/1937] [hearthisat] Add support for more high-quality
 download links

---
 youtube_dl/extractor/hearthisat.py | 49 ++++++++++++++++++++++--------
 youtube_dl/utils.py                |  8 +++++
 2 files changed, 45 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/hearthisat.py b/youtube_dl/extractor/hearthisat.py
index c1177a9c5..d83072c1a 100644
--- a/youtube_dl/extractor/hearthisat.py
+++ b/youtube_dl/extractor/hearthisat.py
@@ -4,10 +4,15 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
+from ..compat import (
+    compat_urllib_request,
+    compat_urlparse,
+)
 from ..utils import (
+    HEADRequest,
     str_to_int,
     urlencode_postdata,
+    urlhandle_detect_ext,
 )
 
 
@@ -16,10 +21,10 @@ class HearThisAtIE(InfoExtractor):
     _PLAYLIST_URL = 'https://hearthis.at/playlist.php'
     _TEST = {
         'url': 'https://hearthis.at/moofi/dr-kreep',
-        'md5': 'd594c573227a89f4256f0b03e68c80cc',
+        'md5': 'ab6ec33c8fed6556029337c7885eb4e0',
         'info_dict': {
             'id': '150939',
-            'ext': 'mp3',
+            'ext': 'wav',
             'title': 'Moofi - Dr. Kreep',
             'thumbnail': 're:^https?://.*\.jpg$',
             'timestamp': 1421564134,
@@ -67,18 +72,38 @@ class HearThisAtIE(InfoExtractor):
         timestamp = str_to_int(self._search_regex(
             r'<span[^>]+class="calctime"[^>]+data-time="(\d+)', webpage, 'timestamp', fatal=False))
 
-        track_url = self._search_regex(
-            r'<a[^>]+data-mp3="([^"]+)"', webpage, 'track URL')
-
-        formats = [{
-            'format_id': 'mp3',
-            'url': track_url,
-            'vcodec': 'none',
-        }]
+        formats = []
+        mp3_url = self._search_regex(
+            r'(?s)<a class="player-link"\s+(?:[a-zA-Z0-9_:-]+="[^"]+"\s+)*?data-mp3="([^"]+)"',
+            webpage, 'title', fatal=False)
+        if mp3_url:
+            formats.append({
+                'format_id': 'mp3',
+                'vcodec': 'none',
+                'acodec': 'mp3',
+                'url': mp3_url,
+            })
+        download_path = self._search_regex(
+            r'<a class="[^"]*download_fct[^"]*"\s+href="([^"]+)"',
+            webpage, 'download URL', default=None)
+        if download_path:
+            download_url = compat_urlparse.urljoin(url, download_path)
+            ext_req = HEADRequest(download_url)
+            ext_handle = self._request_webpage(
+                ext_req, display_id, note='Determining extension')
+            ext = urlhandle_detect_ext(ext_handle)
+            formats.append({
+                'format_id': 'download',
+                'vcodec': 'none',
+                'ext': ext,
+                'url': download_url,
+                'preference': 2,  # Usually better quality
+            })
+        self._sort_formats(formats)
 
         return {
             'id': track_id,
-            'display-id': display_id,
+            'display_id': display_id,
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7832ed87f..764474c33 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1612,6 +1612,14 @@ def urlhandle_detect_ext(url_handle):
     except AttributeError:  # Python < 3
         getheader = url_handle.info().getheader
 
+    cd = getheader('Content-Disposition')
+    if cd:
+        m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
+        if m:
+            e = determine_ext(m.group('filename'), default_ext=None)
+            if e:
+                return e
+
     return getheader('Content-Type').split("/")[1]
 
 

From 898c23c03f634d14fa414f0e0693d5c4033b2671 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 12:04:26 +0100
Subject: [PATCH 1203/1937] release 2015.01.22

---
 README.md             | 3 ++-
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 078e9df82..60bd7e2b6 100644
--- a/README.md
+++ b/README.md
@@ -304,7 +304,8 @@ which means you can modify it, redistribute it or use it however you like.
 
 ## Authentication Options:
     -u, --username USERNAME          login with this account ID
-    -p, --password PASSWORD          account password
+    -p, --password PASSWORD          account password. If this option is left
+                                     out, youtube-dl will ask interactively.
     -2, --twofactor TWOFACTOR        two-factor auth code
     -n, --netrc                      use .netrc authentication data
     --video-password PASSWORD        video password (vimeo, smotri)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 63a79a7ee..f5df7be31 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.16'
+__version__ = '2015.01.22'

From bb6e38787dcc3593a4d9cb66bcf9f736570e944a Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 22 Jan 2015 18:35:53 +0200
Subject: [PATCH 1204/1937] [videomega] Fix extraction (Closes #4763)

---
 youtube_dl/extractor/videomega.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 9fc64d172..fc6e05fe0 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -1,12 +1,15 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
 from ..utils import (
+    ExtractorError,
     remove_start,
 )
 
@@ -35,8 +38,11 @@ class VideoMegaIE(InfoExtractor):
         req.add_header('Referer', url)
         webpage = self._download_webpage(req, video_id)
 
-        escaped_data = self._search_regex(
-            r'unescape\("([^"]+)"\)', webpage, 'escaped data')
+        try:
+            escaped_data = re.findall(r'unescape\("([^"]+)"\)', webpage)[-1]
+        except IndexError:
+            raise ExtractorError('Unable to extract escaped data')
+
         playlist = compat_urllib_parse.unquote(escaped_data)
 
         thumbnail = self._search_regex(

From 240b9b7a5cd5f19f50ae28fb3ef991fede216ef6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Jan 2015 23:11:22 +0600
Subject: [PATCH 1205/1937] [twitch] Add support for streams (Closes #893,
 closes #3693, closes #1884)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/twitch.py   | 97 +++++++++++++++++++++++++++++++-
 2 files changed, 95 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index be3fecd0d..a983fff76 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -466,6 +466,7 @@ from .twitch import (
     TwitchVodIE,
     TwitchProfileIE,
     TwitchPastBroadcastsIE,
+    TwitchStreamIE,
 )
 from .ubu import UbuIE
 from .udemy import (
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 8866e5182..2891c4680 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -6,6 +6,7 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_str,
     compat_urllib_parse,
     compat_urllib_request,
 )
@@ -16,9 +17,10 @@ from ..utils import (
 
 
 class TwitchBaseIE(InfoExtractor):
-    _VALID_URL_BASE = r'http://(?:www\.)?twitch\.tv'
+    _VALID_URL_BASE = r'https?://(?:www\.)?twitch\.tv'
 
     _API_BASE = 'https://api.twitch.tv'
+    _USHER_BASE = 'http://usher.twitch.tv'
     _LOGIN_URL = 'https://secure.twitch.tv/user/login'
 
     def _handle_error(self, response):
@@ -191,8 +193,8 @@ class TwitchVodIE(TwitchItemBaseIE):
             '%s/api/vods/%s/access_token' % (self._API_BASE, item_id), item_id,
             'Downloading %s access token' % self._ITEM_TYPE)
         formats = self._extract_m3u8_formats(
-            'http://usher.twitch.tv/vod/%s?nauth=%s&nauthsig=%s'
-            % (item_id, access_token['token'], access_token['sig']),
+            '%s/vod/%s?nauth=%s&nauthsig=%s'
+            % (self._USHER_BASE, item_id, access_token['token'], access_token['sig']),
             item_id, 'mp4')
         info['formats'] = formats
         return info
@@ -254,3 +256,92 @@ class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
         },
         'playlist_mincount': 54,
     }
+
+
+class TwitchStreamIE(TwitchBaseIE):
+    IE_NAME = 'twitch:stream'
+    _VALID_URL = r'%s/(?P<id>[^/]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+
+    _TEST = {
+        'url': 'http://www.twitch.tv/shroomztv',
+        'info_dict': {
+            'id': '12772022048',
+            'display_id': 'shroomztv',
+            'ext': 'mp4',
+            'title': 're:^ShroomzTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'H1Z1 - lonewolfing with ShroomzTV | A3 Battle Royale later - @ShroomzTV',
+            'is_live': True,
+            'timestamp': 1421928037,
+            'upload_date': '20150122',
+            'uploader': 'ShroomzTV',
+            'uploader_id': 'shroomztv',
+            'view_count': int,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+
+        stream = self._download_json(
+            '%s/kraken/streams/%s' % (self._API_BASE, channel_id), channel_id,
+            'Downloading stream JSON').get('stream')
+
+        # Fallback on profile extraction if stream is offline
+        if not stream:
+            return self.url_result(
+                'http://www.twitch.tv/%s/profile' % channel_id,
+                'TwitchProfile', channel_id)
+
+        access_token = self._download_json(
+            '%s/api/channels/%s/access_token' % (self._API_BASE, channel_id), channel_id,
+            'Downloading channel access token')
+
+        query = {
+            'allow_source': 'true',
+            'p': '9386337',
+            'player': 'twitchweb',
+            'segment_preference': '4',
+            'sig': access_token['sig'],
+            'token': access_token['token'],
+        }
+
+        formats = self._extract_m3u8_formats(
+            '%s/api/channel/hls/%s.m3u8?%s'
+            % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query).encode('utf-8')),
+            channel_id, 'mp4')
+
+        view_count = stream.get('viewers')
+        timestamp = parse_iso8601(stream.get('created_at'))
+
+        channel = stream['channel']
+        title = self._live_title(channel.get('display_name') or channel.get('name'))
+        description = channel.get('status')
+
+        thumbnails = []
+        for thumbnail_key, thumbnail_url in stream['preview'].items():
+            m = re.search(r'(?P<width>\d+)x(?P<height>\d+)\.jpg$', thumbnail_key)
+            if not m:
+                continue
+            thumbnails.append({
+                'url': thumbnail_url,
+                'width': int(m.group('width')),
+                'height': int(m.group('height')),
+            })
+
+        return {
+            'id': compat_str(stream['_id']),
+            'display_id': channel_id,
+            'title': title,
+            'description': description,
+            'thumbnails': thumbnails,
+            'uploader': channel.get('display_name'),
+            'uploader_id': channel.get('name'),
+            'timestamp': timestamp,
+            'view_count': view_count,
+            'formats': formats,
+            'is_live': True,
+        }
\ No newline at end of file

From 7932de6352f1fcb0418e78dc049fe60b92ca6c68 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 18:15:04 +0100
Subject: [PATCH 1206/1937] [hearthisat] Correct error message

---
 youtube_dl/extractor/hearthisat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/hearthisat.py b/youtube_dl/extractor/hearthisat.py
index d83072c1a..a19b31ac0 100644
--- a/youtube_dl/extractor/hearthisat.py
+++ b/youtube_dl/extractor/hearthisat.py
@@ -75,7 +75,7 @@ class HearThisAtIE(InfoExtractor):
         formats = []
         mp3_url = self._search_regex(
             r'(?s)<a class="player-link"\s+(?:[a-zA-Z0-9_:-]+="[^"]+"\s+)*?data-mp3="([^"]+)"',
-            webpage, 'title', fatal=False)
+            webpage, 'mp3 URL', fatal=False)
         if mp3_url:
             formats.append({
                 'format_id': 'mp3',

From 650ab5beebfa555383365602c914729249fad4c6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 18:15:58 +0100
Subject: [PATCH 1207/1937] [comedycentral:shows] Remove references to colbert
 report

---
 test/test_all_urls.py                 | 2 --
 youtube_dl/extractor/comedycentral.py | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index bd4fe17bf..88fda76a4 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -115,8 +115,6 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertMatch(':ythistory', ['youtube:history'])
         self.assertMatch(':thedailyshow', ['ComedyCentralShows'])
         self.assertMatch(':tds', ['ComedyCentralShows'])
-        self.assertMatch(':colbertreport', ['ComedyCentralShows'])
-        self.assertMatch(':cr', ['ComedyCentralShows'])
 
     def test_vimeo_matching(self):
         self.assertMatch('http://vimeo.com/channels/tributes', ['vimeo:channel'])
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 8d27af5e5..15ca361f0 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -34,12 +34,12 @@ class ComedyCentralIE(MTVServicesInfoExtractor):
 
 class ComedyCentralShowsIE(MTVServicesInfoExtractor):
     IE_DESC = 'The Daily Show / The Colbert Report'
-    # urls can be abbreviations like :thedailyshow or :colbert
+    # urls can be abbreviations like :thedailyshow
     # urls for episodes like:
     # or urls for clips like: http://www.thedailyshow.com/watch/mon-december-10-2012/any-given-gun-day
     #                     or: http://www.colbertnation.com/the-colbert-report-videos/421667/november-29-2012/moon-shattering-news
     #                     or: http://www.colbertnation.com/the-colbert-report-collections/422008/festival-of-lights/79524
-    _VALID_URL = r'''(?x)^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport)
+    _VALID_URL = r'''(?x)^(:(?P<shortname>tds|thedailyshow)
                       |https?://(:www\.)?
                           (?P<showname>thedailyshow|thecolbertreport)\.(?:cc\.)?com/
                          ((?:full-)?episodes/(?:[0-9a-z]{6}/)?(?P<episode>.*)|

From 78111136db788918452a80233e74fd4c506eb6f8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 18:18:21 +0100
Subject: [PATCH 1208/1937] [twitch] Move URL matching tests into extractor

---
 test/test_all_urls.py          | 13 -------------
 youtube_dl/extractor/twitch.py |  7 +++++--
 2 files changed, 5 insertions(+), 15 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index 88fda76a4..e66264b4b 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -14,7 +14,6 @@ from test.helper import gettestcases
 from youtube_dl.extractor import (
     FacebookIE,
     gen_extractors,
-    TwitchIE,
     YoutubeIE,
 )
 
@@ -72,18 +71,6 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertMatch('http://www.youtube.com/results?search_query=making+mustard', ['youtube:search_url'])
         self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url'])
 
-    def test_twitch_channelid_matching(self):
-        self.assertTrue(TwitchIE.suitable('twitch.tv/vanillatv'))
-        self.assertTrue(TwitchIE.suitable('www.twitch.tv/vanillatv'))
-        self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv'))
-        self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv/'))
-
-    def test_twitch_videoid_matching(self):
-        self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/vanillatv/b/328087483'))
-
-    def test_twitch_chapterid_matching(self):
-        self.assertTrue(TwitchIE.suitable('http://www.twitch.tv/tsm_theoddone/c/2349361'))
-
     def test_youtube_extract(self):
         assertExtractId = lambda url, id: self.assertEqual(YoutubeIE.extract_id(url), id)
         assertExtractId('http://www.youtube.com/watch?&v=BaW_jenozKc', 'BaW_jenozKc')
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 8866e5182..017eff742 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -148,14 +148,17 @@ class TwitchChapterIE(TwitchItemBaseIE):
     _ITEM_TYPE = 'chapter'
     _ITEM_SHORTCUT = 'c'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.twitch.tv/acracingleague/c/5285812',
         'info_dict': {
             'id': 'c5285812',
             'title': 'ACRL Off Season - Sports Cars @ Nordschleife',
         },
         'playlist_mincount': 3,
-    }
+    }, {
+        'url': 'http://www.twitch.tv/tsm_theoddone/c/2349361',
+        'only_matching': True,
+    }]
 
 
 class TwitchVodIE(TwitchItemBaseIE):

From b4a64c592b5f0f547c11c72e6c840612b336de5d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 18:21:17 +0100
Subject: [PATCH 1209/1937] [README] Add an FAQ entry about destination folder

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index 60bd7e2b6..efa48f280 100644
--- a/README.md
+++ b/README.md
@@ -488,6 +488,10 @@ To make a different directory work - either for ffmpeg, or for youtube-dl, or fo
 
 From then on, after restarting your shell, you will be able to access both youtube-dl and ffmpeg (and youtube-dl will be able to find ffmpeg) by simply typing `youtube-dl` or `ffmpeg`, no matter what directory you're in.
 
+### How do I put downloads into a specific folder?
+
+Use the `-o` to specify an [output template](#output-template), for example `-o "/home/user/videos/%(title)s-%(id)s.%(ext)s"`. If you want this for all of your downloads, put the option into your [configuration file](#configuration).
+
 ### How can I detect whether a given URL is supported by youtube-dl?
 
 For one, have a look at the [list of supported sites](docs/supportedsites). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.

From f353cbdb2f91564d4637462875c1a12bfbcd2ada Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 22 Jan 2015 23:34:40 +0600
Subject: [PATCH 1210/1937] [twitch:stream] Randomize query

---
 youtube_dl/extractor/twitch.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 14314916a..87e3c453d 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 import itertools
 import re
+import random
 
 from .common import InfoExtractor
 from ..compat import (
@@ -305,7 +306,7 @@ class TwitchStreamIE(TwitchBaseIE):
 
         query = {
             'allow_source': 'true',
-            'p': '9386337',
+            'p': random.randint(1000000, 10000000),
             'player': 'twitchweb',
             'segment_preference': '4',
             'sig': access_token['sig'],

From dc1b027cd4672c6b4772c12daf2a6a2a93469270 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 23:06:03 +0100
Subject: [PATCH 1211/1937] [twitch] PEP8

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 14314916a..6831bb596 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -347,4 +347,4 @@ class TwitchStreamIE(TwitchBaseIE):
             'view_count': view_count,
             'formats': formats,
             'is_live': True,
-        }
\ No newline at end of file
+        }

From 50789175edbe3aaad77f45e6fe883ba09580cc6f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 22 Jan 2015 23:48:58 +0100
Subject: [PATCH 1212/1937] [pornhub] Detect private videos and emit an error
 message (Closes #4764)

---
 youtube_dl/extractor/pornhub.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 634142d0d..fb2032832 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -10,6 +10,7 @@ from ..compat import (
     compat_urllib_request,
 )
 from ..utils import (
+    ExtractorError,
     str_to_int,
 )
 from ..aes import (
@@ -44,6 +45,15 @@ class PornHubIE(InfoExtractor):
         req.add_header('Cookie', 'age_verified=1')
         webpage = self._download_webpage(req, video_id)
 
+        error_msg = self._html_search_regex(
+            r'(?s)<div class="userMessageSection[^"]*".*?>(.*?)</div>',
+            webpage, 'error message', default=None)
+        if error_msg:
+            error_msg = re.sub(r'\s+', ' ', error_msg)
+            raise ExtractorError(
+                'PornHub said: %s' % error_msg,
+                expected=True, video_id=video_id)
+
         video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, 'title')
         video_uploader = self._html_search_regex(
             r'(?s)From:&nbsp;.+?<(?:a href="/users/|a href="/channels/|<span class="username)[^>]+>(.+?)<',

From 083c9df93b08a24e967b68fbdd2f4a71ae74c8c8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 00:04:05 +0100
Subject: [PATCH 1213/1937] [YoutubeDL] Allow filtering by properties (Fixes
 #4584)

---
 test/test_YoutubeDL.py  | 55 +++++++++++++++++++++++++++++++++++++++++
 youtube_dl/YoutubeDL.py | 54 ++++++++++++++++++++++++++++++++++++++++
 youtube_dl/options.py   | 11 +++++++++
 3 files changed, 120 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 85d87f2c3..678b9f7d1 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -281,6 +281,61 @@ class TestFormatSelection(unittest.TestCase):
             downloaded = ydl.downloaded_info_dicts[0]
             self.assertEqual(downloaded['format_id'], f1id)
 
+    def test_format_filtering(self):
+        formats = [
+            {'format_id': 'A', 'filesize': 500, 'width': 1000},
+            {'format_id': 'B', 'filesize': 1000, 'width': 500},
+            {'format_id': 'C', 'filesize': 1000, 'width': 400},
+            {'format_id': 'D', 'filesize': 2000, 'width': 600},
+            {'format_id': 'E', 'filesize': 3000},
+            {'format_id': 'F'},
+            {'format_id': 'G', 'filesize': 1000000},
+        ]
+        for f in formats:
+            f['url'] = 'http://_/'
+            f['ext'] = 'unknown'
+        info_dict = _make_result(formats)
+
+        ydl = YDL({'format': 'best[filesize<3000]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'D')
+
+        ydl = YDL({'format': 'best[filesize<=3000]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'E')
+
+        ydl = YDL({'format': 'best[filesize <= ? 3000]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'F')
+
+        ydl = YDL({'format': 'best [filesize = 1000] [width>450]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'B')
+
+        ydl = YDL({'format': 'best [filesize = 1000] [width!=450]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'C')
+
+        ydl = YDL({'format': '[filesize>?1]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'G')
+
+        ydl = YDL({'format': '[filesize<1M]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'E')
+
+        ydl = YDL({'format': '[filesize<1MiB]'})
+        ydl.process_ie_result(info_dict)
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], 'G')
+
     def test_add_extra_info(self):
         test_dict = {
             'extractor': 'Foo',
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 772fddd45..8ef74e414 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -10,6 +10,7 @@ import io
 import itertools
 import json
 import locale
+import operator
 import os
 import platform
 import re
@@ -49,6 +50,7 @@ from .utils import (
     make_HTTPS_handler,
     MaxDownloadsReached,
     PagedList,
+    parse_filesize,
     PostProcessingError,
     platform_name,
     preferredencoding,
@@ -768,7 +770,59 @@ class YoutubeDL(object):
         else:
             raise Exception('Invalid result type: %s' % result_type)
 
+    def _apply_format_filter(self, format_spec, available_formats):
+        " Returns a tuple of the remaining format_spec and filtered formats "
+
+        OPERATORS = {
+            '<': operator.lt,
+            '<=': operator.le,
+            '>': operator.gt,
+            '>=': operator.ge,
+            '=': operator.eq,
+            '!=': operator.ne,
+        }
+        operator_rex = re.compile(r'''(?x)\s*\[
+            (?P<key>width|height|tbr|abr|vbr|filesize)
+            \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
+            (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
+            \]$
+            ''' % '|'.join(map(re.escape, OPERATORS.keys())))
+        m = operator_rex.search(format_spec)
+        if not m:
+            raise ValueError('Invalid format specification %r' % format_spec)
+
+        try:
+            comparison_value = int(m.group('value'))
+        except ValueError:
+            comparison_value = parse_filesize(m.group('value'))
+            if comparison_value is None:
+                comparison_value = parse_filesize(m.group('value') + 'B')
+            if comparison_value is None:
+                raise ValueError(
+                    'Invalid value %r in format specification %r' % (
+                        m.group('value'), format_spec))
+        op = OPERATORS[m.group('op')]
+
+        def _filter(f):
+            actual_value = f.get(m.group('key'))
+            if actual_value is None:
+                return m.group('none_inclusive')
+            return op(actual_value, comparison_value)
+        new_formats = [f for f in available_formats if _filter(f)]
+
+        new_format_spec = format_spec[:-len(m.group(0))]
+        if not new_format_spec:
+            new_format_spec = 'best'
+
+        return (new_format_spec, new_formats)
+
     def select_format(self, format_spec, available_formats):
+        while format_spec.endswith(']'):
+            format_spec, available_formats = self._apply_format_filter(
+                format_spec, available_formats)
+        if not available_formats:
+            return None
+
         if format_spec == 'best' or format_spec is None:
             return available_formats[-1]
         elif format_spec == 'worst':
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index f25c12e52..fd7b400b2 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -289,6 +289,17 @@ def parseOpts(overrideArguments=None):
             'extensions aac, m4a, mp3, mp4, ogg, wav, webm. '
             'You can also use the special names "best",'
             ' "bestvideo", "bestaudio", "worst". '
+            ' You can filter the video results by putting a condition in'
+            ' brackets, as in -f "best[height=720]"'
+            ' (or -f "[filesize>10M]"). '
+            ' This works for filesize, height, width, tbr, abr, and vbr'
+            ' and the comparisons <, <=, >, >=, =, != .'
+            ' Formats for which the value is not known are excluded unless you'
+            ' put a question mark (?) after the operator.'
+            ' You can combine format filters, so  '
+            '-f "[height <=? 720][tbr>500]" '
+            'selects up to 720p videos (or videos where the height is not '
+            'known) with a bitrate of at least 500 KBit/s.'
             ' By default, youtube-dl will pick the best quality.'
             ' Use commas to download multiple audio formats, such as'
             ' -f  136/137/mp4/bestvideo,140/m4a/bestaudio.'

From c994e6bd63feb95e1e385af2333ae2f615314ac5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 00:06:11 +0100
Subject: [PATCH 1214/1937] release 2015.01.23

---
 README.md             | 20 ++++++++++++++++----
 youtube_dl/version.py |  2 +-
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index efa48f280..1408ebba0 100644
--- a/README.md
+++ b/README.md
@@ -267,10 +267,22 @@ which means you can modify it, redistribute it or use it however you like.
                                      by extension for the extensions aac, m4a,
                                      mp3, mp4, ogg, wav, webm. You can also use
                                      the special names "best", "bestvideo",
-                                     "bestaudio", "worst".  By default, youtube-
-                                     dl will pick the best quality. Use commas
-                                     to download multiple audio formats, such as
-                                     -f
+                                     "bestaudio", "worst".  You can filter the
+                                     video results by putting a condition in
+                                     brackets, as in -f "best[height=720]" (or
+                                     -f "[filesize>10M]").  This works for
+                                     filesize, height, width, tbr, abr, and vbr
+                                     and the comparisons <, <=, >, >=, =, != .
+                                     Formats for which the value is not known
+                                     are excluded unless you put a question mark
+                                     (?) after the operator. You can combine
+                                     format filters, so  -f "[height <=?
+                                     720][tbr>500]" selects up to 720p videos
+                                     (or videos where the height is not known)
+                                     with a bitrate of at least 500 KBit/s. By
+                                     default, youtube-dl will pick the best
+                                     quality. Use commas to download multiple
+                                     audio formats, such as -f
                                      136/137/mp4/bestvideo,140/m4a/bestaudio.
                                      You can merge the video and audio of two
                                      formats into a single file using -f <video-
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f5df7be31..6c552064b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.22'
+__version__ = '2015.01.23'

From c2a30b250cdea5a8c73225e6a14707a669640de0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 00:32:35 +0100
Subject: [PATCH 1215/1937] [testtube] Add new extractor (Fixes #4759)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/testtube.py | 60 ++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 youtube_dl/extractor/testtube.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a983fff76..9ab90ac62 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -432,6 +432,7 @@ from .telemb import TeleMBIE
 from .teletask import TeleTaskIE
 from .tenplay import TenPlayIE
 from .testurl import TestURLIE
+from .testtube import TestTubeIE
 from .tf1 import TF1IE
 from .theonion import TheOnionIE
 from .theplatform import ThePlatformIE
diff --git a/youtube_dl/extractor/testtube.py b/youtube_dl/extractor/testtube.py
new file mode 100644
index 000000000..fd47e71a2
--- /dev/null
+++ b/youtube_dl/extractor/testtube.py
@@ -0,0 +1,60 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import int_or_none
+
+
+class TestTubeIE(InfoExtractor):
+    _VALID_URL = r'https?://testtube\.com/[^/?#]+/(?P<id>[^/?#]+)'
+    _TESTS = [{
+        'url': 'https://testtube.com/dnews/5-weird-ways-plants-can-eat-animals?utm_source=FB&utm_medium=DNews&utm_campaign=DNewsSocial',
+        'info_dict': {
+            'id': '60163',
+            'display_id': '5-weird-ways-plants-can-eat-animals',
+            'duration': 275,
+            'ext': 'mp4',
+            'title': '5 Weird Ways Plants Can Eat Animals',
+            'description': 'Why have some plants evolved to eat meat?',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': 'DNews',
+            'uploader_id': 'dnews',
+        },
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._search_regex(
+            r"player\.loadRevision3Item\('video_id',\s*([0-9]+)\);",
+            webpage, 'video ID')
+
+        all_info = self._download_json(
+            'https://testtube.com/api/getPlaylist.json?api_key=ba9c741bce1b9d8e3defcc22193f3651b8867e62&codecs=h264,vp8,theora&video_id=%s' % video_id,
+            video_id)
+        info = all_info['items'][0]
+
+        formats = []
+        for vcodec, fdatas in info['media'].items():
+            for name, fdata in fdatas.items():
+                formats.append({
+                    'format_id': '%s-%s' % (vcodec, name),
+                    'url': fdata['url'],
+                    'vcodec': vcodec,
+                    'tbr': fdata.get('bitrate'),
+                })
+        self._sort_formats(formats)
+
+        duration = int_or_none(info.get('duration'))
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': info['title'],
+            'description': info.get('summary'),
+            'thumbnail': info.get('images', {}).get('large'),
+            'uploader': info.get('show', {}).get('name'),
+            'uploader_id': info.get('show', {}).get('slug'),
+            'duration': duration,
+            'formats': formats,
+        }

From 317239b097a913824e20d436cdb8c74161564268 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 00:33:14 +0100
Subject: [PATCH 1216/1937] release 2015.01.23.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6c552064b..28458fd69 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.23'
+__version__ = '2015.01.23.1'

From 61ca9a80b34b14fa0e5e19e35ee76e0086d49edf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 01:21:30 +0100
Subject: [PATCH 1217/1937] [generic] Add support for BOMs (Fixes #4753)

---
 test/test_utils.py              | 17 +++++++++++++++++
 youtube_dl/extractor/generic.py |  3 ++-
 youtube_dl/utils.py             | 20 ++++++++++++++++++++
 3 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 206760d99..bdd7f268a 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -28,6 +28,7 @@ from youtube_dl.utils import (
     fix_xml_ampersands,
     InAdvancePagedList,
     intlist_to_bytes,
+    is_html,
     js_to_json,
     limit_length,
     OnDemandPagedList,
@@ -417,5 +418,21 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
         self.assertTrue(age_restricted(18, 14))
         self.assertFalse(age_restricted(18, 18))
 
+    def test_is_html(self):
+        self.assertFalse(is_html(b'\x49\x44\x43<html'))
+        self.assertTrue(is_html(b'<!DOCTYPE foo>\xaaa'))
+        self.assertTrue(is_html(  # UTF-8 with BOM
+            b'\xef\xbb\xbf<!DOCTYPE foo>\xaaa'))
+        self.assertTrue(is_html(  # UTF-16-LE
+            b'\xff\xfe<\x00h\x00t\x00m\x00l\x00>\x00\xe4\x00'
+        ))
+        self.assertTrue(is_html(  # UTF-16-BE
+            b'\xfe\xff\x00<\x00h\x00t\x00m\x00l\x00>\x00\xe4'
+        ))
+        self.assertTrue(is_html(  # UTF-32-BE
+            b'\x00\x00\xFE\xFF\x00\x00\x00<\x00\x00\x00h\x00\x00\x00t\x00\x00\x00m\x00\x00\x00l\x00\x00\x00>\x00\x00\x00\xe4'))
+        self.assertTrue(is_html(  # UTF-32-LE
+            b'\xFF\xFE\x00\x00<\x00\x00\x00h\x00\x00\x00t\x00\x00\x00m\x00\x00\x00l\x00\x00\x00>\x00\x00\x00\xe4\x00\x00\x00'))
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7a5bf9392..b893d8149 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -17,6 +17,7 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     HEADRequest,
+    is_html,
     orderedSet,
     parse_xml,
     smuggle_url,
@@ -647,7 +648,7 @@ class GenericIE(InfoExtractor):
         # Maybe it's a direct link to a video?
         # Be careful not to download the whole thing!
         first_bytes = full_response.read(512)
-        if not re.match(r'^\s*<', first_bytes.decode('utf-8', 'replace')):
+        if not is_html(first_bytes):
             self._downloader.report_warning(
                 'URL could be a direct video link, returning it as such.')
             upload_date = unified_strdate(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 764474c33..b433b591b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1631,3 +1631,23 @@ def age_restricted(content_limit, age_limit):
     if content_limit is None:
         return False  # Content available for everyone
     return age_limit < content_limit
+
+
+def is_html(first_bytes):
+    """ Detect whether a file contains HTML by examining its first bytes. """
+
+    BOMS = [
+        (b'\xef\xbb\xbf', 'utf-8'),
+        (b'\x00\x00\xfe\xff', 'utf-32-be'),
+        (b'\xff\xfe\x00\x00', 'utf-32-le'),
+        (b'\xff\xfe', 'utf-16-le'),
+        (b'\xfe\xff', 'utf-16-be'),
+    ]
+    for bom, enc in BOMS:
+        if first_bytes.startswith(bom):
+            s = first_bytes[len(bom):].decode(enc, 'replace')
+            break
+    else:
+        s = first_bytes.decode('utf-8', 'replace')
+
+    return re.match(r'^\s*<', s)

From 08ff6ab07e9d08853ca19cf3b8745fac11abf0f6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 01:34:24 +0100
Subject: [PATCH 1218/1937] [gamestar] Modernize slightly

---
 youtube_dl/extractor/gamestar.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/gamestar.py b/youtube_dl/extractor/gamestar.py
index 50f8fc7e7..7591a151e 100644
--- a/youtube_dl/extractor/gamestar.py
+++ b/youtube_dl/extractor/gamestar.py
@@ -1,8 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -29,9 +27,7 @@ class GameStarIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         og_title = self._og_search_title(webpage)

From d79323136fabc2cd72afc7c124e17797e32df514 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 11:15:18 +0100
Subject: [PATCH 1219/1937] [utils] Simplify HTTPS socket creation

We were duplicating (bad) code and doing crazy things with SSL.
Just use TLSv1 across the board, and do with one implementation of HTTPSConnection.connect.
Fixes #4696.
---
 youtube_dl/utils.py | 24 +++++-------------------
 1 file changed, 5 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 3536a5bd6..4be323926 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -411,25 +411,9 @@ def make_HTTPS_handler(params, **kwargs):
             pass
 
     if sys.version_info < (3, 2):
-        import httplib
-
-        class HTTPSConnectionV3(httplib.HTTPSConnection):
-            def __init__(self, *args, **kwargs):
-                httplib.HTTPSConnection.__init__(self, *args, **kwargs)
-
-            def connect(self):
-                sock = socket.create_connection((self.host, self.port), self.timeout)
-                if getattr(self, '_tunnel_host', False):
-                    self.sock = sock
-                    self._tunnel()
-                try:
-                    self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_TLSv1)
-                except ssl.SSLError:
-                    self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23)
-
-        return YoutubeDLHTTPSHandler(params, https_conn_class=HTTPSConnectionV3, **kwargs)
+        return YoutubeDLHTTPSHandler(params, **kwargs)
     else:  # Python < 3.4
-        context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         context.verify_mode = (ssl.CERT_NONE
                                if opts_no_check_certificate
                                else ssl.CERT_REQUIRED)
@@ -560,7 +544,9 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
                 sock = compat_socket_create_connection(
                     (self.host, self.port), self.timeout, sa)
                 if is_https:
-                    self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+                    self.sock = ssl.wrap_socket(
+                        sock, self.key_file, self.cert_file,
+                        ssl_version=ssl.PROTOCOL_TLSv1)
                 else:
                     self.sock = sock
             hc.connect = functools.partial(_hc_connect, hc)

From ba551681573aafedf9a0a234a9522cc06b12a4b7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 11:20:24 +0100
Subject: [PATCH 1220/1937] release 2015.01.23.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 28458fd69..674c2f279 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.23.1'
+__version__ = '2015.01.23.2'

From 501f13fbf3d1f7225f91e3e0ad008df2cd3219f1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 12:00:25 +0100
Subject: [PATCH 1221/1937] [generic] Add support for Cinerama player (Fixes
 #4752)

---
 youtube_dl/extractor/generic.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b893d8149..a028c4ed4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -489,6 +489,16 @@ class GenericIE(InfoExtractor):
                 'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
             }
         },
+        # Cinerama player
+        {
+            'url': 'http://www.abc.net.au/7.30/content/2015/s4164797.htm',
+            'info_dict': {
+                'id': '730m_DandD_1901_512k',
+                'ext': 'mp4',
+                'uploader': 'www.abc.net.au',
+                'title': 'Game of Thrones with dice - Dungeons and Dragons fantasy role-playing game gets new life - 19/01/2015',
+            }
+        }
     ]
 
     def report_following_redirect(self, new_url):
@@ -1046,6 +1056,10 @@ class GenericIE(InfoExtractor):
                     \s*{[^}]+? ["']?clip["']?\s*:\s*\{\s*
                         ["']?url["']?\s*:\s*["']([^"']+)["']
             ''', webpage))
+        if not found:
+            # Cinerama player
+            found = re.findall(
+                r"cinerama\.embedPlayer\(\s*\'[^']+\',\s*'([^']+)'", webpage)
         if not found:
             # Try to find twitter cards info
             found = filter_video(re.findall(

From 649f7966f73b215f02b9c4db5a07d8c4b7e17acb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 12:07:13 +0100
Subject: [PATCH 1222/1937] Fix --sleep-interval (#3426)

---
 youtube_dl/__init__.py | 6 ------
 youtube_dl/options.py  | 2 +-
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ea1660452..7bd7295e2 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -132,11 +132,6 @@ def _real_main(argv=None):
         if numeric_limit is None:
             parser.error('invalid rate limit specified')
         opts.ratelimit = numeric_limit
-    if opts.sleepinterval is not None:
-        try:
-            opts.sleepinterval = abs(int(opts.sleepinterval))
-        except ValueError:
-            parser.error(u'invalid sleep interval specified')
     if opts.min_filesize is not None:
         numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
         if numeric_limit is None:
@@ -272,7 +267,6 @@ def _real_main(argv=None):
         'restrictfilenames': opts.restrictfilenames,
         'ignoreerrors': opts.ignoreerrors,
         'ratelimit': opts.ratelimit,
-        'sleepinterval': opts.sleepinterval,
         'nooverwrites': opts.nooverwrites,
         'retries': opts.retries,
         'buffersize': opts.buffersize,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 12c9826f8..262c60013 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -423,7 +423,7 @@ def parseOpts(overrideArguments=None):
         help='Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH')
     workarounds.add_option(
         '--sleep-interval', metavar='SECONDS',
-        dest='sleep_interval',
+        dest='sleep_interval', type=float,
         help='Number of seconds to sleep before each download.')
 
     verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')

From aa42e87340e491d0b151e9dad368711fc275c7b9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 12:17:12 +0100
Subject: [PATCH 1223/1937] [utils] Catch strange Windows errors (Closes #4733)

---
 youtube_dl/utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4be323926..463cc20ff 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -863,6 +863,9 @@ def _windows_write_string(s, out):
     except AttributeError:
         # If the output stream doesn't have a fileno, it's virtual
         return False
+    except io.UnsupportedOperation:
+        # Some strange Windows pseudo files?
+        return False
     if fileno not in WIN_OUTPUT_IDS:
         return False
 

From 6f58db89820fe03effb2014ee70e1ffd0b1f3017 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 12:17:19 +0100
Subject: [PATCH 1224/1937] release 2015.01.23.3

---
 README.md             | 2 ++
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1408ebba0..869523720 100644
--- a/README.md
+++ b/README.md
@@ -259,6 +259,8 @@ which means you can modify it, redistribute it or use it however you like.
     --bidi-workaround                Work around terminals that lack
                                      bidirectional text support. Requires bidiv
                                      or fribidi executable in PATH
+    --sleep-interval SECONDS         Number of seconds to sleep before each
+                                     download.
 
 ## Video Format Options:
     -f, --format FORMAT              video format code, specify the order of
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 674c2f279..f39b94361 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.23.2'
+__version__ = '2015.01.23.3'

From bf7fa94ec7202bde963a75ab903996ac575910db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Jan 2015 16:31:52 +0100
Subject: [PATCH 1225/1937] [downloader/f4m] build_fragments_list: Support
 videos with more than 1 segment

---
 youtube_dl/downloader/f4m.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index c460c167a..c68b2c303 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -177,13 +177,12 @@ def build_fragments_list(boot_info):
     """ Return a list of (segment, fragment) for each fragment in the video """
     res = []
     segment_run_table = boot_info['segments'][0]
-    # I've only found videos with one segment
-    segment_run_entry = segment_run_table['segment_run'][0]
-    n_frags = segment_run_entry[1]
     fragment_run_entry_table = boot_info['fragments'][0]['fragments']
     first_frag_number = fragment_run_entry_table[0]['first']
-    for (i, frag_number) in zip(range(1, n_frags + 1), itertools.count(first_frag_number)):
-        res.append((1, frag_number))
+    fragments_counter = itertools.count(first_frag_number)
+    for segment, fragments_count in segment_run_table['segment_run']:
+        for _ in range(fragments_count):
+            res.append((segment, next(fragments_counter)))
     return res
 
 

From 0920e5830f890580ec16cdd10bfe8def73a1a09f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Jan 2015 16:39:23 +0100
Subject: [PATCH 1226/1937] [atresplayer] Don't include f4m formats if they are
 protected by DRM (fixes #4705)

---
 youtube_dl/extractor/atresplayer.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 5db1941b3..37321ef1d 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -107,7 +107,14 @@ class AtresPlayerIE(InfoExtractor):
 
             for _, video_url in fmt_json['resultObject'].items():
                 if video_url.endswith('/Manifest'):
-                    formats.extend(self._extract_f4m_formats(video_url[:-9] + '/manifest.f4m', video_id))
+                    if 'geodeswowsmpra3player' in video_url:
+                        f4m_path = video_url.split('smil:', 1)[-1].split('free_', 1)[0]
+                        f4m_url = 'http://drg.antena3.com/{0}hds/es/sd.f4m'.format(f4m_path)
+                        # this videos are protected by DRM, the f4m downloader doesn't support them
+                        continue
+                    else:
+                        f4m_url = video_url[:-9] + '/manifest.f4m'
+                    formats.extend(self._extract_f4m_formats(f4m_url, video_id))
                 else:
                     formats.append({
                         'url': video_url,

From c2e64f71d075bbc916f343916ff1e679f642a821 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jan 2015 21:58:40 +0600
Subject: [PATCH 1227/1937] [twitch] Add support for bookmarks

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/twitch.py   | 41 ++++++++++++++++++++++++++++----
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9ab90ac62..9e1ce5db3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -467,6 +467,7 @@ from .twitch import (
     TwitchVodIE,
     TwitchProfileIE,
     TwitchPastBroadcastsIE,
+    TwitchBookmarksIE,
     TwitchStreamIE,
 )
 from .ubu import UbuIE
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 340cadcf5..741df7cbc 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -220,12 +220,18 @@ class TwitchPlaylistBaseIE(TwitchBaseIE):
             response = self._download_json(
                 self._PLAYLIST_URL % (channel_id, offset, limit),
                 channel_id, 'Downloading %s videos JSON page %d' % (self._PLAYLIST_TYPE, counter))
-            videos = response['videos']
-            if not videos:
+            page_entries = self._extract_playlist_page(response)
+            if not page_entries:
                 break
-            entries.extend([self.url_result(video['url']) for video in videos])
+            entries.extend(page_entries)
             offset += limit
-        return self.playlist_result(entries, channel_id, channel_name)
+        return self.playlist_result(
+            [self.url_result(entry) for entry in set(entries)],
+            channel_id, channel_name)
+
+    def _extract_playlist_page(self, response):
+        videos = response.get('videos')
+        return [video['url'] for video in videos] if videos else []
 
     def _real_extract(self, url):
         return self._extract_playlist(self._match_id(url))
@@ -262,6 +268,31 @@ class TwitchPastBroadcastsIE(TwitchPlaylistBaseIE):
     }
 
 
+class TwitchBookmarksIE(TwitchPlaylistBaseIE):
+    IE_NAME = 'twitch:bookmarks'
+    _VALID_URL = r'%s/(?P<id>[^/]+)/profile/bookmarks/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
+    _PLAYLIST_URL = '%s/api/bookmark/?user=%%s&offset=%%d&limit=%%d' % TwitchBaseIE._API_BASE
+    _PLAYLIST_TYPE = 'bookmarks'
+
+    _TEST = {
+        'url': 'http://www.twitch.tv/ognos/profile/bookmarks',
+        'info_dict': {
+            'id': 'ognos',
+            'title': 'Ognos',
+        },
+        'playlist_mincount': 3,
+    }
+
+    def _extract_playlist_page(self, response):
+        entries = []
+        for bookmark in response.get('bookmarks', []):
+            video = bookmark.get('video')
+            if not video:
+                continue
+            entries.append(video['url'])
+        return entries
+
+
 class TwitchStreamIE(TwitchBaseIE):
     IE_NAME = 'twitch:stream'
     _VALID_URL = r'%s/(?P<id>[^/]+)/?(?:\#.*)?$' % TwitchBaseIE._VALID_URL_BASE
@@ -348,4 +379,4 @@ class TwitchStreamIE(TwitchBaseIE):
             'view_count': view_count,
             'formats': formats,
             'is_live': True,
-        }
+        }
\ No newline at end of file

From e793f7671c3e303e4fbe2c4cd18e80ad4b5e5417 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 17:09:26 +0100
Subject: [PATCH 1228/1937] [liveleak] Modernize

---
 youtube_dl/extractor/liveleak.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index b04be1e8c..e56a4d6b1 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -8,7 +8,7 @@ from ..utils import int_or_none
 
 
 class LiveLeakIE(InfoExtractor):
-    _VALID_URL = r'^(?:http://)?(?:\w+\.)?liveleak\.com/view\?(?:.*?)i=(?P<video_id>[\w_]+)(?:.*)'
+    _VALID_URL = r'https?://(?:\w+\.)?liveleak\.com/view\?(?:.*?)i=(?P<id>[\w_]+)(?:.*)'
     _TESTS = [{
         'url': 'http://www.liveleak.com/view?i=757_1364311680',
         'md5': '0813c2430bea7a46bf13acf3406992f4',
@@ -43,8 +43,7 @@ class LiveLeakIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         video_title = self._og_search_title(webpage).replace('LiveLeak.com -', '').strip()

From ebd46aed5119899826629cf751ba5abe7a65d50b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jan 2015 22:21:55 +0600
Subject: [PATCH 1229/1937] [atresplayer] Filter URLs and clarify android
 format ids

---
 youtube_dl/extractor/atresplayer.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 37321ef1d..8fd69b971 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -105,7 +105,9 @@ class AtresPlayerIE(InfoExtractor):
                 raise ExtractorError(
                     '%s returned error: %s' % (self.IE_NAME, result), expected=True)
 
-            for _, video_url in fmt_json['resultObject'].items():
+            for format_id, video_url in fmt_json['resultObject'].items():
+                if format_id == 'token' or not video_url.startswith('http'):
+                    continue
                 if video_url.endswith('/Manifest'):
                     if 'geodeswowsmpra3player' in video_url:
                         f4m_path = video_url.split('smil:', 1)[-1].split('free_', 1)[0]
@@ -118,7 +120,7 @@ class AtresPlayerIE(InfoExtractor):
                 else:
                     formats.append({
                         'url': video_url,
-                        'format_id': 'android',
+                        'format_id': 'android-%s' % format_id,
                         'preference': 1,
                     })
         self._sort_formats(formats)

From 26e274666d285026dfb6033cdad74cc2a108e2bc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 17:22:14 +0100
Subject: [PATCH 1230/1937] [liveleak] Add original videos (Fixes #4768)

---
 youtube_dl/extractor/liveleak.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
index e56a4d6b1..35822067f 100644
--- a/youtube_dl/extractor/liveleak.py
+++ b/youtube_dl/extractor/liveleak.py
@@ -11,17 +11,17 @@ class LiveLeakIE(InfoExtractor):
     _VALID_URL = r'https?://(?:\w+\.)?liveleak\.com/view\?(?:.*?)i=(?P<id>[\w_]+)(?:.*)'
     _TESTS = [{
         'url': 'http://www.liveleak.com/view?i=757_1364311680',
-        'md5': '0813c2430bea7a46bf13acf3406992f4',
+        'md5': '50f79e05ba149149c1b4ea961223d5b3',
         'info_dict': {
             'id': '757_1364311680',
-            'ext': 'mp4',
+            'ext': 'flv',
             'description': 'extremely bad day for this guy..!',
             'uploader': 'ljfriel2',
             'title': 'Most unlucky car accident'
         }
     }, {
         'url': 'http://www.liveleak.com/view?i=f93_1390833151',
-        'md5': 'd3f1367d14cc3c15bf24fbfbe04b9abf',
+        'md5': 'b13a29626183c9d33944e6a04f41aafc',
         'info_dict': {
             'id': 'f93_1390833151',
             'ext': 'mp4',
@@ -80,9 +80,19 @@ class LiveLeakIE(InfoExtractor):
         sources = json.loads(sources_json)
 
         formats = [{
+            'format_id': '%s' % i,
             'format_note': s.get('label'),
             'url': s['file'],
-        } for s in sources]
+        } for i, s in enumerate(sources)]
+        for i, s in enumerate(sources):
+            orig_url = s['file'].replace('.h264_base.mp4', '')
+            if s['file'] != orig_url:
+                formats.append({
+                    'format_id': 'original-%s' % i,
+                    'format_note': s.get('label'),
+                    'url': orig_url,
+                    'preference': 1,
+                })
         self._sort_formats(formats)
 
         return {

From 0c17278843a30d4aea1f8334985e6fbae17000d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 23 Jan 2015 22:54:29 +0600
Subject: [PATCH 1231/1937] [atresplayer] Extract subtitles

---
 youtube_dl/extractor/atresplayer.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 8fd69b971..f42862be3 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import time
 import hmac
 
-from .common import InfoExtractor
+from .subtitles import SubtitlesInfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
@@ -17,7 +17,7 @@ from ..utils import (
 )
 
 
-class AtresPlayerIE(InfoExtractor):
+class AtresPlayerIE(SubtitlesInfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?atresplayer\.com/television/[^/]+/[^/]+/[^/]+/(?P<id>.+?)_\d+\.html'
     _TESTS = [
         {
@@ -143,6 +143,15 @@ class AtresPlayerIE(InfoExtractor):
         description = xpath_text(art, './description', 'description')
         thumbnail = xpath_text(episode, './media/asset/files/background', 'thumbnail')
 
+        subtitles = {}
+        subtitle = xpath_text(episode, './media/asset/files/subtitle', 'subtitle')
+        if subtitle:
+            subtitles['es'] = subtitle
+
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, subtitles)
+            return
+
         return {
             'id': video_id,
             'title': title,
@@ -150,4 +159,5 @@ class AtresPlayerIE(InfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
+            'subtitles': self.extract_subtitles(video_id, subtitles),
         }

From 62cd676c7474f696804eda653558ada94c5953a0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 18:39:12 +0100
Subject: [PATCH 1232/1937] [youtube] Fixup DASH m4a headers

This fixes #2288, #2506, #2607, #3681, #4741, #4767.
---
 youtube_dl/YoutubeDL.py              | 28 +++++++++++++++++++++++-----
 youtube_dl/extractor/youtube.py      |  6 +++---
 youtube_dl/postprocessor/__init__.py |  2 ++
 youtube_dl/postprocessor/ffmpeg.py   | 20 +++++++++++++++++++-
 4 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8f34b17b4..521e4055e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -73,6 +73,7 @@ from .extractor import get_info_extractor, gen_extractors
 from .downloader import get_suitable_downloader
 from .downloader.rtmp import rtmpdump_version
 from .postprocessor import (
+    FFmpegFixupM4aPP,
     FFmpegFixupStretchedPP,
     FFmpegMergerPP,
     FFmpegPostProcessor,
@@ -213,7 +214,7 @@ class YoutubeDL(object):
                        - "never": do nothing
                        - "warn": only emit a warning
                        - "detect_or_warn": check whether we can do anything
-                                           about it, warn otherwise
+                                           about it, warn otherwise (default)
     source_address:    (Experimental) Client-side IP address to bind to.
     call_home:         Boolean, true iff we are allowed to contact the
                        youtube-dl servers for debugging.
@@ -1219,11 +1220,12 @@ class YoutubeDL(object):
 
             if success:
                 # Fixup content
+                fixup_policy = self.params.get('fixup')
+                if fixup_policy is None:
+                    fixup_policy = 'detect_or_warn'
+
                 stretched_ratio = info_dict.get('stretched_ratio')
                 if stretched_ratio is not None and stretched_ratio != 1:
-                    fixup_policy = self.params.get('fixup')
-                    if fixup_policy is None:
-                        fixup_policy = 'detect_or_warn'
                     if fixup_policy == 'warn':
                         self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
                             info_dict['id'], stretched_ratio))
@@ -1237,7 +1239,23 @@ class YoutubeDL(object):
                                 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
                                     info_dict['id'], stretched_ratio))
                     else:
-                        assert fixup_policy == 'ignore'
+                        assert fixup_policy in ('ignore', 'never')
+
+                if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
+                    if fixup_policy == 'warn':
+                        self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
+                            info_dict['id']))
+                    elif fixup_policy == 'detect_or_warn':
+                        fixup_pp = FFmpegFixupM4aPP(self)
+                        if fixup_pp.available:
+                            info_dict.setdefault('__postprocessors', [])
+                            info_dict['__postprocessors'].append(fixup_pp)
+                        else:
+                            self.report_warning(
+                                '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
+                                    info_dict['id']))
+                    else:
+                        assert fixup_policy in ('ignore', 'never')
 
                 try:
                     self.post_process(filename, info_dict)
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2a1f8be0a..eb55d24ce 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -264,9 +264,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'h264'},
 
         # Dash mp4 audio
-        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 48, 'preference': -50},
-        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 128, 'preference': -50},
-        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 256, 'preference': -50},
+        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
+        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
+        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
 
         # Dash webm
         '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'VP8', 'preference': -40},
diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index f8507951c..0ffbca258 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -7,6 +7,7 @@ from .ffmpeg import (
     FFmpegEmbedSubtitlePP,
     FFmpegExtractAudioPP,
     FFmpegFixupStretchedPP,
+    FFmpegFixupM4aPP,
     FFmpegMergerPP,
     FFmpegMetadataPP,
     FFmpegVideoConvertorPP,
@@ -25,6 +26,7 @@ __all__ = [
     'FFmpegAudioFixPP',
     'FFmpegEmbedSubtitlePP',
     'FFmpegExtractAudioPP',
+    'FFmpegFixupM4aPP',
     'FFmpegFixupStretchedPP',
     'FFmpegMergerPP',
     'FFmpegMetadataPP',
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index faccdc43d..855d1e6db 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -564,7 +564,7 @@ class FFmpegFixupStretchedPP(FFmpegPostProcessor):
     def run(self, info):
         stretched_ratio = info.get('stretched_ratio')
         if stretched_ratio is None or stretched_ratio == 1:
-            return
+            return True, info
 
         filename = info['filepath']
         temp_filename = prepend_extension(filename, 'temp')
@@ -577,3 +577,21 @@ class FFmpegFixupStretchedPP(FFmpegPostProcessor):
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
         return True, info
+
+
+class FFmpegFixupM4aPP(FFmpegPostProcessor):
+    def run(self, info):
+        if info.get('container') != 'm4a_dash':
+            return True, info
+
+        filename = info['filepath']
+        temp_filename = prepend_extension(filename, 'temp')
+
+        options = ['-c', 'copy', '-f', 'mp4']
+        self._downloader.to_screen('[ffmpeg] Correcting container in "%s"' % filename)
+        self.run_ffmpeg(filename, temp_filename, options)
+
+        os.remove(encodeFilename(filename))
+        os.rename(encodeFilename(temp_filename), encodeFilename(filename))
+
+        return True, info

From 6896a52721ce3a0dc4f33fa5dda508c30c56ce22 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 18:58:32 +0100
Subject: [PATCH 1233/1937] release 2015.01.23.4

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f39b94361..35f3e1b6b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.23.3'
+__version__ = '2015.01.23.4'

From 3e055aa5c3c069476b1ac9fc103dc1386cad763c Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 23 Jan 2015 21:22:49 +0200
Subject: [PATCH 1234/1937] [cliphunter] Fix extraction and update test (Fixes
 #4362)

---
 youtube_dl/extractor/cliphunter.py | 56 ++++++++++++------------------
 1 file changed, 23 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/cliphunter.py b/youtube_dl/extractor/cliphunter.py
index 2edab90a3..d46592cc5 100644
--- a/youtube_dl/extractor/cliphunter.py
+++ b/youtube_dl/extractor/cliphunter.py
@@ -1,9 +1,7 @@
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
+from ..utils import determine_ext
 
 
 _translation_table = {
@@ -27,10 +25,10 @@ class CliphunterIE(InfoExtractor):
     '''
     _TEST = {
         'url': 'http://www.cliphunter.com/w/1012420/Fun_Jynx_Maze_solo',
-        'md5': 'a2ba71eebf523859fe527a61018f723e',
+        'md5': 'b7c9bbd4eb3a226ab91093714dcaa480',
         'info_dict': {
             'id': '1012420',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Fun Jynx Maze solo',
             'thumbnail': 're:^https?://.*\.jpg$',
             'age_limit': 18,
@@ -44,39 +42,31 @@ class CliphunterIE(InfoExtractor):
         video_title = self._search_regex(
             r'mediaTitle = "([^"]+)"', webpage, 'title')
 
-        pl_fiji = self._search_regex(
-            r'pl_fiji = \'([^\']+)\'', webpage, 'video data')
-        pl_c_qual = self._search_regex(
-            r'pl_c_qual = "(.)"', webpage, 'video quality')
-        video_url = _decode(pl_fiji)
-        formats = [{
-            'url': video_url,
-            'format_id': 'default-%s' % pl_c_qual,
-        }]
+        fmts = {}
+        for fmt in ('mp4', 'flv'):
+            fmt_list = self._parse_json(self._search_regex(
+                r'var %sjson\s*=\s*(\[.*?\]);' % fmt, webpage, '%s formats' % fmt), video_id)
+            for f in fmt_list:
+                fmts[f['fname']] = _decode(f['sUrl'])
 
-        qualities_json = self._search_regex(
-            r'var pl_qualities\s*=\s*(.*?);\n', webpage, 'quality info')
-        qualities_data = json.loads(qualities_json)
+        qualities = self._parse_json(self._search_regex(
+            r'var player_btns\s*=\s*(.*?);\n', webpage, 'quality info'), video_id)
 
-        for i, t in enumerate(
-                re.findall(r"pl_fiji_([a-z0-9]+)\s*=\s*'([^']+')", webpage)):
-            quality_id, crypted_url = t
-            video_url = _decode(crypted_url)
+        formats = []
+        for fname, url in fmts.items():
             f = {
-                'format_id': quality_id,
-                'url': video_url,
-                'quality': i,
+                'url': url,
             }
-            if quality_id in qualities_data:
-                qd = qualities_data[quality_id]
-                m = re.match(
-                    r'''(?x)<b>(?P<width>[0-9]+)x(?P<height>[0-9]+)<\\/b>
-                        \s*\(\s*(?P<tbr>[0-9]+)\s*kb\\/s''', qd)
-                if m:
-                    f['width'] = int(m.group('width'))
-                    f['height'] = int(m.group('height'))
-                    f['tbr'] = int(m.group('tbr'))
+            if fname in qualities:
+                qual = qualities[fname]
+                f.update({
+                    'format_id': '%s_%sp' % (determine_ext(url), qual['h']),
+                    'width': qual['w'],
+                    'height': qual['h'],
+                    'tbr': qual['br'],
+                })
             formats.append(f)
+
         self._sort_formats(formats)
 
         thumbnail = self._search_regex(

From 48f00d15b1bdbe05be397849f6659fe5e0daa9c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Jan 2015 21:03:00 +0100
Subject: [PATCH 1235/1937] [auengine] Remove extractor

The test is probably infringing copyright and nobody has provided a new test (see #4643).
---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/auengine.py | 50 --------------------------------
 2 files changed, 51 deletions(-)
 delete mode 100644 youtube_dl/extractor/auengine.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9e1ce5db3..3f7ca6f7d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -29,7 +29,6 @@ from .arte import (
 from .atresplayer import AtresPlayerIE
 from .atttechchannel import ATTTechChannelIE
 from .audiomack import AudiomackIE, AudiomackAlbumIE
-from .auengine import AUEngineIE
 from .azubu import AzubuIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
diff --git a/youtube_dl/extractor/auengine.py b/youtube_dl/extractor/auengine.py
deleted file mode 100644
index a1b666be0..000000000
--- a/youtube_dl/extractor/auengine.py
+++ /dev/null
@@ -1,50 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..compat import compat_urllib_parse
-from ..utils import (
-    determine_ext,
-    ExtractorError,
-    remove_end,
-)
-
-
-class AUEngineIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?auengine\.com/embed\.php\?.*?file=(?P<id>[^&]+).*?'
-
-    _TEST = {
-        'url': 'http://auengine.com/embed.php?file=lfvlytY6&w=650&h=370',
-        'md5': '48972bdbcf1a3a2f5533e62425b41d4f',
-        'info_dict': {
-            'id': 'lfvlytY6',
-            'ext': 'mp4',
-            'title': '[Commie]The Legend of the Legendary Heroes - 03 - Replication Eye (Alpha Stigma)[F9410F5A]'
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-        title = self._html_search_regex(
-            r'<title>\s*(?P<title>.+?)\s*</title>', webpage, 'title')
-        video_urls = re.findall(r'http://\w+.auengine.com/vod/.*[^\W]', webpage)
-        video_url = compat_urllib_parse.unquote(video_urls[0])
-        thumbnails = re.findall(r'http://\w+.auengine.com/thumb/.*[^\W]', webpage)
-        thumbnail = compat_urllib_parse.unquote(thumbnails[0])
-
-        if not video_url:
-            raise ExtractorError('Could not find video URL')
-
-        ext = '.' + determine_ext(video_url)
-        title = remove_end(title, ext)
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-            'thumbnail': thumbnail,
-            'http_referer': 'http://www.auengine.com/flowplayer/flowplayer.commercial-3.2.14.swf',
-        }

From 12d1fb5aa9aabfe48676d6c60bbd114cc53a513f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Jan 2015 21:05:07 +0100
Subject: [PATCH 1236/1937] [twitch] PEP8

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 741df7cbc..87290d002 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -379,4 +379,4 @@ class TwitchStreamIE(TwitchBaseIE):
             'view_count': view_count,
             'formats': formats,
             'is_live': True,
-        }
\ No newline at end of file
+        }

From fdaaaaa878c42975936bf7f6ecf39a97436fefe2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 23 Jan 2015 21:10:10 +0100
Subject: [PATCH 1237/1937] README: Recommend using flake8 instead of pyflake
 and pep8 separately

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 869523720..36b87444e 100644
--- a/README.md
+++ b/README.md
@@ -586,7 +586,7 @@ If you want to add support for a new site, you can follow this quick list (assum
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want.
-8. If you can, check the code with [pyflakes](https://pypi.python.org/pypi/pyflakes) (a good idea) and [pep8](https://pypi.python.org/pypi/pep8) (optional, ignore E501).
+8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/__init__.py

From a055469fafe088b6aa0e569d989cbf7f70535951 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 23 Jan 2015 23:50:31 +0100
Subject: [PATCH 1238/1937] [downloader] Improve downloader selection

---
 youtube_dl/YoutubeDL.py           |  2 +-
 youtube_dl/downloader/__init__.py | 31 +++++++++++++++----------------
 youtube_dl/utils.py               | 22 ++++++++++++++++++++++
 3 files changed, 38 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 521e4055e..e61e6c2a7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1179,7 +1179,7 @@ class YoutubeDL(object):
         if not self.params.get('skip_download', False):
             try:
                 def dl(name, info):
-                    fd = get_suitable_downloader(info)(self, self.params)
+                    fd = get_suitable_downloader(info, self.params)(self, self.params)
                     for ph in self._progress_hooks:
                         fd.add_progress_hook(ph)
                     if self.params.get('verbose'):
diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 31e28df58..2aca3cab5 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -9,27 +9,26 @@ from .rtmp import RtmpFD
 from .f4m import F4mFD
 
 from ..utils import (
-    determine_ext,
+    determine_protocol,
 )
 
+PROTOCOL_MAP = {
+    'rtmp': RtmpFD,
+    'm3u8_native': NativeHlsFD,
+    'm3u8': HlsFD,
+    'mms': MplayerFD,
+    'rtsp': MplayerFD,
+    'f4m': F4mFD,
+}
 
-def get_suitable_downloader(info_dict):
+
+def get_suitable_downloader(info_dict, params={}):
     """Get the downloader class that can handle the info dict."""
-    url = info_dict['url']
-    protocol = info_dict.get('protocol')
+    protocol = determine_protocol(info_dict)
+    info_dict['protocol'] = protocol
+
+    return PROTOCOL_MAP.get(protocol, HttpFD)
 
-    if url.startswith('rtmp'):
-        return RtmpFD
-    if protocol == 'm3u8_native':
-        return NativeHlsFD
-    if (protocol == 'm3u8') or (protocol is None and determine_ext(url) == 'm3u8'):
-        return HlsFD
-    if url.startswith('mms') or url.startswith('rtsp'):
-        return MplayerFD
-    if determine_ext(url) == 'f4m':
-        return F4mFD
-    else:
-        return HttpFD
 
 __all__ = [
     'get_suitable_downloader',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 463cc20ff..2970d02a1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1642,3 +1642,25 @@ def is_html(first_bytes):
         s = first_bytes.decode('utf-8', 'replace')
 
     return re.match(r'^\s*<', s)
+
+
+def determine_protocol(info_dict):
+    protocol = info_dict.get('protocol')
+    if protocol is not None:
+        return protocol
+
+    url = info_dict['url']
+    if url.startswith('rtmp'):
+        return 'rtmp'
+    elif url.startswith('mms'):
+        return 'mms'
+    elif url.startswith('rtsp'):
+        return 'rtsp'
+
+    ext = determine_ext(url)
+    if ext == 'm3u8':
+        return 'm3u8'
+    elif ext == 'f4m':
+        return 'f4m'
+
+    return compat_urllib_parse_urlparse(url).scheme

From 222516d97d5ff9e62f3a9860fe2e65aa99c001b3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 24 Jan 2015 01:38:48 +0100
Subject: [PATCH 1239/1937] [downloader] Lay groundwork for external
 downloaders.

This comes with a very simply implementation for wget; the real work is in setting up the infrastructure.
---
 youtube_dl/YoutubeDL.py           |   1 +
 youtube_dl/__init__.py            |   1 +
 youtube_dl/downloader/__init__.py |   9 +-
 youtube_dl/downloader/common.py   |  21 +++++
 youtube_dl/downloader/external.py | 131 ++++++++++++++++++++++++++++++
 youtube_dl/downloader/rtmp.py     |  14 +---
 youtube_dl/options.py             |   6 ++
 7 files changed, 169 insertions(+), 14 deletions(-)
 create mode 100644 youtube_dl/downloader/external.py

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e61e6c2a7..54e732943 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -219,6 +219,7 @@ class YoutubeDL(object):
     call_home:         Boolean, true iff we are allowed to contact the
                        youtube-dl servers for debugging.
     sleep_interval:    Number of seconds to sleep before each download.
+    external_downloader:  Executable of the external downloader to call.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 7bd7295e2..3fc7dc5c2 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -330,6 +330,7 @@ def _real_main(argv=None):
         'source_address': opts.source_address,
         'call_home': opts.call_home,
         'sleep_interval': opts.sleep_interval,
+        'external_downloader': opts.external_downloader,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 2aca3cab5..eff1122c5 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -1,12 +1,13 @@
 from __future__ import unicode_literals
 
 from .common import FileDownloader
+from .external import get_external_downloader
+from .f4m import F4mFD
 from .hls import HlsFD
 from .hls import NativeHlsFD
 from .http import HttpFD
 from .mplayer import MplayerFD
 from .rtmp import RtmpFD
-from .f4m import F4mFD
 
 from ..utils import (
     determine_protocol,
@@ -27,6 +28,12 @@ def get_suitable_downloader(info_dict, params={}):
     protocol = determine_protocol(info_dict)
     info_dict['protocol'] = protocol
 
+    external_downloader = params.get('external_downloader')
+    if external_downloader is not None:
+        ed = get_external_downloader(external_downloader)
+        if ed.supports(info_dict):
+            return ed
+
     return PROTOCOL_MAP.get(protocol, HttpFD)
 
 
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 82c917d92..c35c42c1d 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -325,3 +325,24 @@ class FileDownloader(object):
         # See YoutubeDl.py (search for progress_hooks) for a description of
         # this interface
         self._progress_hooks.append(ph)
+
+    def _debug_cmd(self, args, subprocess_encoding, exe=None):
+        if not self.params.get('verbose', False):
+            return
+
+        if exe is None:
+            exe = os.path.basename(args[0])
+
+        if subprocess_encoding:
+            str_args = [
+                a.decode(subprocess_encoding) if isinstance(a, bytes) else a
+                for a in args]
+        else:
+            str_args = args
+        try:
+            import pipes
+            shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))
+        except ImportError:
+            shell_quote = repr
+        self.to_screen('[debug] %s command line: %s' % (
+            exe, shell_quote(str_args)))
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
new file mode 100644
index 000000000..c05596255
--- /dev/null
+++ b/youtube_dl/downloader/external.py
@@ -0,0 +1,131 @@
+from __future__ import unicode_literals
+
+import os.path
+import subprocess
+import sys
+
+from .common import FileDownloader
+from ..utils import (
+    encodeFilename,
+    std_headers,
+)
+
+
+class ExternalFD(FileDownloader):
+    def real_download(self, filename, info_dict):
+        self.report_destination(filename)
+        tmpfilename = self.temp_name(filename)
+
+        retval = self._call_downloader(tmpfilename, info_dict)
+        if retval == 0:
+            fsize = os.path.getsize(encodeFilename(tmpfilename))
+            self.to_screen('\r[%s] Downloaded %s bytes' % (self.get_basename(), fsize))
+            self.try_rename(tmpfilename, filename)
+            self._hook_progress({
+                'downloaded_bytes': fsize,
+                'total_bytes': fsize,
+                'filename': filename,
+                'status': 'finished',
+            })
+            return True
+        else:
+            self.to_stderr('\n')
+            self.report_error('%s exited with code %d' % (
+                self.get_basename(), retval))
+            return False
+
+    @classmethod
+    def get_basename(cls):
+        return cls.__name__[:-2].lower()
+
+    @property
+    def exe(self):
+        return self.params.get('external_downloader')
+
+    @classmethod
+    def supports(cls, info_dict):
+        return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
+
+    def _calc_headers(self, info_dict):
+        res = std_headers.copy()
+
+        ua = info_dict.get('user_agent')
+        if ua is not None:
+            res['User-Agent'] = ua
+
+        cookies = self._calc_cookies(info_dict)
+        if cookies:
+            res['Cookie'] = cookies
+
+        return res
+
+    def _calc_cookies(self, info_dict):
+        class _PseudoRequest(object):
+            def __init__(self, url):
+                self.url = url
+                self.headers = {}
+                self.unverifiable = False
+
+            def add_unredirected_header(self, k, v):
+                self.headers[k] = v
+
+            def get_full_url(self):
+                return self.url
+
+            def is_unverifiable(self):
+                return self.unverifiable
+
+            def has_header(self, h):
+                return h in self.headers
+
+        pr = _PseudoRequest(info_dict['url'])
+        self.ydl.cookiejar.add_cookie_header(pr)
+        return pr.headers.get('Cookie')
+
+    def _call_downloader(self, tmpfilename, info_dict):
+        """ Either overwrite this or implement _make_cmd """
+        cmd = self._make_cmd(tmpfilename, info_dict)
+
+        if sys.platform == 'win32' and sys.version_info < (3, 0):
+            # Windows subprocess module does not actually support Unicode
+            # on Python 2.x
+            # See http://stackoverflow.com/a/9951851/35070
+            subprocess_encoding = sys.getfilesystemencoding()
+            cmd = [a.encode(subprocess_encoding, 'ignore') for a in cmd]
+        else:
+            subprocess_encoding = None
+        self._debug_cmd(cmd, subprocess_encoding)
+
+        p = subprocess.Popen(
+            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = p.communicate()
+        if p.returncode != 0:
+            self.to_stderr(stderr)
+        return p.returncode
+
+
+class WgetFD(ExternalFD):
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
+        for key, val in self._calc_headers(info_dict).items():
+            cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += ['--', info_dict['url']]
+        return cmd
+
+
+_BY_NAME = dict(
+    (klass.get_basename(), klass)
+    for name, klass in globals().items()
+    if name.endswith('FD') and name != 'ExternalFD'
+)
+
+
+def list_external_downloaders():
+    return sorted(_BY_NAME.keys())
+
+
+def get_external_downloader(external_downloader):
+    """ Given the name of the executable, see whether we support the given
+        downloader . """
+    bn = os.path.basename(external_downloader)
+    return _BY_NAME[bn]
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 5346cb9a0..6dbbc053c 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -152,19 +152,7 @@ class RtmpFD(FileDownloader):
         else:
             subprocess_encoding = None
 
-        if self.params.get('verbose', False):
-            if subprocess_encoding:
-                str_args = [
-                    a.decode(subprocess_encoding) if isinstance(a, bytes) else a
-                    for a in args]
-            else:
-                str_args = args
-            try:
-                import pipes
-                shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))
-            except ImportError:
-                shell_quote = repr
-            self.to_screen('[debug] rtmpdump command line: ' + shell_quote(str_args))
+        self._debug_cmd(args, subprocess_encoding, exe='rtmpdump')
 
         RD_SUCCESS = 0
         RD_FAILED = 1
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 262c60013..b38b8349f 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -5,6 +5,7 @@ import optparse
 import shlex
 import sys
 
+from .downloader.external import list_external_downloaders
 from .compat import (
     compat_expanduser,
     compat_getenv,
@@ -389,6 +390,11 @@ def parseOpts(overrideArguments=None):
         '--playlist-reverse',
         action='store_true',
         help='Download playlist videos in reverse order')
+    downloader.add_option(
+        '--external-downloader',
+        dest='external_downloader', metavar='COMMAND',
+        help='(experimental) Use the specified external downloader. '
+             'Currently supports %s' % ','.join(list_external_downloaders()))
 
     workarounds = optparse.OptionGroup(parser, 'Workarounds')
     workarounds.add_option(

From 6aa4f54d6660bfb8fefe225e47cba7ade62fe564 Mon Sep 17 00:00:00 2001
From: Irfan Charania <irfan.charania@gmail.com>
Date: Fri, 23 Jan 2015 17:41:07 -0800
Subject: [PATCH 1240/1937] [videott] improve extraction

---
 youtube_dl/extractor/videott.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videott.py b/youtube_dl/extractor/videott.py
index 1f938838c..6f8084140 100644
--- a/youtube_dl/extractor/videott.py
+++ b/youtube_dl/extractor/videott.py
@@ -13,7 +13,7 @@ from ..utils import (
 class VideoTtIE(InfoExtractor):
     ID_NAME = 'video.tt'
     IE_DESC = 'video.tt - Your True Tube'
-    _VALID_URL = r'http://(?:www\.)?video\.tt/(?:video/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'
+    _VALID_URL = r'http://(?:www\.)?video\.tt/(?:video\/|embed\/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'
 
     _TEST = {
         'url': 'http://www.video.tt/watch_video.php?v=amd5YujV8',

From fc2d6abfe7e7a216825296001b889e5455e0412f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Jan 2015 16:11:40 +0600
Subject: [PATCH 1241/1937] [videott] Improve _VALID_URL and add test

---
 youtube_dl/extractor/videott.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/videott.py b/youtube_dl/extractor/videott.py
index 6f8084140..ececc7ee0 100644
--- a/youtube_dl/extractor/videott.py
+++ b/youtube_dl/extractor/videott.py
@@ -13,9 +13,9 @@ from ..utils import (
 class VideoTtIE(InfoExtractor):
     ID_NAME = 'video.tt'
     IE_DESC = 'video.tt - Your True Tube'
-    _VALID_URL = r'http://(?:www\.)?video\.tt/(?:video\/|embed\/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'
+    _VALID_URL = r'http://(?:www\.)?video\.tt/(?:(?:video|embed)/|watch_video\.php\?v=)(?P<id>[\da-zA-Z]{9})'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.video.tt/watch_video.php?v=amd5YujV8',
         'md5': 'b13aa9e2f267effb5d1094443dff65ba',
         'info_dict': {
@@ -26,7 +26,10 @@ class VideoTtIE(InfoExtractor):
             'upload_date': '20130827',
             'uploader': 'joseph313',
         }
-    }
+    }, {
+        'url': 'http://video.tt/embed/amd5YujV8',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From b95aab8482881e8b1f7fba856da816a2dbc50d0f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 24 Jan 2015 11:42:20 +0100
Subject: [PATCH 1242/1937] [youtube:truncated_url] Add x-yt-cl URLs (#4773)

---
 youtube_dl/extractor/youtube.py | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index eb55d24ce..b7b91f354 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1682,11 +1682,17 @@ class YoutubeTruncatedURLIE(InfoExtractor):
     IE_NAME = 'youtube:truncated_url'
     IE_DESC = False  # Do not list
     _VALID_URL = r'''(?x)
-        (?:https?://)?[^/]+/watch\?(?:
+        (?:https?://)?
+        (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
+        (?:watch\?(?:
             feature=[a-z_]+|
-            annotation_id=annotation_[^&]+
-        )?$|
-        (?:https?://)?(?:www\.)?youtube\.com/attribution_link\?a=[^&]+$
+            annotation_id=annotation_[^&]+|
+            x-yt-cl=[0-9]+|
+        )?
+        |
+            attribution_link\?a=[^&]+
+        )
+        $
     '''
 
     _TESTS = [{
@@ -1695,6 +1701,12 @@ class YoutubeTruncatedURLIE(InfoExtractor):
     }, {
         'url': 'http://www.youtube.com/watch?',
         'only_matching': True,
+    }, {
+        'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.youtube.com/watch?feature=foo',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -1710,7 +1722,7 @@ class YoutubeTruncatedURLIE(InfoExtractor):
 class YoutubeTruncatedIDIE(InfoExtractor):
     IE_NAME = 'youtube:truncated_id'
     IE_DESC = False  # Do not list
-    _VALID_URL = r'https?://(?:www\.)youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
+    _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
 
     _TESTS = [{
         'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',

From 384b62028a4c3c35727d714ccfc9944a36934069 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 24 Jan 2015 13:33:45 +0100
Subject: [PATCH 1243/1937] [downloader/external] Add curl and aria2c (Closes
 #182)

---
 youtube_dl/downloader/external.py | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index c05596255..7ebe40096 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -97,13 +97,22 @@ class ExternalFD(FileDownloader):
         self._debug_cmd(cmd, subprocess_encoding)
 
         p = subprocess.Popen(
-            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        stdout, stderr = p.communicate()
+            cmd, stderr=subprocess.PIPE)
+        _, stderr = p.communicate()
         if p.returncode != 0:
             self.to_stderr(stderr)
         return p.returncode
 
 
+class CurlFD(ExternalFD):
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = [self.exe, '-o', tmpfilename]
+        for key, val in self._calc_headers(info_dict).items():
+            cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += ['--', info_dict['url']]
+        return cmd
+
+
 class WgetFD(ExternalFD):
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
@@ -113,6 +122,20 @@ class WgetFD(ExternalFD):
         return cmd
 
 
+class Aria2cFD(ExternalFD):
+    def _make_cmd(self, tmpfilename, info_dict):
+        cmd = [
+            self.exe, '-c',
+            '--min-split-size', '1M', '--max-connection-per-server', '4']
+        dn = os.path.dirname(tmpfilename)
+        if dn:
+            cmd += ['--dir', dn]
+        cmd += ['--out', os.path.basename(tmpfilename)]
+        for key, val in self._calc_headers(info_dict).items():
+            cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += ['--', info_dict['url']]
+        return cmd
+
 _BY_NAME = dict(
     (klass.get_basename(), klass)
     for name, klass in globals().items()

From 3fcfb8e9faf3cf1dcadedd6fecc5158a86d07065 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Jan 2015 18:07:21 +0100
Subject: [PATCH 1244/1937] [utils] YoutubeDLHandler: don't use
 'Youtubedl-user-agent' for overriding the default user agent

Setting the 'User-Agent' header is enough
---
 youtube_dl/downloader/http.py       | 2 +-
 youtube_dl/extractor/atresplayer.py | 2 +-
 youtube_dl/extractor/bliptv.py      | 2 +-
 youtube_dl/extractor/mtv.py         | 2 +-
 youtube_dl/utils.py                 | 5 -----
 5 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index e68f20c9f..90a2e4c53 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -25,7 +25,7 @@ class HttpFD(FileDownloader):
         # Do not include the Accept-Encoding header
         headers = {'Youtubedl-no-compression': 'True'}
         if 'user_agent' in info_dict:
-            headers['Youtubedl-user-agent'] = info_dict['user_agent']
+            headers['User-agent'] = info_dict['user_agent']
         if 'http_referer' in info_dict:
             headers['Referer'] = info_dict['http_referer']
         add_headers = info_dict.get('http_headers')
diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index f42862be3..f016368fa 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -95,7 +95,7 @@ class AtresPlayerIE(SubtitlesInfoExtractor):
         for fmt in ['windows', 'android_tablet']:
             request = compat_urllib_request.Request(
                 self._URL_VIDEO_TEMPLATE.format(fmt, episode_id, timestamp_shifted, token))
-            request.add_header('Youtubedl-user-agent', self._USER_AGENT)
+            request.add_header('User-Agent', self._USER_AGENT)
 
             fmt_json = self._download_json(
                 request, video_id, 'Downloading %s video JSON' % fmt)
diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index 14b814120..436cc5155 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -199,7 +199,7 @@ class BlipTVIE(SubtitlesInfoExtractor):
         # For some weird reason, blip.tv serves a video instead of subtitles
         # when we request with a common UA
         req = compat_urllib_request.Request(url)
-        req.add_header('Youtubedl-user-agent', 'youtube-dl')
+        req.add_header('User-Agent', 'youtube-dl')
         return self._download_webpage(req, None, note=False)
 
 
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 5ebc78033..22a726327 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -53,7 +53,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         webpage_url = self._MOBILE_TEMPLATE % mtvn_id
         req = compat_urllib_request.Request(webpage_url)
         # Otherwise we get a webpage that would execute some javascript
-        req.add_header('Youtubedl-user-agent', 'curl/7')
+        req.add_header('User-Agent', 'curl/7')
         webpage = self._download_webpage(req, mtvn_id,
                                          'Downloading mobile page')
         metrics_url = unescapeHTML(self._search_regex(r'<a href="(http://metrics.+?)"', webpage, 'url'))
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 2970d02a1..d22b03134 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -606,11 +606,6 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
             if 'Accept-encoding' in req.headers:
                 del req.headers['Accept-encoding']
             del req.headers['Youtubedl-no-compression']
-        if 'Youtubedl-user-agent' in req.headers:
-            if 'User-agent' in req.headers:
-                del req.headers['User-agent']
-            req.headers['User-agent'] = req.headers['Youtubedl-user-agent']
-            del req.headers['Youtubedl-user-agent']
 
         if sys.version_info < (2, 7) and '#' in req.get_full_url():
             # Python 2.6 is brain-dead when it comes to fragments

From e1554a407d8953d0ec4fabec7f5a3b3906d1029b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Jan 2015 18:19:58 +0100
Subject: [PATCH 1245/1937] [extractors] Use http_headers for setting the
 User-Agent and the Referer

---
 youtube_dl/downloader/http.py         | 4 ----
 youtube_dl/extractor/appletrailers.py | 4 +++-
 youtube_dl/extractor/common.py        | 1 -
 youtube_dl/extractor/videomega.py     | 4 +++-
 youtube_dl/extractor/wdr.py           | 4 +++-
 5 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 90a2e4c53..4db50ee90 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -24,10 +24,6 @@ class HttpFD(FileDownloader):
 
         # Do not include the Accept-Encoding header
         headers = {'Youtubedl-no-compression': 'True'}
-        if 'user_agent' in info_dict:
-            headers['User-agent'] = info_dict['user_agent']
-        if 'http_referer' in info_dict:
-            headers['Referer'] = info_dict['http_referer']
         add_headers = info_dict.get('http_headers')
         if add_headers:
             headers.update(add_headers)
diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 7cd0482c7..70621946d 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -129,7 +129,9 @@ class AppleTrailersIE(InfoExtractor):
                 'thumbnail': thumbnail,
                 'upload_date': upload_date,
                 'uploader_id': uploader_id,
-                'user_agent': 'QuickTime compatible (youtube-dl)',
+                'http_headers': {
+                    'User-Agent': 'QuickTime compatible (youtube-dl)',
+                },
             })
 
         return {
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 03f3f18c8..523400062 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -108,7 +108,6 @@ class InfoExtractor(object):
                                   (quality takes higher priority)
                                  -1 for default (order by other properties),
                                  -2 or smaller for less than default.
-                    * http_referer  HTTP Referer header value to set.
                     * http_method  HTTP method to use for the download.
                     * http_headers  A dictionary of additional HTTP headers
                                  to add to the request.
diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index fc6e05fe0..273030316 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -62,5 +62,7 @@ class VideoMegaIE(InfoExtractor):
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
-            'http_referer': iframe_url,
+            'http_headers': {
+                'Referer': iframe_url,
+            },
         }
diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 45466e31b..313b9c15d 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -169,7 +169,9 @@ class WDRMobileIE(InfoExtractor):
             'title': mobj.group('title'),
             'age_limit': int(mobj.group('age_limit')),
             'url': url,
-            'user_agent': 'mobile',
+            'http_headers': {
+                'User-Agent': 'mobile',
+            },
         }
 
 

From 587a9c27496979d983296944fee8d1fa589e1b85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Jan 2015 18:25:09 +0100
Subject: [PATCH 1246/1937] [downloader/external] Use the 'http_headers' field

---
 youtube_dl/downloader/external.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 7ebe40096..5bf24ccbb 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -49,9 +49,9 @@ class ExternalFD(FileDownloader):
     def _calc_headers(self, info_dict):
         res = std_headers.copy()
 
-        ua = info_dict.get('user_agent')
-        if ua is not None:
-            res['User-Agent'] = ua
+        add_headers = info_dict.get('http_headers')
+        if add_headers:
+            res.update(add_headers)
 
         cookies = self._calc_cookies(info_dict)
         if cookies:

From 8011fba3ae4301995f77c64ac63a4a467ef73b7e Mon Sep 17 00:00:00 2001
From: David-Development <david-dev@live.de>
Date: Sat, 24 Jan 2015 18:28:16 +0100
Subject: [PATCH 1247/1937] [rtl2] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/rtl2.py     | 98 ++++++++++++++++++++++++++++++++
 2 files changed, 99 insertions(+)
 create mode 100644 youtube_dl/extractor/rtl2.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3f7ca6f7d..03c56156a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -349,6 +349,7 @@ from .rtbf import RTBFIE
 from .rte import RteIE
 from .rtlnl import RtlXlIE
 from .rtlnow import RTLnowIE
+from .rtl2 import RTL2IE
 from .rtp import RTPIE
 from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE
diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
new file mode 100644
index 000000000..2bdbf2ec2
--- /dev/null
+++ b/youtube_dl/extractor/rtl2.py
@@ -0,0 +1,98 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    unified_strdate,
+    int_or_none,
+)
+
+
+class RTL2IE(InfoExtractor):
+    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
+    _VALID_URL = r'http?://(?P<url>(?P<domain>(www\.)?rtl2\.de)/.*/(?P<video_id>.*))'
+    _TEST = {
+        'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
+        'md5': 'dsadasdada',
+        'info_dict': {
+            'id': 'folge-203-0',
+            'ext': 'f4v',
+            'title': 'GRIP sucht den Sommerk\xf6nig',
+	    'description' : 'Matthias, Det und Helge treten gegeneinander an.'
+            # TODO more properties, either as:
+            # * A value
+            # * MD5 checksum; start the string with md5:
+            # * A regular expression; start the string with re:
+            # * Any Python type (for example int or float)
+        },
+	#'params': {
+                # rtmp download
+        #	'skip_download': True,
+	#},
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_page_url = 'http://%s/' % mobj.group('domain')
+        video_id = mobj.group('video_id')
+	
+        webpage = self._download_webpage('http://' + mobj.group('url'), video_id)
+
+	vico_id = self._html_search_regex(r'vico_id: ([0-9]+)', webpage, '%s');
+	vivi_id = self._html_search_regex(r'vivi_id: ([0-9]+)', webpage, '%s');
+
+	info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id
+	webpage = self._download_webpage(info_url, '')
+
+	video_info = json.loads(webpage.decode("latin1"))
+	print video_info
+
+
+	#self._download_webpage('http://cp108781.edgefcs.net/crossdomain.xml', '')
+
+	download_url = video_info["video"]["streamurl"] # self._html_search_regex(r'streamurl\":\"(.*?)\"', webpage, '%s');
+	title = video_info["video"]["titel"] # self._html_search_regex(r'titel\":\"(.*?)\"', webpage, '%s');
+	description = video_info["video"]["beschreibung"] # self._html_search_regex(r'beschreibung\":\"(.*?)\"', webpage, '%s');
+	#ext = self._html_search_regex(r'streamurl\":\".*?(\..{2,4})\"', webpage, '%s');
+
+	thumbnail = video_info["video"]["image"]
+
+	download_url = download_url.replace("\\", "")
+
+	stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, '%s');
+
+	#upload_date = self._html_search_regex(r'property=\"dc:date\".*?datatype=\"xsd:dateTime\".*?content=\"(.*?)\"', webpage, 'title')
+	#download_url += " -y " + stream_url
+	
+	#print stream_url
+	#print download_url
+	#print description
+	#print title
+	#print ext
+
+	formats = []
+
+	fmt = {
+	    'url' : download_url,
+            #'app': 'ondemand?_fcs_vhost=cp108781.edgefcs.net',
+            'play_path': stream_url,
+            #'player_url': 'http://www.cbsnews.com/[[IMPORT]]/vidtech.cbsinteractive.com/player/3_3_0/CBSI_PLAYER_HD.swf',
+            #'page_url': 'http://www.cbsnews.com',
+            #'ext': ext,	
+        }
+
+	formats.append(fmt)
+
+
+        return {
+	    'id': video_id,
+            'title': title,
+	    'thumbnail' : thumbnail,
+	    'description' : description,
+            'formats': formats,
+        }

From e5660ee6aebacee4b149648bee1a7ce1ec72c1e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Jan 2015 18:52:26 +0100
Subject: [PATCH 1248/1937] [YoutubeDL] Fill the info dict 'http_headers' field
 with all the headers available

Useful for external tools using the json output.

The methods '_calc_headers' and '_calc_cookies' have been copied from the downloader/external, now they just use "info_dict['http_headers']".
---
 youtube_dl/YoutubeDL.py           | 36 ++++++++++++++++++++++++++
 youtube_dl/downloader/external.py | 43 +++----------------------------
 2 files changed, 39 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 54e732943..4c3d45f48 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -56,6 +56,7 @@ from .utils import (
     preferredencoding,
     SameFileError,
     sanitize_filename,
+    std_headers,
     subtitles_filename,
     takewhile_inclusive,
     UnavailableVideoError,
@@ -865,6 +866,36 @@ class YoutubeDL(object):
                 return matches[-1]
         return None
 
+    def _calc_headers(self, info_dict):
+        res = std_headers.copy()
+
+        add_headers = info_dict.get('http_headers')
+        if add_headers:
+            res.update(add_headers)
+
+        cookies = self._calc_cookies(info_dict)
+        if cookies:
+            res['Cookie'] = cookies
+
+        return res
+
+    def _calc_cookies(self, info_dict):
+        class _PseudoRequest(object):
+            def __init__(self, url):
+                self.url = url
+                self.headers = {}
+                self.unverifiable = False
+
+            def add_unredirected_header(self, k, v):
+                self.headers[k] = v
+
+            def get_full_url(self):
+                return self.url
+
+        pr = _PseudoRequest(info_dict['url'])
+        self.cookiejar.add_cookie_header(pr)
+        return pr.headers.get('Cookie')
+
     def process_video_result(self, info_dict, download=True):
         assert info_dict.get('_type', 'video') == 'video'
 
@@ -933,6 +964,11 @@ class YoutubeDL(object):
             # Automatically determine file extension if missing
             if 'ext' not in format:
                 format['ext'] = determine_ext(format['url']).lower()
+            # Add HTTP headers, so that external programs can use them from the
+            # json output
+            full_format_info = info_dict.copy()
+            full_format_info.update(format)
+            format['http_headers'] = self._calc_headers(full_format_info)
 
         format_limit = self.params.get('format_limit', None)
         if format_limit:
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 5bf24ccbb..af9fdba75 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -7,7 +7,6 @@ import sys
 from .common import FileDownloader
 from ..utils import (
     encodeFilename,
-    std_headers,
 )
 
 
@@ -46,42 +45,6 @@ class ExternalFD(FileDownloader):
     def supports(cls, info_dict):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
 
-    def _calc_headers(self, info_dict):
-        res = std_headers.copy()
-
-        add_headers = info_dict.get('http_headers')
-        if add_headers:
-            res.update(add_headers)
-
-        cookies = self._calc_cookies(info_dict)
-        if cookies:
-            res['Cookie'] = cookies
-
-        return res
-
-    def _calc_cookies(self, info_dict):
-        class _PseudoRequest(object):
-            def __init__(self, url):
-                self.url = url
-                self.headers = {}
-                self.unverifiable = False
-
-            def add_unredirected_header(self, k, v):
-                self.headers[k] = v
-
-            def get_full_url(self):
-                return self.url
-
-            def is_unverifiable(self):
-                return self.unverifiable
-
-            def has_header(self, h):
-                return h in self.headers
-
-        pr = _PseudoRequest(info_dict['url'])
-        self.ydl.cookiejar.add_cookie_header(pr)
-        return pr.headers.get('Cookie')
-
     def _call_downloader(self, tmpfilename, info_dict):
         """ Either overwrite this or implement _make_cmd """
         cmd = self._make_cmd(tmpfilename, info_dict)
@@ -107,7 +70,7 @@ class ExternalFD(FileDownloader):
 class CurlFD(ExternalFD):
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-o', tmpfilename]
-        for key, val in self._calc_headers(info_dict).items():
+        for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += ['--', info_dict['url']]
         return cmd
@@ -116,7 +79,7 @@ class CurlFD(ExternalFD):
 class WgetFD(ExternalFD):
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
-        for key, val in self._calc_headers(info_dict).items():
+        for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += ['--', info_dict['url']]
         return cmd
@@ -131,7 +94,7 @@ class Aria2cFD(ExternalFD):
         if dn:
             cmd += ['--dir', dn]
         cmd += ['--out', os.path.basename(tmpfilename)]
-        for key, val in self._calc_headers(info_dict).items():
+        for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += ['--', info_dict['url']]
         return cmd

From 4b405cfc6e4f9184567b94ee2f8a4a9851a912b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Jan 2015 20:05:35 +0100
Subject: [PATCH 1249/1937] [YoutubeDL._calc_cookies] Restore the 'has_header'
 method

I didn't copied it from downloader/external
---
 youtube_dl/YoutubeDL.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4c3d45f48..458fd15ea 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -892,6 +892,9 @@ class YoutubeDL(object):
             def get_full_url(self):
                 return self.url
 
+            def has_header(self, h):
+                return h in self.headers
+
         pr = _PseudoRequest(info_dict['url'])
         self.cookiejar.add_cookie_header(pr)
         return pr.headers.get('Cookie')

From 1070711d6003e6750b8cf803c3926b2e273a9e85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 24 Jan 2015 20:12:47 +0100
Subject: [PATCH 1250/1937] [YoutubeDL._calc_cookies] Restore the
 'is_unverifiable'

I should have check everything was copied before commiting 4b405cfc6e4f9184567b94ee2f8a4a9851a912b5.
---
 youtube_dl/YoutubeDL.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 458fd15ea..d6728b2dd 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -892,6 +892,9 @@ class YoutubeDL(object):
             def get_full_url(self):
                 return self.url
 
+            def is_unverifiable(self):
+                return self.unverifiable
+
             def has_header(self, h):
                 return h in self.headers
 

From 7906d199a11c0f8deccc0f87d7ee850106c361dc Mon Sep 17 00:00:00 2001
From: David-Development <david-dev@live.de>
Date: Sat, 24 Jan 2015 18:28:16 +0100
Subject: [PATCH 1251/1937] [rtl2] Add new extractor

---
 youtube_dl/downloader/rtmp.py    |   8 ++-
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/rtl2.py     | 100 +++++++++++++++++++++++++++++++
 3 files changed, 107 insertions(+), 2 deletions(-)
 create mode 100644 youtube_dl/extractor/rtl2.py

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 6dbbc053c..6f7ad588e 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -104,7 +104,8 @@ class RtmpFD(FileDownloader):
         live = info_dict.get('rtmp_live', False)
         conn = info_dict.get('rtmp_conn', None)
         protocol = info_dict.get('rtmp_protocol', None)
-
+        no_resume = info_dict.get('no_resume', False)
+        
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
         test = self.params.get('test', False)
@@ -141,7 +142,10 @@ class RtmpFD(FileDownloader):
             basic_args += ['--conn', conn]
         if protocol is not None:
             basic_args += ['--protocol', protocol]
-        args = basic_args + [[], ['--resume', '--skip', '1']][not live and self.params.get('continuedl', False)]
+        if not no_resume:
+            basic_args += ['--resume']
+        
+        args = basic_args + [[], ['--skip', '1']][not live and self.params.get('continuedl', False)]
 
         if sys.platform == 'win32' and sys.version_info < (3, 0):
             # Windows subprocess module does not actually support Unicode
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3f7ca6f7d..03c56156a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -349,6 +349,7 @@ from .rtbf import RTBFIE
 from .rte import RteIE
 from .rtlnl import RtlXlIE
 from .rtlnow import RTLnowIE
+from .rtl2 import RTL2IE
 from .rtp import RTPIE
 from .rts import RTSIE
 from .rtve import RTVEALaCartaIE, RTVELiveIE
diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
new file mode 100644
index 000000000..14b45e86e
--- /dev/null
+++ b/youtube_dl/extractor/rtl2.py
@@ -0,0 +1,100 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    clean_html,
+    unified_strdate,
+    int_or_none,
+)
+
+
+class RTL2IE(InfoExtractor):
+    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
+    _VALID_URL = r'http?://(?P<url>(?P<domain>(www\.)?rtl2\.de)/.*/(?P<video_id>.*))'
+    _TESTS = [{
+            'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
+            'info_dict': {
+                'id': 'folge-203-0',
+                'ext': 'f4v',
+                'title': 'GRIP sucht den Sommerk\xf6nig',
+                'description' : 'Matthias, Det und Helge treten gegeneinander an.'
+            },
+            'params': {
+                # rtmp download
+                #'skip_download': True,
+            },
+        },
+        {
+            'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/',
+            'info_dict': {
+                'id': '21040-anna-erwischt-alex',
+                'ext': 'f4v',
+                'title': 'GRIP sucht den Sommerk\xf6nig',
+                'description' : 'Matthias, Det und Helge treten gegeneinander an.'
+            },
+            'params': {
+                # rtmp download
+                #'skip_download': True,
+            },
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_page_url = 'http://%s/' % mobj.group('domain')
+        video_id = mobj.group('video_id')
+    
+        webpage = self._download_webpage('http://' + mobj.group('url'), video_id)
+
+        vico_id = self._html_search_regex(r'vico_id\s*:\s*([0-9]+)', webpage, '%s');
+        vivi_id = self._html_search_regex(r'vivi_id\s*:\s*([0-9]+)', webpage, '%s');
+
+        info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id
+        webpage = self._download_webpage(info_url, '')
+
+        video_info = json.loads(webpage)
+
+        download_url = video_info["video"]["streamurl"] # self._html_search_regex(r'streamurl\":\"(.*?)\"', webpage, '%s');
+        title = video_info["video"]["titel"] # self._html_search_regex(r'titel\":\"(.*?)\"', webpage, '%s');
+        description = video_info["video"]["beschreibung"] # self._html_search_regex(r'beschreibung\":\"(.*?)\"', webpage, '%s');
+        #ext = self._html_search_regex(r'streamurl\":\".*?(\..{2,4})\"', webpage, '%s');
+
+        thumbnail = video_info["video"]["image"]
+
+        download_url = download_url.replace("\\", "")
+
+        stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, '%s')
+
+        #print(download_url)
+        #print(stream_url)
+        #print(title)
+        #print(description)
+        #print(video_id)
+        
+        formats = []
+
+        fmt = {
+            'url' : download_url,
+                #'app': 'ondemand?_fcs_vhost=cp108781.edgefcs.net',
+                'play_path': stream_url,
+                'player_url': 'http://www.rtl2.de/flashplayer/vipo_player.swf',
+                'page_url': url,
+                'flash_version' : "LNX 11,2,202,429",
+                'rtmp_conn' : ["S:connect", "O:1", "NS:pageUrl:" + url, "NB:fpad:0", "NN:videoFunction:1", "O:0"],
+                'no_resume' : 1,
+        }
+
+        formats.append(fmt)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail' : thumbnail,
+            'description' : description,
+            'formats': formats,
+        }

From 1e108029907ca28b75f37d2cf0bf25bcabbfbdac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Jan 2015 05:21:39 +0600
Subject: [PATCH 1252/1937] [krasview] Fix extraction

---
 youtube_dl/extractor/krasview.py | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/krasview.py b/youtube_dl/extractor/krasview.py
index 6f3d2345b..e46954b47 100644
--- a/youtube_dl/extractor/krasview.py
+++ b/youtube_dl/extractor/krasview.py
@@ -2,18 +2,17 @@
 from __future__ import unicode_literals
 
 import json
-import re
 
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    unescapeHTML,
+    js_to_json,
 )
 
 
 class KrasViewIE(InfoExtractor):
     IE_DESC = 'Красвью'
-    _VALID_URL = r'https?://krasview\.ru/video/(?P<id>\d+)'
+    _VALID_URL = r'https?://krasview\.ru/(?:video|embed)/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://krasview.ru/video/512228',
@@ -29,20 +28,18 @@ class KrasViewIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
 
-        flashvars = json.loads(self._search_regex(
-            r'flashvars\s*:\s*({.+?})\s*}\);', webpage, 'flashvars'))
+        flashvars = json.loads(js_to_json(self._search_regex(
+            r'video_Init\(({.+?})', webpage, 'flashvars')))
 
         video_url = flashvars['url']
-        title = unescapeHTML(flashvars['title'])
-        description = unescapeHTML(flashvars.get('subtitle') or self._og_search_description(webpage, default=None))
-        thumbnail = flashvars['image']
-        duration = int(flashvars['duration'])
-        filesize = int(flashvars['size'])
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage, default=None)
+        thumbnail = flashvars.get('image') or self._og_search_thumbnail(webpage)
+        duration = int_or_none(flashvars.get('duration'))
         width = int_or_none(self._og_search_property('video:width', webpage, 'video width'))
         height = int_or_none(self._og_search_property('video:height', webpage, 'video height'))
 
@@ -53,7 +50,6 @@ class KrasViewIE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
             'duration': duration,
-            'filesize': filesize,
             'width': width,
             'height': height,
         }

From cfb56d1af38f3e1e0251dbd8a20e3ed8884976ff Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 02:38:47 +0100
Subject: [PATCH 1253/1937] Add --list-thumbnails

---
 test/test_utils.py               | 11 ++++++++++
 youtube_dl/YoutubeDL.py          | 37 ++++++++++++++++++++++++++++----
 youtube_dl/__init__.py           |  1 +
 youtube_dl/extractor/common.py   |  2 ++
 youtube_dl/extractor/testtube.py | 16 ++++++++++++--
 youtube_dl/options.py            | 15 +++++++++----
 youtube_dl/utils.py              |  8 +++++++
 7 files changed, 80 insertions(+), 10 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index bdd7f268a..ebec7986f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -52,6 +52,7 @@ from youtube_dl.utils import (
     urlencode_postdata,
     version_tuple,
     xpath_with_ns,
+    render_table,
 )
 
 
@@ -434,5 +435,15 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
         self.assertTrue(is_html(  # UTF-32-LE
             b'\xFF\xFE\x00\x00<\x00\x00\x00h\x00\x00\x00t\x00\x00\x00m\x00\x00\x00l\x00\x00\x00>\x00\x00\x00\xe4\x00\x00\x00'))
 
+    def test_render_table(self):
+        self.assertEqual(
+            render_table(
+                ['a', 'bcd'],
+                [[123, 4], [9999, 51]]),
+            'a    bcd\n'
+            '123  4\n'
+            '9999 51')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d6728b2dd..e0f5a0d74 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -54,6 +54,7 @@ from .utils import (
     PostProcessingError,
     platform_name,
     preferredencoding,
+    render_table,
     SameFileError,
     sanitize_filename,
     std_headers,
@@ -221,6 +222,8 @@ class YoutubeDL(object):
                        youtube-dl servers for debugging.
     sleep_interval:    Number of seconds to sleep before each download.
     external_downloader:  Executable of the external downloader to call.
+    listformats:       Print an overview of available video formats and exit.
+    list_thumbnails:   Print a table of all thumbnails and exit.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
@@ -916,9 +919,14 @@ class YoutubeDL(object):
             info_dict['playlist_index'] = None
 
         thumbnails = info_dict.get('thumbnails')
+        if thumbnails is None:
+            thumbnail = info_dict.get('thumbnail')
+            if thumbnail:
+                thumbnails = [{'url': thumbnail}]
         if thumbnails:
             thumbnails.sort(key=lambda t: (
-                t.get('width'), t.get('height'), t.get('url')))
+                t.get('preference'), t.get('width'), t.get('height'),
+                t.get('id'), t.get('url')))
             for t in thumbnails:
                 if 'width' in t and 'height' in t:
                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
@@ -990,9 +998,12 @@ class YoutubeDL(object):
             # element in the 'formats' field in info_dict is info_dict itself,
             # wich can't be exported to json
             info_dict['formats'] = formats
-        if self.params.get('listformats', None):
+        if self.params.get('listformats'):
             self.list_formats(info_dict)
             return
+        if self.params.get('list_thumbnails'):
+            self.list_thumbnails(info_dict)
+            return
 
         req_format = self.params.get('format')
         if req_format is None:
@@ -1500,8 +1511,26 @@ class YoutubeDL(object):
         header_line = line({
             'format_id': 'format code', 'ext': 'extension',
             'resolution': 'resolution', 'format_note': 'note'}, idlen=idlen)
-        self.to_screen('[info] Available formats for %s:\n%s\n%s' %
-                       (info_dict['id'], header_line, '\n'.join(formats_s)))
+        self.to_screen(
+            '[info] Available formats for %s:\n%s\n%s' %
+            (info_dict['id'], header_line, '\n'.join(formats_s)))
+
+    def list_thumbnails(self, info_dict):
+        thumbnails = info_dict.get('thumbnails')
+        if not thumbnails:
+            tn_url = info_dict.get('thumbnail')
+            if tn_url:
+                thumbnails = [{'id': '0', 'url': tn_url}]
+            else:
+                self.to_screen(
+                    '[info] No thumbnails present for %s' % info_dict['id'])
+                return
+
+        self.to_screen(
+            '[info] Thumbnails for %s:' % info_dict['id'])
+        self.to_screen(render_table(
+            ['ID', 'width', 'height', 'URL'],
+            [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
 
     def urlopen(self, req):
         """ Start an HTTP download """
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 3fc7dc5c2..a3f82612c 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -331,6 +331,7 @@ def _real_main(argv=None):
         'call_home': opts.call_home,
         'sleep_interval': opts.sleep_interval,
         'external_downloader': opts.external_downloader,
+        'list_thumbnails': opts.list_thumbnails,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 523400062..7b7a832dc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -129,7 +129,9 @@ class InfoExtractor(object):
                     something like "4234987", title "Dancing naked mole rats",
                     and display_id "dancing-naked-mole-rats"
     thumbnails:     A list of dictionaries, with the following entries:
+                        * "id" (optional, string) - Thumbnail format ID
                         * "url"
+                        * "preference" (optional, int) - quality of the image
                         * "width" (optional, int)
                         * "height" (optional, int)
                         * "resolution" (optional, string "{width}x{height"},
diff --git a/youtube_dl/extractor/testtube.py b/youtube_dl/extractor/testtube.py
index fd47e71a2..6a7b5e49d 100644
--- a/youtube_dl/extractor/testtube.py
+++ b/youtube_dl/extractor/testtube.py
@@ -1,7 +1,10 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    qualities,
+)
 
 
 class TestTubeIE(InfoExtractor):
@@ -46,13 +49,22 @@ class TestTubeIE(InfoExtractor):
         self._sort_formats(formats)
 
         duration = int_or_none(info.get('duration'))
+        images = info.get('images')
+        thumbnails = None
+        preference = qualities(['mini', 'small', 'medium', 'large'])
+        if images:
+            thumbnails = [{
+                'id': thumbnail_id,
+                'url': img_url,
+                'preference': preference(thumbnail_id)
+            } for thumbnail_id, img_url in images.items()]
 
         return {
             'id': video_id,
             'display_id': display_id,
             'title': info['title'],
             'description': info.get('summary'),
-            'thumbnail': info.get('images', {}).get('large'),
+            'thumbnails': thumbnails,
             'uploader': info.get('show', {}).get('name'),
             'uploader_id': info.get('show', {}).get('slug'),
             'duration': duration,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index b38b8349f..e3b4b8a8a 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -614,10 +614,6 @@ def parseOpts(overrideArguments=None):
         '--write-annotations',
         action='store_true', dest='writeannotations', default=False,
         help='write video annotations to a .annotation file')
-    filesystem.add_option(
-        '--write-thumbnail',
-        action='store_true', dest='writethumbnail', default=False,
-        help='write thumbnail image to disk')
     filesystem.add_option(
         '--load-info',
         dest='load_info_filename', metavar='FILE',
@@ -637,6 +633,16 @@ def parseOpts(overrideArguments=None):
         action='store_true', dest='rm_cachedir',
         help='Delete all filesystem cache files')
 
+    thumbnail = optparse.OptionGroup(parser, 'Thumbnail images')
+    thumbnail.add_option(
+        '--write-thumbnail',
+        action='store_true', dest='writethumbnail', default=False,
+        help='write thumbnail image to disk')
+    thumbnail.add_option(
+        '--list-thumbnails',
+        action='store_true', dest='list_thumbnails', default=False,
+        help='Simulate and list all available thumbnail formats')
+
     postproc = optparse.OptionGroup(parser, 'Post-processing Options')
     postproc.add_option(
         '-x', '--extract-audio',
@@ -702,6 +708,7 @@ def parseOpts(overrideArguments=None):
     parser.add_option_group(selection)
     parser.add_option_group(downloader)
     parser.add_option_group(filesystem)
+    parser.add_option_group(thumbnail)
     parser.add_option_group(verbosity)
     parser.add_option_group(workarounds)
     parser.add_option_group(video_format)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d22b03134..b8c52af74 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1659,3 +1659,11 @@ def determine_protocol(info_dict):
         return 'f4m'
 
     return compat_urllib_parse_urlparse(url).scheme
+
+
+def render_table(header_row, data):
+    """ Render a list of rows, each as a list of values """
+    table = [header_row] + data
+    max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
+    format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
+    return '\n'.join(format_str % tuple(row) for row in table)

From ec82d85acdc497436e7e2e767088d3ecb5947f68 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 03:11:12 +0100
Subject: [PATCH 1254/1937] [YoutubeDL] Implement --write-all-thumbnails
 (Closes #2269)

---
 youtube_dl/YoutubeDL.py | 57 +++++++++++++++++++++++++++--------------
 youtube_dl/__init__.py  |  1 +
 youtube_dl/options.py   |  4 +++
 3 files changed, 43 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e0f5a0d74..0e73dc8ff 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -146,6 +146,7 @@ class YoutubeDL(object):
     writeinfojson:     Write the video description to a .info.json file
     writeannotations:  Write the video annotations to a .annotations.xml file
     writethumbnail:    Write the thumbnail image to a file
+    write_all_thumbnails:  Write all thumbnail formats to files
     writesubtitles:    Write the video subtitles to a file
     writeautomaticsub: Write the automatic subtitles to a file
     allsubtitles:      Downloads all the subtitles of the video
@@ -1210,25 +1211,7 @@ class YoutubeDL(object):
                     self.report_error('Cannot write metadata to JSON file ' + infofn)
                     return
 
-        if self.params.get('writethumbnail', False):
-            if info_dict.get('thumbnail') is not None:
-                thumb_format = determine_ext(info_dict['thumbnail'], 'jpg')
-                thumb_filename = os.path.splitext(filename)[0] + '.' + thumb_format
-                if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
-                    self.to_screen('[%s] %s: Thumbnail is already present' %
-                                   (info_dict['extractor'], info_dict['id']))
-                else:
-                    self.to_screen('[%s] %s: Downloading thumbnail ...' %
-                                   (info_dict['extractor'], info_dict['id']))
-                    try:
-                        uf = self.urlopen(info_dict['thumbnail'])
-                        with open(thumb_filename, 'wb') as thumbf:
-                            shutil.copyfileobj(uf, thumbf)
-                        self.to_screen('[%s] %s: Writing thumbnail to: %s' %
-                                       (info_dict['extractor'], info_dict['id'], thumb_filename))
-                    except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
-                        self.report_warning('Unable to download thumbnail "%s": %s' %
-                                            (info_dict['thumbnail'], compat_str(err)))
+        self._write_thumbnails(info_dict, filename)
 
         if not self.params.get('skip_download', False):
             try:
@@ -1676,3 +1659,39 @@ class YoutubeDL(object):
         if encoding is None:
             encoding = preferredencoding()
         return encoding
+
+    def _write_thumbnails(self, info_dict, filename):
+        if self.params.get('writethumbnail', False):
+            thumbnails = info_dict.get('thumbnails')
+            if thumbnails:
+                thumbnails = [thumbnails[-1]]
+        elif self.params.get('write_all_thumbnails', False):
+            thumbnails = info_dict.get('thumbnails')
+        else:
+            return
+
+        if not thumbnails:
+            # No thumbnails present, so return immediately
+            return
+
+        for t in thumbnails:
+            thumb_ext = determine_ext(t['url'], 'jpg')
+            suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
+            thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
+            thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
+
+            if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
+                self.to_screen('[%s] %s: Thumbnail %sis already present' %
+                               (info_dict['extractor'], info_dict['id'], thumb_display_id))
+            else:
+                self.to_screen('[%s] %s: Downloading thumbnail %s...' %
+                               (info_dict['extractor'], info_dict['id'], thumb_display_id))
+                try:
+                    uf = self.urlopen(t['url'])
+                    with open(thumb_filename, 'wb') as thumbf:
+                        shutil.copyfileobj(uf, thumbf)
+                    self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
+                                   (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
+                except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+                    self.report_warning('Unable to download thumbnail "%s": %s' %
+                                        (t['url'], compat_str(err)))
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index a3f82612c..04f668334 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -286,6 +286,7 @@ def _real_main(argv=None):
         'writeannotations': opts.writeannotations,
         'writeinfojson': opts.writeinfojson,
         'writethumbnail': opts.writethumbnail,
+        'write_all_thumbnails': opts.write_all_thumbnails,
         'writesubtitles': opts.writesubtitles,
         'writeautomaticsub': opts.writeautomaticsub,
         'allsubtitles': opts.allsubtitles,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index e3b4b8a8a..a3b012ddb 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -638,6 +638,10 @@ def parseOpts(overrideArguments=None):
         '--write-thumbnail',
         action='store_true', dest='writethumbnail', default=False,
         help='write thumbnail image to disk')
+    thumbnail.add_option(
+        '--write-all-thumbnails',
+        action='store_true', dest='write_all_thumbnails', default=False,
+        help='write all thumbnail image formats to disk')
     thumbnail.add_option(
         '--list-thumbnails',
         action='store_true', dest='list_thumbnails', default=False,

From c14e88f0f561c5ac0a1cb9e6764fe4702bd9f7ca Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 04:24:55 +0100
Subject: [PATCH 1255/1937] [YoutubeDL] Add --playlist-items option (Fixes
 #2662)

---
 AUTHORS                 |  1 +
 youtube_dl/YoutubeDL.py | 38 +++++++++++++++++++++++++++++++++-----
 youtube_dl/__init__.py  |  1 +
 youtube_dl/options.py   |  4 ++++
 4 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/AUTHORS b/AUTHORS
index b8bf3cb6f..8362b6d8a 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -104,3 +104,4 @@ Ondřej Caletka
 Dinesh S
 Johan K. Jensen
 Yen Chi Hsuan
+Enam Mijbah Noor
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0e73dc8ff..b7e93b8dd 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -137,6 +137,7 @@ class YoutubeDL(object):
     nooverwrites:      Prevent overwriting files.
     playliststart:     Playlist item to start at.
     playlistend:       Playlist item to end at.
+    playlist_items:    Specific indices of playlist to download.
     playlistreverse:   Download playlist items in reverse order.
     matchtitle:        Download only matching titles.
     rejecttitle:       Reject downloads for matching titles.
@@ -703,24 +704,51 @@ class YoutubeDL(object):
             if playlistend == -1:
                 playlistend = None
 
+            playlistitems_str = self.params.get('playlist_items', None)
+            playlistitems = None
+            if playlistitems_str is not None:
+                def iter_playlistitems(format):
+                    for string_segment in format.split(','):
+                        if '-' in string_segment:
+                            start, end = string_segment.split('-')
+                            for item in range(int(start), int(end) + 1):
+                                yield int(item)
+                        else:
+                            yield int(string_segment)
+                playlistitems = iter_playlistitems(playlistitems_str)
+
             ie_entries = ie_result['entries']
             if isinstance(ie_entries, list):
                 n_all_entries = len(ie_entries)
-                entries = ie_entries[playliststart:playlistend]
+                if playlistitems:
+                    entries = [ie_entries[i - 1] for i in playlistitems]
+                else:
+                    entries = ie_entries[playliststart:playlistend]
                 n_entries = len(entries)
                 self.to_screen(
                     "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
                     (ie_result['extractor'], playlist, n_all_entries, n_entries))
             elif isinstance(ie_entries, PagedList):
-                entries = ie_entries.getslice(
-                    playliststart, playlistend)
+                if playlistitems:
+                    entries = []
+                    for item in playlistitems:
+                        entries.extend(ie_entries.getslice(
+                            item - 1, item
+                        ))
+                else:
+                    entries = ie_entries.getslice(
+                        playliststart, playlistend)
                 n_entries = len(entries)
                 self.to_screen(
                     "[%s] playlist %s: Downloading %d videos" %
                     (ie_result['extractor'], playlist, n_entries))
             else:  # iterable
-                entries = list(itertools.islice(
-                    ie_entries, playliststart, playlistend))
+                if playlistitems:
+                    entry_list = list(ie_entries)
+                    entries = [entry_list[i - 1] for i in playlistitems]
+                else:
+                    entries = list(itertools.islice(
+                        ie_entries, playliststart, playlistend))
                 n_entries = len(entries)
                 self.to_screen(
                     "[%s] playlist %s: Downloading %d videos" %
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 04f668334..0bd7b68c3 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -333,6 +333,7 @@ def _real_main(argv=None):
         'sleep_interval': opts.sleep_interval,
         'external_downloader': opts.external_downloader,
         'list_thumbnails': opts.list_thumbnails,
+        'playlist_items': opts.playlist_items,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index a3b012ddb..872835295 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -200,6 +200,10 @@ def parseOpts(overrideArguments=None):
         '--playlist-end',
         dest='playlistend', metavar='NUMBER', default=None, type=int,
         help='playlist video to end at (default is last)')
+    selection.add_option(
+        '--playlist-items',
+        dest='playlist_items', metavar='ITEM_SPEC', default=None,
+        help='playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8" if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.')
     selection.add_option(
         '--match-title',
         dest='matchtitle', metavar='REGEX',

From baeaeffce550b23848983cd281f173a7906cd7f9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 04:34:38 +0100
Subject: [PATCH 1256/1937] [options] Add support for infinite retries (Fixes
 #507)

---
 youtube_dl/__init__.py | 13 ++++++++-----
 youtube_dl/options.py  |  2 +-
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 0bd7b68c3..09da8802d 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -143,10 +143,13 @@ def _real_main(argv=None):
             parser.error('invalid max_filesize specified')
         opts.max_filesize = numeric_limit
     if opts.retries is not None:
-        try:
-            opts.retries = int(opts.retries)
-        except (TypeError, ValueError):
-            parser.error('invalid retry count specified')
+        if opts.retries in ('inf', 'infinite'):
+            opts_retries = float('inf')
+        else:
+            try:
+                opts_retries = int(opts.retries)
+            except (TypeError, ValueError):
+                parser.error('invalid retry count specified')
     if opts.buffersize is not None:
         numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
         if numeric_buffersize is None:
@@ -268,7 +271,7 @@ def _real_main(argv=None):
         'ignoreerrors': opts.ignoreerrors,
         'ratelimit': opts.ratelimit,
         'nooverwrites': opts.nooverwrites,
-        'retries': opts.retries,
+        'retries': opts_retries,
         'buffersize': opts.buffersize,
         'noresizebuffer': opts.noresizebuffer,
         'continuedl': opts.continue_dl,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 872835295..1ddbdbc78 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -377,7 +377,7 @@ def parseOpts(overrideArguments=None):
     downloader.add_option(
         '-R', '--retries',
         dest='retries', metavar='RETRIES', default=10,
-        help='number of retries (default is %default)')
+        help='number of retries (default is %default), or "infinite".')
     downloader.add_option(
         '--buffer-size',
         dest='buffersize', metavar='SIZE', default='1024',

From 881e6a1f5c47a65348879f817ad833081e8c5ada Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 04:49:44 +0100
Subject: [PATCH 1257/1937] Add --xattr-set-filesize option (Fixes #1348)

---
 youtube_dl/YoutubeDL.py         |  3 ++-
 youtube_dl/__init__.py          |  6 ++++++
 youtube_dl/downloader/common.py | 32 +++++++++++++++++---------------
 youtube_dl/downloader/http.py   |  8 ++++++++
 youtube_dl/options.py           |  4 ++++
 5 files changed, 37 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b7e93b8dd..0241f7e3c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -231,7 +231,8 @@ class YoutubeDL(object):
     The following parameters are not used by YoutubeDL itself, they are used by
     the FileDownloader:
     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
-    noresizebuffer, retries, continuedl, noprogress, consoletitle
+    noresizebuffer, retries, continuedl, noprogress, consoletitle,
+    xattr_set_filesize.
 
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 09da8802d..112a8ba60 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -241,6 +241,11 @@ def _real_main(argv=None):
             'verboseOutput': opts.verbose,
             'exec_cmd': opts.exec_cmd,
         })
+    if opts.xattr_set_filesize:
+        try:
+            import xattr
+        except ImportError:
+            parser.error('setting filesize xattr requested but python-xattr is not available')
 
     ydl_opts = {
         'usenetrc': opts.usenetrc,
@@ -337,6 +342,7 @@ def _real_main(argv=None):
         'external_downloader': opts.external_downloader,
         'list_thumbnails': opts.list_thumbnails,
         'playlist_items': opts.playlist_items,
+        'xattr_set_filesize': opts.xattr_set_filesize,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index c35c42c1d..7bb3a948d 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -25,21 +25,23 @@ class FileDownloader(object):
 
     Available options:
 
-    verbose:           Print additional info to stdout.
-    quiet:             Do not print messages to stdout.
-    ratelimit:         Download speed limit, in bytes/sec.
-    retries:           Number of times to retry for HTTP error 5xx
-    buffersize:        Size of download buffer in bytes.
-    noresizebuffer:    Do not automatically resize the download buffer.
-    continuedl:        Try to continue downloads if possible.
-    noprogress:        Do not print the progress bar.
-    logtostderr:       Log messages to stderr instead of stdout.
-    consoletitle:      Display progress in console window's titlebar.
-    nopart:            Do not use temporary .part files.
-    updatetime:        Use the Last-modified header to set output file timestamps.
-    test:              Download only first bytes to test the downloader.
-    min_filesize:      Skip files smaller than this size
-    max_filesize:      Skip files larger than this size
+    verbose:            Print additional info to stdout.
+    quiet:              Do not print messages to stdout.
+    ratelimit:          Download speed limit, in bytes/sec.
+    retries:            Number of times to retry for HTTP error 5xx
+    buffersize:         Size of download buffer in bytes.
+    noresizebuffer:     Do not automatically resize the download buffer.
+    continuedl:         Try to continue downloads if possible.
+    noprogress:         Do not print the progress bar.
+    logtostderr:        Log messages to stderr instead of stdout.
+    consoletitle:       Display progress in console window's titlebar.
+    nopart:             Do not use temporary .part files.
+    updatetime:         Use the Last-modified header to set output file timestamps.
+    test:               Download only first bytes to test the downloader.
+    min_filesize:       Skip files smaller than this size
+    max_filesize:       Skip files larger than this size
+    xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
+                        (experimenatal)
 
     Subclasses of this one must re-define the real_download method.
     """
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 4db50ee90..8a1d578d5 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -157,6 +157,14 @@ class HttpFD(FileDownloader):
                 except (OSError, IOError) as err:
                     self.report_error('unable to open for writing: %s' % str(err))
                     return False
+
+                if self.params.get('xattr_set_filesize', False) and data_len is not None:
+                    try:
+                        import xattr
+                        xattr.setxattr(tmpfilename, 'user.ytdl.filesize', str(data_len))
+                    except(OSError, IOError, ImportError) as err:
+                        self.report_error('unable to set filesize xattr: %s' % str(err))
+
             try:
                 stream.write(data_block)
             except (IOError, OSError) as err:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 1ddbdbc78..dbc6f5528 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -394,6 +394,10 @@ def parseOpts(overrideArguments=None):
         '--playlist-reverse',
         action='store_true',
         help='Download playlist videos in reverse order')
+    downloader.add_option(
+        '--xattr-set-filesize',
+        dest='xattr_set_filesize', action='store_true',
+        help='(experimental) set file xattribute ytdl.filesize with expected filesize')
     downloader.add_option(
         '--external-downloader',
         dest='external_downloader', metavar='COMMAND',

From e1ccc04e9f68988df0520b7502edc6479b62378f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 04:56:32 +0100
Subject: [PATCH 1258/1937] Test rtmpdump on travis (Fixes #1601)

---
 .travis.yml                         | 3 +++
 youtube_dl/extractor/folketinget.py | 4 +---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index f14014414..fb34299fc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,6 +4,9 @@ python:
   - "2.7"
   - "3.3"
   - "3.4"
+before_install:
+  - sudo apt-get update -qq
+  - sudo apt-get install -yqq rtmpdump
 script: nosetests test --verbose
 notifications:
   email:
diff --git a/youtube_dl/extractor/folketinget.py b/youtube_dl/extractor/folketinget.py
index 68e2db943..0fb29de75 100644
--- a/youtube_dl/extractor/folketinget.py
+++ b/youtube_dl/extractor/folketinget.py
@@ -16,6 +16,7 @@ class FolketingetIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?ft\.dk/webtv/video/[^?#]*?\.(?P<id>[0-9]+)\.aspx'
     _TEST = {
         'url': 'http://www.ft.dk/webtv/video/20141/eru/td.1165642.aspx?as=1#player',
+        'md5': '6269e8626fa1a891bf5369b386ae996a',
         'info_dict': {
             'id': '1165642',
             'ext': 'mp4',
@@ -29,9 +30,6 @@ class FolketingetIE(InfoExtractor):
             'upload_date': '20141120',
             'duration': 3960,
         },
-        'params': {
-            'skip_download': 'rtmpdump required',
-        }
     }
 
     def _real_extract(self, url):

From 7d346331b53498b5966790677e0b41631edd8a4e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 05:15:47 +0100
Subject: [PATCH 1259/1937] [audiomack:album] Update testcase

---
 test/helper.py                    |  2 +-
 youtube_dl/extractor/audiomack.py | 13 +++++++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index c416f388c..27a68091f 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -140,7 +140,7 @@ def expect_info_dict(self, got_dict, expected_dict):
     # Are checkable fields missing from the test case definition?
     test_info_dict = dict((key, value if not isinstance(value, compat_str) or len(value) < 250 else 'md5:' + md5(value))
                           for key, value in got_dict.items()
-                          if value and key in ('title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location'))
+                          if value and key in ('id', 'title', 'description', 'uploader', 'upload_date', 'timestamp', 'uploader_id', 'location'))
     missing_keys = set(test_info_dict.keys()) - set(expected_dict.keys())
     if missing_keys:
         def _repr(v):
diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 8bfe50214..693ba22c6 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -88,16 +88,21 @@ class AudiomackAlbumIE(InfoExtractor):
         # Album playlist ripped from fakeshoredrive with no metadata
         {
             'url': 'http://www.audiomack.com/album/fakeshoredrive/ppp-pistol-p-project',
+            'info_dict': {
+                'title': 'PPP (Pistol P Project)',
+                'id': '837572',
+            },
             'playlist': [{
                 'info_dict': {
-                    'title': '9.-heaven-or-hell-chimaca-ft-zuse-prod-by-dj-fu',
-                    'id': '9.-heaven-or-hell-chimaca-ft-zuse-prod-by-dj-fu',
+                    'title': 'PPP (Pistol P Project) - 9. Heaven or Hell (CHIMACA) ft Zuse (prod by DJ FU)',
+                    'id': '837577',
                     'ext': 'mp3',
+                    'uploader': 'Lil Herb a.k.a. G Herbo',
                 }
             }],
             'params': {
-                'playliststart': 8,
-                'playlistend': 8,
+                'playliststart': 9,
+                'playlistend': 9,
             }
         }
     ]

From 37f4ce538ab572812465fa6d0e429bf767a030fd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 05:17:15 +0100
Subject: [PATCH 1260/1937] [smotri] Fix test case

---
 youtube_dl/extractor/smotri.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index 26f361c93..e94f41362 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -102,6 +102,7 @@ class SmotriIE(InfoExtractor):
                 'uploader_id': 'mopeder',
                 'duration': 71,
                 'thumbnail': 'http://frame9.loadup.ru/d7/32/2888853.2.3.jpg',
+                'upload_date': '20150114',
             },
         },
         # swf player

From cc1237f4845e108017687d6727c3dfe6efc3cb4d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 05:17:38 +0100
Subject: [PATCH 1261/1937] [__init__] Work around flake8 false positive

---
 youtube_dl/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 112a8ba60..71d2c6f35 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -244,6 +244,7 @@ def _real_main(argv=None):
     if opts.xattr_set_filesize:
         try:
             import xattr
+            xattr  # Confuse flake8
         except ImportError:
             parser.error('setting filesize xattr requested but python-xattr is not available')
 

From 8604e882a8f99fcd632efbb94e449477d860218d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 05:23:21 +0100
Subject: [PATCH 1262/1937] [ubu] Fix test and modernize

---
 youtube_dl/extractor/ubu.py | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/ubu.py b/youtube_dl/extractor/ubu.py
index 0182d67ec..d50237758 100644
--- a/youtube_dl/extractor/ubu.py
+++ b/youtube_dl/extractor/ubu.py
@@ -3,50 +3,51 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    qualities,
+)
 
 
 class UbuIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?ubu\.com/film/(?P<id>[\da-z_-]+)\.html'
     _TEST = {
         'url': 'http://ubu.com/film/her_noise.html',
-        'md5': '8edd46ee8aa6b265fb5ed6cf05c36bc9',
+        'md5': '138d5652618bf0f03878978db9bef1ee',
         'info_dict': {
             'id': 'her_noise',
-            'ext': 'mp4',
+            'ext': 'm4v',
             'title': 'Her Noise - The Making Of (2007)',
             'duration': 3600,
         },
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(
             r'<title>.+?Film &amp; Video: ([^<]+)</title>', webpage, 'title')
 
         duration = int_or_none(self._html_search_regex(
-            r'Duration: (\d+) minutes', webpage, 'duration', fatal=False, default=None))
-        if duration:
-            duration *= 60
+            r'Duration: (\d+) minutes', webpage, 'duration', fatal=False),
+            invscale=60)
 
         formats = []
-
         FORMAT_REGEXES = [
-            ['sq', r"'flashvars'\s*,\s*'file=([^']+)'"],
-            ['hq', r'href="(http://ubumexico\.centro\.org\.mx/video/[^"]+)"']
+            ('sq', r"'flashvars'\s*,\s*'file=([^']+)'"),
+            ('hq', r'href="(http://ubumexico\.centro\.org\.mx/video/[^"]+)"'),
         ]
-
+        preference = qualities([fid for fid, _ in FORMAT_REGEXES])
         for format_id, format_regex in FORMAT_REGEXES:
             m = re.search(format_regex, webpage)
             if m:
                 formats.append({
                     'url': m.group(1),
                     'format_id': format_id,
+                    'preference': preference(format_id),
                 })
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From ee69b99af6d00202a5ad8510604c807876d1b7b1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 06:15:51 +0100
Subject: [PATCH 1263/1937] [YoutubeDL] clarify hook documentation

---
 youtube_dl/YoutubeDL.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0241f7e3c..b772f87f1 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -198,11 +198,12 @@ class YoutubeDL(object):
                        postprocessor.
     progress_hooks:    A list of functions that get called on download
                        progress, with a dictionary with the entries
-                       * filename: The final filename
-                       * status: One of "downloading" and "finished"
-
-                       The dict may also have some of the following entries:
+                       * status: One of "downloading" and "finished".
+                                 Check this first and ignore unknown values.
 
+                       If status is one of "downloading" or "finished", the
+                       following properties may also be present:
+                       * filename: The final filename (always present)
                        * downloaded_bytes: Bytes on disk
                        * total_bytes: Size of the whole file, None if unknown
                        * tmpfilename: The filename we're currently writing to
@@ -1251,6 +1252,7 @@ class YoutubeDL(object):
                     if self.params.get('verbose'):
                         self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
                     return fd.download(name, info)
+
                 if info_dict.get('requested_formats') is not None:
                     downloaded = []
                     success = True

From fe41ddbb285abccc3c4d7a3ebc1238c13ec72577 Mon Sep 17 00:00:00 2001
From: David Development <david-dev@live.de>
Date: Sun, 25 Jan 2015 11:53:53 +0100
Subject: [PATCH 1264/1937] refactoring - bug fixes

---
 youtube_dl/extractor/rtl2.py | 66 ++++++++++++++++++------------------
 1 file changed, 33 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
index 14b45e86e..7086d698f 100644
--- a/youtube_dl/extractor/rtl2.py
+++ b/youtube_dl/extractor/rtl2.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..utils import (
@@ -14,82 +13,83 @@ from ..utils import (
 
 
 class RTL2IE(InfoExtractor):
-    """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW"""
-    _VALID_URL = r'http?://(?P<url>(?P<domain>(www\.)?rtl2\.de)/.*/(?P<video_id>.*))'
+    """Information Extractor for RTL2"""
+    _VALID_URL = r'http?://(?P<url>(?P<domain>(www\.)?rtl2\.de)/.*/(?P<video_id>.*))/'
     _TESTS = [{
             'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
             'info_dict': {
                 'id': 'folge-203-0',
                 'ext': 'f4v',
-                'title': 'GRIP sucht den Sommerk\xf6nig',
+                'title': 'GRIP sucht den Sommerkönig',
                 'description' : 'Matthias, Det und Helge treten gegeneinander an.'
             },
             'params': {
                 # rtmp download
-                #'skip_download': True,
+                'skip_download': True,
             },
         },
         {
             'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/',
             'info_dict': {
                 'id': '21040-anna-erwischt-alex',
-                'ext': 'f4v',
-                'title': 'GRIP sucht den Sommerk\xf6nig',
-                'description' : 'Matthias, Det und Helge treten gegeneinander an.'
+                'ext': 'mp4',
+                'title': 'Anna erwischt Alex!',
+                'description' : 'Anna ist Alex\' Tochter bei Köln 50667.'
             },
             'params': {
                 # rtmp download
-                #'skip_download': True,
+                'skip_download': True,
             },
         },
     ]
 
     def _real_extract(self, url):
+        
+        #Some rtl2 urls have no slash at the end, so append it.
+        if not url.endswith("/"):
+            url += '/'
+        
         mobj = re.match(self._VALID_URL, url)
-        video_page_url = 'http://%s/' % mobj.group('domain')
         video_id = mobj.group('video_id')
-    
-        webpage = self._download_webpage('http://' + mobj.group('url'), video_id)
 
-        vico_id = self._html_search_regex(r'vico_id\s*:\s*([0-9]+)', webpage, '%s');
-        vivi_id = self._html_search_regex(r'vivi_id\s*:\s*([0-9]+)', webpage, '%s');
+        webpage = self._download_webpage(url, video_id)
+
+        vico_id = self._html_search_regex(r'vico_id\s*:\s*([0-9]+)', webpage, 'vico_id not found');
+        vivi_id = self._html_search_regex(r'vivi_id\s*:\s*([0-9]+)', webpage, 'vivi_id not found');
 
         info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id
         webpage = self._download_webpage(info_url, '')
 
-        video_info = json.loads(webpage)
+        video_info = self._download_json(info_url, video_id)
 
-        download_url = video_info["video"]["streamurl"] # self._html_search_regex(r'streamurl\":\"(.*?)\"', webpage, '%s');
-        title = video_info["video"]["titel"] # self._html_search_regex(r'titel\":\"(.*?)\"', webpage, '%s');
-        description = video_info["video"]["beschreibung"] # self._html_search_regex(r'beschreibung\":\"(.*?)\"', webpage, '%s');
-        #ext = self._html_search_regex(r'streamurl\":\".*?(\..{2,4})\"', webpage, '%s');
+        download_url = video_info["video"]["streamurl"]
+        title = video_info["video"]["titel"]
+        description = video_info["video"]["beschreibung"]
 
         thumbnail = video_info["video"]["image"]
 
         download_url = download_url.replace("\\", "")
 
         stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, '%s')
-
-        #print(download_url)
-        #print(stream_url)
-        #print(title)
-        #print(description)
-        #print(video_id)
         
-        formats = []
-
-        fmt = {
-            'url' : download_url,
+        #Debug output
+        #print('URL: '        + url)
+        #print('DL URL: '     + download_url)
+        #print('Stream URL: ' + stream_url)
+        #print('Title: '      + title)
+        #print('Description: '+ description)
+        #print('Video ID: '   + video_id)
+                
+        formats = [{
+                'url' : download_url,
                 #'app': 'ondemand?_fcs_vhost=cp108781.edgefcs.net',
                 'play_path': stream_url,
                 'player_url': 'http://www.rtl2.de/flashplayer/vipo_player.swf',
                 'page_url': url,
                 'flash_version' : "LNX 11,2,202,429",
                 'rtmp_conn' : ["S:connect", "O:1", "NS:pageUrl:" + url, "NB:fpad:0", "NN:videoFunction:1", "O:0"],
-                'no_resume' : 1,
-        }
-
-        formats.append(fmt)
+                'no_resume' : True,
+            }]
 
         return {
             'id': video_id,

From c9326b38b824eb8babbf2306b3bfe4bd5e683cfe Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 18:09:09 +0100
Subject: [PATCH 1265/1937] flake8: Ignore .git

---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 13dcd8af6..02aa56257 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -2,5 +2,5 @@
 universal = True
 
 [flake8]
-exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,setup.py,build
+exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,setup.py,build,.git
 ignore = E501

From 3dee7826e7133fb4a73aa0aabaee78499e49264f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 18:09:48 +0100
Subject: [PATCH 1266/1937] [rtl2] PEP8, simplify, make rtmp tests run (#470)

---
 youtube_dl/downloader/rtmp.py  |  13 ++--
 youtube_dl/extractor/common.py |   7 +-
 youtube_dl/extractor/rtl2.py   | 116 +++++++++++++--------------------
 3 files changed, 57 insertions(+), 79 deletions(-)

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 6f7ad588e..e06ebe826 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -105,7 +105,8 @@ class RtmpFD(FileDownloader):
         conn = info_dict.get('rtmp_conn', None)
         protocol = info_dict.get('rtmp_protocol', None)
         no_resume = info_dict.get('no_resume', False)
-        
+        continue_dl = info_dict.get('continuedl', False)
+
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
         test = self.params.get('test', False)
@@ -142,10 +143,12 @@ class RtmpFD(FileDownloader):
             basic_args += ['--conn', conn]
         if protocol is not None:
             basic_args += ['--protocol', protocol]
-        if not no_resume:
-            basic_args += ['--resume']
-        
-        args = basic_args + [[], ['--skip', '1']][not live and self.params.get('continuedl', False)]
+
+        args = basic_args
+        if not no_resume and continue_dl and not live:
+            args += ['--resume']
+        if not live and continue_dl:
+            args += ['--skip', '1']
 
         if sys.platform == 'win32' and sys.version_info < (3, 0):
             # Windows subprocess module does not actually support Unicode
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7b7a832dc..388c55e99 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -114,8 +114,11 @@ class InfoExtractor(object):
                     * http_post_data  Additional data to send with a POST
                                  request.
                     * stretched_ratio  If given and not 1, indicates that the
-                                       video's pixels are not square.
-                                       width : height ratio as float.
+                                 video's pixels are not square.
+                                 width : height ratio as float.
+                    * no_resume  The server does not support resuming the
+                                 (HTTP or RTMP) download. Boolean.
+
     url:            Final video URL.
     ext:            Video filename extension.
     format:         The video format, defaults to ext (used for --get-format)
diff --git a/youtube_dl/extractor/rtl2.py b/youtube_dl/extractor/rtl2.py
index 7086d698f..72cd80498 100644
--- a/youtube_dl/extractor/rtl2.py
+++ b/youtube_dl/extractor/rtl2.py
@@ -1,100 +1,72 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    clean_html,
-    unified_strdate,
-    int_or_none,
-)
 
 
 class RTL2IE(InfoExtractor):
-    """Information Extractor for RTL2"""
-    _VALID_URL = r'http?://(?P<url>(?P<domain>(www\.)?rtl2\.de)/.*/(?P<video_id>.*))/'
+    _VALID_URL = r'http?://(?:www\.)?rtl2\.de/[^?#]*?/(?P<id>[^?#/]*?)(?:$|/(?:$|[?#]))'
     _TESTS = [{
-            'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
-            'info_dict': {
-                'id': 'folge-203-0',
-                'ext': 'f4v',
-                'title': 'GRIP sucht den Sommerkönig',
-                'description' : 'Matthias, Det und Helge treten gegeneinander an.'
-            },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
+        'url': 'http://www.rtl2.de/sendung/grip-das-motormagazin/folge/folge-203-0',
+        'md5': 'bfcc179030535b08dc2b36b469b5adc7',
+        'info_dict': {
+            'id': 'folge-203-0',
+            'ext': 'f4v',
+            'title': 'GRIP sucht den Sommerkönig',
+            'description': 'Matthias, Det und Helge treten gegeneinander an.'
         },
-        {
-            'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/',
-            'info_dict': {
-                'id': '21040-anna-erwischt-alex',
-                'ext': 'mp4',
-                'title': 'Anna erwischt Alex!',
-                'description' : 'Anna ist Alex\' Tochter bei Köln 50667.'
-            },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
+    }, {
+        'url': 'http://www.rtl2.de/sendung/koeln-50667/video/5512-anna/21040-anna-erwischt-alex/',
+        'md5': 'ffcd517d2805b57ce11a58a2980c2b02',
+        'info_dict': {
+            'id': '21040-anna-erwischt-alex',
+            'ext': 'mp4',
+            'title': 'Anna erwischt Alex!',
+            'description': 'Anna ist Alex\' Tochter bei Köln 50667.'
         },
-    ]
+    }]
 
     def _real_extract(self, url):
-        
-        #Some rtl2 urls have no slash at the end, so append it.
-        if not url.endswith("/"):
+        # Some rtl2 urls have no slash at the end, so append it.
+        if not url.endswith('/'):
             url += '/'
-        
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
 
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        vico_id = self._html_search_regex(r'vico_id\s*:\s*([0-9]+)', webpage, 'vico_id not found');
-        vivi_id = self._html_search_regex(r'vivi_id\s*:\s*([0-9]+)', webpage, 'vivi_id not found');
-
+        vico_id = self._html_search_regex(
+            r'vico_id\s*:\s*([0-9]+)', webpage, 'vico_id')
+        vivi_id = self._html_search_regex(
+            r'vivi_id\s*:\s*([0-9]+)', webpage, 'vivi_id')
         info_url = 'http://www.rtl2.de/video/php/get_video.php?vico_id=' + vico_id + '&vivi_id=' + vivi_id
         webpage = self._download_webpage(info_url, '')
 
-        video_info = self._download_json(info_url, video_id)
+        info = self._download_json(info_url, video_id)
+        video_info = info['video']
+        title = video_info['titel']
+        description = video_info.get('beschreibung')
+        thumbnail = video_info.get('image')
 
-        download_url = video_info["video"]["streamurl"]
-        title = video_info["video"]["titel"]
-        description = video_info["video"]["beschreibung"]
+        download_url = video_info['streamurl']
+        download_url = download_url.replace('\\', '')
+        stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, 'stream URL')
+        rtmp_conn = ["S:connect", "O:1", "NS:pageUrl:" + url, "NB:fpad:0", "NN:videoFunction:1", "O:0"]
 
-        thumbnail = video_info["video"]["image"]
-
-        download_url = download_url.replace("\\", "")
-
-        stream_url = 'mp4:' + self._html_search_regex(r'ondemand/(.*)', download_url, '%s')
-        
-        #Debug output
-        #print('URL: '        + url)
-        #print('DL URL: '     + download_url)
-        #print('Stream URL: ' + stream_url)
-        #print('Title: '      + title)
-        #print('Description: '+ description)
-        #print('Video ID: '   + video_id)
-                
         formats = [{
-                'url' : download_url,
-                #'app': 'ondemand?_fcs_vhost=cp108781.edgefcs.net',
-                'play_path': stream_url,
-                'player_url': 'http://www.rtl2.de/flashplayer/vipo_player.swf',
-                'page_url': url,
-                'flash_version' : "LNX 11,2,202,429",
-                'rtmp_conn' : ["S:connect", "O:1", "NS:pageUrl:" + url, "NB:fpad:0", "NN:videoFunction:1", "O:0"],
-                'no_resume' : True,
-            }]
+            'url': download_url,
+            'play_path': stream_url,
+            'player_url': 'http://www.rtl2.de/flashplayer/vipo_player.swf',
+            'page_url': url,
+            'flash_version': 'LNX 11,2,202,429',
+            'rtmp_conn': rtmp_conn,
+            'no_resume': True,
+        }]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
-            'thumbnail' : thumbnail,
-            'description' : description,
+            'thumbnail': thumbnail,
+            'description': description,
             'formats': formats,
         }

From b1b0b1ca3030a2b04dd94ec28d5ece3fda88a282 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 18:14:59 +0100
Subject: [PATCH 1267/1937] [generic] Improve description testcase in rss test

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a028c4ed4..ad16b8330 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -362,7 +362,7 @@ class GenericIE(InfoExtractor):
             'info_dict': {
                 'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
                 'title': 'Zero Punctuation',
-                'description': 're:'
+                'description': 're:.*groundbreaking video review series.*'
             },
             'playlist_mincount': 11,
         },

From 6d2749aac407df1e039f5b61a294991c1e810cff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 25 Jan 2015 18:56:04 +0100
Subject: [PATCH 1268/1937] [drtv] Prefer the version without spoken subtitles
 (fixes #4779)

For example for http://www.dr.dk/tv/se/moderne-klassikere/moderne-klassikere-one-republic-apologize#!/, there's a version where everytime someone speaks in English a computer voice translates it.
---
 youtube_dl/extractor/drtv.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index c44adb109..510ef04b0 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -48,14 +48,20 @@ class DRTVIE(SubtitlesInfoExtractor):
             elif asset['Kind'] == 'VideoResource':
                 duration = asset['DurationInMilliseconds'] / 1000.0
                 restricted_to_denmark = asset['RestrictedToDenmark']
+                spoken_subtitles = asset['Target'] == 'SpokenSubtitles'
                 for link in asset['Links']:
                     target = link['Target']
                     uri = link['Uri']
+                    format_id = target
+                    preference = -1 if target == 'HDS' else -2
+                    if spoken_subtitles:
+                        preference -= 2
+                        format_id += '-spoken-subtitles'
                     formats.append({
                         'url': uri + '?hdcore=3.3.0&plugin=aasp-3.3.0.99.43' if target == 'HDS' else uri,
-                        'format_id': target,
+                        'format_id': format_id,
                         'ext': link['FileFormat'],
-                        'preference': -1 if target == 'HDS' else -2,
+                        'preference': preference,
                     })
                 subtitles_list = asset.get('SubtitlesList')
                 if isinstance(subtitles_list, list):

From 96a53167fa64293506f446d0c2bf3e0db6c8df31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Jan 2015 00:32:31 +0600
Subject: [PATCH 1269/1937] [common] Generalize URLs' HTTP errors pre-testing

---
 youtube_dl/extractor/common.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 388c55e99..478232682 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -14,6 +14,7 @@ import xml.etree.ElementTree
 
 from ..compat import (
     compat_cookiejar,
+    compat_HTTPError,
     compat_http_client,
     compat_urllib_error,
     compat_urllib_parse_urlparse,
@@ -26,6 +27,7 @@ from ..utils import (
     compiled_regex_type,
     ExtractorError,
     float_or_none,
+    HEADRequest,
     int_or_none,
     RegexNotFoundError,
     sanitize_filename,
@@ -716,6 +718,27 @@ class InfoExtractor(object):
             )
         formats.sort(key=_formats_key)
 
+    def _check_formats(self, formats, video_id):
+        if formats:
+            formats[:] = filter(
+                lambda f: self._is_valid_url(
+                    f['url'], video_id,
+                    item='%s video format' % f.get('format_id') if f.get('format_id') else 'video'),
+                formats)
+
+    def _is_valid_url(self, url, video_id, item='video'):
+        try:
+            self._request_webpage(
+                HEADRequest(url), video_id,
+                'Checking %s URL' % item)
+            return True
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError):
+                self.report_warning(
+                    '%s URL is invalid, skipping' % item, video_id)
+                return False
+            raise
+
     def http_scheme(self):
         """ Either "http:" or "https:", depending on the user's preferences """
         return (

From a57e8ce6580202c179c38a15abc31f84ca471521 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Jan 2015 00:33:42 +0600
Subject: [PATCH 1270/1937] [lynda] Pre-test video URLs for HTTP errors (Closes
 #2185, closes #4782)

---
 youtube_dl/extractor/lynda.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 26e84970d..762cefa34 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -85,6 +85,7 @@ class LyndaIE(SubtitlesInfoExtractor):
                 } for format_id, video_url in prioritized_streams['0'].items()
             ])
 
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         if self._downloader.params.get('listsubtitles', False):

From d862a4f94fac46e5c6de790a9b66d78d463d666d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Jan 2015 00:34:31 +0600
Subject: [PATCH 1271/1937] [spiegel] Use generalized formats pre-testing

---
 youtube_dl/extractor/spiegel.py | 20 ++------------------
 1 file changed, 2 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index f345883c7..b868241d5 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -4,14 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urlparse,
-    compat_HTTPError,
-)
-from ..utils import (
-    HEADRequest,
-    ExtractorError,
-)
+from ..compat import compat_urlparse
 from .spiegeltv import SpiegeltvIE
 
 
@@ -72,16 +65,6 @@ class SpiegelIE(InfoExtractor):
             if n.tag.startswith('type') and n.tag != 'type6':
                 format_id = n.tag.rpartition('type')[2]
                 video_url = base_url + n.find('./filename').text
-                # Test video URLs beforehand as some of them are invalid
-                try:
-                    self._request_webpage(
-                        HEADRequest(video_url), video_id,
-                        'Checking %s video URL' % format_id)
-                except ExtractorError as e:
-                    if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
-                        self.report_warning(
-                            '%s video URL is invalid, skipping' % format_id, video_id)
-                        continue
                 formats.append({
                     'format_id': format_id,
                     'url': video_url,
@@ -94,6 +77,7 @@ class SpiegelIE(InfoExtractor):
                 })
         duration = float(idoc[0].findall('./duration')[0].text)
 
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         return {

From 80a49d3d7bcd235ba15bd491cc62a0345c9abce1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Jan 2015 02:08:29 +0600
Subject: [PATCH 1272/1937] Credit @David-Development for rtl2 (#4780)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 8362b6d8a..1596a7548 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -105,3 +105,4 @@ Dinesh S
 Johan K. Jensen
 Yen Chi Hsuan
 Enam Mijbah Noor
+David Luhmer

From 2b1bd292ae72fdea4ab731f93777a99b3729b31d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 25 Jan 2015 21:40:43 +0100
Subject: [PATCH 1273/1937] release 2015.01.25

---
 README.md             | 23 +++++++++++++++++++++--
 youtube_dl/version.py |  2 +-
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 36b87444e..0580ab2a3 100644
--- a/README.md
+++ b/README.md
@@ -93,6 +93,14 @@ which means you can modify it, redistribute it or use it however you like.
 ## Video Selection:
     --playlist-start NUMBER          playlist video to start at (default is 1)
     --playlist-end NUMBER            playlist video to end at (default is last)
+    --playlist-items ITEM_SPEC       playlist video items to download. Specify
+                                     indices of the videos in the playlist
+                                     seperated by commas like: "--playlist-items
+                                     1,2,5,8" if you want to download videos
+                                     indexed 1, 2, 5, 8 in the playlist. You can
+                                     specify range: "--playlist-items
+                                     1-3,7,10-13", it will download the videos
+                                     at index 1, 2, 3, 7, 10, 11, 12 and 13.
     --match-title REGEX              download only matching titles (regex or
                                      caseless sub-string)
     --reject-title REGEX             skip download for matching titles (regex or
@@ -124,7 +132,8 @@ which means you can modify it, redistribute it or use it however you like.
 ## Download Options:
     -r, --rate-limit LIMIT           maximum download rate in bytes per second
                                      (e.g. 50K or 4.2M)
-    -R, --retries RETRIES            number of retries (default is 10)
+    -R, --retries RETRIES            number of retries (default is 10), or
+                                     "infinite".
     --buffer-size SIZE               size of download buffer (e.g. 1024 or 16K)
                                      (default is 1024)
     --no-resize-buffer               do not automatically adjust the buffer
@@ -132,6 +141,11 @@ which means you can modify it, redistribute it or use it however you like.
                                      automatically resized from an initial value
                                      of SIZE.
     --playlist-reverse               Download playlist videos in reverse order
+    --xattr-set-filesize             (experimental) set file xattribute
+                                     ytdl.filesize with expected filesize
+    --external-downloader COMMAND    (experimental) Use the specified external
+                                     downloader. Currently supports
+                                     aria2c,curl,wget
 
 ## Filesystem Options:
     -a, --batch-file FILE            file containing URLs to download ('-' for
@@ -191,7 +205,6 @@ which means you can modify it, redistribute it or use it however you like.
     --write-info-json                write video metadata to a .info.json file
     --write-annotations              write video annotations to a .annotation
                                      file
-    --write-thumbnail                write thumbnail image to disk
     --load-info FILE                 json file containing the video information
                                      (created with the "--write-json" option)
     --cookies FILE                   file to read cookies from and dump cookie
@@ -206,6 +219,12 @@ which means you can modify it, redistribute it or use it however you like.
     --no-cache-dir                   Disable filesystem caching
     --rm-cache-dir                   Delete all filesystem cache files
 
+## Thumbnail images:
+    --write-thumbnail                write thumbnail image to disk
+    --write-all-thumbnails           write all thumbnail image formats to disk
+    --list-thumbnails                Simulate and list all available thumbnail
+                                     formats
+
 ## Verbosity / Simulation Options:
     -q, --quiet                      activates quiet mode
     --no-warnings                    Ignore warnings
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 35f3e1b6b..0d59cddd7 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.23.4'
+__version__ = '2015.01.25'

From e72c7e41238681338c92427f6731c74947645308 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 26 Jan 2015 12:01:43 +0100
Subject: [PATCH 1274/1937] [YoutubeDL] Always set the '_filename' field in the
 info_dict (reported in #4053)

It's also useful when you use the '--write-info-json' option.
---
 youtube_dl/YoutubeDL.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b772f87f1..ae62432c6 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1130,7 +1130,7 @@ class YoutubeDL(object):
 
         self._num_downloads += 1
 
-        filename = self.prepare_filename(info_dict)
+        info_dict['_filename'] = filename = self.prepare_filename(info_dict)
 
         # Forced printings
         if self.params.get('forcetitle', False):
@@ -1155,10 +1155,7 @@ class YoutubeDL(object):
         if self.params.get('forceformat', False):
             self.to_stdout(info_dict['format'])
         if self.params.get('forcejson', False):
-            info_dict['_filename'] = filename
             self.to_stdout(json.dumps(info_dict))
-        if self.params.get('dump_single_json', False):
-            info_dict['_filename'] = filename
 
         # Do nothing else if in simulate mode
         if self.params.get('simulate', False):

From 9f0df77ab13ca19bc074b501296bb6a494193cd1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 26 Jan 2015 14:36:22 +0100
Subject: [PATCH 1275/1937] [YoutubeDL] Allow format filtering by fps

---
 youtube_dl/YoutubeDL.py | 2 +-
 youtube_dl/options.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ae62432c6..b5dd77e3f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -820,7 +820,7 @@ class YoutubeDL(object):
             '!=': operator.ne,
         }
         operator_rex = re.compile(r'''(?x)\s*\[
-            (?P<key>width|height|tbr|abr|vbr|filesize)
+            (?P<key>width|height|tbr|abr|vbr|filesize|fps)
             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
             \]$
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index dbc6f5528..5e4a1ad9b 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -297,7 +297,7 @@ def parseOpts(overrideArguments=None):
             ' You can filter the video results by putting a condition in'
             ' brackets, as in -f "best[height=720]"'
             ' (or -f "[filesize>10M]"). '
-            ' This works for filesize, height, width, tbr, abr, and vbr'
+            ' This works for filesize, height, width, tbr, abr, vbr, and fps'
             ' and the comparisons <, <=, >, >=, =, != .'
             ' Formats for which the value is not known are excluded unless you'
             ' put a question mark (?) after the operator.'

From 6ca85be6f8697dc8cb0378854c2c6cdc154593d4 Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Wed, 28 May 2014 18:19:23 +0200
Subject: [PATCH 1276/1937] Filter DRM protected media in f4m downloader

---
 youtube_dl/downloader/f4m.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index c68b2c303..29de7630d 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -230,6 +230,23 @@ class F4mFD(FileDownloader):
     A downloader for f4m manifests or AdobeHDS.
     """
 
+    def _get_unencrypted_media(self, doc):
+        media=doc.findall(_add_ns('media'))
+        if not media:
+            self.report_error('No media found')
+        for e in (doc.findall(_add_ns('drmAdditionalHeader')) +
+                  doc.findall(_add_ns('drmAdditionalHeaderSet'))):
+            # If id attribute is missing it's valid for all media nodes
+            # without drmAdditionalHeaderId or drmAdditionalHeaderSetId attribute
+            if not 'id' in e.attrib:
+                self.report_error('Media is DRM protected')
+        media = list(filter(lambda e: 'drmAdditionalHeaderId' not in e.attrib and
+                                      'drmAdditionalHeaderSetId' not in e.attrib,
+                            media))
+        if not media:
+            self.report_error('Media is DRM protected')
+        return media
+
     def real_download(self, filename, info_dict):
         man_url = info_dict['url']
         requested_bitrate = info_dict.get('tbr')
@@ -248,7 +265,8 @@ class F4mFD(FileDownloader):
         )
 
         doc = etree.fromstring(manifest)
-        formats = [(int(f.attrib.get('bitrate', -1)), f) for f in doc.findall(_add_ns('media'))]
+        formats = [(int(f.attrib.get('bitrate', -1)), f)
+                   for f in self._get_unencrypted_media(doc)]
         if requested_bitrate is None:
             # get the best format
             formats = sorted(formats, key=lambda f: f[0])

From 2103d038b3dac6f489d13282dcd2a6f4b74975c3 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 27 Jan 2015 15:38:54 +0200
Subject: [PATCH 1277/1937] [lnkgo] Adapt to website changes

---
 youtube_dl/extractor/lnkgo.py | 61 ++++++++++++++---------------------
 1 file changed, 25 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/lnkgo.py b/youtube_dl/extractor/lnkgo.py
index a8e357859..fd23b0b43 100644
--- a/youtube_dl/extractor/lnkgo.py
+++ b/youtube_dl/extractor/lnkgo.py
@@ -6,13 +6,12 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    js_to_json,
     unified_strdate,
 )
 
 
 class LnkGoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?lnkgo\.alfa\.lt/visi\-video/(?P<show>[^/]+)/ziurek\-(?P<display_id>[A-Za-z0-9\-]+)'
+    _VALID_URL = r'https?://(?:www\.)?lnkgo\.alfa\.lt/visi-video/(?P<show>[^/]+)/ziurek-(?P<id>[A-Za-z0-9-]+)'
     _TESTS = [{
         'url': 'http://lnkgo.alfa.lt/visi-video/yra-kaip-yra/ziurek-yra-kaip-yra-162',
         'info_dict': {
@@ -51,8 +50,7 @@ class LnkGoIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
+        display_id = self._match_id(url)
 
         webpage = self._download_webpage(
             url, display_id, 'Downloading player webpage')
@@ -61,6 +59,8 @@ class LnkGoIE(InfoExtractor):
             r'data-ep="([^"]+)"', webpage, 'video ID')
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage)
+        upload_date = unified_strdate(self._search_regex(
+            r'class="[^"]*meta-item[^"]*air-time[^"]*">.*?<strong>([^<]+)</strong>', webpage, 'upload date', fatal=False))
 
         thumbnail_w = int_or_none(
             self._og_search_property('image:width', webpage, 'thumbnail width', fatal=False))
@@ -75,39 +75,28 @@ class LnkGoIE(InfoExtractor):
                 'height': thumbnail_h,
             })
 
-        upload_date = unified_strdate(self._search_regex(
-            r'class="meta-item\sair-time">.*?<strong>([^<]+)</strong>', webpage, 'upload date', fatal=False))
-        duration = int_or_none(self._search_regex(
-            r'VideoDuration = "([^"]+)"', webpage, 'duration', fatal=False))
+        config = self._parse_json(self._search_regex(
+            r'episodePlayer\((\{.*?\}),\s*\{', webpage, 'sources'), video_id)
 
-        pg_rating = self._search_regex(
-            r'pgrating="([^"]+)"', webpage, 'PG rating', fatal=False, default='')
-        age_limit = self._AGE_LIMITS.get(pg_rating.upper(), 0)
+        if config.get('pGeo'):
+            self.report_warning(
+                'This content might not be available in your country due to copyright reasons')
 
-        sources_js = self._search_regex(
-            r'(?s)sources:\s(\[.*?\]),', webpage, 'sources')
-        sources = self._parse_json(
-            sources_js, video_id, transform_source=js_to_json)
+        formats = [{
+            'format_id': 'hls',
+            'ext': 'mp4',
+            'url': config['EpisodeVideoLink_HLS'],
+        }]
 
-        formats = []
-        for source in sources:
-            if source.get('provider') == 'rtmp':
-                m = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<play_path>.+)$', source['file'])
-                if not m:
-                    continue
-                formats.append({
-                    'format_id': 'rtmp',
-                    'ext': 'flv',
-                    'url': m.group('url'),
-                    'play_path': m.group('play_path'),
-                    'page_url': url,
-                })
-            elif source.get('file').endswith('.m3u8'):
-                formats.append({
-                    'format_id': 'hls',
-                    'ext': source.get('type', 'mp4'),
-                    'url': source['file'],
-                })
+        m = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<play_path>.+)$', config['EpisodeVideoLink'])
+        if m:
+            formats.append({
+                'format_id': 'rtmp',
+                'ext': 'flv',
+                'url': m.group('url'),
+                'play_path': m.group('play_path'),
+                'page_url': url,
+            })
 
         self._sort_formats(formats)
 
@@ -117,8 +106,8 @@ class LnkGoIE(InfoExtractor):
             'title': title,
             'formats': formats,
             'thumbnails': [thumbnail],
-            'duration': duration,
+            'duration': int_or_none(config.get('VideoTime')),
             'description': description,
-            'age_limit': age_limit,
+            'age_limit': self._AGE_LIMITS.get(config.get('PGRating'), 0),
             'upload_date': upload_date,
         }

From fe7710cbccdade0b66ef48a7e2eedad71a0702cc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 27 Jan 2015 23:55:22 +0800
Subject: [PATCH 1278/1937] [xuite] Add new extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/xuite.py    | 147 +++++++++++++++++++++++++++++++
 2 files changed, 148 insertions(+)
 create mode 100644 youtube_dl/extractor/xuite.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 03c56156a..09070daa4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -546,6 +546,7 @@ from .xminus import XMinusIE
 from .xnxx import XNXXIE
 from .xvideos import XVideosIE
 from .xtube import XTubeUserIE, XTubeIE
+from .xuite import XuiteIE
 from .xxxymovies import XXXYMoviesIE
 from .yahoo import (
     YahooIE,
diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
new file mode 100644
index 000000000..943757c27
--- /dev/null
+++ b/youtube_dl/extractor/xuite.py
@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import base64
+from .common import InfoExtractor
+from ..compat import (
+    compat_urlparse,
+    compat_urllib_parse_unquote,
+    compat_parse_qs
+)
+from ..utils import (
+    ExtractorError,
+    parse_iso8601,
+    parse_duration
+)
+
+# ref: http://stackoverflow.com/questions/475074/regex-to-parse-or-validate-base64-data
+REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'
+
+CST_ZONE = +8  # China Standard Time
+
+
+class XuiteIE(InfoExtractor):
+    _VALID_URL = r'http://vlog.xuite.net/play/(?P<id>%s)(/.*)?' % REGEX_BASE64
+    _TESTS = [{
+        # Audio
+        'url': 'http://vlog.xuite.net/play/RGkzc1ZULTM4NjA5MTQuZmx2',
+        'md5': '63a42c705772aa53fd4c1a0027f86adf',
+        'info_dict': {
+            'id': 'RGkzc1ZULTM4NjA5MTQuZmx2',
+            'ext': 'mp3',
+            'upload_date': '20110902',
+            'uploader_id': '15973816',
+            'uploader': '阿能',
+            'timestamp': 1314932940,
+            'title': '孤單南半球-歐德陽'
+        }
+    }, {
+        # Audio with alternative form of url
+        'url': 'http://vlog.xuite.net/play/S1dDUjdyLTMyOTc3NjcuZmx2/%E5%AD%AB%E7%87%95%E5%A7%BF-%E7%9C%BC%E6%B7%9A%E6%88%90%E8%A9%A9',
+        'md5': 'c91645f324de53d82ebb80930e1a73d2',
+        'info_dict': {
+            'id': 'S1dDUjdyLTMyOTc3NjcuZmx2',
+            'ext': 'mp3',
+            'upload_date': '20101226',
+            'uploader_id': '10102699',
+            'uploader': '蠍',
+            'timestamp': 1293367080,
+            'title': '孫燕姿-眼淚成詩',
+        }
+    }, {
+        # Video with only one format
+        'url': 'http://vlog.xuite.net/play/TkRZNjhULTM0NDE2MjkuZmx2',
+        'md5': 'c45737fc8ac5dc8ac2f92ecbcecf505e',
+        'info_dict': {
+            'id': 'TkRZNjhULTM0NDE2MjkuZmx2',
+            'ext': 'mp4',
+            'upload_date': '20110306',
+            'uploader_id': '10400126',
+            'uploader': 'Valen',
+            'timestamp': 1299383640,
+            'title': '孫燕姿 - 眼淚成詩',
+        }
+    }, {
+        # Video with two formats
+        'url': 'http://vlog.xuite.net/play/bWo1N1pLLTIxMzAxMTcwLmZsdg==',
+        'md5': '1166e0f461efe55b62e26a2d2a68e6de',
+        'info_dict': {
+            'id': 'bWo1N1pLLTIxMzAxMTcwLmZsdg==',
+            'ext': 'mp4',
+            'upload_date': '20150117',
+            'uploader_id': '242127761',
+            'uploader': '我只是想認真點',
+            'timestamp': 1421481240,
+            'title': '暗殺教室 02',
+        }
+    }]
+
+    def _flv_config(self, media_id):
+        base64_media_id = base64.b64encode(media_id.encode('utf-8')).decode('utf-8')
+        flv_config_url = 'http://vlog.xuite.net/flash/player?media=' + base64_media_id
+        flv_config = self._download_xml(flv_config_url, 'flv config')
+
+        prop_dict = {}
+        for prop in flv_config.findall('./property'):
+            prop_id = base64.b64decode(prop.attrib['id']).decode('utf-8')
+
+            if not prop.text:
+                continue  # CDATA may be empty in flv config
+
+            encoded_content = base64.b64decode(prop.text).decode('utf-8')
+            prop_dict[prop_id] = compat_urllib_parse_unquote(encoded_content)
+
+        return prop_dict
+
+    def _type_string(self, media_url):
+        query_string = compat_urlparse.urlparse(media_url).query
+        type_string = compat_parse_qs(query_string)['q'][0]
+        return type_string
+
+    def _guess_ext(self, media_url):
+        type_string = self._type_string(media_url)
+        if type_string == 'mp3':
+            return 'mp3'
+        elif type_string == '360' or type_string == '720':
+            return 'mp4'
+        else:
+            raise ExtractorError('Unknown type string %s' % type_string)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        page = self._download_webpage(url, video_id)
+        media_id = self._html_search_regex(r'data-mediaid="(\d+)"', page, 'media id')
+        flv_config = self._flv_config(media_id)
+
+        timestamp_local = parse_iso8601(flv_config['publish_datetime'], ' ')
+        timestamp_gmt = timestamp_local - CST_ZONE * 3600
+
+        ret_attrs = {
+            'id': video_id,
+            'title': flv_config['title'],
+            'thumbnail': flv_config['thumb'],
+            'uploader': flv_config['author_name'],
+            'timestamp': timestamp_gmt,
+            'uploader_id': flv_config['author_id'],
+            'duration': parse_duration(flv_config['duration']),
+            'categories': [flv_config['category']]
+        }
+
+        if 'hq_src' in flv_config:
+            src = flv_config['src']
+            src_hq = flv_config['hq_src']
+            ret_attrs['formats'] = [{
+                'url': src,
+                'ext': self._guess_ext(src),
+                'format_id': self._type_string(src)
+            }, {
+                'url': src_hq,
+                'ext': self._guess_ext(src_hq),
+                'format_id': self._type_string(src_hq)
+            }]
+        else:
+            ret_attrs['url'] = flv_config['src']
+            ret_attrs['ext'] = self._guess_ext(flv_config['src'])
+
+        return ret_attrs

From 6348ad12a057cc1c454488ce7c37070cf39a8f06 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 28 Jan 2015 00:13:40 +0800
Subject: [PATCH 1279/1937] [xuite] Add height information for the two formats

---
 youtube_dl/extractor/xuite.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index 943757c27..aef9a14a3 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -129,17 +129,17 @@ class XuiteIE(InfoExtractor):
         }
 
         if 'hq_src' in flv_config:
-            src = flv_config['src']
-            src_hq = flv_config['hq_src']
-            ret_attrs['formats'] = [{
-                'url': src,
-                'ext': self._guess_ext(src),
-                'format_id': self._type_string(src)
-            }, {
-                'url': src_hq,
-                'ext': self._guess_ext(src_hq),
-                'format_id': self._type_string(src_hq)
-            }]
+            urls = [flv_config['src'], flv_config['hq_src']]
+
+            ret_attrs['formats'] = []
+
+            for url in urls:
+                ret_attrs['formats'].append({
+                    'url': url,
+                    'ext': self._guess_ext(url),
+                    'format_id': self._type_string(url),
+                    'height': int(self._type_string(url))
+                })
         else:
             ret_attrs['url'] = flv_config['src']
             ret_attrs['ext'] = self._guess_ext(flv_config['src'])

From 3a0d2f520a0f95c2f87b1c95049135b10206f97f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 27 Jan 2015 22:38:28 +0600
Subject: [PATCH 1280/1937] [YoutubeDL] Temporary fix for subprocess encoding
 issues on python2 @ Windows (Closes #4787)

For now filenames will be encoded with preferrefencoding before written to disk
---
 youtube_dl/YoutubeDL.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b5dd77e3f..7f054cdff 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -543,6 +543,11 @@ class YoutubeDL(object):
             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
             tmpl = compat_expanduser(outtmpl)
             filename = tmpl % template_dict
+            # Temporary fix for #4787
+            # 'Treat' all problem characters by passing filename through preferredencoding
+            # to workaround encoding issues with subprocess on python2 @ Windows
+            if sys.version_info < (3, 0) and sys.platform == 'win32':
+                filename = encodeFilename(filename, True).decode(preferredencoding())
             return filename
         except ValueError as err:
             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')

From a28383834b07787b22a837bd6028bf24bea7ec94 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 28 Jan 2015 01:30:14 +0800
Subject: [PATCH 1281/1937] [xuite] Update tests

---
 youtube_dl/extractor/xuite.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index aef9a14a3..8466d4bc5 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -33,7 +33,10 @@ class XuiteIE(InfoExtractor):
             'uploader_id': '15973816',
             'uploader': '阿能',
             'timestamp': 1314932940,
-            'title': '孤單南半球-歐德陽'
+            'title': '孤單南半球-歐德陽',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'categories': ['個人短片'],
+            'duration': 247.246
         }
     }, {
         # Audio with alternative form of url
@@ -47,6 +50,9 @@ class XuiteIE(InfoExtractor):
             'uploader': '蠍',
             'timestamp': 1293367080,
             'title': '孫燕姿-眼淚成詩',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'categories': ['個人短片'],
+            'duration': 223.19
         }
     }, {
         # Video with only one format
@@ -60,6 +66,9 @@ class XuiteIE(InfoExtractor):
             'uploader': 'Valen',
             'timestamp': 1299383640,
             'title': '孫燕姿 - 眼淚成詩',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'categories': ['影視娛樂'],
+            'duration': 217.399
         }
     }, {
         # Video with two formats
@@ -73,6 +82,9 @@ class XuiteIE(InfoExtractor):
             'uploader': '我只是想認真點',
             'timestamp': 1421481240,
             'title': '暗殺教室 02',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'categories': ['電玩動漫'],
+            'duration': 1384.907
         }
     }]
 

From 796df3c631dcf28f7ebdf256e57fcfa3b6463abe Mon Sep 17 00:00:00 2001
From: Shaya G <hyshai29@gmail.com>
Date: Wed, 28 Jan 2015 00:08:19 -0500
Subject: [PATCH 1282/1937] fixed viddler support - needed a Referer header;
 also added a viddler generic extractor

---
 youtube_dl/extractor/generic.py | 20 ++++++++++++++++++
 youtube_dl/extractor/viddler.py | 36 +++++++++++++++++++++++++++++----
 2 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index ad16b8330..a937a1e09 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -498,6 +498,19 @@ class GenericIE(InfoExtractor):
                 'uploader': 'www.abc.net.au',
                 'title': 'Game of Thrones with dice - Dungeons and Dragons fantasy role-playing game gets new life - 19/01/2015',
             }
+        },
+        # embedded viddler video
+        {
+            'url': 'http://deadspin.com/i-cant-stop-watching-john-wall-chop-the-nuggets-with-th-1681801597',
+            'info_dict': {
+                'id': '4d03aad9',
+                'ext': 'mp4',
+                'uploader': 'deadspin',
+                'title': 'WALL-TO-GORTAT',
+                'timestamp': 1422285291,
+                'upload_date': '20150126',
+            },
+            'add_ie': ['Viddler'],
         }
     ]
 
@@ -860,6 +873,13 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'))
 
+        # Look for embedded Viddler player
+        mobj = (re.search(r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/embed/.+?)\1', webpage) or
+                re.search(r'<param[^>]+?value=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/player/.+?)\1', webpage))
+
+        if mobj is not None:
+            return self.url_result(mobj.group('url'))
+
         # Look for Ooyala videos
         mobj = (re.search(r'player.ooyala.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
                 re.search(r'OO.Player.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage))
diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index 0faa729c6..bacbad50a 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -5,11 +5,14 @@ from ..utils import (
     float_or_none,
     int_or_none,
 )
+from ..compat import (
+   compat_urllib_request
+)
 
 
 class ViddlerIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?viddler\.com/(?:v|embed|player)/(?P<id>[a-z0-9]+)'
-    _TEST = {
+    _TESTS = [{
         "url": "http://www.viddler.com/v/43903784",
         'md5': 'ae43ad7cb59431ce043f0ff7fa13cbf4',
         'info_dict': {
@@ -25,7 +28,30 @@ class ViddlerIE(InfoExtractor):
             'view_count': int,
             'categories': ['video content', 'high quality video', 'video made easy', 'how to produce video with limited resources', 'viddler'],
         }
-    }
+    }, {
+        "url": "http://www.viddler.com/v/4d03aad9/",
+        "file": "4d03aad9.mp4",
+        "md5": "faa71fbf70c0bee7ab93076fd007f4b0",
+        "info_dict": {
+            'upload_date': '20150126',
+            'uploader': 'deadspin',
+            'id': '4d03aad9',
+            'timestamp': 1422285291,
+            'title': 'WALL-TO-GORTAT',
+        }
+    }, {
+        "url": "http://www.viddler.com/player/221ebbbd/0/",
+        "file": "221ebbbd.mp4",
+        "md5": "0defa2bd0ea613d14a6e9bd1db6be326",
+        "info_dict": {
+            'upload_date': '20140929',
+            'uploader': 'BCLETeens',
+            'id': '221ebbbd',
+            'timestamp': 1411997190,
+            'title': 'LETeens-Grammar-snack-third-conditional',
+            'description': ' '
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -33,7 +59,9 @@ class ViddlerIE(InfoExtractor):
         json_url = (
             'http://api.viddler.com/api/v2/viddler.videos.getPlaybackDetails.json?video_id=%s&key=v0vhrt7bg2xq1vyxhkct' %
             video_id)
-        data = self._download_json(json_url, video_id)['video']
+        headers = {'Referer': 'http://static.cdn-ec.viddler.com/js/arpeggio/v2/embed.html'}
+        request = compat_urllib_request.Request(json_url, None, headers)
+        data = self._download_json(request, video_id)['video']
 
         formats = []
         for filed in data['files']:
@@ -53,7 +81,7 @@ class ViddlerIE(InfoExtractor):
 
             if filed.get('cdn_url'):
                 f = f.copy()
-                f['url'] = self._proto_relative_url(filed['cdn_url'])
+                f['url'] = self._proto_relative_url(filed['cdn_url'], 'http:')
                 f['format_id'] = filed['profile_id'] + '-cdn'
                 f['source_preference'] = 1
                 formats.append(f)

From 0865f397ae6c875ec4194093af5c8f53e75f2285 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Wed, 28 Jan 2015 08:21:04 +0100
Subject: [PATCH 1283/1937] added extractor for dctp.tv

---
 youtube_dl/downloader/rtmp.py    |  3 +++
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/dctp.py     | 41 ++++++++++++++++++++++++++++++++
 3 files changed, 45 insertions(+)
 create mode 100644 youtube_dl/extractor/dctp.py

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index e06ebe826..6cb1bfc50 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -106,6 +106,7 @@ class RtmpFD(FileDownloader):
         protocol = info_dict.get('rtmp_protocol', None)
         no_resume = info_dict.get('no_resume', False)
         continue_dl = info_dict.get('continuedl', False)
+        real_time = info_dict.get('real_time', False)
 
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
@@ -143,6 +144,8 @@ class RtmpFD(FileDownloader):
             basic_args += ['--conn', conn]
         if protocol is not None:
             basic_args += ['--protocol', protocol]
+        if real_time:
+            basic_args += ['--realtime']
 
         args = basic_args
         if not no_resume and continue_dl and not live:
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 03c56156a..873ae69d3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -89,6 +89,7 @@ from .dailymotion import (
 )
 from .daum import DaumIE
 from .dbtv import DBTVIE
+from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
 from .dotsub import DotsubIE
diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
new file mode 100644
index 000000000..5382255f0
--- /dev/null
+++ b/youtube_dl/extractor/dctp.py
@@ -0,0 +1,41 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+class DctpTvIE(InfoExtractor):
+    _VALID_URL = r'^http://www.dctp.tv/(#/)?filme/(?P<id>.+?)/$'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        base_url = 'http://dctp-ivms2-restapi.s3.amazonaws.com/'
+        version_json = self._download_json(base_url + 'version.json', video_id)
+        version = version_json['version_name']
+        info_json = self._download_json(
+            '{}{}/restapi/slugs/{}.json'.format(base_url, version, video_id), video_id)
+        object_id = info_json['object_id']
+        meta_json = self._download_json(
+            '{}{}/restapi/media/{}.json'.format(base_url, version, object_id), video_id)
+        uuid = meta_json['uuid']
+        title = meta_json['title']
+        wide = meta_json['is_wide']
+        if wide:
+            ratio = '16x9'
+        else:
+            ratio = '4x3'
+        play_path = 'mp4:{}_dctp_0500_{}.m4v'.format(uuid, ratio)
+
+        servers_json = self._download_json('http://www.dctp.tv/streaming_servers/', video_id)
+        url = servers_json[0]['endpoint']
+
+        return {
+            'id': video_id,
+            'title': title,
+            'format': 'rtmp',
+            'url': url,
+            'play_path': play_path,
+            'real_time': True,
+            'ext': 'flv'
+        }
+
+

From 48a1e5141ad9f6b5c4ce8a355dcd7bf99c80e333 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Wed, 28 Jan 2015 08:59:58 +0100
Subject: [PATCH 1284/1937] added test for dctp

---
 youtube_dl/extractor/dctp.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 5382255f0..9b687ef43 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -3,8 +3,16 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 
+
 class DctpTvIE(InfoExtractor):
     _VALID_URL = r'^http://www.dctp.tv/(#/)?filme/(?P<id>.+?)/$'
+    _TEST = {
+        'url': 'http://www.dctp.tv/filme/videoinstallation-fuer-eine-kaufhausfassade/',
+        'info_dict': {
+            'id': 'videoinstallation-fuer-eine-kaufhausfassade',
+            'ext': 'flv',
+            'title': 'Videoinstallation für eine Kaufhausfassade'}
+        }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -38,4 +46,3 @@ class DctpTvIE(InfoExtractor):
             'ext': 'flv'
         }
 
-

From b04fbd789c1efd5f918f81b5a5b9b6dafa806900 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Jan 2015 22:49:42 +0600
Subject: [PATCH 1285/1937] [viddler] Modernize

---
 youtube_dl/extractor/viddler.py | 46 ++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index bacbad50a..9caee94e7 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -13,43 +13,43 @@ from ..compat import (
 class ViddlerIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?viddler\.com/(?:v|embed|player)/(?P<id>[a-z0-9]+)'
     _TESTS = [{
-        "url": "http://www.viddler.com/v/43903784",
+        'url': 'http://www.viddler.com/v/43903784',
         'md5': 'ae43ad7cb59431ce043f0ff7fa13cbf4',
         'info_dict': {
             'id': '43903784',
             'ext': 'mp4',
-            "title": "Video Made Easy",
-            'description': 'You don\'t need to be a professional to make high-quality video content. Viddler provides some quick and easy tips on how to produce great video content with limited resources. ',
-            "uploader": "viddler",
+            'title': 'Video Made Easy',
+            'description': 'md5:6a697ebd844ff3093bd2e82c37b409cd',
+            'uploader': 'viddler',
             'timestamp': 1335371429,
             'upload_date': '20120425',
-            "duration": 100.89,
+            'duration': 100.89,
             'thumbnail': 're:^https?://.*\.jpg$',
             'view_count': int,
             'categories': ['video content', 'high quality video', 'video made easy', 'how to produce video with limited resources', 'viddler'],
         }
     }, {
-        "url": "http://www.viddler.com/v/4d03aad9/",
-        "file": "4d03aad9.mp4",
-        "md5": "faa71fbf70c0bee7ab93076fd007f4b0",
-        "info_dict": {
+        'url': 'http://www.viddler.com/v/4d03aad9/',
+        'md5': 'faa71fbf70c0bee7ab93076fd007f4b0',
+        'info_dict': {
+            'id': '4d03aad9',
+            'ext': 'mp4',
+            'title': 'WALL-TO-GORTAT',
             'upload_date': '20150126',
             'uploader': 'deadspin',
-            'id': '4d03aad9',
             'timestamp': 1422285291,
-            'title': 'WALL-TO-GORTAT',
         }
     }, {
-        "url": "http://www.viddler.com/player/221ebbbd/0/",
-        "file": "221ebbbd.mp4",
-        "md5": "0defa2bd0ea613d14a6e9bd1db6be326",
-        "info_dict": {
+        'url': 'http://www.viddler.com/player/221ebbbd/0/',
+        'md5': '0defa2bd0ea613d14a6e9bd1db6be326',
+        'info_dict': {
+            'id': '221ebbbd',
+            'ext': 'mp4',
+            'title': 'LETeens-Grammar-snack-third-conditional',
+            'description': ' ',
             'upload_date': '20140929',
             'uploader': 'BCLETeens',
-            'id': '221ebbbd',
             'timestamp': 1411997190,
-            'title': 'LETeens-Grammar-snack-third-conditional',
-            'description': ' '
         }
     }]
 
@@ -68,7 +68,7 @@ class ViddlerIE(InfoExtractor):
             if filed.get('status', 'ready') != 'ready':
                 continue
             f = {
-                'format_id': filed['profile_id'],
+                'format_id': filed['profile_id'] or filed['profile_name'],
                 'format_note': filed['profile_name'],
                 'url': self._proto_relative_url(filed['url']),
                 'width': int_or_none(filed.get('width')),
@@ -82,15 +82,14 @@ class ViddlerIE(InfoExtractor):
             if filed.get('cdn_url'):
                 f = f.copy()
                 f['url'] = self._proto_relative_url(filed['cdn_url'], 'http:')
-                f['format_id'] = filed['profile_id'] + '-cdn'
+                f['format_id'] = (filed['profile_id'] or filed['profile_name']) + '-cdn'
                 f['source_preference'] = 1
                 formats.append(f)
 
             if filed.get('html5_video_source'):
                 f = f.copy()
-                f['url'] = self._proto_relative_url(
-                    filed['html5_video_source'])
-                f['format_id'] = filed['profile_id'] + '-html5'
+                f['url'] = self._proto_relative_url(filed['html5_video_source'])
+                f['format_id'] = (filed['profile_id'] or filed['profile_name']) + '-html5'
                 f['source_preference'] = 0
                 formats.append(f)
         self._sort_formats(formats)
@@ -99,7 +98,6 @@ class ViddlerIE(InfoExtractor):
             t.get('text') for t in data.get('tags', []) if 'text' in t]
 
         return {
-            '_type': 'video',
             'id': video_id,
             'title': data['title'],
             'formats': formats,

From e0d9f85aee92ebcb40ce688ed43172c718751e27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Jan 2015 22:56:06 +0600
Subject: [PATCH 1286/1937] Credit @HyShai for viddler fixes (#4794)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 1596a7548..20e620c35 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -106,3 +106,4 @@ Johan K. Jensen
 Yen Chi Hsuan
 Enam Mijbah Noor
 David Luhmer
+Shaya Goldberg

From cb454b333d91718a0c2b36c34c8b0d6858ff9505 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Jan 2015 23:07:37 +0600
Subject: [PATCH 1287/1937] [generic] Improve some regexes

---
 youtube_dl/extractor/generic.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a937a1e09..41884ed7a 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -874,15 +874,15 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'))
 
         # Look for embedded Viddler player
-        mobj = (re.search(r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/embed/.+?)\1', webpage) or
-                re.search(r'<param[^>]+?value=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/player/.+?)\1', webpage))
-
+        mobj = re.search(
+            r'<(?:iframe[^>]+?src|param[^>]+?value)=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/(?:embed|player)/.+?)\1',
+            webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'))
 
         # Look for Ooyala videos
-        mobj = (re.search(r'player.ooyala.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
-                re.search(r'OO.Player.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage))
+        mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
+                re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage))
         if mobj is not None:
             return OoyalaIE._build_url_result(mobj.group('ec'))
 

From 18b4e9e79d6cc1ed21df7479273915ca3a763b80 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Jan 2015 23:21:17 +0600
Subject: [PATCH 1288/1937] [viddler] Extract comment count

---
 youtube_dl/extractor/viddler.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index 9caee94e7..ef104dc29 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -26,6 +26,7 @@ class ViddlerIE(InfoExtractor):
             'duration': 100.89,
             'thumbnail': 're:^https?://.*\.jpg$',
             'view_count': int,
+            'comment_count': int,
             'categories': ['video content', 'high quality video', 'video made easy', 'how to produce video with limited resources', 'viddler'],
         }
     }, {
@@ -38,6 +39,8 @@ class ViddlerIE(InfoExtractor):
             'upload_date': '20150126',
             'uploader': 'deadspin',
             'timestamp': 1422285291,
+            'view_count': int,
+            'comment_count': int,
         }
     }, {
         'url': 'http://www.viddler.com/player/221ebbbd/0/',
@@ -50,6 +53,8 @@ class ViddlerIE(InfoExtractor):
             'upload_date': '20140929',
             'uploader': 'BCLETeens',
             'timestamp': 1411997190,
+            'view_count': int,
+            'comment_count': int,
         }
     }]
 
@@ -67,8 +72,9 @@ class ViddlerIE(InfoExtractor):
         for filed in data['files']:
             if filed.get('status', 'ready') != 'ready':
                 continue
+            format_id = filed.get('profile_id') or filed['profile_name']
             f = {
-                'format_id': filed['profile_id'] or filed['profile_name'],
+                'format_id': format_id,
                 'format_note': filed['profile_name'],
                 'url': self._proto_relative_url(filed['url']),
                 'width': int_or_none(filed.get('width')),
@@ -82,14 +88,14 @@ class ViddlerIE(InfoExtractor):
             if filed.get('cdn_url'):
                 f = f.copy()
                 f['url'] = self._proto_relative_url(filed['cdn_url'], 'http:')
-                f['format_id'] = (filed['profile_id'] or filed['profile_name']) + '-cdn'
+                f['format_id'] = format_id + '-cdn'
                 f['source_preference'] = 1
                 formats.append(f)
 
             if filed.get('html5_video_source'):
                 f = f.copy()
                 f['url'] = self._proto_relative_url(filed['html5_video_source'])
-                f['format_id'] = (filed['profile_id'] or filed['profile_name']) + '-html5'
+                f['format_id'] = format_id + '-html5'
                 f['source_preference'] = 0
                 formats.append(f)
         self._sort_formats(formats)
@@ -107,5 +113,6 @@ class ViddlerIE(InfoExtractor):
             'uploader': data.get('author'),
             'duration': float_or_none(data.get('length')),
             'view_count': int_or_none(data.get('view_count')),
+            'comment_count': int_or_none(data.get('comment_count')),
             'categories': categories,
         }

From 63be3b89894f26caf69d9667f8e7db5ecbcb03ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 28 Jan 2015 23:58:14 +0600
Subject: [PATCH 1289/1937] [ivi] Modernize

---
 youtube_dl/extractor/ivi.py | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py
index 7a400323d..e82594444 100644
--- a/youtube_dl/extractor/ivi.py
+++ b/youtube_dl/extractor/ivi.py
@@ -16,7 +16,7 @@ from ..utils import (
 class IviIE(InfoExtractor):
     IE_DESC = 'ivi.ru'
     IE_NAME = 'ivi'
-    _VALID_URL = r'https?://(?:www\.)?ivi\.ru/(?:watch/(?:[^/]+/)?|video/player\?.*?videoId=)(?P<videoid>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?ivi\.ru/(?:watch/(?:[^/]+/)?|video/player\?.*?videoId=)(?P<id>\d+)'
 
     _TESTS = [
         # Single movie
@@ -63,29 +63,34 @@ class IviIE(InfoExtractor):
         return int(m.group('commentcount')) if m is not None else 0
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
+        video_id = self._match_id(url)
 
         api_url = 'http://api.digitalaccess.ru/api/json/'
 
-        data = {'method': 'da.content.get',
-                'params': [video_id, {'site': 's183',
-                                      'referrer': 'http://www.ivi.ru/watch/%s' % video_id,
-                                      'contentid': video_id
-                                      }
-                           ]
+        data = {
+            'method': 'da.content.get',
+            'params': [
+                video_id, {
+                    'site': 's183',
+                    'referrer': 'http://www.ivi.ru/watch/%s' % video_id,
+                    'contentid': video_id
                 }
+            ]
+        }
 
         request = compat_urllib_request.Request(api_url, json.dumps(data))
 
-        video_json_page = self._download_webpage(request, video_id, 'Downloading video JSON')
+        video_json_page = self._download_webpage(
+            request, video_id, 'Downloading video JSON')
         video_json = json.loads(video_json_page)
 
         if 'error' in video_json:
             error = video_json['error']
             if error['origin'] == 'NoRedisValidData':
                 raise ExtractorError('Video %s does not exist' % video_id, expected=True)
-            raise ExtractorError('Unable to download video %s: %s' % (video_id, error['message']), expected=True)
+            raise ExtractorError(
+                'Unable to download video %s: %s' % (video_id, error['message']),
+                expected=True)
 
         result = video_json['result']
 

From dcf53d440801505a27ee5615e3fb58b6a794bc73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Jan 2015 00:44:40 +0600
Subject: [PATCH 1290/1937] [YoutubeDL] Set format_id for video+audio (Closes
 #3634)

---
 youtube_dl/YoutubeDL.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 7f054cdff..4c238c555 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1074,6 +1074,7 @@ class YoutubeDL(object):
                             selected_format = {
                                 'requested_formats': formats_info,
                                 'format': rf,
+                                'format_id': rf,
                                 'ext': formats_info[0]['ext'],
                                 'width': formats_info[0].get('width'),
                                 'height': formats_info[0].get('height'),

From 206dba27a4d134cf9a65b93f7a12bee033a80b18 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 29 Jan 2015 03:18:53 +0800
Subject: [PATCH 1291/1937] [NextMedia] Add new extractor

---
 youtube_dl/extractor/__init__.py  |   6 ++
 youtube_dl/extractor/nextmedia.py | 163 ++++++++++++++++++++++++++++++
 2 files changed, 169 insertions(+)
 create mode 100644 youtube_dl/extractor/nextmedia.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 03c56156a..444af8b6e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -284,6 +284,12 @@ from .netzkino import NetzkinoIE
 from .nerdcubed import NerdCubedFeedIE
 from .newgrounds import NewgroundsIE
 from .newstube import NewstubeIE
+from .nextmedia import (
+    NextMediaIE,
+    NextMediaActionNewsIE,
+    AppleDailyRealtimeNewsIE,
+    AppleDailyAnimationNewsIE
+)
 from .nfb import NFBIE
 from .nfl import NFLIE
 from .nhl import NHLIE, NHLVideocenterIE
diff --git a/youtube_dl/extractor/nextmedia.py b/youtube_dl/extractor/nextmedia.py
new file mode 100644
index 000000000..02dba4ef6
--- /dev/null
+++ b/youtube_dl/extractor/nextmedia.py
@@ -0,0 +1,163 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import parse_iso8601
+
+
+class NextMediaIE(InfoExtractor):
+    _VALID_URL = r'http://hk.apple.nextmedia.com/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://hk.apple.nextmedia.com/realtime/news/20141108/53109199',
+        'md5': 'dff9fad7009311c421176d1ac90bfe4f',
+        'info_dict': {
+            'id': '53109199',
+            'ext': 'mp4',
+            'title': '【佔領金鐘】50外國領事議員撐場 讚學生勇敢香港有希望',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:28222b9912b6665a21011b034c70fcc7',
+            'timestamp': 1415456273,
+            'upload_date': '20141108',
+        }
+    }]
+
+    _URL_PATTERN = r'\{ url: \'(.+)\' \}'
+
+    def _real_extract(self, url):
+        news_id = self._match_id(url)
+        page = self._download_webpage(url, news_id)
+        return self._extract_from_nextmedia_page(news_id, url, page)
+
+    def _extract_from_nextmedia_page(self, news_id, url, page):
+        title = self._fetch_title(page)
+        video_url = self._search_regex(self._URL_PATTERN, page, 'video url')
+
+        attrs = {
+            'id': news_id,
+            'title': title,
+            'url': video_url,  # ext can be inferred from url
+            'thumbnail': self._fetch_thumbnail(page),
+            'description': self._fetch_description(page),
+        }
+
+        timestamp = self._fetch_timestamp(page)
+        if timestamp:
+            attrs['timestamp'] = timestamp
+        else:
+            attrs['upload_date'] = self._fetch_upload_date(url)
+
+        return attrs
+
+    def _fetch_title(self, page):
+        return self._og_search_title(page)
+
+    def _fetch_thumbnail(self, page):
+        return self._og_search_thumbnail(page)
+
+    def _fetch_timestamp(self, page):
+        dateCreated = self._search_regex('"dateCreated":"([^"]+)"', page, 'created time')
+        return parse_iso8601(dateCreated)
+
+    def _fetch_upload_date(self, url):
+        return self._search_regex(self._VALID_URL, url, 'upload date', group='date')
+
+    def _fetch_description(self, page):
+        return self._og_search_property('description', page)
+
+
+class NextMediaActionNewsIE(NextMediaIE):
+    _VALID_URL = r'http://hk.dv.nextmedia.com/actionnews/[^/]+/(?P<date>\d+)/(?P<id>\d+)/\d+'
+    _TESTS = [{
+        'url': 'http://hk.dv.nextmedia.com/actionnews/hit/20150121/19009428/20061460',
+        'md5': '05fce8ffeed7a5e00665d4b7cf0f9201',
+        'info_dict': {
+            'id': '19009428',
+            'ext': 'mp4',
+            'title': '【壹週刊】細10年男友偷食　50歲邵美琪再失戀',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:cd802fad1f40fd9ea178c1e2af02d659',
+            'timestamp': 1421791200,
+            'upload_date': '20150120',
+        }
+    }]
+
+    def _real_extract(self, url):
+        news_id = self._match_id(url)
+        actionnews_page = self._download_webpage(url, news_id)
+        article_url = self._og_search_url(actionnews_page)
+        article_page = self._download_webpage(article_url, news_id)
+        return self._extract_from_nextmedia_page(news_id, url, article_page)
+
+
+class AppleDailyRealtimeNewsIE(NextMediaIE):
+    _VALID_URL = r'http://(www|ent).appledaily.com.tw/(realtimenews|enews)/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
+    _TESTS = [{
+        'url': 'http://ent.appledaily.com.tw/enews/article/entertainment/20150128/36354694',
+        'md5': 'a843ab23d150977cc55ef94f1e2c1e4d',
+        'info_dict': {
+            'id': '36354694',
+            'ext': 'mp4',
+            'title': '周亭羽走過摩鐵陰霾2男陪吃 九把刀孤寒看醫生',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:b23787119933404ce515c6356a8c355c',
+            'upload_date': '20150128',
+        }
+    }, {
+        'url': 'http://www.appledaily.com.tw/realtimenews/article/strange/20150128/550549/%E4%B8%8D%E6%BB%BF%E8%A2%AB%E8%B8%A9%E8%85%B3%E3%80%80%E5%B1%B1%E6%9D%B1%E5%85%A9%E5%A4%A7%E5%AA%BD%E4%B8%80%E8%B7%AF%E6%89%93%E4%B8%8B%E8%BB%8A',
+        'md5': '86b4e9132d158279c7883822d94ccc49',
+        'info_dict': {
+            'id': '550549',
+            'ext': 'mp4',
+            'title': '不滿被踩腳　山東兩大媽一路打下車',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:2648aaf6fc4f401f6de35a91d111aa1d',
+            'upload_date': '20150128',
+        }
+    }]
+
+    _URL_PATTERN = r'\{url: \'(.+)\'\}'
+
+    def _fetch_title(self, page):
+        return self._html_search_regex(r'<h1 id="h1">([^<>]+)</h1>', page, 'news title')
+
+    def _fetch_thumbnail(self, page):
+        return self._html_search_regex(r"setInitialImage\(\'([^']+)'\)", page, 'video thumbnail', fatal=False)
+
+    def _fetch_timestamp(self, page):
+        return None
+
+
+class AppleDailyAnimationNewsIE(AppleDailyRealtimeNewsIE):
+    _VALID_URL = 'http://www.appledaily.com.tw/animation/[^/]+/[^/]+/(?P<date>\d+)/(?P<id>\d+)(/.*)?'
+    _TESTS = [{
+        'url': 'http://www.appledaily.com.tw/animation/realtimenews/new/20150128/5003671',
+        'md5': '03df296d95dedc2d5886debbb80cb43f',
+        'info_dict': {
+            'id': '5003671',
+            'ext': 'mp4',
+            'title': '20正妹熱舞　《刀龍傳說Online》火辣上市',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:23c0aac567dc08c9c16a3161a2c2e3cd',
+            'upload_date': '20150128',
+        }
+    }, {
+        # No thumbnail
+        'url': 'http://www.appledaily.com.tw/animation/realtimenews/new/20150128/5003673/',
+        'md5': 'b06182cd386ea7bc6115ec7ff0f72aeb',
+        'info_dict': {
+            'id': '5003673',
+            'ext': 'mp4',
+            'title': '半夜尿尿　好像會看到___',
+            'description': 'md5:61d2da7fe117fede148706cdb85ac066',
+            'upload_date': '20150128',
+        },
+        'expected_warnings': [
+            'video thumbnail',
+        ]
+    }]
+
+    def _fetch_title(self, page):
+        return self._html_search_meta('description', page, 'news title')
+
+    def _fetch_description(self, page):
+        return self._html_search_meta('description', page, 'news description')

From 367cc95aa76731aa951e949b325ac9c909af639d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 29 Jan 2015 03:49:56 +0800
Subject: [PATCH 1292/1937] [CtsNews] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ctsnews.py  | 51 ++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 youtube_dl/extractor/ctsnews.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 03c56156a..72c5e4973 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -82,6 +82,7 @@ from .crunchyroll import (
     CrunchyrollShowPlaylistIE
 )
 from .cspan import CSpanIE
+from .ctsnews import CtsNewsIE
 from .dailymotion import (
     DailymotionIE,
     DailymotionPlaylistIE,
diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py
new file mode 100644
index 000000000..e1d8c814e
--- /dev/null
+++ b/youtube_dl/extractor/ctsnews.py
@@ -0,0 +1,51 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import parse_iso8601, ExtractorError
+
+
+class CtsNewsIE(InfoExtractor):
+    _VALID_URL = r'http://news\.cts\.com\.tw/[a-z]+/[a-z]+/\d+/(?P<id>\d+)\.html'
+    _TESTS = [{
+        'url': 'http://news.cts.com.tw/cts/international/201309/201309031304098.html',
+        'md5': '3aee7e0df7cdff94e43581f54c22619e',
+        'info_dict': {
+            'id': '201309031304098',
+            'ext': 'mp4',
+            'title': '韓國31歲童顏男 貌如十多歲小孩',
+            'description': 'md5:f183feeba3752b683827aab71adad584',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'timestamp': 1378205880,
+            'upload_date': '20130903',
+        }
+    }]
+
+    def _real_extract(self, url):
+        news_id = self._match_id(url)
+        page = self._download_webpage(url, news_id)
+
+        if not self._search_regex(r'(CTSPlayer2)', page, 'CTSPlayer2 identifier', fatal=False):
+            raise ExtractorError('The news includes no videos!')
+
+        feed_pattern = r'(http://news.cts.com.tw/action/mp4feed.php\?news_id=\d+)'
+        feed_url = self._html_search_regex(feed_pattern, page, 'feed url')
+        feed_page = self._download_webpage(feed_url, news_id)
+
+        description = self._html_search_meta('description', page)
+        title = self._html_search_meta('title', page)
+        thumbnail = self._html_search_meta('image', page)
+
+        datetime_pattern = r'(\d{4}/\d{2}/\d{2} \d{2}:\d{2})'
+        datetime_str = self._html_search_regex(datetime_pattern, page, 'date and time')
+        time = (datetime_str + ':00+08:00').replace('/', '-')
+        timestamp = parse_iso8601(time, delimiter=' ')
+
+        return {
+            'id': news_id,
+            'title': title,
+            'description': description,
+            'url': feed_page,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp
+        }

From d20547610314e0821542fecd1872b5851bd8d5d7 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Thu, 29 Jan 2015 01:36:15 +0100
Subject: [PATCH 1293/1937] fix srmediathek description

---
 youtube_dl/extractor/srmediathek.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/srmediathek.py b/youtube_dl/extractor/srmediathek.py
index 666a7dcc8..5d583c720 100644
--- a/youtube_dl/extractor/srmediathek.py
+++ b/youtube_dl/extractor/srmediathek.py
@@ -8,7 +8,7 @@ from ..utils import js_to_json
 
 
 class SRMediathekIE(InfoExtractor):
-    IE_DESC = 'Süddeutscher Rundfunk'
+    IE_DESC = 'Saarländischer Rundfunk'
     _VALID_URL = r'https?://sr-mediathek\.sr-online\.de/index\.php\?.*?&id=(?P<id>[0-9]+)'
 
     _TEST = {

From 4f264c02c73c8f7551bcdbc960c9635860003196 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 29 Jan 2015 13:37:17 +0100
Subject: [PATCH 1294/1937] [utils] YoutubeDLHTTPSHandler.https_open: pass all
 required arguments to do_open

With this change the '--no-check-certificate' works again (#4807).
---
 youtube_dl/utils.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b8c52af74..a4c9813ec 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -654,9 +654,14 @@ class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
         self._params = params
 
     def https_open(self, req):
+        kwargs = {}
+        if hasattr(self, '_context'):  # python > 2.6
+            kwargs['context'] = self._context
+        if hasattr(self, '_check_hostname'):  # python 3.x
+            kwargs['check_hostname'] = self._check_hostname
         return self.do_open(functools.partial(
             _create_http_connection, self, self._https_conn_class, True),
-            req)
+            req, **kwargs)
 
 
 def parse_iso8601(date_str, delimiter='T'):

From affd04a45dcc46f21b55e4d9419812c33b9aa9ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Jan 2015 22:09:59 +0600
Subject: [PATCH 1295/1937] [xuite] Simplify and improve

---
 youtube_dl/extractor/xuite.py | 181 +++++++++++++++-------------------
 1 file changed, 82 insertions(+), 99 deletions(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index 8466d4bc5..a9dbf8c2d 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -2,158 +2,141 @@
 from __future__ import unicode_literals
 
 import base64
+
 from .common import InfoExtractor
-from ..compat import (
-    compat_urlparse,
-    compat_urllib_parse_unquote,
-    compat_parse_qs
-)
+from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     ExtractorError,
     parse_iso8601,
-    parse_duration
+    parse_duration,
 )
 
-# ref: http://stackoverflow.com/questions/475074/regex-to-parse-or-validate-base64-data
-REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'
-
-CST_ZONE = +8  # China Standard Time
-
 
 class XuiteIE(InfoExtractor):
-    _VALID_URL = r'http://vlog.xuite.net/play/(?P<id>%s)(/.*)?' % REGEX_BASE64
+    _REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'
+    _VALID_URL = r'http://vlog.xuite.net/(?:play|embed)/(?P<id>%s)' % _REGEX_BASE64
     _TESTS = [{
         # Audio
         'url': 'http://vlog.xuite.net/play/RGkzc1ZULTM4NjA5MTQuZmx2',
         'md5': '63a42c705772aa53fd4c1a0027f86adf',
         'info_dict': {
-            'id': 'RGkzc1ZULTM4NjA5MTQuZmx2',
+            'id': '3860914',
             'ext': 'mp3',
-            'upload_date': '20110902',
-            'uploader_id': '15973816',
-            'uploader': '阿能',
-            'timestamp': 1314932940,
             'title': '孤單南半球-歐德陽',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 247.246,
+            'timestamp': 1314932940,
+            'upload_date': '20110902',
+            'uploader': '阿能',
+            'uploader_id': '15973816',
             'categories': ['個人短片'],
-            'duration': 247.246
-        }
-    }, {
-        # Audio with alternative form of url
-        'url': 'http://vlog.xuite.net/play/S1dDUjdyLTMyOTc3NjcuZmx2/%E5%AD%AB%E7%87%95%E5%A7%BF-%E7%9C%BC%E6%B7%9A%E6%88%90%E8%A9%A9',
-        'md5': 'c91645f324de53d82ebb80930e1a73d2',
-        'info_dict': {
-            'id': 'S1dDUjdyLTMyOTc3NjcuZmx2',
-            'ext': 'mp3',
-            'upload_date': '20101226',
-            'uploader_id': '10102699',
-            'uploader': '蠍',
-            'timestamp': 1293367080,
-            'title': '孫燕姿-眼淚成詩',
-            'thumbnail': 're:^https?://.*\.jpg$',
-            'categories': ['個人短片'],
-            'duration': 223.19
-        }
+        },
     }, {
         # Video with only one format
         'url': 'http://vlog.xuite.net/play/TkRZNjhULTM0NDE2MjkuZmx2',
         'md5': 'c45737fc8ac5dc8ac2f92ecbcecf505e',
         'info_dict': {
-            'id': 'TkRZNjhULTM0NDE2MjkuZmx2',
+            'id': '3441629',
             'ext': 'mp4',
-            'upload_date': '20110306',
-            'uploader_id': '10400126',
-            'uploader': 'Valen',
-            'timestamp': 1299383640,
             'title': '孫燕姿 - 眼淚成詩',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 217.399,
+            'timestamp': 1299383640,
+            'upload_date': '20110306',
+            'uploader': 'Valen',
+            'uploader_id': '10400126',
             'categories': ['影視娛樂'],
-            'duration': 217.399
-        }
+        },
     }, {
         # Video with two formats
         'url': 'http://vlog.xuite.net/play/bWo1N1pLLTIxMzAxMTcwLmZsdg==',
         'md5': '1166e0f461efe55b62e26a2d2a68e6de',
         'info_dict': {
-            'id': 'bWo1N1pLLTIxMzAxMTcwLmZsdg==',
+            'id': '21301170',
             'ext': 'mp4',
-            'upload_date': '20150117',
-            'uploader_id': '242127761',
-            'uploader': '我只是想認真點',
-            'timestamp': 1421481240,
             'title': '暗殺教室 02',
+            'description': '字幕:【極影字幕社】',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 1384.907,
+            'timestamp': 1421481240,
+            'upload_date': '20150117',
+            'uploader': '我只是想認真點',
+            'uploader_id': '242127761',
             'categories': ['電玩動漫'],
-            'duration': 1384.907
-        }
+        },
+    }, {
+        'url': 'http://vlog.xuite.net/play/S1dDUjdyLTMyOTc3NjcuZmx2/%E5%AD%AB%E7%87%95%E5%A7%BF-%E7%9C%BC%E6%B7%9A%E6%88%90%E8%A9%A9',
+        'only_matching': True,
     }]
 
-    def _flv_config(self, media_id):
+    def _extract_flv_config(self, media_id):
         base64_media_id = base64.b64encode(media_id.encode('utf-8')).decode('utf-8')
-        flv_config_url = 'http://vlog.xuite.net/flash/player?media=' + base64_media_id
-        flv_config = self._download_xml(flv_config_url, 'flv config')
-
+        flv_config = self._download_xml(
+            'http://vlog.xuite.net/flash/player?media=%s' % base64_media_id,
+            'flv config')
         prop_dict = {}
         for prop in flv_config.findall('./property'):
             prop_id = base64.b64decode(prop.attrib['id']).decode('utf-8')
-
+            # CDATA may be empty in flv config
             if not prop.text:
-                continue  # CDATA may be empty in flv config
-
+                continue
             encoded_content = base64.b64decode(prop.text).decode('utf-8')
             prop_dict[prop_id] = compat_urllib_parse_unquote(encoded_content)
-
         return prop_dict
 
-    def _type_string(self, media_url):
-        query_string = compat_urlparse.urlparse(media_url).query
-        type_string = compat_parse_qs(query_string)['q'][0]
-        return type_string
-
-    def _guess_ext(self, media_url):
-        type_string = self._type_string(media_url)
-        if type_string == 'mp3':
-            return 'mp3'
-        elif type_string == '360' or type_string == '720':
-            return 'mp4'
-        else:
-            raise ExtractorError('Unknown type string %s' % type_string)
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        page = self._download_webpage(url, video_id)
-        media_id = self._html_search_regex(r'data-mediaid="(\d+)"', page, 'media id')
-        flv_config = self._flv_config(media_id)
+        webpage = self._download_webpage(url, video_id)
 
-        timestamp_local = parse_iso8601(flv_config['publish_datetime'], ' ')
-        timestamp_gmt = timestamp_local - CST_ZONE * 3600
+        error_msg = self._search_regex(
+            r'<div id="error-message-content">([^<]+)',
+            webpage, 'error message', default=None)
+        if error_msg:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error_msg),
+                expected=True)
 
-        ret_attrs = {
-            'id': video_id,
-            'title': flv_config['title'],
-            'thumbnail': flv_config['thumb'],
-            'uploader': flv_config['author_name'],
-            'timestamp': timestamp_gmt,
-            'uploader_id': flv_config['author_id'],
-            'duration': parse_duration(flv_config['duration']),
-            'categories': [flv_config['category']]
+        video_id = self._html_search_regex(
+            r'data-mediaid="(\d+)"', webpage, 'media id')
+        flv_config = self._extract_flv_config(video_id)
+
+        FORMATS = {
+            'audio': 'mp3',
+            'video': 'mp4',
         }
 
-        if 'hq_src' in flv_config:
-            urls = [flv_config['src'], flv_config['hq_src']]
+        formats = []
+        for format_tag in ('src', 'hq_src'):
+            video_url = flv_config.get(format_tag)
+            if not video_url:
+                continue
+            format_id = self._search_regex(
+                r'\bq=(.+?)\b', video_url, 'format id', default=format_tag)
+            formats.append({
+                'url': video_url,
+                'ext': FORMATS.get(flv_config['type'], 'mp4'),
+                'format_id': format_id,
+                'height': int(format_id) if format_id.isnumeric() else None,
+            })
+        self._sort_formats(formats)
 
-            ret_attrs['formats'] = []
+        timestamp = flv_config.get('publish_datetime')
+        if timestamp:
+            timestamp = parse_iso8601(timestamp + ' +0800', ' ')
 
-            for url in urls:
-                ret_attrs['formats'].append({
-                    'url': url,
-                    'ext': self._guess_ext(url),
-                    'format_id': self._type_string(url),
-                    'height': int(self._type_string(url))
-                })
-        else:
-            ret_attrs['url'] = flv_config['src']
-            ret_attrs['ext'] = self._guess_ext(flv_config['src'])
+        category = flv_config.get('category')
+        categories = [category] if category else []
 
-        return ret_attrs
+        return {
+            'id': video_id,
+            'title': flv_config['title'],
+            'description': flv_config.get('description'),
+            'thumbnail': flv_config.get('thumb'),
+            'timestamp': timestamp,
+            'uploader': flv_config.get('author_name'),
+            'uploader_id': flv_config.get('author_id'),
+            'duration': parse_duration(flv_config.get('duration')),
+            'categories': categories,
+            'formats': formats,
+        }

From 219337990b8008eab81e1f9adf9eba20757f3cad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 29 Jan 2015 23:11:13 +0600
Subject: [PATCH 1296/1937] [xuite] Fix _VALID_URL

---
 youtube_dl/extractor/xuite.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py
index a9dbf8c2d..4971965f9 100644
--- a/youtube_dl/extractor/xuite.py
+++ b/youtube_dl/extractor/xuite.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 class XuiteIE(InfoExtractor):
     _REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?'
-    _VALID_URL = r'http://vlog.xuite.net/(?:play|embed)/(?P<id>%s)' % _REGEX_BASE64
+    _VALID_URL = r'https?://vlog\.xuite\.net/(?:play|embed)/(?P<id>%s)' % _REGEX_BASE64
     _TESTS = [{
         # Audio
         'url': 'http://vlog.xuite.net/play/RGkzc1ZULTM4NjA5MTQuZmx2',

From a7a14d958604f5334413e2fc1872a8317d5e4884 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 29 Jan 2015 20:15:38 +0100
Subject: [PATCH 1297/1937] [YoutubeDL] set the 'thumbnails' field if the
 info_dict has the 'thumbnails' field

Since the '--write-thumbnail' uses the 'thumbnails' field and we didn't updated the info_dict, it wouldn't detect the thumbnail. (fixes #4812)
---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4c238c555..14e92ddcf 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -958,7 +958,7 @@ class YoutubeDL(object):
         if thumbnails is None:
             thumbnail = info_dict.get('thumbnail')
             if thumbnail:
-                thumbnails = [{'url': thumbnail}]
+                info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
         if thumbnails:
             thumbnails.sort(key=lambda t: (
                 t.get('preference'), t.get('width'), t.get('height'),

From e683a48d0e8650e1d9e8c323cacb468b104ea7bc Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 30 Jan 2015 05:38:05 +0800
Subject: [PATCH 1298/1937] [ctsnews] Detect youtube embedde videos

---
 youtube_dl/extractor/ctsnews.py | 75 +++++++++++++++++++++++++++------
 1 file changed, 63 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py
index e1d8c814e..35f3756f5 100644
--- a/youtube_dl/extractor/ctsnews.py
+++ b/youtube_dl/extractor/ctsnews.py
@@ -1,13 +1,27 @@
-# coding: utf-8
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_request
 from ..utils import parse_iso8601, ExtractorError
 
 
 class CtsNewsIE(InfoExtractor):
+    # https connection failed (Connection reset)
     _VALID_URL = r'http://news\.cts\.com\.tw/[a-z]+/[a-z]+/\d+/(?P<id>\d+)\.html'
     _TESTS = [{
+        'url': 'http://news.cts.com.tw/cts/international/201501/201501291578109.html',
+        'md5': 'a9875cb790252b08431186d741beaabe',
+        'info_dict': {
+            'id': '201501291578109',
+            'ext': 'mp4',
+            'title': '以色列.真主黨交火 3人死亡',
+            'description': 'md5:95e9b295c898b7ff294f09d450178d7d',
+            'timestamp': 1422528540,
+            'upload_date': '20150129',
+        }
+    }, {
+        # News count not appear on page but still available in database
         'url': 'http://news.cts.com.tw/cts/international/201309/201309031304098.html',
         'md5': '3aee7e0df7cdff94e43581f54c22619e',
         'info_dict': {
@@ -19,33 +33,70 @@ class CtsNewsIE(InfoExtractor):
             'timestamp': 1378205880,
             'upload_date': '20130903',
         }
+    }, {
+        # With Youtube embedded video
+        'url': 'http://news.cts.com.tw/cts/money/201501/201501291578003.html',
+        'md5': '1d842c771dc94c8c3bca5af2cc1db9c5',
+        'add_ie': ['Youtube'],
+        'info_dict': {
+            'id': 'OVbfO7d0_hQ',
+            'ext': 'mp4',
+            'title': 'iPhone6熱銷 蘋果財報亮眼',
+            'description': 'md5:f395d4f485487bb0f992ed2c4b07aa7d',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20150128',
+            'uploader_id': 'TBSCTS',
+            'uploader': '中華電視公司',
+        }
     }]
 
     def _real_extract(self, url):
         news_id = self._match_id(url)
         page = self._download_webpage(url, news_id)
 
-        if not self._search_regex(r'(CTSPlayer2)', page, 'CTSPlayer2 identifier', fatal=False):
-            raise ExtractorError('The news includes no videos!')
+        if self._search_regex(r'(CTSPlayer2)', page, 'CTSPlayer2 identifier', default=None):
+            feed_url = self._html_search_regex(
+                r'(http://news\.cts\.com\.tw/action/mp4feed\.php\?news_id=\d+)',
+                page, 'feed url')
+            video_url = self._download_webpage(feed_url, news_id)
+        else:
+            self.to_screen('Not CTSPlayer video, trying Youtube...')
+            youtube_url = self._search_regex(
+                r'src="(//www\.youtube\.com/embed/[^"]+)"', page, 'youtube url',
+                default=None)
+            if not youtube_url:
+                raise ExtractorError('The news includes no videos!', expected=True)
 
-        feed_pattern = r'(http://news.cts.com.tw/action/mp4feed.php\?news_id=\d+)'
-        feed_url = self._html_search_regex(feed_pattern, page, 'feed url')
-        feed_page = self._download_webpage(feed_url, news_id)
+            return {
+                '_type': 'url',
+                'url': youtube_url,
+                'ie_key': 'Youtube',
+            }
 
         description = self._html_search_meta('description', page)
         title = self._html_search_meta('title', page)
         thumbnail = self._html_search_meta('image', page)
 
-        datetime_pattern = r'(\d{4}/\d{2}/\d{2} \d{2}:\d{2})'
-        datetime_str = self._html_search_regex(datetime_pattern, page, 'date and time')
-        time = (datetime_str + ':00+08:00').replace('/', '-')
-        timestamp = parse_iso8601(time, delimiter=' ')
+        datetime_str = self._html_search_regex(
+            r'(\d{4}/\d{2}/\d{2} \d{2}:\d{2})', page, 'date and time')
+        # Transform into ISO 8601 format with timezone info
+        datetime_str = datetime_str.replace('/', '-') + ':00+0800'
+        timestamp = parse_iso8601(datetime_str, delimiter=' ')
+
+        # Note: the news count may decrease as time goes by
+        # It should be a bug in CTS website
+        req = compat_urllib_request.Request(
+            'http://news.cts.com.tw/action/news_count.php?callback=cb&news_id=' + news_id)
+        req.add_header('Referer', url)
+        newscount_page = self._download_webpage(req, news_id)
+        news_count = self._search_regex(r'cb\((\d+)\)', newscount_page, 'news count')
 
         return {
             'id': news_id,
+            'url': video_url,
             'title': title,
             'description': description,
-            'url': feed_page,
             'thumbnail': thumbnail,
-            'timestamp': timestamp
+            'timestamp': timestamp,
+            'view_count': news_count,
         }

From f345fe9db77a900a603d753c73c5566d62c1cff9 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Thu, 29 Jan 2015 23:32:23 +0100
Subject: [PATCH 1299/1937] [dctp] fix python 2.6 compatibility

---
 youtube_dl/extractor/dctp.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 9b687ef43..31bcd35c3 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -20,10 +20,10 @@ class DctpTvIE(InfoExtractor):
         version_json = self._download_json(base_url + 'version.json', video_id)
         version = version_json['version_name']
         info_json = self._download_json(
-            '{}{}/restapi/slugs/{}.json'.format(base_url, version, video_id), video_id)
+            '{0}{1}/restapi/slugs/{2}.json'.format(base_url, version, video_id), video_id)
         object_id = info_json['object_id']
         meta_json = self._download_json(
-            '{}{}/restapi/media/{}.json'.format(base_url, version, object_id), video_id)
+            '{0}{1}/restapi/media/{2}.json'.format(base_url, version, object_id), video_id)
         uuid = meta_json['uuid']
         title = meta_json['title']
         wide = meta_json['is_wide']
@@ -31,7 +31,7 @@ class DctpTvIE(InfoExtractor):
             ratio = '16x9'
         else:
             ratio = '4x3'
-        play_path = 'mp4:{}_dctp_0500_{}.m4v'.format(uuid, ratio)
+        play_path = 'mp4:{0}_dctp_0500_{1}.m4v'.format(uuid, ratio)
 
         servers_json = self._download_json('http://www.dctp.tv/streaming_servers/', video_id)
         url = servers_json[0]['endpoint']

From 87673cd4389f6798bb438bb46a306dc08c906cf5 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Thu, 29 Jan 2015 23:34:56 +0100
Subject: [PATCH 1300/1937] [dctp] follow id conventions

---
 youtube_dl/extractor/dctp.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 31bcd35c3..752cff978 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -37,12 +37,13 @@ class DctpTvIE(InfoExtractor):
         url = servers_json[0]['endpoint']
 
         return {
-            'id': video_id,
+            'id': object_id,
             'title': title,
             'format': 'rtmp',
             'url': url,
             'play_path': play_path,
             'real_time': True,
-            'ext': 'flv'
+            'ext': 'flv',
+            'display_id': video_id
         }
 

From 75a4fc5b7233d4cb4376b205c585eb4d3d486241 Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Fri, 30 Jan 2015 00:35:53 +0100
Subject: [PATCH 1301/1937] [dctp] fix test

---
 youtube_dl/extractor/dctp.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 752cff978..c80d28865 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -9,7 +9,8 @@ class DctpTvIE(InfoExtractor):
     _TEST = {
         'url': 'http://www.dctp.tv/filme/videoinstallation-fuer-eine-kaufhausfassade/',
         'info_dict': {
-            'id': 'videoinstallation-fuer-eine-kaufhausfassade',
+            'id': '1324',
+            'display_id': 'videoinstallation-fuer-eine-kaufhausfassade',
             'ext': 'flv',
             'title': 'Videoinstallation für eine Kaufhausfassade'}
         }
@@ -21,7 +22,7 @@ class DctpTvIE(InfoExtractor):
         version = version_json['version_name']
         info_json = self._download_json(
             '{0}{1}/restapi/slugs/{2}.json'.format(base_url, version, video_id), video_id)
-        object_id = info_json['object_id']
+        object_id = str(info_json['object_id'])
         meta_json = self._download_json(
             '{0}{1}/restapi/media/{2}.json'.format(base_url, version, object_id), video_id)
         uuid = meta_json['uuid']

From 7bb3ceb4c731c9783d2c565204cbc1071c3545ed Mon Sep 17 00:00:00 2001
From: Paul Hartmann <phaaurlt@gmail.com>
Date: Fri, 30 Jan 2015 00:47:43 +0100
Subject: [PATCH 1302/1937] [dctp] prefix real_time parameter with rtmp_

---
 youtube_dl/downloader/rtmp.py | 2 +-
 youtube_dl/extractor/dctp.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 6cb1bfc50..f7eeb6f43 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -104,9 +104,9 @@ class RtmpFD(FileDownloader):
         live = info_dict.get('rtmp_live', False)
         conn = info_dict.get('rtmp_conn', None)
         protocol = info_dict.get('rtmp_protocol', None)
+        real_time = info_dict.get('rtmp_real_time', False)
         no_resume = info_dict.get('no_resume', False)
         continue_dl = info_dict.get('continuedl', False)
-        real_time = info_dict.get('real_time', False)
 
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index c80d28865..8a77f2b66 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -43,7 +43,7 @@ class DctpTvIE(InfoExtractor):
             'format': 'rtmp',
             'url': url,
             'play_path': play_path,
-            'real_time': True,
+            'rtmp_real_time': True,
             'ext': 'flv',
             'display_id': video_id
         }

From adc0ae3ceb9371eb23a9a4d81d0e134ac861723e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 01:56:15 +0100
Subject: [PATCH 1303/1937] [__init__] Provide a better error messages if URLs
 are missing (Closes #4813)

---
 youtube_dl/__init__.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 71d2c6f35..e90679ff9 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -361,7 +361,9 @@ def _real_main(argv=None):
                 sys.exit()
 
             ydl.warn_if_short_id(sys.argv[1:] if argv is None else argv)
-            parser.error('you must provide at least one URL')
+            parser.error(
+                'You must provide at least one URL.\n'
+                'Type youtube-dl --help to see a list of all options.')
 
         try:
             if opts.load_info_filename is not None:

From 4fe8495a23271e164a1ce618682d50f927adc075 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 02:13:37 +0100
Subject: [PATCH 1304/1937] [viddler] PEP8

---
 youtube_dl/extractor/viddler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py
index ef104dc29..8516a2940 100644
--- a/youtube_dl/extractor/viddler.py
+++ b/youtube_dl/extractor/viddler.py
@@ -6,7 +6,7 @@ from ..utils import (
     int_or_none,
 )
 from ..compat import (
-   compat_urllib_request
+    compat_urllib_request
 )
 
 

From 83fda3c000a680317cfc9bb6fec899beb8bca773 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 02:57:37 +0100
Subject: [PATCH 1305/1937] Add a test for --no-check-certificate

---
 test/test_http.py    | 71 ++++++++++++++++++++++++++++++++++++++++++++
 test/testcert.pem    | 52 ++++++++++++++++++++++++++++++++
 youtube_dl/compat.py |  6 ++++
 3 files changed, 129 insertions(+)
 create mode 100644 test/test_http.py
 create mode 100644 test/testcert.pem

diff --git a/test/test_http.py b/test/test_http.py
new file mode 100644
index 000000000..5cce5b3ae
--- /dev/null
+++ b/test/test_http.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl import YoutubeDL
+from youtube_dl.compat import compat_http_server
+import ssl
+import threading
+
+TEST_DIR = os.path.dirname(os.path.abspath(__file__))
+
+class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
+    def log_message(self, format, *args):
+        pass
+
+    def do_GET(self):
+        if self.path == '/video.html':
+            self.send_response(200)
+            self.send_header('Content-Type', 'text/html; charset=utf-8')
+            self.end_headers()
+            self.wfile.write(b'<html><video src="/vid.mp4" /></html>')
+        elif self.path == '/vid.mp4':
+            self.send_response(200)
+            self.send_header('Content-Type', 'video/mp4')
+            self.end_headers()
+            self.wfile.write(b'\x00\x00\x00\x00\x20\x66\x74[video]')
+        else:
+            assert False
+
+
+class FakeLogger(object):
+    def debug(self, msg):
+        pass
+
+    def warning(self, msg):
+        pass
+
+    def error(self, msg):
+        pass
+
+
+class TestHTTP(unittest.TestCase):
+    def setUp(self):
+        certfn = os.path.join(TEST_DIR, 'testcert.pem')
+        self.httpd = compat_http_server.HTTPServer(
+            ('localhost', 0), HTTPTestRequestHandler)
+        self.httpd.socket = ssl.wrap_socket(
+            self.httpd.socket, certfile=certfn, server_side=True)
+        self.port = self.httpd.socket.getsockname()[1]
+        self.server_thread = threading.Thread(target=self.httpd.serve_forever)
+        self.server_thread.daemon = True
+        self.server_thread.start()
+
+    def test_nocheckcertificate(self):
+        if sys.version_info >= (2, 7, 9):  # No certificate checking anyways
+            ydl = YoutubeDL({'logger': FakeLogger()})
+            self.assertRaises(
+                Exception,
+                ydl.extract_info, 'https://localhost:%d/video.html' % self.port)
+
+        ydl = YoutubeDL({'logger': FakeLogger(), 'nocheckcertificate': True})
+        r = ydl.extract_info('https://localhost:%d/video.html' % self.port)
+        self.assertEqual(r['url'], 'https://localhost:%d/vid.mp4' % self.port)
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/testcert.pem b/test/testcert.pem
new file mode 100644
index 000000000..b3e0f00c7
--- /dev/null
+++ b/test/testcert.pem
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDMF0bAzaHAdIyB
+HRmnIp4vv40lGqEePmWqicCl0QZ0wsb5dNysSxSa7330M2QeQopGfdaUYF1uTcNp
+Qx6ECgBSfg+RrOBI7r/u4F+sKX8MUXVaf/5QoBUrGNGSn/pp7HMGOuQqO6BVg4+h
+A1ySSwUG8mZItLRry1ISyErmW8b9xlqfd97uLME/5tX+sMelRFjUbAx8A4CK58Ev
+mMguHVTlXzx5RMdYcf1VScYcjlV/qA45uzP8zwI5aigfcmUD+tbGuQRhKxUhmw0J
+aobtOR6+JSOAULW5gYa/egE4dWLwbyM6b6eFbdnjlQzEA1EW7ChMPAW/Mo83KyiP
+tKMCSQulAgMBAAECggEALCfBDAexPjU5DNoh6bIorUXxIJzxTNzNHCdvgbCGiA54
+BBKPh8s6qwazpnjT6WQWDIg/O5zZufqjE4wM9x4+0Zoqfib742ucJO9wY4way6x4
+Clt0xzbLPabB+MoZ4H7ip+9n2+dImhe7pGdYyOHoNYeOL57BBi1YFW42Hj6u/8pd
+63YCXisto3Rz1YvRQVjwsrS+cRKZlzAFQRviL30jav7Wh1aWEfcXxjj4zhm8pJdk
+ITGtq6howz57M0NtX6hZnfe8ywzTnDFIGKIMA2cYHuYJcBh9bc4tCGubTvTKK9UE
+8fM+f6UbfGqfpKCq1mcgs0XMoFDSzKS9+mSJn0+5JQKBgQD+OCKaeH3Yzw5zGnlw
+XuQfMJGNcgNr+ImjmvzUAC2fAZUJLAcQueE5kzMv5Fmd+EFE2CEX1Vit3tg0SXvA
+G+bq609doILHMA03JHnV1npO/YNIhG3AAtJlKYGxQNfWH9mflYj9mEui8ZFxG52o
+zWhHYuifOjjZszUR+/eio6NPzwKBgQDNhUBTrT8LIX4SE/EFUiTlYmWIvOMgXYvN
+8Cm3IRNQ/yyphZaXEU0eJzfX5uCDfSVOgd6YM/2pRah+t+1Hvey4H8e0GVTu5wMP
+gkkqwKPGIR1YOmlw6ippqwvoJD7LuYrm6Q4D6e1PvkjwCq6lEndrOPmPrrXNd0JJ
+XO60y3U2SwKBgQDLkyZarryQXxcCI6Q10Tc6pskYDMIit095PUbTeiUOXNT9GE28
+Hi32ziLCakk9kCysNasii81MxtQ54tJ/f5iGbNMMddnkKl2a19Hc5LjjAm4cJzg/
+98KGEhvyVqvAo5bBDZ06/rcrD+lZOzUglQS5jcIcqCIYa0LHWQ/wJLxFzwKBgFcZ
+1SRhdSmDfUmuF+S4ZpistflYjC3IV5rk4NkS9HvMWaJS0nqdw4A3AMzItXgkjq4S
+DkOVLTkTI5Do5HAWRv/VwC5M2hkR4NMu1VGAKSisGiKtRsirBWSZMEenLNHshbjN
+Jrpz5rZ4H7NT46ZkCCZyFBpX4gb9NyOedjA7Via3AoGARF8RxbYjnEGGFuhnbrJB
+FTPR0vaL4faY3lOgRZ8jOG9V2c9Hzi/y8a8TU4C11jnJSDqYCXBTd5XN28npYxtD
+pjRsCwy6ze+yvYXPO7C978eMG3YRyj366NXUxnXN59ibwe/lxi2OD9z8J1LEdF6z
+VJua1Wn8HKxnXMI61DhTCSo=
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIIEEzCCAvugAwIBAgIJAK1haYi6gmSKMA0GCSqGSIb3DQEBCwUAMIGeMQswCQYD
+VQQGEwJERTEMMAoGA1UECAwDTlJXMRQwEgYDVQQHDAtEdWVzc2VsZG9yZjEbMBkG
+A1UECgwSeW91dHViZS1kbCBwcm9qZWN0MRkwFwYDVQQLDBB5b3V0dWJlLWRsIHRl
+c3RzMRIwEAYDVQQDDAlsb2NhbGhvc3QxHzAdBgkqhkiG9w0BCQEWEHBoaWhhZ0Bw
+aGloYWcuZGUwIBcNMTUwMTMwMDExNTA4WhgPMjExNTAxMDYwMTE1MDhaMIGeMQsw
+CQYDVQQGEwJERTEMMAoGA1UECAwDTlJXMRQwEgYDVQQHDAtEdWVzc2VsZG9yZjEb
+MBkGA1UECgwSeW91dHViZS1kbCBwcm9qZWN0MRkwFwYDVQQLDBB5b3V0dWJlLWRs
+IHRlc3RzMRIwEAYDVQQDDAlsb2NhbGhvc3QxHzAdBgkqhkiG9w0BCQEWEHBoaWhh
+Z0BwaGloYWcuZGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDMF0bA
+zaHAdIyBHRmnIp4vv40lGqEePmWqicCl0QZ0wsb5dNysSxSa7330M2QeQopGfdaU
+YF1uTcNpQx6ECgBSfg+RrOBI7r/u4F+sKX8MUXVaf/5QoBUrGNGSn/pp7HMGOuQq
+O6BVg4+hA1ySSwUG8mZItLRry1ISyErmW8b9xlqfd97uLME/5tX+sMelRFjUbAx8
+A4CK58EvmMguHVTlXzx5RMdYcf1VScYcjlV/qA45uzP8zwI5aigfcmUD+tbGuQRh
+KxUhmw0JaobtOR6+JSOAULW5gYa/egE4dWLwbyM6b6eFbdnjlQzEA1EW7ChMPAW/
+Mo83KyiPtKMCSQulAgMBAAGjUDBOMB0GA1UdDgQWBBTBUZoqhQkzHQ6xNgZfFxOd
+ZEVt8TAfBgNVHSMEGDAWgBTBUZoqhQkzHQ6xNgZfFxOdZEVt8TAMBgNVHRMEBTAD
+AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCUOCl3T/J9B08Z+ijfOJAtkbUaEHuVZb4x
+5EpZSy2ZbkLvtsftMFieHVNXn9dDswQc5qjYStCC4o60LKw4M6Y63FRsAZ/DNaqb
+PY3jyCyuugZ8/sNf50vHYkAcF7SQYqOQFQX4TQsNUk2xMJIt7H0ErQFmkf/u3dg6
+cy89zkT462IwxzSG7NNhIlRkL9o5qg+Y1mF9eZA1B0rcL6hO24PPTHOd90HDChBu
+SZ6XMi/LzYQSTf0Vg2R+uMIVlzSlkdcZ6sqVnnqeLL8dFyIa4e9sj/D4ZCYP8Mqe
+Z73H5/NNhmwCHRqVUTgm307xblQaWGhwAiDkaRvRW2aJQ0qGEdZK
+-----END CERTIFICATE-----
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 4453b34fc..497ca52de 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -71,6 +71,11 @@ try:
 except ImportError:
     compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
 
+try:
+    import http.server as compat_http_server
+except ImportError:
+    import BaseHTTPServer as compat_http_server
+
 try:
     from urllib.parse import unquote as compat_urllib_parse_unquote
 except ImportError:
@@ -365,6 +370,7 @@ __all__ = [
     'compat_html_entities',
     'compat_html_parser',
     'compat_http_client',
+    'compat_http_server',
     'compat_kwargs',
     'compat_ord',
     'compat_parse_qs',

From 03d8d4df385bbf1e4cf1f63609d06dc4ad11313a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:06:40 +0100
Subject: [PATCH 1306/1937] [test_http] PEP8

---
 test/test_http.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/test_http.py b/test/test_http.py
index 5cce5b3ae..bd4d46fef 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -14,6 +14,7 @@ import threading
 
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 
+
 class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
     def log_message(self, format, *args):
         pass

From ec4161a57d66c92ea7460ac44dda13aadfb85f21 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:06:51 +0100
Subject: [PATCH 1307/1937] [ctsnews] Remove news count check (#4802)

That's the number of articles (or at least some constant value); view count is the number of views of the video!
Also add a better description so that we can distinguish the two downloads.
---
 youtube_dl/extractor/ctsnews.py | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py
index 35f3756f5..f769dc031 100644
--- a/youtube_dl/extractor/ctsnews.py
+++ b/youtube_dl/extractor/ctsnews.py
@@ -58,7 +58,8 @@ class CtsNewsIE(InfoExtractor):
             feed_url = self._html_search_regex(
                 r'(http://news\.cts\.com\.tw/action/mp4feed\.php\?news_id=\d+)',
                 page, 'feed url')
-            video_url = self._download_webpage(feed_url, news_id)
+            video_url = self._download_webpage(
+                feed_url, news_id, note='Fetching feed')
         else:
             self.to_screen('Not CTSPlayer video, trying Youtube...')
             youtube_url = self._search_regex(
@@ -83,14 +84,6 @@ class CtsNewsIE(InfoExtractor):
         datetime_str = datetime_str.replace('/', '-') + ':00+0800'
         timestamp = parse_iso8601(datetime_str, delimiter=' ')
 
-        # Note: the news count may decrease as time goes by
-        # It should be a bug in CTS website
-        req = compat_urllib_request.Request(
-            'http://news.cts.com.tw/action/news_count.php?callback=cb&news_id=' + news_id)
-        req.add_header('Referer', url)
-        newscount_page = self._download_webpage(req, news_id)
-        news_count = self._search_regex(r'cb\((\d+)\)', newscount_page, 'news count')
-
         return {
             'id': news_id,
             'url': video_url,
@@ -98,5 +91,4 @@ class CtsNewsIE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
             'timestamp': timestamp,
-            'view_count': news_count,
         }

From 3198291f26e215a7b26906a630d42a30615869b4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:09:51 +0100
Subject: [PATCH 1308/1937] [ctsnews] Remove unused import

---
 youtube_dl/extractor/ctsnews.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py
index f769dc031..0226f8036 100644
--- a/youtube_dl/extractor/ctsnews.py
+++ b/youtube_dl/extractor/ctsnews.py
@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_request
 from ..utils import parse_iso8601, ExtractorError
 
 

From f452f72c6ba0cc3fe560e7449fcb95e804da5b69 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:12:12 +0100
Subject: [PATCH 1309/1937] Credit @yan12125 for ctsnews and nextmedia

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 20e620c35..2203c4b63 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -107,3 +107,4 @@ Yen Chi Hsuan
 Enam Mijbah Noor
 David Luhmer
 Shaya Goldberg
+Yen Chi Hsuan

From 6ebb0dca9f40ecf9fb82882c097f42f497c70cda Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:15:34 +0100
Subject: [PATCH 1310/1937] [dctp] Improve output (#4796)

---
 youtube_dl/extractor/dctp.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 8a77f2b66..dbedafa7d 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -2,10 +2,11 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import compat_str
 
 
 class DctpTvIE(InfoExtractor):
-    _VALID_URL = r'^http://www.dctp.tv/(#/)?filme/(?P<id>.+?)/$'
+    _VALID_URL = r'http://www.dctp.tv/(#/)?filme/(?P<id>.+?)/$'
     _TEST = {
         'url': 'http://www.dctp.tv/filme/videoinstallation-fuer-eine-kaufhausfassade/',
         'info_dict': {
@@ -18,13 +19,17 @@ class DctpTvIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         base_url = 'http://dctp-ivms2-restapi.s3.amazonaws.com/'
-        version_json = self._download_json(base_url + 'version.json', video_id)
+        version_json = self._download_json(
+            base_url + 'version.json',
+            video_id, note='Determining file version')
         version = version_json['version_name']
         info_json = self._download_json(
-            '{0}{1}/restapi/slugs/{2}.json'.format(base_url, version, video_id), video_id)
-        object_id = str(info_json['object_id'])
+            '{0}{1}/restapi/slugs/{2}.json'.format(base_url, version, video_id),
+            video_id, note='Fetching object ID')
+        object_id = compat_str(info_json['object_id'])
         meta_json = self._download_json(
-            '{0}{1}/restapi/media/{2}.json'.format(base_url, version, object_id), video_id)
+            '{0}{1}/restapi/media/{2}.json'.format(base_url, version, object_id),
+            video_id, note='Downloading metadata')
         uuid = meta_json['uuid']
         title = meta_json['title']
         wide = meta_json['is_wide']
@@ -34,7 +39,9 @@ class DctpTvIE(InfoExtractor):
             ratio = '4x3'
         play_path = 'mp4:{0}_dctp_0500_{1}.m4v'.format(uuid, ratio)
 
-        servers_json = self._download_json('http://www.dctp.tv/streaming_servers/', video_id)
+        servers_json = self._download_json(
+            'http://www.dctp.tv/streaming_servers/',
+            video_id, note='Downloading server list')
         url = servers_json[0]['endpoint']
 
         return {

From 1798791df11920928818161c48fc2b12334e1e52 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:16:36 +0100
Subject: [PATCH 1311/1937] Credit @bastik for dctp.tv (#4796)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 2203c4b63..4a70e7cca 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -108,3 +108,4 @@ Enam Mijbah Noor
 David Luhmer
 Shaya Goldberg
 Yen Chi Hsuan
+Paul Hartmann

From 83fddfd493ffcbb7256f0f1a467cc48ca2191090 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:45:06 +0100
Subject: [PATCH 1312/1937] [dctp] PEP8

---
 youtube_dl/extractor/dctp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index dbedafa7d..6ed354379 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -13,8 +13,9 @@ class DctpTvIE(InfoExtractor):
             'id': '1324',
             'display_id': 'videoinstallation-fuer-eine-kaufhausfassade',
             'ext': 'flv',
-            'title': 'Videoinstallation für eine Kaufhausfassade'}
+            'title': 'Videoinstallation für eine Kaufhausfassade'
         }
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -54,4 +55,3 @@ class DctpTvIE(InfoExtractor):
             'ext': 'flv',
             'display_id': video_id
         }
-

From c1708b89c09d7288213110fba859f4c5e0339c31 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:45:29 +0100
Subject: [PATCH 1313/1937] [youtube:truncated_url] Add hl= forms (#4799)

---
 youtube_dl/extractor/youtube.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b7b91f354..bc5fa0b39 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1688,6 +1688,7 @@ class YoutubeTruncatedURLIE(InfoExtractor):
             feature=[a-z_]+|
             annotation_id=annotation_[^&]+|
             x-yt-cl=[0-9]+|
+            hl=[^&]*|
         )?
         |
             attribution_link\?a=[^&]+
@@ -1707,6 +1708,9 @@ class YoutubeTruncatedURLIE(InfoExtractor):
     }, {
         'url': 'https://www.youtube.com/watch?feature=foo',
         'only_matching': True,
+    }, {
+        'url': 'https://www.youtube.com/watch?hl=en-GB',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 5a8a29cfea8da1afaa94bd2d5be5c1bf72b2d17c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:46:57 +0100
Subject: [PATCH 1314/1937] release 2015.01.30

---
 README.md             | 6 +++---
 youtube_dl/version.py | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 0580ab2a3..314f9a8b2 100644
--- a/README.md
+++ b/README.md
@@ -292,9 +292,9 @@ which means you can modify it, redistribute it or use it however you like.
                                      video results by putting a condition in
                                      brackets, as in -f "best[height=720]" (or
                                      -f "[filesize>10M]").  This works for
-                                     filesize, height, width, tbr, abr, and vbr
-                                     and the comparisons <, <=, >, >=, =, != .
-                                     Formats for which the value is not known
+                                     filesize, height, width, tbr, abr, vbr, and
+                                     fps and the comparisons <, <=, >, >=, =, !=
+                                     . Formats for which the value is not known
                                      are excluded unless you put a question mark
                                      (?) after the operator. You can combine
                                      format filters, so  -f "[height <=?
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0d59cddd7..4c75db08f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.25'
+__version__ = '2015.01.30'

From 5d6677ca28147495ffbeaf5bd49647e611cf50fc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 03:50:40 +0100
Subject: [PATCH 1315/1937] release 2015.01.30.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4c75db08f..a5d558520 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.30'
+__version__ = '2015.01.30.1'

From e2a08185c6b9b3f1f4b7e78138b2b6735d6b0184 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 04:17:44 +0100
Subject: [PATCH 1316/1937] [README] Add an FAQ for YouTube IDs starting with a
 dash (Closes #4800)

---
 README.md | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/README.md b/README.md
index 314f9a8b2..09da0129e 100644
--- a/README.md
+++ b/README.md
@@ -525,6 +525,13 @@ From then on, after restarting your shell, you will be able to access both youtu
 
 Use the `-o` to specify an [output template](#output-template), for example `-o "/home/user/videos/%(title)s-%(id)s.%(ext)s"`. If you want this for all of your downloads, put the option into your [configuration file](#configuration).
 
+### How do I download a video starting with a `-` ?
+
+Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the options with `--`:
+
+    youtube-dl -- -wNyEUrxzFU
+    youtube-dl "http://www.youtube.com/watch?v=-wNyEUrxzFU"
+
 ### How can I detect whether a given URL is supported by youtube-dl?
 
 For one, have a look at the [list of supported sites](docs/supportedsites). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.

From 6449cd807e07e04eb88b1a34063e00677de0713e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 04:43:50 +0100
Subject: [PATCH 1317/1937] [youtube] Fall back to embed webpage when content
 is blocked on main (Fixes #4717)

---
 youtube_dl/extractor/youtube.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index bc5fa0b39..a9e444351 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -28,6 +28,7 @@ from ..utils import (
     get_element_by_attribute,
     get_element_by_id,
     int_or_none,
+    js_to_json,
     OnDemandPagedList,
     orderedSet,
     unescapeHTML,
@@ -809,6 +810,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             player_url = None
 
         # Get video info
+        embed_webpage = None
         if re.search(r'player-age-gate-content">', video_webpage) is not None:
             age_gate = True
             # We simulate the access to the video from www.youtube.com/v/{video_id}
@@ -1016,10 +1018,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                     url += '&signature=' + url_data['sig'][0]
                 elif 's' in url_data:
                     encrypted_sig = url_data['s'][0]
+                    ASSETS_RE = r'"assets":.+?"js":\s*("[^"]+")'
 
                     jsplayer_url_json = self._search_regex(
-                        r'"assets":.+?"js":\s*("[^"]+")',
-                        embed_webpage if age_gate else video_webpage, 'JS player URL')
+                        ASSETS_RE,
+                        embed_webpage if age_gate else video_webpage,
+                        'JS player URL (1)', default=None)
+                    if not jsplayer_url_json and not age_gate:
+                        # We need the embed website after all
+                        if embed_webpage is None:
+                            embed_url = proto + '://www.youtube.com/embed/%s' % video_id
+                            embed_webpage = self._download_webpage(
+                                embed_url, video_id, 'Downloading embed webpage')
+                        jsplayer_url_json = self._search_regex(
+                            ASSETS_RE, embed_webpage, 'JS player URL')
+
                     player_url = json.loads(jsplayer_url_json)
                     if player_url is None:
                         player_url_json = self._search_regex(

From c03844a4ec0bffef4803394545c7a5cc16dc70e1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 04:44:00 +0100
Subject: [PATCH 1318/1937] release 2015.01.30.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a5d558520..5e6288e8e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.30.1'
+__version__ = '2015.01.30.2'

From 52585fd6dcd5020f2800d133588d7232419dddc0 Mon Sep 17 00:00:00 2001
From: codesparkle <adam.thalhammer@outlook.com>
Date: Fri, 30 Jan 2015 18:41:40 +1100
Subject: [PATCH 1319/1937] The opening curly brace `{` is a regex reserved
 [control character](http://stackoverflow.com/a/400316/1106367), so it needs
 to be escaped.

---
 youtube_dl/extractor/generic.py     | 2 +-
 youtube_dl/extractor/grooveshark.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 41884ed7a..a6f102a4e 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1073,7 +1073,7 @@ class GenericIE(InfoExtractor):
             found = filter_video(re.findall(r'''(?xs)
                 flowplayer\("[^"]+",\s*
                     \{[^}]+?\}\s*,
-                    \s*{[^}]+? ["']?clip["']?\s*:\s*\{\s*
+                    \s*\{[^}]+? ["']?clip["']?\s*:\s*\{\s*
                         ["']?url["']?\s*:\s*["']([^"']+)["']
             ''', webpage))
         if not found:
diff --git a/youtube_dl/extractor/grooveshark.py b/youtube_dl/extractor/grooveshark.py
index fff74a70a..848d17beb 100644
--- a/youtube_dl/extractor/grooveshark.py
+++ b/youtube_dl/extractor/grooveshark.py
@@ -83,7 +83,7 @@ class GroovesharkIE(InfoExtractor):
         return compat_urlparse.urlunparse((uri.scheme, uri.netloc, obj['attrs']['data'], None, None, None))
 
     def _transform_bootstrap(self, js):
-        return re.split('(?m)^\s*try\s*{', js)[0] \
+        return re.split('(?m)^\s*try\s*\{', js)[0] \
                  .split(' = ', 1)[1].strip().rstrip(';')
 
     def _transform_meta(self, js):

From 1d2daaea6372f5119be8fc21637a275fe93a52ff Mon Sep 17 00:00:00 2001
From: codesparkle <adam.thalhammer@outlook.com>
Date: Fri, 30 Jan 2015 20:10:12 +1100
Subject: [PATCH 1320/1937] Simplify list creation

---
 youtube_dl/extractor/cinchcast.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/cinchcast.py b/youtube_dl/extractor/cinchcast.py
index 0c9a24bef..e23c515d9 100644
--- a/youtube_dl/extractor/cinchcast.py
+++ b/youtube_dl/extractor/cinchcast.py
@@ -28,12 +28,7 @@ class CinchcastIE(InfoExtractor):
             item, './{http://developer.longtailvideo.com/trac/}date')
         upload_date = unified_strdate(date_str, day_first=False)
         # duration is present but wrong
-        formats = []
-        formats.append({
-            'format_id': 'main',
-            'url': item.find(
-                './{http://search.yahoo.com/mrss/}content').attrib['url'],
-        })
+        formats = [{'format_id': 'main', 'url': item.find('./{http://search.yahoo.com/mrss/}content').attrib['url']}]
         backup_url = xpath_text(
             item, './{http://developer.longtailvideo.com/trac/}backupContent')
         if backup_url:

From 5468ff4d917eeb292ce0bfe7cf13eef75cccae19 Mon Sep 17 00:00:00 2001
From: codesparkle <adam.thalhammer@outlook.com>
Date: Fri, 30 Jan 2015 20:11:51 +1100
Subject: [PATCH 1321/1937] Remove duplicate dictionary keys

---
 youtube_dl/YoutubeDL.py               | 1 -
 youtube_dl/extractor/appletrailers.py | 1 -
 2 files changed, 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 14e92ddcf..ce3abaee3 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1075,7 +1075,6 @@ class YoutubeDL(object):
                                 'requested_formats': formats_info,
                                 'format': rf,
                                 'format_id': rf,
-                                'ext': formats_info[0]['ext'],
                                 'width': formats_info[0].get('width'),
                                 'height': formats_info[0].get('height'),
                                 'resolution': formats_info[0].get('resolution'),
diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 70621946d..287f71e07 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -122,7 +122,6 @@ class AppleTrailersIE(InfoExtractor):
             playlist.append({
                 '_type': 'video',
                 'id': video_id,
-                'title': title,
                 'formats': formats,
                 'title': title,
                 'duration': duration,

From 749f2ca044dc32c6a3ae9e9c55e4a22b091e2635 Mon Sep 17 00:00:00 2001
From: codesparkle <adam.thalhammer@outlook.com>
Date: Fri, 30 Jan 2015 20:35:20 +1100
Subject: [PATCH 1322/1937] Smotri info extractor: removed unreachable code and
 updated old md5 for test video

---
 youtube_dl/extractor/smotri.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py
index e94f41362..24746a09a 100644
--- a/youtube_dl/extractor/smotri.py
+++ b/youtube_dl/extractor/smotri.py
@@ -108,7 +108,7 @@ class SmotriIE(InfoExtractor):
         # swf player
         {
             'url': 'http://pics.smotri.com/scrubber_custom8.swf?file=v9188090500',
-            'md5': '4d47034979d9390d14acdf59c4935bc2',
+            'md5': '31099eeb4bc906712c5f40092045108d',
             'info_dict': {
                 'id': 'v9188090500',
                 'ext': 'mp4',
@@ -139,9 +139,6 @@ class SmotriIE(InfoExtractor):
     def _search_meta(self, name, html, display_name=None):
         if display_name is None:
             display_name = name
-        return self._html_search_regex(
-            r'<meta itemprop="%s" content="([^"]+)" />' % re.escape(name),
-            html, display_name, fatal=False)
         return self._html_search_meta(name, html, display_name)
 
     def _real_extract(self, url):

From 83e7a314b4974d6d5674d88b72223d6785f29af7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 10:48:39 +0100
Subject: [PATCH 1323/1937] dedup AUTHORS

---
 AUTHORS | 1 -
 1 file changed, 1 deletion(-)

diff --git a/AUTHORS b/AUTHORS
index 4a70e7cca..83d8161a9 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -107,5 +107,4 @@ Yen Chi Hsuan
 Enam Mijbah Noor
 David Luhmer
 Shaya Goldberg
-Yen Chi Hsuan
 Paul Hartmann

From a52c6335361d0c86ddc8798d291139faa15de54c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 10:59:07 +0100
Subject: [PATCH 1324/1937] [cinchcast] Wrap overly long lines (#4820)

---
 youtube_dl/extractor/cinchcast.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cinchcast.py b/youtube_dl/extractor/cinchcast.py
index e23c515d9..562c9bbbb 100644
--- a/youtube_dl/extractor/cinchcast.py
+++ b/youtube_dl/extractor/cinchcast.py
@@ -28,7 +28,10 @@ class CinchcastIE(InfoExtractor):
             item, './{http://developer.longtailvideo.com/trac/}date')
         upload_date = unified_strdate(date_str, day_first=False)
         # duration is present but wrong
-        formats = [{'format_id': 'main', 'url': item.find('./{http://search.yahoo.com/mrss/}content').attrib['url']}]
+        formats = [{
+            'format_id': 'main',
+            'url': item.find('./{http://search.yahoo.com/mrss/}content').attrib['url'],
+        }]
         backup_url = xpath_text(
             item, './{http://developer.longtailvideo.com/trac/}backupContent')
         if backup_url:

From 70fca8d694e6f611384fdcabad748fb7a65235e4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 10:59:19 +0100
Subject: [PATCH 1325/1937] [youtube] Remove unused import

---
 youtube_dl/extractor/youtube.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a9e444351..3b3678c6e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -28,7 +28,6 @@ from ..utils import (
     get_element_by_attribute,
     get_element_by_id,
     int_or_none,
-    js_to_json,
     OnDemandPagedList,
     orderedSet,
     unescapeHTML,

From dc35bfd2d50728d04c1b8213524f6edd2a06513c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 15:52:31 +0100
Subject: [PATCH 1326/1937] [test/helper] Clarify which keys have to be added

---
 test/helper.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 27a68091f..805467c82 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -148,9 +148,15 @@ def expect_info_dict(self, got_dict, expected_dict):
                 return "'%s'" % v.replace('\\', '\\\\').replace("'", "\\'").replace('\n', '\\n')
             else:
                 return repr(v)
-        info_dict_str = ''.join(
-            '    %s: %s,\n' % (_repr(k), _repr(v))
-            for k, v in test_info_dict.items())
+        info_dict_str = ''
+        if len(missing_keys) != len(expected_dict):
+            info_dict_str += ''.join(
+                '    %s: %s,\n' % (_repr(k), _repr(v))
+                for k, v in test_info_dict.items() if k not in missing_keys)
+            info_dict_str += '\n'
+        info_dict_str += ''.join(
+            '    %s: %s,\n' % (_repr(k), _repr(test_info_dict[k]))
+            for k in missing_keys)
         write_string(
             '\n\'info_dict\': {\n' + info_dict_str + '}\n', out=sys.stderr)
         self.assertFalse(

From b04b8852710834bfbfdd38722e2cc212eab2a9ec Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 15:53:16 +0100
Subject: [PATCH 1327/1937] [extractor/common] Document all protocol values

---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 478232682..8b4ef3f09 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -89,7 +89,8 @@ class InfoExtractor(object):
                     * player_url SWF Player URL (used for rtmpdump).
                     * protocol   The protocol that will be used for the actual
                                  download, lower-case.
-                                 "http", "https", "rtsp", "rtmp", "m3u8" or so.
+                                 "http", "https", "rtsp", "rtmp", "rtmpe",
+                                 "m3u8", or "m3u8_native".
                     * preference Order number of this format. If this field is
                                  present and not None, the formats get sorted
                                  by this field, regardless of all other values.

From 01c46659c4aa312a6d7c935d3a92d9fa07d66c00 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 15:53:58 +0100
Subject: [PATCH 1328/1937] [washingtonpost] Catch more UUIDs

---
 youtube_dl/extractor/washingtonpost.py | 34 +++++++++++++++++++++++---
 1 file changed, 30 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/washingtonpost.py b/youtube_dl/extractor/washingtonpost.py
index c17bebd6e..72eb010f8 100644
--- a/youtube_dl/extractor/washingtonpost.py
+++ b/youtube_dl/extractor/washingtonpost.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -11,9 +12,10 @@ from ..utils import (
 
 class WashingtonPostIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?washingtonpost\.com/.*?/(?P<id>[^/]+)/(?:$|[?#])'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.washingtonpost.com/sf/national/2014/03/22/sinkhole-of-bureaucracy/',
         'info_dict': {
+            'id': 'sinkhole-of-bureaucracy',
             'title': 'Sinkhole of bureaucracy',
         },
         'playlist': [{
@@ -40,15 +42,38 @@ class WashingtonPostIE(InfoExtractor):
                 'upload_date': '20140322',
                 'uploader': 'The Washington Post',
             },
+        }],
+    }, {
+        'url': 'http://www.washingtonpost.com/blogs/wonkblog/wp/2014/12/31/one-airline-figured-out-how-to-make-sure-its-airplanes-never-disappear/',
+        'info_dict': {
+            'id': 'one-airline-figured-out-how-to-make-sure-its-airplanes-never-disappear',
+            'title': 'One airline figured out how to make sure its airplanes never disappear',
+        },
+        'playlist': [{
+            'md5': 'a7c1b5634ba5e57a6a82cdffa5b1e0d0',
+            'info_dict': {
+                'id': '0e4bb54c-9065-11e4-a66f-0ca5037a597d',
+                'ext': 'mp4',
+                'description': 'Washington Post transportation reporter Ashley Halsey III explains why a plane\'s black box needs to be recovered from a crash site instead of having its information streamed in real time throughout the flight.',
+                'upload_date': '20141230',
+                'uploader': 'The Washington Post',
+                'timestamp': 1419974765,
+                'title': 'Why black boxes don’t transmit data in real time',
+            }
         }]
-    }
+    }]
 
     def _real_extract(self, url):
         page_id = self._match_id(url)
         webpage = self._download_webpage(url, page_id)
 
         title = self._og_search_title(webpage)
-        uuids = re.findall(r'data-video-uuid="([^"]+)"', webpage)
+
+        uuids = re.findall(r'''(?x)
+            (?:
+                <div\s+class="posttv-video-embed[^>]*?data-uuid=|
+                data-video-uuid=
+            )"([^"]+)"''', webpage)
         entries = []
         for i, uuid in enumerate(uuids, start=1):
             vinfo_all = self._download_json(
@@ -75,10 +100,11 @@ class WashingtonPostIE(InfoExtractor):
                 'filesize': s.get('fileSize'),
                 'url': s.get('url'),
                 'ext': 'mp4',
+                'preference': -100 if s.get('type') == 'smil' else None,
                 'protocol': {
                     'MP4': 'http',
                     'F4F': 'f4m',
-                }.get(s.get('type'))
+                }.get(s.get('type')),
             } for s in vinfo.get('streams', [])]
             source_media_url = vinfo.get('sourceMediaURL')
             if source_media_url:

From d543bdc351c073621e2f8647fad0f9c8da1cf5b6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 16:06:55 +0100
Subject: [PATCH 1329/1937] [downloader/f4m] Clarify that we should eventually
 just implement the DRM scheme (#3000)

---
 youtube_dl/downloader/f4m.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 29de7630d..13055a67e 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -238,13 +238,13 @@ class F4mFD(FileDownloader):
                   doc.findall(_add_ns('drmAdditionalHeaderSet'))):
             # If id attribute is missing it's valid for all media nodes
             # without drmAdditionalHeaderId or drmAdditionalHeaderSetId attribute
-            if not 'id' in e.attrib:
-                self.report_error('Media is DRM protected')
+            if 'id' not in e.attrib:
+                self.report_error('Missing ID in f4m DRM')
         media = list(filter(lambda e: 'drmAdditionalHeaderId' not in e.attrib and
                                       'drmAdditionalHeaderSetId' not in e.attrib,
                             media))
         if not media:
-            self.report_error('Media is DRM protected')
+            self.report_error('Unsupported DRM')
         return media
 
     def real_download(self, filename, info_dict):

From dee3f73787226e3faaf409f8c355b5f8d5dc5632 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 21:54:14 +0600
Subject: [PATCH 1330/1937] [spike] Modernize

---
 youtube_dl/extractor/spike.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index a3adf54e3..abba66aee 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -25,8 +25,7 @@ class SpikeIE(MTVServicesInfoExtractor):
     _MOBILE_TEMPLATE = 'http://m.spike.com/videos/video.rbml?id=%s'
 
     def _real_extract(self, url):
-        mobj = re.search(self._VALID_URL, url)
-        mobile_id = mobj.group('mobile_id')
-        if mobile_id is not None:
+        mobile_id = self._match_id(url)
+        if mobile_id:
             url = 'http://www.spike.com/video-clips/%s' % mobile_id
         return super(SpikeIE, self)._real_extract(url)

From 20b4492c718a5c3cfb2aa493134b9ba8acb0f750 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 21:54:48 +0600
Subject: [PATCH 1331/1937] [spike] Improve _VALID_URL

---
 youtube_dl/extractor/spike.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index abba66aee..1af2e9725 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -7,8 +7,8 @@ from .mtv import MTVServicesInfoExtractor
 
 class SpikeIE(MTVServicesInfoExtractor):
     _VALID_URL = r'''(?x)https?://
-        (www\.spike\.com/(video-clips|episodes)/.+|
-         m\.spike\.com/videos/video.rbml\?id=(?P<mobile_id>[^&]+))
+        (?:www\.spike\.com/(?:video-clips|(?:full-)?episodes)/.+|
+         m\.spike\.com/videos/video\.rbml\?id=(?P<id>[^&]+))
         '''
     _TEST = {
         'url': 'http://www.spike.com/video-clips/lhtu8m/auction-hunters-can-allen-ride-a-hundred-year-old-motorcycle',

From e525d9a3dfb03152e133b8c0ccc8a104289cf5cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 21:57:59 +0600
Subject: [PATCH 1332/1937] [mtv] Extract subtitles (Closes #4811)

---
 youtube_dl/extractor/mtv.py | 34 +++++++++++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 22a726327..7a3b62ebe 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -2,10 +2,11 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .subtitles import SubtitlesInfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
+    compat_str,
 )
 from ..utils import (
     ExtractorError,
@@ -22,7 +23,7 @@ def _media_xml_tag(tag):
     return '{http://search.yahoo.com/mrss/}%s' % tag
 
 
-class MTVServicesInfoExtractor(InfoExtractor):
+class MTVServicesInfoExtractor(SubtitlesInfoExtractor):
     _MOBILE_TEMPLATE = None
 
     @staticmethod
@@ -89,6 +90,28 @@ class MTVServicesInfoExtractor(InfoExtractor):
         self._sort_formats(formats)
         return formats
 
+    def _extract_subtitles(self, mdoc, mtvn_id):
+        subtitles = {}
+        FORMATS = {
+            'scc': 'cea-608',
+            'eia-608': 'cea-608',
+            'xml': 'ttml',
+        }
+        subtitles_format = FORMATS.get(
+            self._downloader.params.get('subtitlesformat'), 'ttml')
+        for transcript in mdoc.findall('.//transcript'):
+            if transcript.get('kind') != 'captions':
+                continue
+            lang = transcript.get('srclang')
+            for typographic in transcript.findall('./typographic'):
+                captions_format = typographic.get('format')
+                if captions_format == subtitles_format:
+                    subtitles[lang] = compat_str(typographic.get('src'))
+                    break
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(mtvn_id, subtitles)
+        return self.extract_subtitles(mtvn_id, subtitles)
+
     def _get_video_info(self, itemdoc):
         uri = itemdoc.find('guid').text
         video_id = self._id_from_uri(uri)
@@ -135,6 +158,7 @@ class MTVServicesInfoExtractor(InfoExtractor):
         return {
             'title': title,
             'formats': self._extract_video_formats(mediagen_doc, mtvn_id),
+            'subtitles': self._extract_subtitles(mediagen_doc, mtvn_id),
             'id': video_id,
             'thumbnail': self._get_thumbnail_url(uri, itemdoc),
             'description': description,
@@ -167,7 +191,11 @@ class MTVServicesInfoExtractor(InfoExtractor):
             mgid = self._search_regex(
                 [r'data-mgid="(.*?)"', r'swfobject.embedSWF\(".*?(mgid:.*?)"'],
                 webpage, 'mgid')
-        return self._get_videos_info(mgid)
+
+        videos_info = self._get_videos_info(mgid)
+        if self._downloader.params.get('listsubtitles', False):
+            return
+        return videos_info
 
 
 class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):

From ef1269fb07833a06a15d0601e54d58e1678bc06a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 22:42:11 +0600
Subject: [PATCH 1333/1937] [drtv] Improve _VALID_URL (#4814)

---
 youtube_dl/extractor/drtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index 510ef04b0..8d113a5e8 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -6,7 +6,7 @@ from ..utils import parse_iso8601
 
 
 class DRTVIE(SubtitlesInfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?dr\.dk/tv/se/(?:[^/]+/)+(?P<id>[\da-z-]+)(?:[/#?]|$)'
+    _VALID_URL = r'https?://(?:www\.)?dr\.dk/tv/se/(?:[^/]+/)*(?P<id>[\da-z-]+)(?:[/#?]|$)'
 
     _TEST = {
         'url': 'http://www.dr.dk/tv/se/partiets-mand/partiets-mand-7-8',

From 2c58674e0ede5d656cb559a18a3b86ffbb21b3a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 22:46:53 +0600
Subject: [PATCH 1334/1937] [nhl] Improve _VALID_URL (#4805)

---
 youtube_dl/extractor/nhl.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index d3a4fc513..cb279074e 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -54,7 +54,7 @@ class NHLBaseInfoExtractor(InfoExtractor):
 
 class NHLIE(NHLBaseInfoExtractor):
     IE_NAME = 'nhl.com'
-    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/console(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
+    _VALID_URL = r'https?://video(?P<team>\.[^.]*)?\.nhl\.com/videocenter/(?:console)?(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
 
     _TESTS = [{
         'url': 'http://video.canucks.nhl.com/videocenter/console?catid=6?id=453614',
@@ -92,6 +92,9 @@ class NHLIE(NHLBaseInfoExtractor):
     }, {
         'url': 'http://video.flames.nhl.com/videocenter/console?id=630616',
         'only_matching': True,
+    }, {
+        'url': 'http://video.nhl.com/videocenter/?id=736722',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From e4c17d7274235ec880857d7691bc7c818e2c7a01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 23:12:27 +0600
Subject: [PATCH 1335/1937] [nhl:news] Add extractor (Closes #4805)

---
 youtube_dl/extractor/__init__.py |  6 ++++-
 youtube_dl/extractor/nhl.py      | 41 +++++++++++++++++++++++++++-----
 2 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d3f51d8c4..9c14b096f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -294,7 +294,11 @@ from .nextmedia import (
 )
 from .nfb import NFBIE
 from .nfl import NFLIE
-from .nhl import NHLIE, NHLVideocenterIE
+from .nhl import (
+    NHLIE,
+    NHLNewsIE,
+    NHLVideocenterIE,
+)
 from .niconico import NiconicoIE, NiconicoPlaylistIE
 from .ninegag import NineGagIE
 from .noco import NocoIE
diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index cb279074e..407465998 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -20,6 +20,12 @@ class NHLBaseInfoExtractor(InfoExtractor):
     def _fix_json(json_string):
         return json_string.replace('\\\'', '\'')
 
+    def _real_extract_video(self, video_id):
+        json_url = 'http://video.nhl.com/videocenter/servlets/playlist?ids=%s&format=json' % video_id
+        data = self._download_json(
+            json_url, video_id, transform_source=self._fix_json)
+        return self._extract_video(data[0])
+
     def _extract_video(self, info):
         video_id = info['id']
         self.report_extraction(video_id)
@@ -98,12 +104,35 @@ class NHLIE(NHLBaseInfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        json_url = 'http://video.nhl.com/videocenter/servlets/playlist?ids=%s&format=json' % video_id
-        data = self._download_json(
-            json_url, video_id, transform_source=self._fix_json)
-        return self._extract_video(data[0])
+        video_id = self._match_id(url)
+        return self._real_extract_video(video_id)
+
+
+class NHLNewsIE(NHLBaseInfoExtractor):
+    IE_NAME = 'nhl.com:news'
+    IE_DESC = 'NHL news'
+    _VALID_URL = r'https?://(?:www\.)?nhl\.com/ice/news\.html?(?:\?(?:.*?[?&])?)id=(?P<id>[-0-9a-zA-Z]+)'
+
+    _TEST = {
+        'url': 'http://www.nhl.com/ice/news.htm?id=750727',
+        'md5': '4b3d1262e177687a3009937bd9ec0be8',
+        'info_dict': {
+            'id': '736722',
+            'ext': 'mp4',
+            'title': 'Cal Clutterbuck has been fined $2,000',
+            'description': 'md5:45fe547d30edab88b23e0dd0ab1ed9e6',
+            'duration': 37,
+            'upload_date': '20150128',
+        },
+    }
+
+    def _real_extract(self, url):
+        news_id = self._match_id(url)
+        webpage = self._download_webpage(url, news_id)
+        video_id = self._search_regex(
+            [r'pVid(\d+)', r"nlid\s*:\s*'(\d+)'"],
+            webpage, 'video id')
+        return self._real_extract_video(video_id)
 
 
 class NHLVideocenterIE(NHLBaseInfoExtractor):

From b80505a40938444c6f7b0b80c445cc0726c39bb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 23:21:44 +0600
Subject: [PATCH 1336/1937] [mixcloud] Fix extraction (Closes #4784)

---
 youtube_dl/extractor/mixcloud.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 07d194562..162046fe6 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -10,6 +10,7 @@ from ..utils import (
     ExtractorError,
     HEADRequest,
     int_or_none,
+    str_to_int,
     parse_iso8601,
 )
 
@@ -85,15 +86,17 @@ class MixcloudIE(InfoExtractor):
         uploader_id = self._search_regex(
             r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False)
         description = self._og_search_description(webpage)
-        like_count = int_or_none(self._search_regex(
-            r'<meta itemprop="interactionCount" content="UserLikes:([0-9]+)"',
+        like_count = str_to_int(self._search_regex(
+            [r'<meta itemprop="interactionCount" content="UserLikes:([0-9]+)"',
+             r'/favorites/?">([0-9]+)<'],
             webpage, 'like count', fatal=False))
-        view_count = int_or_none(self._search_regex(
-            r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
+        view_count = str_to_int(self._search_regex(
+            [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
+             r'/listeners/?">([0-9,.]+)</a>'],
             webpage, 'play count', fatal=False))
         timestamp = parse_iso8601(self._search_regex(
             r'<time itemprop="dateCreated" datetime="([^"]+)">',
-            webpage, 'upload date'))
+            webpage, 'upload date', default=None))
 
         return {
             'id': track_id,

From 439d9be27dc3cf811f3337e40c3222bc8d2ed339 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 23:21:58 +0600
Subject: [PATCH 1337/1937] [mixcloud] Remove unused import

---
 youtube_dl/extractor/mixcloud.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 162046fe6..141bb463e 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -9,7 +9,6 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     HEADRequest,
-    int_or_none,
     str_to_int,
     parse_iso8601,
 )

From 0692ef86ef85da69074b00b5525c13bfde9159de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 30 Jan 2015 23:47:09 +0600
Subject: [PATCH 1338/1937] [bbccouk] Improve _VALID_URL

---
 youtube_dl/extractor/bbccouk.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 1cf48fe0d..126c8824c 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -10,7 +10,7 @@ from ..compat import compat_HTTPError
 class BBCCoUkIE(SubtitlesInfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
-    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
+    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
 
     _TESTS = [
         {
@@ -118,6 +118,9 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
         }, {
             'url': 'http://www.bbc.co.uk/music/clips#p02frcc3',
             'only_matching': True,
+        }, {
+            'url': 'http://www.bbc.co.uk/iplayer/cbeebies/episode/b0480276/bing-14-atchoo',
+            'only_matching': True,
         }
     ]
 

From 91755ee384ddee314ba40748a44ad1289df8b86c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 30 Jan 2015 19:43:46 +0100
Subject: [PATCH 1339/1937] [comedycentral:shows] Generate better IDs and add a
 test for that

---
 youtube_dl/extractor/comedycentral.py | 34 ++++++++++++++++++++++++++-
 1 file changed, 33 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 15ca361f0..f9256dcf4 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -49,7 +49,9 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
                               |(watch/(?P<date>[^/]*)/(?P<tdstitle>.*))
                           )|
                           (?P<interview>
-                              extended-interviews/(?P<interID>[0-9a-z]+)/(?:playlist_tds_extended_)?(?P<interview_title>.*?)(/.*?)?)))
+                              extended-interviews/(?P<interID>[0-9a-z]+)/
+                              (?:playlist_tds_extended_)?(?P<interview_title>[^/?#]*?)
+                              (?:/[^/?#]?|[?#]|$))))
                      '''
     _TESTS = [{
         'url': 'http://thedailyshow.cc.com/watch/thu-december-13-2012/kristen-stewart',
@@ -62,6 +64,35 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
             'uploader': 'thedailyshow',
             'title': 'thedailyshow kristen-stewart part 1',
         }
+    }, {
+        'url': 'http://thedailyshow.cc.com/extended-interviews/b6364d/sarah-chayes-extended-interview',
+        'info_dict': {
+            'id': 'sarah-chayes-extended-interview',
+            'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
+            'title': 'thedailyshow Sarah Chayes Extended Interview',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': '0baad492-cbec-4ec1-9e50-ad91c291127f',
+                'ext': 'mp4',
+                'upload_date': '20150129',
+                'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
+                'uploader': 'thedailyshow',
+                'title': 'thedailyshow sarah-chayes-extended-interview part 1',
+            }
+         }, {
+            'info_dict': {
+                'id': '1e4fb91b-8ce7-4277-bd7c-98c9f1bbd283',
+                'ext': 'mp4',
+                'upload_date': '20150129',
+                'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
+                'uploader': 'thedailyshow',
+                'title': 'thedailyshow sarah-chayes-extended-interview part 2',
+            }
+        }],
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'http://thedailyshow.cc.com/extended-interviews/xm3fnq/andrew-napolitano-extended-interview',
         'only_matching': True,
@@ -230,6 +261,7 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
 
         return {
             '_type': 'playlist',
+            'id': epTitle,
             'entries': entries,
             'title': show_name + ' ' + title,
             'description': description,

From 6d593c3276da6a0a1ad738c779b87a1998c5b06b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 31 Jan 2015 03:50:11 +0600
Subject: [PATCH 1340/1937] [YoutubeDL] Fix video+audio format_id (Closes
 #4824)

---
 youtube_dl/YoutubeDL.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ce3abaee3..e5a96cad5 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1074,7 +1074,8 @@ class YoutubeDL(object):
                             selected_format = {
                                 'requested_formats': formats_info,
                                 'format': rf,
-                                'format_id': rf,
+                                'format_id': '%s+%s' % (formats_info[0].get('format_id'),
+                                                        formats_info[1].get('format_id')),
                                 'width': formats_info[0].get('width'),
                                 'height': formats_info[0].get('height'),
                                 'resolution': formats_info[0].get('resolution'),

From cc57bd33a803a061ac0bbe971088fb1ef78be82b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 30 Jan 2015 23:05:06 +0100
Subject: [PATCH 1341/1937] [rtve] Fix extraction

Skip live stream test, we can't use the f4m manifest yet
---
 youtube_dl/extractor/rtve.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 0ce22d60c..9e8688fbf 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -74,7 +74,8 @@ class RTVEALaCartaIE(InfoExtractor):
             'id': '1694255',
             'ext': 'flv',
             'title': 'TODO',
-        }
+        },
+        'skip': 'The f4m manifest can\'t be used yet',
     }]
 
     def _real_extract(self, url):
@@ -86,6 +87,18 @@ class RTVEALaCartaIE(InfoExtractor):
         png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % video_id
         png = self._download_webpage(png_url, video_id, 'Downloading url information')
         video_url = _decrypt_url(png)
+        if not video_url.endswith('.f4m'):
+            auth_url = video_url.replace(
+                'resources/', 'auth/resources/'
+            ).replace('.net.rtve', '.multimedia.cdn.rtve')
+            video_path = self._download_webpage(
+                auth_url, video_id, 'Getting video url')
+            # Use mvod.akcdn instead of flash.akamaihd.multimedia.cdn to get
+            # the right Content-Length header and the mp4 format
+            video_url = (
+                'http://mvod.akcdn.rtve.es/{0}&v=2.6.8'
+                '&fp=MAC%2016,0,0,296&r=MRUGG&g=OEOJWFXNFGCP'.format(video_path)
+            )
 
         return {
             'id': video_id,

From cd596028d6c873ac7e9a61902bbe8f86d517b5b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 30 Jan 2015 23:12:53 +0100
Subject: [PATCH 1342/1937] [rtve] Recognize mobile urls (fixes #4823)

---
 youtube_dl/extractor/rtve.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 9e8688fbf..3469d9578 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -57,7 +57,7 @@ def _decrypt_url(png):
 class RTVEALaCartaIE(InfoExtractor):
     IE_NAME = 'rtve.es:alacarta'
     IE_DESC = 'RTVE a la carta'
-    _VALID_URL = r'http://www\.rtve\.es/alacarta/videos/[^/]+/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'http://www\.rtve\.es/(m/)?alacarta/videos/[^/]+/[^/]+/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/',
@@ -76,6 +76,9 @@ class RTVEALaCartaIE(InfoExtractor):
             'title': 'TODO',
         },
         'skip': 'The f4m manifest can\'t be used yet',
+    }, {
+        'url': 'http://www.rtve.es/m/alacarta/videos/cuentame-como-paso/cuentame-como-paso-t16-ultimo-minuto-nuestra-vida-capitulo-276/2969138/?media=tve',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From e41b1f73854d8a422235a3b92ca4255affbcd39e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 31 Jan 2015 10:51:39 +0100
Subject: [PATCH 1343/1937] Fix flake8 errors

---
 youtube_dl/downloader/f4m.py          |  2 +-
 youtube_dl/extractor/comedycentral.py | 41 ++++++++++++++-------------
 youtube_dl/extractor/spike.py         |  2 --
 3 files changed, 23 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 13055a67e..0e7a1c200 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -231,7 +231,7 @@ class F4mFD(FileDownloader):
     """
 
     def _get_unencrypted_media(self, doc):
-        media=doc.findall(_add_ns('media'))
+        media = doc.findall(_add_ns('media'))
         if not media:
             self.report_error('No media found')
         for e in (doc.findall(_add_ns('drmAdditionalHeader')) +
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index f9256dcf4..b24538981 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -71,25 +71,28 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
             'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
             'title': 'thedailyshow Sarah Chayes Extended Interview',
         },
-        'playlist': [{
-            'info_dict': {
-                'id': '0baad492-cbec-4ec1-9e50-ad91c291127f',
-                'ext': 'mp4',
-                'upload_date': '20150129',
-                'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
-                'uploader': 'thedailyshow',
-                'title': 'thedailyshow sarah-chayes-extended-interview part 1',
-            }
-         }, {
-            'info_dict': {
-                'id': '1e4fb91b-8ce7-4277-bd7c-98c9f1bbd283',
-                'ext': 'mp4',
-                'upload_date': '20150129',
-                'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
-                'uploader': 'thedailyshow',
-                'title': 'thedailyshow sarah-chayes-extended-interview part 2',
-            }
-        }],
+        'playlist': [
+            {
+                'info_dict': {
+                    'id': '0baad492-cbec-4ec1-9e50-ad91c291127f',
+                    'ext': 'mp4',
+                    'upload_date': '20150129',
+                    'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
+                    'uploader': 'thedailyshow',
+                    'title': 'thedailyshow sarah-chayes-extended-interview part 1',
+                },
+            },
+            {
+                'info_dict': {
+                    'id': '1e4fb91b-8ce7-4277-bd7c-98c9f1bbd283',
+                    'ext': 'mp4',
+                    'upload_date': '20150129',
+                    'description': 'Carnegie Endowment Senior Associate Sarah Chayes discusses how corrupt institutions function throughout the world in her book "Thieves of State: Why Corruption Threatens Global Security."',
+                    'uploader': 'thedailyshow',
+                    'title': 'thedailyshow sarah-chayes-extended-interview part 2',
+                },
+            },
+        ],
         'params': {
             'skip_download': True,
         },
diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index 1af2e9725..e529bb55c 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .mtv import MTVServicesInfoExtractor
 
 

From e77d2975afda3750104c90b7c0012c02a2a00b10 Mon Sep 17 00:00:00 2001
From: light94 <mishra.rahul1712@gmail.com>
Date: Sun, 1 Feb 2015 00:10:58 +0530
Subject: [PATCH 1344/1937] Handling Connection Reset by Peer Error

---
 youtube_dl/downloader/http.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 8a1d578d5..9b023c5c1 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -3,6 +3,9 @@ from __future__ import unicode_literals
 import os
 import time
 
+from socket import error as SocketError
+import errno
+
 from .common import FileDownloader
 from ..compat import (
     compat_urllib_request,
@@ -99,6 +102,11 @@ class HttpFD(FileDownloader):
                             resume_len = 0
                             open_mode = 'wb'
                             break
+
+            except SocketError as e:
+                if e.errno != errno.ECONNRESET:
+                    raise # Not error we are looking for
+                pass
             # Retry
             count += 1
             if count <= retries:

From 1ee94db2d03770c0280047c974f0e6886d2a753f Mon Sep 17 00:00:00 2001
From: kinetoskombi <filipeamiranda@gmail.com>
Date: Sat, 31 Jan 2015 20:07:43 -0200
Subject: [PATCH 1345/1937] [globo] Fix error on some globo videos

---
 youtube_dl/extractor/globo.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 6949a57c7..d936be384 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -70,6 +70,19 @@ class GloboIE(InfoExtractor):
                 'like_count': int,
             }
         },
+        {
+            'url': 'http://globotv.globo.com/canal-brasil/sangue-latino/t/todos-os-videos/v/ator-e-diretor-argentino-ricado-darin-fala-sobre-utopias-e-suas-perdas/3928201/',
+            'md5': 'c1defca721ce25b2354e927d3e4b3dec',
+            'info_dict': {
+                'id': '3928201',
+                'ext': 'mp4',
+                'title': 'Ator e diretor argentino, Ricado Darín fala sobre utopias e suas perdas',
+                'duration': 1472.906,
+                'uploader': 'Canal Brasil',
+                'uploader_id': 705,
+                'like_count': int,
+            }
+        },
     ]
 
     class MD5():
@@ -355,7 +368,9 @@ class GloboIE(InfoExtractor):
 
         for resource in video['resources']:
             resource_id = resource.get('_id')
-            if not resource_id:
+            resource_height = resource.get('height')
+
+            if not (resource_id or resource_height):
                 continue
 
             security = self._download_json(
@@ -384,7 +399,7 @@ class GloboIE(InfoExtractor):
             formats.append({
                 'url': '%s?h=%s&k=%s' % (resource['url'], signed_hash, 'flash'),
                 'format_id': resource_id,
-                'height': resource['height']
+                'height': resource_height
             })
 
         self._sort_formats(formats)

From 8c72beb25eaa1f1960a3b50295bcdcfd50c450af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Feb 2015 04:36:24 +0600
Subject: [PATCH 1346/1937] [globo] Properly extract m3u8 formats (#4346 #4832)

---
 youtube_dl/extractor/globo.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index d936be384..29638a194 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -368,9 +368,7 @@ class GloboIE(InfoExtractor):
 
         for resource in video['resources']:
             resource_id = resource.get('_id')
-            resource_height = resource.get('height')
-
-            if not (resource_id or resource_height):
+            if not resource_id:
                 continue
 
             security = self._download_json(
@@ -396,11 +394,16 @@ class GloboIE(InfoExtractor):
             signed_md5 = self.MD5.b64_md5(received_md5 + compat_str(sign_time) + padding)
             signed_hash = hash_code + compat_str(received_time) + received_random + compat_str(sign_time) + padding + signed_md5
 
-            formats.append({
-                'url': '%s?h=%s&k=%s' % (resource['url'], signed_hash, 'flash'),
-                'format_id': resource_id,
-                'height': resource_height
-            })
+            resource_url = resource['url']
+            signed_url = '%s?h=%s&k=%s' % (resource_url, signed_hash, 'flash')
+            if resource_id.endswith('m3u8') or resource_url.endswith('.m3u8'):
+                formats.extend(self._extract_m3u8_formats(signed_url, resource_id, 'mp4'))
+            else:
+                formats.append({
+                    'url': signed_url,
+                    'format_id': resource_id,
+                    'height': resource.get('height'),
+                })
 
         self._sort_formats(formats)
 

From 8f9312c38779f4e0746c01ef9a8a93ab4767699b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 11:30:56 +0100
Subject: [PATCH 1347/1937] Appease pyflakes8-3

---
 youtube_dl/YoutubeDL.py |  3 ++-
 youtube_dl/compat.py    | 37 +++++++++++++++++++++----------------
 youtube_dl/utils.py     |  7 ++++---
 3 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e5a96cad5..c18ce9660 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -25,6 +25,7 @@ if os.name == 'nt':
     import ctypes
 
 from .compat import (
+    compat_basestring,
     compat_cookiejar,
     compat_expanduser,
     compat_http_client,
@@ -1558,7 +1559,7 @@ class YoutubeDL(object):
         # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
         # To work around aforementioned issue we will replace request's original URL with
         # percent-encoded one
-        req_is_string = isinstance(req, basestring if sys.version_info < (3, 0) else compat_str)
+        req_is_string = isinstance(req, compat_basestring)
         url = req if req_is_string else req.get_full_url()
         url_escaped = escape_url(url)
 
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 497ca52de..ec685d50e 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -114,6 +114,26 @@ except ImportError:
             string += pct_sequence.decode(encoding, errors)
         return string
 
+try:
+    compat_str = unicode  # Python 2
+except NameError:
+    compat_str = str
+
+try:
+    compat_basestr = basestring  # Python 2
+except NameError:
+    compat_basestr = str
+
+try:
+    compat_chr = unichr  # Python 2
+except NameError:
+    compat_chr = chr
+
+try:
+    from xml.etree.ElementTree import ParseError as compat_xml_parse_error
+except ImportError:  # Python 2.6
+    from xml.parsers.expat import ExpatError as compat_xml_parse_error
+
 
 try:
     from urllib.parse import parse_qs as compat_parse_qs
@@ -123,7 +143,7 @@ except ImportError:  # Python 2
 
     def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
                    encoding='utf-8', errors='replace'):
-        qs, _coerce_result = qs, unicode
+        qs, _coerce_result = qs, compat_str
         pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
         r = []
         for name_value in pairs:
@@ -162,21 +182,6 @@ except ImportError:  # Python 2
                 parsed_result[name] = [value]
         return parsed_result
 
-try:
-    compat_str = unicode  # Python 2
-except NameError:
-    compat_str = str
-
-try:
-    compat_chr = unichr  # Python 2
-except NameError:
-    compat_chr = chr
-
-try:
-    from xml.etree.ElementTree import ParseError as compat_xml_parse_error
-except ImportError:  # Python 2.6
-    from xml.parsers.expat import ExpatError as compat_xml_parse_error
-
 try:
     from shlex import quote as shlex_quote
 except ImportError:  # Python < 3.3
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a4c9813ec..4ade0554e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -32,6 +32,7 @@ import xml.etree.ElementTree
 import zlib
 
 from .compat import (
+    compat_basestring,
     compat_chr,
     compat_getenv,
     compat_html_entities,
@@ -140,7 +141,7 @@ else:
     def find_xpath_attr(node, xpath, key, val):
         # Here comes the crazy part: In 2.6, if the xpath is a unicode,
         # .//node does not match if a node is a direct child of . !
-        if isinstance(xpath, unicode):
+        if isinstance(xpath, compat_str):
             xpath = xpath.encode('ascii')
 
         for f in node.findall(xpath):
@@ -1262,7 +1263,7 @@ def float_or_none(v, scale=1, invscale=1, default=None):
 
 
 def parse_duration(s):
-    if not isinstance(s, basestring if sys.version_info < (3, 0) else compat_str):
+    if not isinstance(s, compat_basestring):
         return None
 
     s = s.strip()
@@ -1426,7 +1427,7 @@ def uppercase_escape(s):
 
 def escape_rfc3986(s):
     """Escape non-ASCII characters as suggested by RFC 3986"""
-    if sys.version_info < (3, 0) and isinstance(s, unicode):
+    if sys.version_info < (3, 0) and isinstance(s, compat_str):
         s = s.encode('utf-8')
     return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
 

From 0196149c5bf6b72c1d268a4beb2c50e4ecb063d7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 11:36:59 +0100
Subject: [PATCH 1348/1937] [compat] Correct compat_basestring definition

---
 youtube_dl/compat.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index ec685d50e..e989cdbbd 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -120,9 +120,9 @@ except NameError:
     compat_str = str
 
 try:
-    compat_basestr = basestring  # Python 2
+    compat_basestring = basestring  # Python 2
 except NameError:
-    compat_basestr = str
+    compat_basestring = str
 
 try:
     compat_chr = unichr  # Python 2
@@ -367,6 +367,7 @@ def workaround_optparse_bug9161():
 
 __all__ = [
     'compat_HTTPError',
+    'compat_basestring',
     'compat_chr',
     'compat_cookiejar',
     'compat_expanduser',

From 9158b2b301f83a1cec13a6340f354646434c69c1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 11:58:37 +0100
Subject: [PATCH 1349/1937] [mpora] Modernize

---
 youtube_dl/extractor/mpora.py | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/mpora.py b/youtube_dl/extractor/mpora.py
index 88c9501cd..6db3c67a5 100644
--- a/youtube_dl/extractor/mpora.py
+++ b/youtube_dl/extractor/mpora.py
@@ -1,21 +1,19 @@
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
 from ..utils import int_or_none
 
 
 class MporaIE(InfoExtractor):
-    _VALID_URL = r'^https?://(www\.)?mpora\.(?:com|de)/videos/(?P<id>[^?#/]+)'
+    _VALID_URL = r'https?://(www\.)?mpora\.(?:com|de)/videos/(?P<id>[^?#/]+)'
     IE_NAME = 'MPORA'
 
     _TEST = {
         'url': 'http://mpora.de/videos/AAdo8okx4wiz/embed?locale=de',
-        'file': 'AAdo8okx4wiz.mp4',
         'md5': 'a7a228473eedd3be741397cf452932eb',
         'info_dict': {
+            'id': 'AAdo8okx4wiz',
+            'ext': 'mp4',
             'title': 'Katy Curd -  Winter in the Forest',
             'duration': 416,
             'uploader': 'Peter Newman Media',
@@ -23,14 +21,12 @@ class MporaIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+
         data_json = self._search_regex(
             r"new FM\.Player\('[^']+',\s*(\{.*?)\).player;", webpage, 'json')
-
-        data = json.loads(data_json)
+        data = self._parse_json(data_json, video_id)
 
         uploader = data['info_overlay'].get('username')
         duration = data['video']['duration'] // 1000

From 29aef5a33c46ceaa5884a3f074fbb5af4fa44f32 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 12:00:47 +0100
Subject: [PATCH 1350/1937] [ard] Remove deleted video test case

---
 youtube_dl/extractor/ard.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 967bd865c..783b53e23 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -23,13 +23,7 @@ class ARDMediathekIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht',
-        'file': '22429276.mp4',
-        'md5': '469751912f1de0816a9fc9df8336476c',
-        'info_dict': {
-            'title': 'Vertrauen ist gut, Spionieren ist besser - Geht so deutsch-amerikanische Freundschaft?',
-            'description': 'Das Erste Mediathek [ARD]: Vertrauen ist gut, Spionieren ist besser - Geht so deutsch-amerikanische Freundschaft?, Anne Will, Über die Spionage-Affäre diskutieren Clemens Binninger, Katrin Göring-Eckardt, Georg Mascolo, Andrew B. Denison und Constanze Kurz.. Das Video zur Sendung Anne Will am Mittwoch, 16.07.2014',
-        },
-        'skip': 'Blocked outside of Germany',
+        'only_matching': True,
     }, {
         'url': 'http://www.ardmediathek.de/tv/Tatort/Das-Wunder-von-Wolbeck-Video-tgl-ab-20/Das-Erste/Video?documentId=22490580&bcastId=602916',
         'info_dict': {

From ca0f500ecfde0a0ada7002348e8b04ac5c79d4b4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 12:08:21 +0100
Subject: [PATCH 1351/1937] [mtv] Modernize and clean up test

---
 youtube_dl/extractor/mtv.py | 34 +++++++++++++---------------------
 1 file changed, 13 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 7a3b62ebe..bc7f49ebb 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -79,12 +79,15 @@ class MTVServicesInfoExtractor(SubtitlesInfoExtractor):
             try:
                 _, _, ext = rendition.attrib['type'].partition('/')
                 rtmp_video_url = rendition.find('./src').text
-                formats.append({'ext': ext,
-                                'url': self._transform_rtmp_url(rtmp_video_url),
-                                'format_id': rendition.get('bitrate'),
-                                'width': int(rendition.get('width')),
-                                'height': int(rendition.get('height')),
-                                })
+                if rtmp_video_url.endswith('siteunavail.png'):
+                    continue
+                formats.append({
+                    'ext': ext,
+                    'url': self._transform_rtmp_url(rtmp_video_url),
+                    'format_id': rendition.get('bitrate'),
+                    'width': int(rendition.get('width')),
+                    'height': int(rendition.get('height')),
+                })
             except (KeyError, TypeError):
                 raise ExtractorError('Invalid rendition field.')
         self._sort_formats(formats)
@@ -240,25 +243,14 @@ class MTVIE(MTVServicesInfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.mtv.com/videos/misc/853555/ours-vh1-storytellers.jhtml',
-            'file': '853555.mp4',
             'md5': '850f3f143316b1e71fa56a4edfd6e0f8',
             'info_dict': {
+                'id': '853555',
+                'ext': 'mp4',
                 'title': 'Taylor Swift - "Ours (VH1 Storytellers)"',
                 'description': 'Album: Taylor Swift performs "Ours" for VH1 Storytellers at Harvey Mudd College.',
             },
         },
-        {
-            'add_ie': ['Vevo'],
-            'url': 'http://www.mtv.com/videos/taylor-swift/916187/everything-has-changed-ft-ed-sheeran.jhtml',
-            'file': 'USCJY1331283.mp4',
-            'md5': '73b4e7fcadd88929292fe52c3ced8caf',
-            'info_dict': {
-                'title': 'Everything Has Changed',
-                'upload_date': '20130606',
-                'uploader': 'Taylor Swift',
-            },
-            'skip': 'VEVO is only available in some countries',
-        },
     ]
 
     def _get_thumbnail_url(self, uri, itemdoc):
@@ -272,8 +264,8 @@ class MTVIE(MTVServicesInfoExtractor):
             webpage = self._download_webpage(url, video_id)
 
             # Some videos come from Vevo.com
-            m_vevo = re.search(r'isVevoVideo = true;.*?vevoVideoId = "(.*?)";',
-                               webpage, re.DOTALL)
+            m_vevo = re.search(
+                r'(?s)isVevoVideo = true;.*?vevoVideoId = "(.*?)";', webpage)
             if m_vevo:
                 vevo_id = m_vevo.group(1)
                 self.to_screen('Vevo video detected: %s' % vevo_id)

From 2886be15aa00e064c494eb0b6977246cdb905b1f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 12:10:15 +0100
Subject: [PATCH 1352/1937] [defense] Modernize

---
 youtube_dl/extractor/defense.py | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/defense.py b/youtube_dl/extractor/defense.py
index 5e50c63d9..1637fa4a7 100644
--- a/youtube_dl/extractor/defense.py
+++ b/youtube_dl/extractor/defense.py
@@ -8,33 +8,34 @@ from .common import InfoExtractor
 
 class DefenseGouvFrIE(InfoExtractor):
     IE_NAME = 'defense.gouv.fr'
-    _VALID_URL = (r'http://.*?\.defense\.gouv\.fr/layout/set/'
-                  r'ligthboxvideo/base-de-medias/webtv/(.*)')
+    _VALID_URL = r'http://.*?\.defense\.gouv\.fr/layout/set/ligthboxvideo/base-de-medias/webtv/(?P<id>[^/?#]*)'
 
     _TEST = {
         'url': 'http://www.defense.gouv.fr/layout/set/ligthboxvideo/base-de-medias/webtv/attaque-chimique-syrienne-du-21-aout-2013-1',
-        'file': '11213.mp4',
         'md5': '75bba6124da7e63d2d60b5244ec9430c',
-        "info_dict": {
-            "title": "attaque-chimique-syrienne-du-21-aout-2013-1"
+        'info_dict': {
+            'id': '11213',
+            'ext': 'mp4',
+            'title': 'attaque-chimique-syrienne-du-21-aout-2013-1'
         }
     }
 
     def _real_extract(self, url):
-        title = re.match(self._VALID_URL, url).group(1)
+        title = self._match_id(url)
         webpage = self._download_webpage(url, title)
+
         video_id = self._search_regex(
             r"flashvars.pvg_id=\"(\d+)\";",
             webpage, 'ID')
 
         json_url = ('http://static.videos.gouv.fr/brightcovehub/export/json/'
                     + video_id)
-        info = self._download_webpage(json_url, title,
-                                      'Downloading JSON config')
-        video_url = json.loads(info)['renditions'][0]['url']
+        info = self._download_json(json_url, title, 'Downloading JSON config')
+        video_url = info['renditions'][0]['url']
 
-        return {'id': video_id,
-                'ext': 'mp4',
-                'url': video_url,
-                'title': title,
-                }
+        return {
+            'id': video_id,
+            'ext': 'mp4',
+            'url': video_url,
+            'title': title,
+        }

From 96aded8d3d48c5987a687b9233351ccbfc9c9237 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 12:11:14 +0100
Subject: [PATCH 1353/1937] [rottentomatoes] Modernize test definition

---
 youtube_dl/extractor/rottentomatoes.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rottentomatoes.py b/youtube_dl/extractor/rottentomatoes.py
index c1500b82f..e8bb20a08 100644
--- a/youtube_dl/extractor/rottentomatoes.py
+++ b/youtube_dl/extractor/rottentomatoes.py
@@ -10,8 +10,9 @@ class RottenTomatoesIE(VideoDetectiveIE):
 
     _TEST = {
         'url': 'http://www.rottentomatoes.com/m/toy_story_3/trailers/11028566/',
-        'file': '613340.mp4',
         'info_dict': {
+            'id': '613340',
+            'ext': 'mp4',
             'title': 'TOY STORY 3',
             'description': 'From the creators of the beloved TOY STORY films, comes a story that will reunite the gang in a whole new way.',
         },

From 146303136fa30ef0edf2dfbac5acd69fb2f0dbcc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 12:11:20 +0100
Subject: [PATCH 1354/1937] [nerdcubed] Modernize test definition

---
 youtube_dl/extractor/nerdcubed.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/nerdcubed.py b/youtube_dl/extractor/nerdcubed.py
index efc903afa..dff78e486 100644
--- a/youtube_dl/extractor/nerdcubed.py
+++ b/youtube_dl/extractor/nerdcubed.py
@@ -11,6 +11,7 @@ class NerdCubedFeedIE(InfoExtractor):
     _TEST = {
         'url': 'http://www.nerdcubed.co.uk/feed.json',
         'info_dict': {
+            'id': 'nerdcubed-feed',
             'title': 'nerdcubed.co.uk feed',
         },
         'playlist_mincount': 1300,

From fc09240e24c2455fa9373f77a13bf4649b90ca87 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 12:12:27 +0100
Subject: [PATCH 1355/1937] [vimeo] Modernize test definition

---
 youtube_dl/extractor/vimeo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 06b0bed41..1bb473514 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -501,9 +501,10 @@ class VimeoReviewIE(InfoExtractor):
     _VALID_URL = r'https?://vimeo\.com/[^/]+/review/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'https://vimeo.com/user21297594/review/75524534/3c257a1b5d',
-        'file': '75524534.mp4',
         'md5': 'c507a72f780cacc12b2248bb4006d253',
         'info_dict': {
+            'id': '75524534',
+            'ext': 'mp4',
             'title': "DICK HARDWICK 'Comedian'",
             'uploader': 'Richard Hardwick',
         }

From 17d2712d9c58af788f234a2682d486f1d0a3ab78 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:00:54 +0100
Subject: [PATCH 1356/1937] [teamcoco] Modernize and fix extraction

---
 youtube_dl/extractor/teamcoco.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 5fa67eb8d..18a823719 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -10,17 +10,19 @@ class TeamcocoIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://teamcoco.com/video/80187/conan-becomes-a-mary-kay-beauty-consultant',
-            'file': '80187.mp4',
             'md5': '3f7746aa0dc86de18df7539903d399ea',
             'info_dict': {
+                'id': '80187',
+                'ext': 'mp4',
                 'title': 'Conan Becomes A Mary Kay Beauty Consultant',
                 'description': 'Mary Kay is perhaps the most trusted name in female beauty, so of course Conan is a natural choice to sell their products.'
             }
         }, {
             'url': 'http://teamcoco.com/video/louis-ck-interview-george-w-bush',
-            'file': '19705.mp4',
             'md5': 'cde9ba0fa3506f5f017ce11ead928f9a',
             'info_dict': {
+                'id': '19705',
+                'ext': 'mp4',
                 "description": "Louis C.K. got starstruck by George W. Bush, so what? Part one.",
                 "title": "Louis C.K. Interview Pt. 1 11/3/11"
             }
@@ -36,7 +38,7 @@ class TeamcocoIE(InfoExtractor):
         video_id = mobj.group("video_id")
         if not video_id:
             video_id = self._html_search_regex(
-                r'data-node-id="(\d+?)"',
+                r'<div\s+class="player".*?data-id="(\d+?)"',
                 webpage, 'video id')
 
         data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id

From 057c0609fc6620017ff4a5b30847e0dc2644df9f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:01:33 +0100
Subject: [PATCH 1357/1937] [toutv] Modernize test definition

---
 youtube_dl/extractor/toutv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py
index 2837f9c8e..4797d1310 100644
--- a/youtube_dl/extractor/toutv.py
+++ b/youtube_dl/extractor/toutv.py
@@ -16,8 +16,9 @@ class TouTvIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.tou.tv/30-vies/S04E41',
-        'file': '30-vies_S04E41.mp4',
         'info_dict': {
+            'id': '30-vies_S04E41',
+            'ext': 'mp4',
             'title': '30 vies Saison 4 / Épisode 41',
             'description': 'md5:da363002db82ccbe4dafeb9cab039b09',
             'age_limit': 8,

From 228d30ed0616f6c1b5cff335b8b56f4fde48b76d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:03:03 +0100
Subject: [PATCH 1358/1937] [la7] Modernize

---
 youtube_dl/extractor/la7.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/la7.py b/youtube_dl/extractor/la7.py
index db2028e9f..60625c42d 100644
--- a/youtube_dl/extractor/la7.py
+++ b/youtube_dl/extractor/la7.py
@@ -20,9 +20,10 @@ class LA7IE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.la7.tv/richplayer/?assetid=50355319',
-        'file': '50355319.mp4',
         'md5': 'ec7d1f0224d20ba293ab56cf2259651f',
         'info_dict': {
+            'id': '50355319',
+            'ext': 'mp4',
             'title': 'IL DIVO',
             'description': 'Un film di Paolo Sorrentino con Toni Servillo, Anna Bonaiuto, Giulio Bosetti  e Flavio Bucci',
             'duration': 6254,
@@ -31,9 +32,7 @@ class LA7IE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         xml_url = 'http://www.la7.tv/repliche/content/index.php?contentId=%s' % video_id
         doc = self._download_xml(xml_url, video_id)
 

From 9a3e5e69554c06fbf8bb7fd248c2e921593f8046 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:03:55 +0100
Subject: [PATCH 1359/1937] [kankan] Modernize

---
 youtube_dl/extractor/kankan.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/kankan.py b/youtube_dl/extractor/kankan.py
index dbfe4cc03..364dc878e 100644
--- a/youtube_dl/extractor/kankan.py
+++ b/youtube_dl/extractor/kankan.py
@@ -13,17 +13,17 @@ class KankanIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://yinyue.kankan.com/vod/48/48863.shtml',
-        'file': '48863.flv',
         'md5': '29aca1e47ae68fc28804aca89f29507e',
         'info_dict': {
+            'id': '48863',
+            'ext': 'flv',
             'title': 'Ready To Go',
         },
         'skip': 'Only available from China',
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         title = self._search_regex(r'(?:G_TITLE=|G_MOVIE_TITLE = )[\'"](.+?)[\'"]', webpage, 'video title')

From c4e817ce4aeeb4012b0e128afe6c138d3d905a24 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:06:55 +0100
Subject: [PATCH 1360/1937] [france2.fr:generation-quoi] Modernize

---
 youtube_dl/extractor/francetv.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index bbc760a49..170d68075 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -230,12 +230,13 @@ class FranceTVIE(FranceTVBaseInfoExtractor):
 
 class GenerationQuoiIE(InfoExtractor):
     IE_NAME = 'france2.fr:generation-quoi'
-    _VALID_URL = r'https?://generation-quoi\.france2\.fr/portrait/(?P<name>.*)(\?|$)'
+    _VALID_URL = r'https?://generation-quoi\.france2\.fr/portrait/(?P<id>[^/?#]+)'
 
     _TEST = {
         'url': 'http://generation-quoi.france2.fr/portrait/garde-a-vous',
-        'file': 'k7FJX8VBcvvLmX4wA5Q.mp4',
         'info_dict': {
+            'id': 'k7FJX8VBcvvLmX4wA5Q',
+            'ext': 'mp4',
             'title': 'Génération Quoi - Garde à Vous',
             'uploader': 'Génération Quoi',
         },
@@ -243,14 +244,12 @@ class GenerationQuoiIE(InfoExtractor):
             # It uses Dailymotion
             'skip_download': True,
         },
-        'skip': 'Only available from France',
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        name = mobj.group('name')
-        info_url = compat_urlparse.urljoin(url, '/medias/video/%s.json' % name)
-        info_json = self._download_webpage(info_url, name)
+        display_id = self._match_id(url)
+        info_url = compat_urlparse.urljoin(url, '/medias/video/%s.json' % display_id)
+        info_json = self._download_webpage(info_url, display_id)
         info = json.loads(info_json)
         return self.url_result('http://www.dailymotion.com/video/%s' % info['id'],
                                ie='Dailymotion')

From c1f06d6307109acfa4434a38131c7d9a07d58ab6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:08:33 +0100
Subject: [PATCH 1361/1937] [macgamestore] Modernize

---
 youtube_dl/extractor/macgamestore.py | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/macgamestore.py b/youtube_dl/extractor/macgamestore.py
index b818cf50c..3cd4a3a19 100644
--- a/youtube_dl/extractor/macgamestore.py
+++ b/youtube_dl/extractor/macgamestore.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import ExtractorError
 
@@ -13,21 +11,22 @@ class MacGameStoreIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://www.macgamestore.com/mediaviewer.php?trailer=2450',
-        'file': '2450.m4v',
         'md5': '8649b8ea684b6666b4c5be736ecddc61',
         'info_dict': {
+            'id': '2450',
+            'ext': 'm4v',
             'title': 'Crow',
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url, video_id, 'Downloading trailer page')
 
-        webpage = self._download_webpage(url, video_id, 'Downloading trailer page')
-
-        if re.search(r'>Missing Media<', webpage) is not None:
-            raise ExtractorError('Trailer %s does not exist' % video_id, expected=True)
+        if '>Missing Media<' in webpage:
+            raise ExtractorError(
+                'Trailer %s does not exist' % video_id, expected=True)
 
         video_title = self._html_search_regex(
             r'<title>MacGameStore: (.*?) Trailer</title>', webpage, 'title')

From 748a0fab8a35fdfde03407d49eb4b41e4be3a1fc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:08:50 +0100
Subject: [PATCH 1362/1937] Remove unused imports

---
 youtube_dl/extractor/defense.py | 3 ---
 youtube_dl/extractor/la7.py     | 2 --
 2 files changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/defense.py b/youtube_dl/extractor/defense.py
index 1637fa4a7..2b90bf4fc 100644
--- a/youtube_dl/extractor/defense.py
+++ b/youtube_dl/extractor/defense.py
@@ -1,8 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-import json
-
 from .common import InfoExtractor
 
 
diff --git a/youtube_dl/extractor/la7.py b/youtube_dl/extractor/la7.py
index 60625c42d..b08f6e3c9 100644
--- a/youtube_dl/extractor/la7.py
+++ b/youtube_dl/extractor/la7.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     parse_duration,

From 9f281cacd2bb1de0dc11dac35685f04dcb5f447c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:13:41 +0100
Subject: [PATCH 1363/1937] [keezmovies] Fix extraction and modernize test

---
 youtube_dl/extractor/keezmovies.py | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/keezmovies.py b/youtube_dl/extractor/keezmovies.py
index 97dcb518a..82eddec51 100644
--- a/youtube_dl/extractor/keezmovies.py
+++ b/youtube_dl/extractor/keezmovies.py
@@ -7,10 +7,6 @@ from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
-    compat_urllib_parse,
-)
-from ..aes import (
-    aes_decrypt_text
 )
 
 
@@ -18,9 +14,10 @@ class KeezMoviesIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?keezmovies\.com/video/.+?(?P<id>[0-9]+)(?:[/?&]|$)'
     _TEST = {
         'url': 'http://www.keezmovies.com/video/petite-asian-lady-mai-playing-in-bathtub-1214711',
-        'file': '1214711.mp4',
         'md5': '6e297b7e789329923fcf83abb67c9289',
         'info_dict': {
+            'id': '1214711',
+            'ext': 'mp4',
             'title': 'Petite Asian Lady Mai Playing In Bathtub',
             'age_limit': 18,
         }
@@ -39,11 +36,10 @@ class KeezMoviesIE(InfoExtractor):
             embedded_url = mobj.group(1)
             return self.url_result(embedded_url)
 
-        video_title = self._html_search_regex(r'<h1 [^>]*>([^<]+)', webpage, 'title')
-        video_url = compat_urllib_parse.unquote(self._html_search_regex(r'video_url=(.+?)&amp;', webpage, 'video_url'))
-        if 'encrypted=true' in webpage:
-            password = self._html_search_regex(r'video_title=(.+?)&amp;', webpage, 'password')
-            video_url = aes_decrypt_text(video_url, password, 32).decode('utf-8')
+        video_title = self._html_search_regex(
+            r'<h1 [^>]*>([^<]+)', webpage, 'title')
+        video_url = self._html_search_regex(
+            r'(?s)html5VideoPlayer = .*?src="([^"]+)"', webpage, 'video URL')
         path = compat_urllib_parse_urlparse(video_url).path
         extension = os.path.splitext(path)[1][1:]
         format = path.split('/')[4].split('_')[:2]

From 437cac8cc18e541dd1c4c55ef1b19de7d6529ae9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:16:35 +0100
Subject: [PATCH 1364/1937] [sina] Modernize and simplify

---
 youtube_dl/extractor/sina.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/sina.py b/youtube_dl/extractor/sina.py
index a63d126d4..0891a441f 100644
--- a/youtube_dl/extractor/sina.py
+++ b/youtube_dl/extractor/sina.py
@@ -11,7 +11,7 @@ from ..compat import (
 
 
 class SinaIE(InfoExtractor):
-    _VALID_URL = r'''https?://(.*?\.)?video\.sina\.com\.cn/
+    _VALID_URL = r'''(?x)https?://(.*?\.)?video\.sina\.com\.cn/
                         (
                             (.+?/(((?P<pseudo_id>\d+).html)|(.*?(\#|(vid=)|b/)(?P<id>\d+?)($|&|\-))))
                             |
@@ -23,9 +23,10 @@ class SinaIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://video.sina.com.cn/news/vlist/zt/chczlj2013/?opsubject_id=top12#110028898',
-            'file': '110028898.flv',
             'md5': 'd65dd22ddcf44e38ce2bf58a10c3e71f',
             'info_dict': {
+                'id': '110028898',
+                'ext': 'flv',
                 'title': '《中国新闻》 朝鲜要求巴拿马立即释放被扣船员',
             }
         },
@@ -39,10 +40,6 @@ class SinaIE(InfoExtractor):
         },
     ]
 
-    @classmethod
-    def suitable(cls, url):
-        return re.match(cls._VALID_URL, url, flags=re.VERBOSE) is not None
-
     def _extract_video(self, video_id):
         data = compat_urllib_parse.urlencode({'vid': video_id})
         url_doc = self._download_xml('http://v.iask.com/v_play.php?%s' % data,
@@ -59,7 +56,7 @@ class SinaIE(InfoExtractor):
                 }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
+        mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         if mobj.group('token') is not None:
             # The video id is in the redirected url

From e332772531d15bf15ccfc807085c6a1bdae163f3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:18:52 +0100
Subject: [PATCH 1365/1937] [servingsys] Modernize

---
 youtube_dl/extractor/servingsys.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/servingsys.py b/youtube_dl/extractor/servingsys.py
index 16dc3736b..c013d678f 100644
--- a/youtube_dl/extractor/servingsys.py
+++ b/youtube_dl/extractor/servingsys.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -13,10 +11,15 @@ class ServingSysIE(InfoExtractor):
 
     _TEST = {
         'url': 'http://bs.serving-sys.com/BurstingPipe/adServer.bs?cn=is&c=23&pl=VAST&pli=5349193&PluID=0&pos=7135&ord=[timestamp]&cim=1?',
+        'info_dict': {
+            'id': '5349193',
+            'title': 'AdAPPter_Hyundai_demo',
+        },
         'playlist': [{
-            'file': '29955898.flv',
             'md5': 'baed851342df6846eb8677a60a011a0f',
             'info_dict': {
+                'id': '29955898',
+                'ext': 'flv',
                 'title': 'AdAPPter_Hyundai_demo (1)',
                 'duration': 74,
                 'tbr': 1378,
@@ -24,9 +27,10 @@ class ServingSysIE(InfoExtractor):
                 'height': 400,
             },
         }, {
-            'file': '29907998.flv',
             'md5': '979b4da2655c4bc2d81aeb915a8c5014',
             'info_dict': {
+                'id': '29907998',
+                'ext': 'flv',
                 'title': 'AdAPPter_Hyundai_demo (2)',
                 'duration': 34,
                 'width': 854,
@@ -37,14 +41,13 @@ class ServingSysIE(InfoExtractor):
         'params': {
             'playlistend': 2,
         },
-        'skip': 'Blocked in the US [sic]',
+        '_skip': 'Blocked in the US [sic]',
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        pl_id = mobj.group('id')
-
+        pl_id = self._match_id(url)
         vast_doc = self._download_xml(url, pl_id)
+
         title = vast_doc.find('.//AdTitle').text
         media = vast_doc.find('.//MediaFile').text
         info_url = self._search_regex(r'&adData=([^&]+)&', media, 'info URL')

From c172440ac50b75eaee660a5c00f44c9c8a2f37c0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:21:02 +0100
Subject: [PATCH 1366/1937] [ringtv] Modernize test definition

---
 youtube_dl/extractor/ringtv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ringtv.py b/youtube_dl/extractor/ringtv.py
index 59dc137cc..efa4afeb6 100644
--- a/youtube_dl/extractor/ringtv.py
+++ b/youtube_dl/extractor/ringtv.py
@@ -6,12 +6,13 @@ from .common import InfoExtractor
 
 
 class RingTVIE(InfoExtractor):
-    _VALID_URL = r'(?:http://)?(?:www\.)?ringtv\.craveonline\.com/(?P<type>news|videos/video)/(?P<id>[^/?#]+)'
+    _VALID_URL = r'http://(?:www\.)?ringtv\.craveonline\.com/(?P<type>news|videos/video)/(?P<id>[^/?#]+)'
     _TEST = {
         "url": "http://ringtv.craveonline.com/news/310833-luis-collazo-says-victor-ortiz-better-not-quit-on-jan-30",
-        "file": "857645.mp4",
         "md5": "d25945f5df41cdca2d2587165ac28720",
         "info_dict": {
+            'id': '857645',
+            'ext': 'mp4',
             "title": 'Video: Luis Collazo says Victor Ortiz "better not quit on Jan. 30" - Ring TV',
             "description": 'Luis Collazo is excited about his Jan. 30 showdown with fellow former welterweight titleholder Victor Ortiz at Barclays Center in his hometown of Brooklyn. The SuperBowl week fight headlines a Golden Boy Live! card on Fox Sports 1.',
         }

From 4e980275b517b101d5cd3646fee6ced5f627da8a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:21:18 +0100
Subject: [PATCH 1367/1937] [test_download] Remove references to "file" in test
 definitions

---
 test/test_download.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_download.py b/test/test_download.py
index 412f3dbce..6a149ae4f 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -89,7 +89,7 @@ def generator(test_case):
 
         for tc in test_cases:
             info_dict = tc.get('info_dict', {})
-            if not tc.get('file') and not (info_dict.get('id') and info_dict.get('ext')):
+            if not (info_dict.get('id') and info_dict.get('ext')):
                 raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?')
 
         if 'skip' in test_case:
@@ -116,7 +116,7 @@ def generator(test_case):
         expect_warnings(ydl, test_case.get('expected_warnings', []))
 
         def get_tc_filename(tc):
-            return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {}))
+            return ydl.prepare_filename(tc.get('info_dict', {}))
 
         res_dict = None
 

From a9551e90207a2bb2ef8f35279fc525846e21cbe4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:24:38 +0100
Subject: [PATCH 1368/1937] [soundcloud:set] Fix test definition

---
 youtube_dl/extractor/soundcloud.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 5d60c4939..c5284fa67 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -246,6 +246,7 @@ class SoundcloudSetIE(SoundcloudIE):
     _TESTS = [{
         'url': 'https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep',
         'info_dict': {
+            'id': '2284613',
             'title': 'The Royal Concept EP',
         },
         'playlist_mincount': 6,
@@ -279,7 +280,7 @@ class SoundcloudSetIE(SoundcloudIE):
         return {
             '_type': 'playlist',
             'entries': [self._extract_info_dict(track, secret_token=token) for track in info['tracks']],
-            'id': info['id'],
+            'id': '%s' % info['id'],
             'title': info['title'],
         }
 

From 654bd52f5847dc3f1604611f7f1ca3beafee7046 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:25:33 +0100
Subject: [PATCH 1369/1937] [teletask] Fix test definition

---
 youtube_dl/extractor/teletask.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/teletask.py b/youtube_dl/extractor/teletask.py
index e54145105..b9e2ef8ca 100644
--- a/youtube_dl/extractor/teletask.py
+++ b/youtube_dl/extractor/teletask.py
@@ -11,6 +11,7 @@ class TeleTaskIE(InfoExtractor):
     _TEST = {
         'url': 'http://www.tele-task.de/archive/video/html5/26168/',
         'info_dict': {
+            'id': '26168',
             'title': 'Duplicate Detection',
         },
         'playlist': [{
@@ -34,7 +35,6 @@ class TeleTaskIE(InfoExtractor):
 
     def _real_extract(self, url):
         lecture_id = self._match_id(url)
-
         webpage = self._download_webpage(url, lecture_id)
 
         title = self._html_search_regex(

From dd8982f19c97a539fc11ee0ee74bb6c6fb6beb57 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:27:16 +0100
Subject: [PATCH 1370/1937] [wdr] Complete test definition

---
 youtube_dl/extractor/wdr.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 313b9c15d..c90488500 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -71,6 +71,9 @@ class WDRIE(InfoExtractor):
         {
             'url': 'http://www1.wdr.de/mediathek/video/sendungen/quarks_und_co/filterseite-quarks-und-co100.html',
             'playlist_mincount': 146,
+            'info_dict': {
+                'id': 'mediathek/video/sendungen/quarks_und_co/filterseite-quarks-und-co100',
+            }
         }
     ]
 

From acf757f42eea16266a539fa0eb871dae422a1d22 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 15:33:32 +0100
Subject: [PATCH 1371/1937] [youtube] Complete test definition

---
 youtube_dl/extractor/youtube.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3b3678c6e..e4b26b84f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1160,6 +1160,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
     }, {
         'url': 'https://www.youtube.com/playlist?list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
         'info_dict': {
+            'id': 'PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
             'title': 'YDL_Empty_List',
         },
         'playlist_count': 0,
@@ -1168,6 +1169,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
         'info_dict': {
             'title': '29C3: Not my department',
+            'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
         },
         'playlist_count': 95,
     }, {
@@ -1175,6 +1177,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'url': 'PLBB231211A4F62143',
         'info_dict': {
             'title': '[OLD]Team Fortress 2 (Class-based LP)',
+            'id': 'PLBB231211A4F62143',
         },
         'playlist_mincount': 26,
     }, {
@@ -1182,12 +1185,14 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
         'info_dict': {
             'title': 'Uploads from Cauchemar',
+            'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
         },
         'playlist_mincount': 799,
     }, {
         'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
         'info_dict': {
             'title': 'YDL_safe_search',
+            'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
         },
         'playlist_count': 2,
     }, {
@@ -1196,6 +1201,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'playlist_count': 4,
         'info_dict': {
             'title': 'JODA15',
+            'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
         }
     }, {
         'note': 'Embedded SWF player',
@@ -1203,12 +1209,14 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
         'playlist_count': 4,
         'info_dict': {
             'title': 'JODA7',
+            'id': 'YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ',
         }
     }, {
         'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
         'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
         'info_dict': {
-                'title': 'Uploads from Interstellar Movie',
+            'title': 'Uploads from Interstellar Movie',
+            'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
         },
         'playlist_mincout': 21,
     }]
@@ -1314,6 +1322,9 @@ class YoutubeChannelIE(InfoExtractor):
         'note': 'paginated channel',
         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
         'playlist_mincount': 91,
+        'info_dict': {
+            'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+        }
     }]
 
     def extract_videos_from_page(self, page):

From c6df692466aebed1b86c02233b579c03f0888704 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Feb 2015 23:48:23 +0600
Subject: [PATCH 1372/1937] [rutv] Extract all m3u8 formats

---
 youtube_dl/extractor/rutv.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index a73e6f331..ef766237b 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -162,10 +162,8 @@ class RUTVIE(InfoExtractor):
                         'vbr': int(quality),
                     }
                 elif transport == 'm3u8':
-                    fmt = {
-                        'url': url,
-                        'ext': 'mp4',
-                    }
+                    formats.extend(self._extract_m3u8_formats(url, video_id, 'mp4'))
+                    continue
                 else:
                     fmt = {
                         'url': url

From 027008b14ebe3d41ac7b9940ab73eaed120dab5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Feb 2015 23:49:23 +0600
Subject: [PATCH 1373/1937] [hls] Fix encode issues on python2 @ Windows

---
 youtube_dl/downloader/hls.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index aa58b52ab..b642786cc 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -11,6 +11,7 @@ from ..compat import (
     compat_urllib_request,
 )
 from ..utils import (
+    encodeArgument,
     encodeFilename,
 )
 
@@ -21,18 +22,17 @@ class HlsFD(FileDownloader):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
-        args = [
-            '-y', '-i', url, '-f', 'mp4', '-c', 'copy',
-            '-bsf:a', 'aac_adtstoasc',
-            encodeFilename(tmpfilename, for_subprocess=True)]
-
         ffpp = FFmpegPostProcessor(downloader=self)
         program = ffpp._executable
         if program is None:
             self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
             return False
         ffpp.check_version()
-        cmd = [program] + args
+
+        args = [encodeArgument(opt) for opt in ('-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
+        args.append(encodeFilename(tmpfilename, True))
+
+        cmd = [encodeArgument(program)] + args
 
         retval = subprocess.call(cmd)
         if retval == 0:

From 4a3da4ebdb1de5e35b47176822ab9ed3b2762c3f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Feb 2015 23:54:38 +0600
Subject: [PATCH 1374/1937] [hls] Simplify

---
 youtube_dl/downloader/hls.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index b642786cc..e527ee425 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -29,15 +29,15 @@ class HlsFD(FileDownloader):
             return False
         ffpp.check_version()
 
-        args = [encodeArgument(opt) for opt in ('-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
+        args = [
+            encodeArgument(opt)
+            for opt in (program, '-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
         args.append(encodeFilename(tmpfilename, True))
 
-        cmd = [encodeArgument(program)] + args
-
-        retval = subprocess.call(cmd)
+        retval = subprocess.call(args)
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
-            self.to_screen('\r[%s] %s bytes' % (cmd[0], fsize))
+            self.to_screen('\r[%s] %s bytes' % (args[0], fsize))
             self.try_rename(tmpfilename, filename)
             self._hook_progress({
                 'downloaded_bytes': fsize,

From 9e3f19919aa3e152760f4cd0a7cad348428f3b35 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 22:38:26 +0100
Subject: [PATCH 1375/1937] [jsinterp] Beef up and add tests

In preparation for #4822, extend jsinterp by a lot. (We may even have to/want to write a proper interpreter with actual parsing)
---
 test/test_jsinterp.py  |  95 +++++++++++++++++++++++
 youtube_dl/jsinterp.py | 172 ++++++++++++++++++++++++++++++-----------
 2 files changed, 221 insertions(+), 46 deletions(-)
 create mode 100644 test/test_jsinterp.py

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
new file mode 100644
index 000000000..b91b8c492
--- /dev/null
+++ b/test/test_jsinterp.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.jsinterp import JSInterpreter
+
+
+class TestJSInterpreter(unittest.TestCase):
+    def test_basic(self):
+        jsi = JSInterpreter('function x(){;}')
+        self.assertEqual(jsi.call_function('x'), None)
+
+        jsi = JSInterpreter('function x3(){return 42;}')
+        self.assertEqual(jsi.call_function('x3'), 42)
+
+    def test_calc(self):
+        jsi = JSInterpreter('function x4(a){return 2*a+1;}')
+        self.assertEqual(jsi.call_function('x4', 3), 7)
+
+    def test_empty_return(self):
+        jsi = JSInterpreter('function f(){return; y()}')
+        self.assertEqual(jsi.call_function('f'), None)
+
+    def test_morespace(self):
+        jsi = JSInterpreter('function x (a) { return 2 * a + 1 ; }')
+        self.assertEqual(jsi.call_function('x', 3), 7)
+
+        jsi = JSInterpreter('function f () { x =  2  ; return x; }')
+        self.assertEqual(jsi.call_function('f'), 2)
+
+    def test_strange_chars(self):
+        jsi = JSInterpreter('function $_xY1 ($_axY1) { var $_axY2 = $_axY1 + 1; return $_axY2; }')
+        self.assertEqual(jsi.call_function('$_xY1', 20), 21)
+
+    def test_operators(self):
+        jsi = JSInterpreter('function f(){return 1 << 5;}')
+        self.assertEqual(jsi.call_function('f'), 32)
+
+        jsi = JSInterpreter('function f(){return 19 & 21;}')
+        self.assertEqual(jsi.call_function('f'), 17)
+
+        jsi = JSInterpreter('function f(){return 11 >> 2;}')
+        self.assertEqual(jsi.call_function('f'), 2)
+
+    def test_array_access(self):
+        jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2] = 7; return x;}')
+        self.assertEqual(jsi.call_function('f'), [5, 2, 7])
+
+    def test_parens(self):
+        jsi = JSInterpreter('function f(){return (1) + (2) * ((( (( (((((3)))))) )) ));}')
+        self.assertEqual(jsi.call_function('f'), 7)
+
+        jsi = JSInterpreter('function f(){return (1 + 2) * 3;}')
+        self.assertEqual(jsi.call_function('f'), 9)
+
+    def test_assignments(self):
+        jsi = JSInterpreter('function f(){var x = 20; x = 30 + 1; return x;}')
+        self.assertEqual(jsi.call_function('f'), 31)
+
+        jsi = JSInterpreter('function f(){var x = 20; x += 30 + 1; return x;}')
+        self.assertEqual(jsi.call_function('f'), 51)
+
+        jsi = JSInterpreter('function f(){var x = 20; x -= 30 + 1; return x;}')
+        self.assertEqual(jsi.call_function('f'), -11)
+
+    def test_comments(self):
+        jsi = JSInterpreter('''
+        function x() {
+            var x = /* 1 + */ 2;
+            var y = /* 30
+            * 40 */ 50;
+            return x + y;
+        }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 52)
+
+    def test_precedence(self):
+        jsi = JSInterpreter('''
+        function x() {
+            var a = [10, 20, 30, 40, 50];
+            var b = 6;
+            a[0]=a[b%a.length];
+            return a;
+        }''')
+        self.assertEqual(jsi.call_function('x'), [20, 20, 30, 40, 50])
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index b4617fbad..49364786b 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -1,59 +1,122 @@
 from __future__ import unicode_literals
 
 import json
+import operator
 import re
 
 from .utils import (
     ExtractorError,
 )
 
+_OPERATORS = [
+    ('|', operator.or_),
+    ('^', operator.xor),
+    ('&', operator.and_),
+    ('>>', operator.rshift),
+    ('<<', operator.lshift),
+    ('-', operator.sub),
+    ('+', operator.add),
+    ('%', operator.mod),
+    ('/', operator.div),
+    ('*', operator.mul),
+]
+_ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS]
+_ASSIGN_OPERATORS.append(('=', lambda cur, right: right))
+
+_NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'
+
 
 class JSInterpreter(object):
-    def __init__(self, code):
-        self.code = code
+    def __init__(self, code, objects=None):
+        if objects is None:
+            objects = {}
+        self.code = self._remove_comments(code)
         self._functions = {}
-        self._objects = {}
+        self._objects = objects
 
-    def interpret_statement(self, stmt, local_vars, allow_recursion=20):
+    def _remove_comments(self, code):
+        return re.sub(r'(?s)/\*.*?\*/', '', code)
+
+    def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if allow_recursion < 0:
             raise ExtractorError('Recursion limit reached')
 
-        if stmt.startswith('var '):
-            stmt = stmt[len('var '):]
-        ass_m = re.match(r'^(?P<out>[a-z]+)(?:\[(?P<index>[^\]]+)\])?' +
-                         r'=(?P<expr>.*)$', stmt)
-        if ass_m:
-            if ass_m.groupdict().get('index'):
-                def assign(val):
-                    lvar = local_vars[ass_m.group('out')]
-                    idx = self.interpret_expression(
-                        ass_m.group('index'), local_vars, allow_recursion)
-                    assert isinstance(idx, int)
-                    lvar[idx] = val
-                    return val
-                expr = ass_m.group('expr')
-            else:
-                def assign(val):
-                    local_vars[ass_m.group('out')] = val
-                    return val
-                expr = ass_m.group('expr')
-        elif stmt.startswith('return '):
-            assign = lambda v: v
-            expr = stmt[len('return '):]
+        should_abort = False
+        stmt = stmt.lstrip()
+        stmt_m = re.match(r'var\s', stmt)
+        if stmt_m:
+            expr = stmt[len(stmt_m.group(0)):]
         else:
-            # Try interpreting it as an expression
-            expr = stmt
-            assign = lambda v: v
+            return_m = re.match(r'return(?:\s+|$)', stmt)
+            if return_m:
+                expr = stmt[len(return_m.group(0)):]
+                should_abort = True
+            else:
+                # Try interpreting it as an expression
+                expr = stmt
 
         v = self.interpret_expression(expr, local_vars, allow_recursion)
-        return assign(v)
+        return v, should_abort
 
     def interpret_expression(self, expr, local_vars, allow_recursion):
+        expr = expr.strip()
+
+        if expr == '':  # Empty expression
+            return None
+
+        if expr.startswith('('):
+            parens_count = 0
+            for m in re.finditer(r'[()]', expr):
+                if m.group(0) == '(':
+                    parens_count += 1
+                else:
+                    parens_count -= 1
+                    if parens_count == 0:
+                        sub_expr = expr[1:m.start()]
+                        sub_result = self.interpret_expression(
+                            sub_expr, local_vars, allow_recursion)
+                        remaining_expr = expr[m.end():].strip()
+                        if not remaining_expr:
+                            return sub_result
+                        else:
+                            expr = json.dumps(sub_result) + remaining_expr
+                        break
+            else:
+                raise ExtractorError('Premature end of parens in %r' % expr)
+
+        for op, opfunc in _ASSIGN_OPERATORS:
+            m = re.match(r'''(?x)
+                (?P<out>%s)(?:\[(?P<index>[^\]]+?)\])?
+                \s*%s
+                (?P<expr>.*)$''' % (_NAME_RE, re.escape(op)), expr)
+            if not m:
+                continue
+            right_val = self.interpret_expression(
+                m.group('expr'), local_vars, allow_recursion - 1)
+
+            if m.groupdict().get('index'):
+                lvar = local_vars[m.group('out')]
+                idx = self.interpret_expression(
+                    m.group('index'), local_vars, allow_recursion)
+                assert isinstance(idx, int)
+                cur = lvar[idx]
+                val = opfunc(cur, right_val)
+                lvar[idx] = val
+                return val
+            else:
+                cur = local_vars.get(m.group('out'))
+                val = opfunc(cur, right_val)
+                local_vars[m.group('out')] = val
+                return val
+
         if expr.isdigit():
             return int(expr)
 
-        if expr.isalpha():
-            return local_vars[expr]
+        var_m = re.match(
+            r'(?!if|return|true|false)(?P<name>%s)$' % _NAME_RE,
+            expr)
+        if var_m:
+            return local_vars[var_m.group('name')]
 
         try:
             return json.loads(expr)
@@ -61,7 +124,7 @@ class JSInterpreter(object):
             pass
 
         m = re.match(
-            r'^(?P<var>[$a-zA-Z0-9_]+)\.(?P<member>[^(]+)(?:\(+(?P<args>[^()]*)\))?$',
+            r'(?P<var>%s)\.(?P<member>[^(]+)(?:\(+(?P<args>[^()]*)\))?$' % _NAME_RE,
             expr)
         if m:
             variable = m.group('var')
@@ -114,23 +177,31 @@ class JSInterpreter(object):
             return obj[member](argvals)
 
         m = re.match(
-            r'^(?P<in>[a-z]+)\[(?P<idx>.+)\]$', expr)
+            r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
         if m:
             val = local_vars[m.group('in')]
             idx = self.interpret_expression(
                 m.group('idx'), local_vars, allow_recursion - 1)
             return val[idx]
 
-        m = re.match(r'^(?P<a>.+?)(?P<op>[%])(?P<b>.+?)$', expr)
-        if m:
-            a = self.interpret_expression(
-                m.group('a'), local_vars, allow_recursion)
-            b = self.interpret_expression(
-                m.group('b'), local_vars, allow_recursion)
-            return a % b
+        for op, opfunc in _OPERATORS:
+            m = re.match(r'(?P<x>.+?)%s(?P<y>.+)' % re.escape(op), expr)
+            if not m:
+                continue
+            x, abort = self.interpret_statement(
+                m.group('x'), local_vars, allow_recursion - 1)
+            if abort:
+                raise ExtractorError(
+                    'Premature left-side return of %s in %r' % (op, expr))
+            y, abort = self.interpret_statement(
+                m.group('y'), local_vars, allow_recursion - 1)
+            if abort:
+                raise ExtractorError(
+                    'Premature right-side return of %s in %r' % (op, expr))
+            return opfunc(x, y)
 
         m = re.match(
-            r'^(?P<func>[a-zA-Z$]+)\((?P<args>[a-z0-9,]+)\)$', expr)
+            r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]+)\)$' % _NAME_RE, expr)
         if m:
             fname = m.group('func')
             argvals = tuple([
@@ -139,6 +210,7 @@ class JSInterpreter(object):
             if fname not in self._functions:
                 self._functions[fname] = self.extract_function(fname)
             return self._functions[fname](argvals)
+
         raise ExtractorError('Unsupported JS expression %r' % expr)
 
     def extract_object(self, objname):
@@ -162,9 +234,11 @@ class JSInterpreter(object):
 
     def extract_function(self, funcname):
         func_m = re.search(
-            (r'(?:function %s|[{;]%s\s*=\s*function)' % (
-                re.escape(funcname), re.escape(funcname))) +
-            r'\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}',
+            r'''(?x)
+                (?:function\s+%s|[{;]%s\s*=\s*function)\s*
+                \((?P<args>[^)]*)\)\s*
+                \{(?P<code>[^}]+)\}''' % (
+                re.escape(funcname), re.escape(funcname)),
             self.code)
         if func_m is None:
             raise ExtractorError('Could not find JS function %r' % funcname)
@@ -172,10 +246,16 @@ class JSInterpreter(object):
 
         return self.build_function(argnames, func_m.group('code'))
 
+    def call_function(self, funcname, *args):
+        f = self.extract_function(funcname)
+        return f(args)
+
     def build_function(self, argnames, code):
         def resf(args):
             local_vars = dict(zip(argnames, args))
             for stmt in code.split(';'):
-                res = self.interpret_statement(stmt, local_vars)
+                res, abort = self.interpret_statement(stmt, local_vars)
+                if abort:
+                    break
             return res
         return resf

From 0a7055c90dac10e7f4c3393922bccf546081262b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Feb 2015 22:42:19 +0100
Subject: [PATCH 1376/1937] release 2015.02.01

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5e6288e8e..e46ca56a9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.01.30.2'
+__version__ = '2015.02.01'

From 044131ba213be1bf277561432bcb98a07bbdb265 Mon Sep 17 00:00:00 2001
From: Sahebjot singh <sahebjot94@gmail.com>
Date: Mon, 2 Feb 2015 03:49:28 +0530
Subject: [PATCH 1377/1937] supported sites was not opening

required a .md at the end.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 09da0129e..68b41970a 100644
--- a/README.md
+++ b/README.md
@@ -534,7 +534,7 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
 
 ### How can I detect whether a given URL is supported by youtube-dl?
 
-For one, have a look at the [list of supported sites](docs/supportedsites). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
+For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
 
 It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
 

From 8cfb6efe6f4a007a4faab132d08717cea0cdfbaf Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 01:49:32 +0100
Subject: [PATCH 1378/1937] [jsinterp] Correct div command

---
 youtube_dl/jsinterp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 49364786b..453e2732c 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -17,7 +17,7 @@ _OPERATORS = [
     ('-', operator.sub),
     ('+', operator.add),
     ('%', operator.mod),
-    ('/', operator.div),
+    ('/', operator.truediv),
     ('*', operator.mul),
 ]
 _ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS]

From acf2a6e97b53b191edf3f92af1eae300633ea2d9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 01:49:40 +0100
Subject: [PATCH 1379/1937] release 2015.02.02

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e46ca56a9..1e469d93b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.01'
+__version__ = '2015.02.02'

From 12832049170e8b3bd457706f49e8e09de12a368c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 12:05:39 +0100
Subject: [PATCH 1380/1937] [http] PEP8 (#4831)

---
 youtube_dl/downloader/http.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 9b023c5c1..49170cf9d 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -102,11 +102,11 @@ class HttpFD(FileDownloader):
                             resume_len = 0
                             open_mode = 'wb'
                             break
-
             except SocketError as e:
                 if e.errno != errno.ECONNRESET:
-                    raise # Not error we are looking for
-                pass
+                    # Connection reset is no problem, just retry
+                    raise
+
             # Retry
             count += 1
             if count <= retries:

From 92bf0bcdf847682cee9f66a5cef4c96c19a100d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Feb 2015 20:52:37 +0600
Subject: [PATCH 1381/1937] [historicfilms] Add extractor (Closes #4825)

---
 youtube_dl/extractor/__init__.py      |  1 +
 youtube_dl/extractor/historicfilms.py | 46 +++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 youtube_dl/extractor/historicfilms.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9c14b096f..dabf86ec8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -182,6 +182,7 @@ from .heise import HeiseIE
 from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
+from .historicfilms import HistoricFilmsIE
 from .hitbox import HitboxIE, HitboxLiveIE
 from .hornbunny import HornBunnyIE
 from .hostingbulk import HostingBulkIE
diff --git a/youtube_dl/extractor/historicfilms.py b/youtube_dl/extractor/historicfilms.py
new file mode 100644
index 000000000..40afbe537
--- /dev/null
+++ b/youtube_dl/extractor/historicfilms.py
@@ -0,0 +1,46 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import parse_duration
+
+
+class HistoricFilmsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?historicfilms\.com/(?:tapes/|play)(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.historicfilms.com/tapes/4728',
+        'md5': 'd4a437aec45d8d796a38a215db064e9a',
+        'info_dict': {
+            'id': '4728',
+            'ext': 'mov',
+            'title': 'Historic Films: GP-7',
+            'description': 'md5:1a86a0f3ac54024e419aba97210d959a',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 2096,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        tape_id = self._search_regex(
+            r'class="tapeId">([^<]+)<', webpage, 'tape id')
+
+        title = self._og_search_title(webpage)
+        description = self._og_search_description(webpage)
+        thumbnail = self._html_search_meta(
+            'thumbnailUrl', webpage, 'thumbnails') or self._og_search_thumbnail(webpage)
+        duration = parse_duration(self._html_search_meta(
+            'duration', webpage, 'duration'))
+
+        video_url = 'http://www.historicfilms.com/video/%s_%s_web.mov' % (tape_id, video_id)
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+        }

From 78271e33197e56452dc60f075bd326cb908e8e7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Feb 2015 21:11:25 +0600
Subject: [PATCH 1382/1937] [drtv] Extract material id (Closes #4814)

---
 youtube_dl/extractor/drtv.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index 8d113a5e8..d5df18d7c 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -25,9 +25,15 @@ class DRTVIE(SubtitlesInfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        programcard = self._download_json(
-            'http://www.dr.dk/mu/programcard/expanded/%s' % video_id, video_id, 'Downloading video JSON')
+        webpage = self._download_webpage(url, video_id)
 
+        video_id = self._search_regex(
+            r'data-(?:material-identifier|episode-slug)="([^"]+)"',
+            webpage, 'video id')
+
+        programcard = self._download_json(
+            'http://www.dr.dk/mu/programcard/expanded/%s' % video_id,
+            video_id, 'Downloading video JSON')
         data = programcard['Data'][0]
 
         title = data['Title']

From 6a66904f8ed469a547d8781fd61b26a97a6ae7aa Mon Sep 17 00:00:00 2001
From: vijayanand nandam <vijay@cybrilla.com>
Date: Mon, 2 Feb 2015 20:51:40 +0530
Subject: [PATCH 1383/1937] passing source address option to external
 downloaders

---
 youtube_dl/downloader/external.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index af9fdba75..3d6554fe4 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -45,6 +45,12 @@ class ExternalFD(FileDownloader):
     def supports(cls, info_dict):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
 
+    def _source_address(self, command_option):
+        command_part = []
+        if self.ydl.params['source_address'] is not None:
+            command_part = [command_option, self.ydl.params['source_address']]
+        return command_part
+
     def _call_downloader(self, tmpfilename, info_dict):
         """ Either overwrite this or implement _make_cmd """
         cmd = self._make_cmd(tmpfilename, info_dict)
@@ -72,6 +78,7 @@ class CurlFD(ExternalFD):
         cmd = [self.exe, '-o', tmpfilename]
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += self._source_address('--interface')
         cmd += ['--', info_dict['url']]
         return cmd
 
@@ -81,6 +88,7 @@ class WgetFD(ExternalFD):
         cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += self._source_address('--bind-address')
         cmd += ['--', info_dict['url']]
         return cmd
 
@@ -96,6 +104,7 @@ class Aria2cFD(ExternalFD):
         cmd += ['--out', os.path.basename(tmpfilename)]
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += self._source_address('--interface')
         cmd += ['--', info_dict['url']]
         return cmd
 

From 469a64cebf534b702e50ad9585e63ae3f8ed5d16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Feb 2015 21:40:52 +0600
Subject: [PATCH 1384/1937] [downloader/external] Simplify

---
 youtube_dl/downloader/external.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 3d6554fe4..012057ecb 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -47,8 +47,9 @@ class ExternalFD(FileDownloader):
 
     def _source_address(self, command_option):
         command_part = []
-        if self.ydl.params['source_address'] is not None:
-            command_part = [command_option, self.ydl.params['source_address']]
+        source_address = self.ydl.params.get('source_address')
+        if source_address:
+            command_part = [command_option, source_address]
         return command_part
 
     def _call_downloader(self, tmpfilename, info_dict):

From ae7773942ecf816e9592e55bdef7234f41c89c85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Feb 2015 21:51:38 +0600
Subject: [PATCH 1385/1937] [downloader/external] Simplify

---
 youtube_dl/downloader/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 012057ecb..faac11805 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -47,7 +47,7 @@ class ExternalFD(FileDownloader):
 
     def _source_address(self, command_option):
         command_part = []
-        source_address = self.ydl.params.get('source_address')
+        source_address = self.params.get('source_address')
         if source_address:
             command_part = [command_option, source_address]
         return command_part

From 9fffd0469f1353c5f5e96b91a87e34d02ae7491a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 20:28:18 +0100
Subject: [PATCH 1386/1937] [options] Mark --fixup as non-experimental and
 correct its help

---
 youtube_dl/downloader/external.py | 2 +-
 youtube_dl/options.py             | 5 ++---
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 012057ecb..e47e3bf4c 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -48,7 +48,7 @@ class ExternalFD(FileDownloader):
     def _source_address(self, command_option):
         command_part = []
         source_address = self.ydl.params.get('source_address')
-        if source_address:
+        if source_address is None:
             command_part = [command_option, source_address]
         return command_part
 
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 5e4a1ad9b..d5e575964 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -698,10 +698,9 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--fixup',
         metavar='POLICY', dest='fixup', default='detect_or_warn',
-        help='(experimental) Automatically correct known faults of the file. '
+        help='Automatically correct known faults of the file. '
              'One of never (do nothing), warn (only emit a warning), '
-             'detect_or_warn(check whether we can do anything about it, warn '
-             'otherwise')
+             'detect_or_warn(the default; fix file if we can, warn otherwise)')
     postproc.add_option(
         '--prefer-avconv',
         action='store_false', dest='prefer_ffmpeg',

From cc9ca3ba6e0ba24b318ef191fdf84927e773008d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 20:33:23 +0100
Subject: [PATCH 1387/1937] [downloader/external] Simplify source_address

'' might actually be passed in, so check for None.
---
 youtube_dl/downloader/external.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index faac11805..ff031d2e0 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -46,11 +46,10 @@ class ExternalFD(FileDownloader):
         return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')
 
     def _source_address(self, command_option):
-        command_part = []
         source_address = self.params.get('source_address')
-        if source_address:
-            command_part = [command_option, source_address]
-        return command_part
+        if source_address is None:
+            return []
+        return [command_option, source_address]
 
     def _call_downloader(self, tmpfilename, info_dict):
         """ Either overwrite this or implement _make_cmd """

From 869b4aeff4239b02ff8f699741902fe7f91b999a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 20:35:04 +0100
Subject: [PATCH 1388/1937] release 2015.02.02.1

---
 README.md             | 10 +++++-----
 youtube_dl/version.py |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 68b41970a..d432904fa 100644
--- a/README.md
+++ b/README.md
@@ -368,11 +368,11 @@ which means you can modify it, redistribute it or use it however you like.
     --add-metadata                   write metadata to the video file
     --xattrs                         write metadata to the video file's xattrs
                                      (using dublin core and xdg standards)
-    --fixup POLICY                   (experimental) Automatically correct known
-                                     faults of the file. One of never (do
-                                     nothing), warn (only emit a warning),
-                                     detect_or_warn(check whether we can do
-                                     anything about it, warn otherwise
+    --fixup POLICY                   Automatically correct known faults of the
+                                     file. One of never (do nothing), warn (only
+                                     emit a warning), detect_or_warn(the
+                                     default; fix file if we can, warn
+                                     otherwise)
     --prefer-avconv                  Prefer avconv over ffmpeg for running the
                                      postprocessors (default)
     --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1e469d93b..94c181a46 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.02'
+__version__ = '2015.02.02.1'

From 3fd45e03bf86c2ad2ef17362452a9a745dc10755 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 20:43:02 +0100
Subject: [PATCH 1389/1937] [ntvru] Rename from NTV to clarify the difference
 between n-tv.de and ntv.ru

---
 youtube_dl/extractor/__init__.py          | 2 +-
 youtube_dl/extractor/{ntv.py => ntvru.py} | 6 ++----
 2 files changed, 3 insertions(+), 5 deletions(-)
 rename youtube_dl/extractor/{ntv.py => ntvru.py} (97%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index dabf86ec8..bd707e028 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -317,7 +317,7 @@ from .nrk import (
     NRKIE,
     NRKTVIE,
 )
-from .ntv import NTVIE
+from .ntvru import NTVRuIE
 from .nytimes import NYTimesIE
 from .nuvid import NuvidIE
 from .oktoberfesttv import OktoberfestTVIE
diff --git a/youtube_dl/extractor/ntv.py b/youtube_dl/extractor/ntvru.py
similarity index 97%
rename from youtube_dl/extractor/ntv.py
rename to youtube_dl/extractor/ntvru.py
index ee740cd9c..8b9166d48 100644
--- a/youtube_dl/extractor/ntv.py
+++ b/youtube_dl/extractor/ntvru.py
@@ -9,7 +9,7 @@ from ..utils import (
 )
 
 
-class NTVIE(InfoExtractor):
+class NTVRuIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?ntv\.ru/(?P<id>.+)'
 
     _TESTS = [
@@ -92,9 +92,7 @@ class NTVIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         page = self._download_webpage(url, video_id)
 
         video_id = self._html_search_regex(self._VIDEO_ID_REGEXES, page, 'video id')

From 8f4b58d70e29df13d6002579b5c9448acc89d6c1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 21:48:54 +0100
Subject: [PATCH 1390/1937] [ntvde] Add new extractor (Fixes #4850)

---
 test/test_utils.py               | 12 ++++++
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ntvde.py    | 65 ++++++++++++++++++++++++++++++++
 youtube_dl/extractor/ntvru.py    |  1 +
 youtube_dl/utils.py              |  9 ++++-
 5 files changed, 86 insertions(+), 2 deletions(-)
 create mode 100644 youtube_dl/extractor/ntvde.py

diff --git a/test/test_utils.py b/test/test_utils.py
index ebec7986f..0ffccd35f 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -238,6 +238,8 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_duration('5 s'), 5)
         self.assertEqual(parse_duration('3 min'), 180)
         self.assertEqual(parse_duration('2.5 hours'), 9000)
+        self.assertEqual(parse_duration('02:03:04'), 7384)
+        self.assertEqual(parse_duration('01:02:03:04'), 93784)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
@@ -371,6 +373,16 @@ class TestUtil(unittest.TestCase):
         on = js_to_json('{"abc": true}')
         self.assertEqual(json.loads(on), {'abc': True})
 
+        # Ignore JavaScript code as well
+        on = js_to_json('''{
+            "x": 1,
+            y: "a",
+            z: some.code
+        }''')
+        d = json.loads(on)
+        self.assertEqual(d['x'], 1)
+        self.assertEqual(d['y'], 'a')
+
     def test_clean_html(self):
         self.assertEqual(clean_html('a:\nb'), 'a: b')
         self.assertEqual(clean_html('a:\n   "b"'), 'a:    "b"')
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index bd707e028..205bd4338 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -317,6 +317,7 @@ from .nrk import (
     NRKIE,
     NRKTVIE,
 )
+from .ntvde import NTVDeIE
 from .ntvru import NTVRuIE
 from .nytimes import NYTimesIE
 from .nuvid import NuvidIE
diff --git a/youtube_dl/extractor/ntvde.py b/youtube_dl/extractor/ntvde.py
new file mode 100644
index 000000000..a4f628fcd
--- /dev/null
+++ b/youtube_dl/extractor/ntvde.py
@@ -0,0 +1,65 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    parse_duration,
+)
+
+
+class NTVDeIE(InfoExtractor):
+    IE_NAME = 'n-tv.de'
+    _VALID_URL = r'https?://(?:www\.)?n-tv\.de/mediathek/videos/[^/?#]+/[^/?#]+-article(?P<id>.+)\.html'
+
+    _TESTS = [{
+        'url': 'http://www.n-tv.de/mediathek/videos/panorama/Schnee-und-Glaette-fuehren-zu-zahlreichen-Unfaellen-und-Staus-article14438086.html',
+        'md5': 'd37b7df1eea32265c51a062499ca488f',
+        'info_dict': {
+            'id': '14438086',
+            'ext': 'mp4',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'title': 'Schnee und Glätte führen zu zahlreichen Unfällen und Staus',
+            'alt_title': 'Winterchaos auf deutschen Straßen',
+            'description': 'Schnee und Glätte sorgen deutschlandweit für einen chaotischen Start in die Woche: Auf den Straßen kommt es zu kilometerlangen Staus und Dutzenden Glätteunfällen. In Düsseldorf und München wirbelt der Schnee zudem den Flugplan durcheinander. Dutzende Flüge landen zu spät, einige fallen ganz aus.',
+            'duration': 4020,
+            'timestamp': 1422892797,
+            'upload_date': '20150202',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        info = self._parse_json(self._search_regex(
+            r'(?s)ntv.pageInfo.article =\s(\{.*?\});', webpage, 'info'),
+            video_id, transform_source=js_to_json)
+        timestamp = int_or_none(info.get('publishedDateAsUnixTimeStamp'))
+        vdata = self._parse_json(self._search_regex(
+            r'(?s)\$\(\s*"\#player"\s*\)\s*\.data\(\s*"player",\s*(\{.*?\})\);',
+            webpage, 'player data'),
+            video_id, transform_source=js_to_json)
+        duration = parse_duration(vdata.get('duration'))
+        formats = [{
+            'format_id': 'flash',
+            'url': 'rtmp://fms.n-tv.de/' + vdata['video'],
+        }, {
+            'format_id': 'mp4',
+            'url': 'http://video.n-tv.de' + vdata['videoMp4'],
+        }]
+        m3u8_url = 'http://video.n-tv.de' + vdata['videoM3u8']
+        formats.extend(self._extract_m3u8_formats(m3u8_url, video_id))
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': info['headline'],
+            'description': info.get('intro'),
+            'alt_title': info.get('kicker'),
+            'timestamp': timestamp,
+            'thumbnail': vdata.get('html5VideoPoster'),
+            'duration': duration,
+            'formats': formats,
+        }
diff --git a/youtube_dl/extractor/ntvru.py b/youtube_dl/extractor/ntvru.py
index 8b9166d48..963bb715a 100644
--- a/youtube_dl/extractor/ntvru.py
+++ b/youtube_dl/extractor/ntvru.py
@@ -10,6 +10,7 @@ from ..utils import (
 
 
 class NTVRuIE(InfoExtractor):
+    IE_NAME = 'ntv.ru'
     _VALID_URL = r'http://(?:www\.)?ntv\.ru/(?P<id>.+)'
 
     _TESTS = [
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4ade0554e..251074bf5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1275,7 +1275,10 @@ def parse_duration(s):
             (?P<only_hours>[0-9.]+)\s*(?:hours?)|
 
             (?:
-                (?:(?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*)?
+                (?:
+                    (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
+                    (?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*
+                )?
                 (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
             )?
             (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
@@ -1293,6 +1296,8 @@ def parse_duration(s):
         res += int(m.group('mins')) * 60
     if m.group('hours'):
         res += int(m.group('hours')) * 60 * 60
+    if m.group('days'):
+        res += int(m.group('days')) * 24 * 60 * 60
     if m.group('ms'):
         res += float(m.group('ms'))
     return res
@@ -1543,7 +1548,7 @@ def js_to_json(code):
     res = re.sub(r'''(?x)
         "(?:[^"\\]*(?:\\\\|\\")?)*"|
         '(?:[^'\\]*(?:\\\\|\\')?)*'|
-        [a-zA-Z_][a-zA-Z_0-9]*
+        [a-zA-Z_][.a-zA-Z_0-9]*
         ''', fix_kv, code)
     res = re.sub(r',(\s*\])', lambda m: m.group(1), res)
     return res

From 7a7bd19c4537a4abd0ccea166610c9f2353046a4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 21:57:48 +0100
Subject: [PATCH 1391/1937] [n-tv.de] Use native m3u8 as best format

---
 youtube_dl/extractor/ntvde.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ntvde.py b/youtube_dl/extractor/ntvde.py
index a4f628fcd..d2cfe0961 100644
--- a/youtube_dl/extractor/ntvde.py
+++ b/youtube_dl/extractor/ntvde.py
@@ -15,7 +15,7 @@ class NTVDeIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.n-tv.de/mediathek/videos/panorama/Schnee-und-Glaette-fuehren-zu-zahlreichen-Unfaellen-und-Staus-article14438086.html',
-        'md5': 'd37b7df1eea32265c51a062499ca488f',
+        'md5': '6ef2514d4b1e8e03ca24b49e2f167153',
         'info_dict': {
             'id': '14438086',
             'ext': 'mp4',
@@ -46,11 +46,14 @@ class NTVDeIE(InfoExtractor):
             'format_id': 'flash',
             'url': 'rtmp://fms.n-tv.de/' + vdata['video'],
         }, {
-            'format_id': 'mp4',
+            'format_id': 'mobile',
             'url': 'http://video.n-tv.de' + vdata['videoMp4'],
+            'tbr': 400,  # estimation
         }]
         m3u8_url = 'http://video.n-tv.de' + vdata['videoM3u8']
-        formats.extend(self._extract_m3u8_formats(m3u8_url, video_id))
+        formats.extend(self._extract_m3u8_formats(
+            m3u8_url, video_id, ext='mp4',
+            entry_protocol='m3u8_native', preference=0))
         self._sort_formats(formats)
 
         return {

From f920ce295e65dc1446120c9375ef11bfd6cede26 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 21:58:17 +0100
Subject: [PATCH 1392/1937] [ntvru] Remove unused import

---
 youtube_dl/extractor/ntvru.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/ntvru.py b/youtube_dl/extractor/ntvru.py
index 963bb715a..0ab8d5100 100644
--- a/youtube_dl/extractor/ntvru.py
+++ b/youtube_dl/extractor/ntvru.py
@@ -1,8 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     unescapeHTML

From c21b1fbeeb0305fbf7804fcad9d4aa2dc6288de6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 21:58:58 +0100
Subject: [PATCH 1393/1937] release 2015.02.02.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 94c181a46..c1d17844e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.02.1'
+__version__ = '2015.02.02.2'

From 7d3d06a16cb585be35c2152f4bd4cba9c32081a3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 22:48:12 +0100
Subject: [PATCH 1394/1937] [vevo] Restore SMIL support (#3656)

---
 youtube_dl/extractor/vevo.py | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 43f6b029d..c17094f81 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -9,6 +9,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    int_or_none,
 )
 
 
@@ -192,9 +193,29 @@ class VevoIE(InfoExtractor):
         # Download via HLS API
         formats.extend(self._download_api_formats(video_id))
 
+        # Download SMIL
+        smil_blocks = sorted((
+            f for f in video_info['videoVersions']
+            if f['sourceType'] == 13),
+            key=lambda f: f['version'])
+        smil_url = '%s/Video/V2/VFILE/%s/%sr.smil' % (
+            self._SMIL_BASE_URL, video_id, video_id.lower())
+        if smil_blocks:
+            smil_url_m = self._search_regex(
+                r'url="([^"]+)"', smil_blocks[-1]['data'], 'SMIL URL',
+                default=None)
+            if smil_url_m is not None:
+                smil_url = smil_url_m
+        if smil_url:
+            smil_xml = self._download_webpage(
+                smil_url, video_id, 'Downloading SMIL info', fatal=False)
+            if smil_xml:
+                formats.extend(self._formats_from_smil(smil_xml))
+
         self._sort_formats(formats)
-        timestamp_ms = int(self._search_regex(
-            r'/Date\((\d+)\)/', video_info['launchDate'], 'launch date'))
+        timestamp_ms = int_or_none(self._search_regex(
+            r'/Date\((\d+)\)/',
+            video_info['launchDate'], 'launch date', fatal=False))
 
         return {
             'id': video_id,

From 3a4cca687f674fd5b93fafbe80453b23ef7ea748 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 22:56:15 +0100
Subject: [PATCH 1395/1937] release 2015.02.02.3

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c1d17844e..dadec36d3 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.02.2'
+__version__ = '2015.02.02.3'

From 531572578e542fcee26f1a30fd9f87ad158e6594 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 23:04:39 +0100
Subject: [PATCH 1396/1937] [normalboots] Modernize

---
 youtube_dl/extractor/normalboots.py | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/normalboots.py b/youtube_dl/extractor/normalboots.py
index c13ff0d65..78e75266f 100644
--- a/youtube_dl/extractor/normalboots.py
+++ b/youtube_dl/extractor/normalboots.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class NormalbootsIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?normalboots\.com/video/(?P<videoid>[0-9a-z-]*)/?$'
+    _VALID_URL = r'http://(?:www\.)?normalboots\.com/video/(?P<id>[0-9a-z-]*)/?$'
     _TEST = {
         'url': 'http://normalboots.com/video/home-alone-games-jontron/',
         'md5': '8bf6de238915dd501105b44ef5f1e0f6',
@@ -30,19 +30,22 @@ class NormalbootsIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('videoid')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        video_uploader = self._html_search_regex(r'Posted\sby\s<a\shref="[A-Za-z0-9/]*">(?P<uploader>[A-Za-z]*)\s</a>',
-                                                 webpage, 'uploader')
-        raw_upload_date = self._html_search_regex('<span style="text-transform:uppercase; font-size:inherit;">[A-Za-z]+, (?P<date>.*)</span>',
-                                                  webpage, 'date')
-        video_upload_date = unified_strdate(raw_upload_date)
 
-        player_url = self._html_search_regex(r'<iframe\swidth="[0-9]+"\sheight="[0-9]+"\ssrc="(?P<url>[\S]+)"', webpage, 'url')
+        video_uploader = self._html_search_regex(
+            r'Posted\sby\s<a\shref="[A-Za-z0-9/]*">(?P<uploader>[A-Za-z]*)\s</a>',
+            webpage, 'uploader', fatal=False)
+        video_upload_date = unified_strdate(self._html_search_regex(
+            r'<span style="text-transform:uppercase; font-size:inherit;">[A-Za-z]+, (?P<date>.*)</span>',
+            webpage, 'date', fatal=False))
+
+        player_url = self._html_search_regex(
+            r'<iframe\swidth="[0-9]+"\sheight="[0-9]+"\ssrc="(?P<url>[\S]+)"',
+            webpage, 'player url')
         player_page = self._download_webpage(player_url, video_id)
-        video_url = self._html_search_regex(r"file:\s'(?P<file>[^']+\.mp4)'", player_page, 'file')
+        video_url = self._html_search_regex(
+            r"file:\s'(?P<file>[^']+\.mp4)'", player_page, 'file')
 
         return {
             'id': video_id,

From a57b562cff57a5cbd06cb22b37962861a423e428 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 2 Feb 2015 23:13:28 +0100
Subject: [PATCH 1397/1937] [nfl] Add support for articles pages (fixes #4848)

---
 youtube_dl/extractor/nfl.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nfl.py b/youtube_dl/extractor/nfl.py
index 606e2294e..2684dd250 100644
--- a/youtube_dl/extractor/nfl.py
+++ b/youtube_dl/extractor/nfl.py
@@ -46,7 +46,18 @@ class NFLIE(InfoExtractor):
                 'timestamp': 1388354455,
                 'thumbnail': 're:^https?://.*\.jpg$',
             }
-        }
+        },
+        {
+            'url': 'http://www.nfl.com/news/story/0ap3000000467586/article/patriots-seahawks-involved-in-lategame-skirmish',
+            'info_dict': {
+                'id': '0ap3000000467607',
+                'ext': 'mp4',
+                'title': 'Frustrations flare on the field',
+                'description': 'Emotions ran high at the end of the Super Bowl on both sides of the ball after a dramatic finish.',
+                'timestamp': 1422850320,
+                'upload_date': '20150202',
+            },
+        },
     ]
 
     @staticmethod
@@ -80,7 +91,11 @@ class NFLIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         config_url = NFLIE.prepend_host(host, self._search_regex(
-            r'(?:config|configURL)\s*:\s*"([^"]+)"', webpage, 'config URL'))
+            r'(?:config|configURL)\s*:\s*"([^"]+)"', webpage, 'config URL',
+            default='static/content/static/config/video/config.json'))
+        # For articles, the id in the url is not the video id
+        video_id = self._search_regex(
+            r'contentId\s*:\s*"([^"]+)"', webpage, 'video id', default=video_id)
         config = self._download_json(config_url, video_id,
                                      note='Downloading player config')
         url_template = NFLIE.prepend_host(

From 995029a142cd0046ee43b583f2d09fc3d5fa321a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 23:38:35 +0100
Subject: [PATCH 1398/1937] [nerdist] Add new extractor (Fixes #4851)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/common.py   |  7 ++-
 youtube_dl/extractor/nerdist.py  | 80 ++++++++++++++++++++++++++++++++
 3 files changed, 86 insertions(+), 2 deletions(-)
 create mode 100644 youtube_dl/extractor/nerdist.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 205bd4338..5dcb14feb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -285,6 +285,7 @@ from .ndr import NDRIE
 from .ndtv import NDTVIE
 from .netzkino import NetzkinoIE
 from .nerdcubed import NerdCubedFeedIE
+from .nerdist import NerdistIE
 from .newgrounds import NewgroundsIE
 from .newstube import NewstubeIE
 from .nextmedia import (
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8b4ef3f09..9e517e3ac 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -860,10 +860,13 @@ class InfoExtractor(object):
         return formats
 
     # TODO: improve extraction
-    def _extract_smil_formats(self, smil_url, video_id):
+    def _extract_smil_formats(self, smil_url, video_id, fatal=True):
         smil = self._download_xml(
             smil_url, video_id, 'Downloading SMIL file',
-            'Unable to download SMIL file')
+            'Unable to download SMIL file', fatal=fatal)
+        if smil is False:
+            assert not fatal
+            return []
 
         base = smil.find('./head/meta').get('base')
 
diff --git a/youtube_dl/extractor/nerdist.py b/youtube_dl/extractor/nerdist.py
new file mode 100644
index 000000000..c6dc34be4
--- /dev/null
+++ b/youtube_dl/extractor/nerdist.py
@@ -0,0 +1,80 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import (
+    determine_ext,
+    parse_iso8601,
+    xpath_text,
+)
+
+
+class NerdistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?nerdist\.com/vepisode/(?P<id>[^/?#]+)'
+    _TEST = {
+        'url': 'http://www.nerdist.com/vepisode/exclusive-which-dc-characters-w',
+        'md5': '3698ed582931b90d9e81e02e26e89f23',
+        'info_dict': {
+            'display_id': 'exclusive-which-dc-characters-w',
+            'id': 'RPHpvJyr',
+            'ext': 'mp4',
+            'title': 'Your TEEN TITANS Revealed! Who\'s on the show?',
+            'thumbnail': 're:^https?://.*/thumbs/.*\.jpg$',
+            'description': 'Exclusive: Find out which DC Comics superheroes will star in TEEN TITANS Live-Action TV Show on Nerdist News with Jessica Chobot!',
+            'uploader': 'Eric Diaz',
+            'upload_date': '20150202',
+            'timestamp': 1422892808,
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._search_regex(
+            r'''(?x)<script\s+(?:type="text/javascript"\s+)?
+                src="https?://content\.nerdist\.com/players/([a-zA-Z0-9_]+)-''',
+            webpage, 'video ID')
+        timestamp = parse_iso8601(self._html_search_meta(
+            'shareaholic:article_published_time', webpage, 'upload date'))
+        uploader = self._html_search_meta(
+            'shareaholic:article_author_name', webpage, 'article author')
+
+        doc = self._download_xml(
+            'http://content.nerdist.com/jw6/%s.xml' % video_id, video_id)
+        video_info = doc.find('.//item')
+        title = xpath_text(video_info, './title', fatal=True)
+        description = xpath_text(video_info, './description')
+        thumbnail = xpath_text(
+            video_info, './{http://rss.jwpcdn.com/}image', 'thumbnail')
+
+        formats = []
+        for source in video_info.findall('./{http://rss.jwpcdn.com/}source'):
+            vurl = source.attrib['file']
+            ext = determine_ext(vurl)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    vurl, video_id, entry_protocol='m3u8_native', ext='mp4',
+                    preference=0))
+            elif ext == 'smil':
+                formats.extend(self._extract_smil_formats(
+                    vurl, video_id, fatal=False
+                ))
+            else:
+                formats.append({
+                    'format_id': ext,
+                    'url': vurl,
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+            'formats': formats,
+            'uploader': uploader,
+        }

From 5c4862bad4ba38f4ddd86d4fb1305aa5b82b2bd6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 23:38:45 +0100
Subject: [PATCH 1399/1937] [normalboots] Remove unused import

---
 youtube_dl/extractor/normalboots.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/youtube_dl/extractor/normalboots.py b/youtube_dl/extractor/normalboots.py
index 78e75266f..5952d136f 100644
--- a/youtube_dl/extractor/normalboots.py
+++ b/youtube_dl/extractor/normalboots.py
@@ -1,8 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 from ..utils import (

From 55898ad2cf4a5cf7628cf921fb5785ed0c7dceff Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 23:39:03 +0100
Subject: [PATCH 1400/1937] release 2015.02.02.4

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index dadec36d3..22464a358 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.02.3'
+__version__ = '2015.02.02.4'

From c1147c05e172f8d8c2b2b8d752eda3f385f1bed1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 23:47:14 +0100
Subject: [PATCH 1401/1937] [brightcove] Fix up more generically invalid XML
 (Fixes #4849)

---
 youtube_dl/extractor/brightcove.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 003152c4e..ea0969d4d 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -108,7 +108,7 @@ class BrightcoveIE(InfoExtractor):
         """
 
         # Fix up some stupid HTML, see https://github.com/rg3/youtube-dl/issues/1553
-        object_str = re.sub(r'(<param name="[^"]+" value="[^"]+")>',
+        object_str = re.sub(r'(<param(?:\s+[a-zA-Z0-9_]+="[^"]*")*)>',
                             lambda m: m.group(1) + '/>', object_str)
         # Fix up some stupid XML, see https://github.com/rg3/youtube-dl/issues/1608
         object_str = object_str.replace('<--', '<!--')

From d0eca21021b6321197caccab110de13843be6e04 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Feb 2015 23:47:19 +0100
Subject: [PATCH 1402/1937] release 2015.02.02.5

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 22464a358..46af42171 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.02.4'
+__version__ = '2015.02.02.5'

From d200b11c7ea34138d9b8a778fd2dd9226308b48f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 00:14:42 +0100
Subject: [PATCH 1403/1937] [Makefile] Simplify clean/cleanall

---
 Makefile              | 5 +----
 devscripts/release.sh | 2 +-
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 578079879..0636fc4cb 100644
--- a/Makefile
+++ b/Makefile
@@ -1,10 +1,7 @@
 all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json CONTRIBUTING.md.tmp
-
-cleanall: clean
-	rm -f youtube-dl youtube-dl.exe
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe
 
 PREFIX ?= /usr/local
 BINDIR ?= $(PREFIX)/bin
diff --git a/devscripts/release.sh b/devscripts/release.sh
index 691517ceb..47f54c414 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -35,7 +35,7 @@ if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in youtube_dl: $us
 if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi
 
 /bin/echo -e "\n### First of all, testing..."
-make cleanall
+make clean
 if $skip_tests ; then
     echo 'SKIPPING TESTS'
 else

From 2cf0ecac7b11e68a94990ce7c0e4a661cc2d8c74 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 00:16:45 +0100
Subject: [PATCH 1404/1937] [ffmpeg] --add-metadata: Set comment and purl
 fields (Fixes #4847)

---
 youtube_dl/postprocessor/ffmpeg.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 855d1e6db..4a4422c5a 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -511,8 +511,9 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
             metadata['artist'] = info['uploader_id']
         if info.get('description') is not None:
             metadata['description'] = info['description']
+            metadata['comment'] = info['description']
         if info.get('webpage_url') is not None:
-            metadata['comment'] = info['webpage_url']
+            metadata['purl'] = info['webpage_url']
 
         if not metadata:
             self._downloader.to_screen('[ffmpeg] There isn\'t any metadata to add')

From 330537d08aa9875c9d66354f2c9bed7aeaf86297 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 00:20:57 +0100
Subject: [PATCH 1405/1937] [README] typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d432904fa..c1221417e 100644
--- a/README.md
+++ b/README.md
@@ -728,7 +728,7 @@ In particular, every site support request issue should only pertain to services
 
 ###  Is anyone going to need the feature?
 
-Only post features that you (or an incapicated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them.
+Only post features that you (or an incapacitated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them.
 
 ###  Is your question about youtube-dl?
 

From ee623d9247752e11e98829834ba0e3308a21df52 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 00:22:17 +0100
Subject: [PATCH 1406/1937] [descripts/release] Regenerate auxiliary
 documentation on build as well

---
 devscripts/release.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/devscripts/release.sh b/devscripts/release.sh
index 47f54c414..61806961c 100755
--- a/devscripts/release.sh
+++ b/devscripts/release.sh
@@ -45,9 +45,9 @@ fi
 /bin/echo -e "\n### Changing version in version.py..."
 sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py
 
-/bin/echo -e "\n### Committing README.md and youtube_dl/version.py..."
-make README.md
-git add README.md youtube_dl/version.py
+/bin/echo -e "\n### Committing documentation and youtube_dl/version.py..."
+make README.md CONTRIBUTING.md supportedsites
+git add README.md CONTRIBUTING.md docs/supportedsites.md youtube_dl/version.py
 git commit -m "release $version"
 
 /bin/echo -e "\n### Now tagging, signing and pushing..."

From 4c1a017e6961c18fdef13d49bcdd0daa08ab0575 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 00:22:52 +0100
Subject: [PATCH 1407/1937] release 2015.02.03

---
 CONTRIBUTING.md        |  6 +++--
 docs/supportedsites.md | 59 +++++++++++++++++++++++++++++++++++++-----
 youtube_dl/version.py  |  2 +-
 3 files changed, 58 insertions(+), 9 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 7917abfc6..351229f21 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,4 +1,6 @@
-Please include the full output of the command when run with `--verbose`. The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
+**Please include the full output of youtube-dl when run with `-v`**.
+
+The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
 
 Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist):
 
@@ -122,7 +124,7 @@ If you want to add support for a new site, you can follow this quick list (assum
 5. Add an import in [`youtube_dl/extractor/__init__.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/__init__.py).
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will be then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc.
 7. Have a look at [`youtube_dl/common/extractor/common.py`](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should return](https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/common.py#L38). Add tests and code for as many as you want.
-8. If you can, check the code with [pyflakes](https://pypi.python.org/pypi/pyflakes) (a good idea) and [pep8](https://pypi.python.org/pypi/pep8) (optional, ignore E501).
+8. If you can, check the code with [flake8](https://pypi.python.org/pypi/flake8).
 9. When the tests pass, [add](http://git-scm.com/docs/git-add) the new files and [commit](http://git-scm.com/docs/git-commit) them and [push](http://git-scm.com/docs/git-push) the result, like this:
 
         $ git add youtube_dl/extractor/__init__.py
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index dbbf4a797..0cebe3981 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -9,6 +9,7 @@
  - **8tracks**
  - **9gag**
  - **abc.net.au**
+ - **Abc7News**
  - **AcademicEarth:Course**
  - **AddAnime**
  - **AdobeTV**
@@ -16,9 +17,12 @@
  - **Aftonbladet**
  - **AlJazeera**
  - **Allocine**
+ - **AlphaPorno**
  - **anitube.se**
  - **AnySex**
  - **Aparat**
+ - **AppleDailyAnimationNews**
+ - **AppleDailyRealtimeNews**
  - **AppleTrailers**
  - **archive.org**: archive.org videos
  - **ARD**
@@ -30,8 +34,10 @@
  - **arte.tv:ddc**
  - **arte.tv:embed**
  - **arte.tv:future**
+ - **AtresPlayer**
+ - **ATTTechChannel**
  - **audiomack**
- - **AUEngine**
+ - **audiomack:album**
  - **Azubu**
  - **bambuser**
  - **bambuser:channel**
@@ -71,8 +77,10 @@
  - **cmt.com**
  - **CNET**
  - **CNN**
+ - **CNNArticle**
  - **CNNBlogs**
  - **CollegeHumor**
+ - **CollegeRama**
  - **ComCarCoff**
  - **ComedyCentral**
  - **ComedyCentralShows**: The Daily Show / The Colbert Report
@@ -82,23 +90,27 @@
  - **Crunchyroll**
  - **crunchyroll:playlist**
  - **CSpan**: C-SPAN
+ - **CtsNews**
  - **culturebox.francetvinfo.fr**
  - **dailymotion**
  - **dailymotion:playlist**
  - **dailymotion:user**
  - **daum.net**
  - **DBTV**
+ - **DctpTv**
  - **DeezerPlaylist**
  - **defense.gouv.fr**
  - **Discovery**
  - **divxstage**: DivxStage
  - **Dotsub**
+ - **DRBonanza**
  - **Dropbox**
  - **DrTuber**
  - **DRTV**
  - **Dump**
  - **dvtv**: http://video.aktualne.cz/
  - **EbaumsWorld**
+ - **EchoMsk**
  - **eHow**
  - **Einthusan**
  - **eitb.tv**
@@ -108,6 +120,7 @@
  - **EMPFlix**
  - **Engadget**
  - **Eporner**
+ - **EroProfile**
  - **Escapist**
  - **EveryonesMixtape**
  - **exfm**: ex.fm
@@ -143,6 +156,7 @@
  - **GDCVault**
  - **generic**: Generic downloader that works on some sites
  - **GiantBomb**
+ - **Giga**
  - **Glide**: Glide mobile video messages (glide.me)
  - **Globo**
  - **GodTube**
@@ -153,9 +167,14 @@
  - **Grooveshark**
  - **Groupon**
  - **Hark**
+ - **HearThisAt**
  - **Heise**
+ - **HellPorno**
  - **Helsinki**: helsinki.fi
  - **HentaiStigma**
+ - **HistoricFilms**
+ - **hitbox**
+ - **hitbox:live**
  - **HornBunny**
  - **HostingBulk**
  - **HotNewHipHop**
@@ -182,6 +201,7 @@
  - **jpopsuki.tv**
  - **Jukebox**
  - **Kankan**
+ - **Karaoketv**
  - **keek**
  - **KeezMovies**
  - **KhanAcademy**
@@ -195,6 +215,7 @@
  - **LiveLeak**
  - **livestream**
  - **livestream:original**
+ - **LnkGo**
  - **lrt.lt**
  - **lynda**: lynda.com videos
  - **lynda:course**: lynda.com online courses
@@ -235,6 +256,7 @@
  - **MySpass**
  - **myvideo**
  - **MyVidster**
+ - **n-tv.de**
  - **Naver**
  - **NBA**
  - **NBC**
@@ -242,11 +264,16 @@
  - **ndr**: NDR.de - Mediathek
  - **NDTV**
  - **NerdCubedFeed**
+ - **Nerdist**
+ - **Netzkino**
  - **Newgrounds**
  - **Newstube**
+ - **NextMedia**
+ - **NextMediaActionNews**
  - **nfb**: National Film Board of Canada
  - **nfl.com**
  - **nhl.com**
+ - **nhl.com:news**: NHL news
  - **nhl.com:videocenter**: NHL videocenter category
  - **niconico**: ニコニコ動画
  - **NiconicoPlaylist**
@@ -257,18 +284,20 @@
  - **Nowness**
  - **nowvideo**: NowVideo
  - **npo.nl**
+ - **npo.nl:live**
  - **NRK**
  - **NRKTV**
- - **NTV**
+ - **ntv.ru**
  - **Nuvid**
  - **NYTimes**
  - **ocw.mit.edu**
  - **OktoberfestTV**
  - **on.aol.com**
  - **Ooyala**
+ - **OpenFilm**
+ - **orf:fm4**: radio FM4
  - **orf:oe1**: Radio Österreich 1
  - **orf:tvthek**: ORF TVthek
- - **ORFFM4**: radio FM4
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
  - **PBS**
@@ -290,6 +319,7 @@
  - **Pyvideo**
  - **QuickVid**
  - **radio.de**
+ - **radiobremen**
  - **radiofrance**
  - **Rai**
  - **RBMARadio**
@@ -300,6 +330,8 @@
  - **RottenTomatoes**
  - **Roxwel**
  - **RTBF**
+ - **Rte**
+ - **RTL2**
  - **RTLnow**
  - **rtlxl.nl**
  - **RTP**
@@ -309,6 +341,7 @@
  - **RUHD**
  - **rutube**: Rutube videos
  - **rutube:channel**: Rutube channels
+ - **rutube:embed**: Rutube embedded videos
  - **rutube:movie**: Rutube movies
  - **rutube:person**: Rutube person videos
  - **RUTV**: RUTV.RU
@@ -351,11 +384,12 @@
  - **Sport5**
  - **SportBox**
  - **SportDeutschland**
- - **SRMediathek**: Süddeutscher Rundfunk
+ - **SRMediathek**: Saarländischer Rundfunk
  - **stanfordoc**: Stanford Open ClassRoom
  - **Steam**
  - **streamcloud.eu**
  - **StreamCZ**
+ - **StreetVoice**
  - **SunPorno**
  - **SWRMediathek**
  - **Syfy**
@@ -375,7 +409,9 @@
  - **TeleBruxelles**
  - **telecinco.es**
  - **TeleMB**
+ - **TeleTask**
  - **TenPlay**
+ - **TestTube**
  - **TF1**
  - **TheOnion**
  - **ThePlatform**
@@ -403,8 +439,15 @@
  - **tv.dfb.de**
  - **tvigle**: Интернет-телевидение Tvigle.ru
  - **tvp.pl**
+ - **tvp.pl:Series**
  - **TVPlay**: TV3Play and related services
- - **Twitch**
+ - **twitch:bookmarks**
+ - **twitch:chapter**
+ - **twitch:past_broadcasts**
+ - **twitch:profile**
+ - **twitch:stream**
+ - **twitch:video**
+ - **twitch:vod**
  - **Ubu**
  - **udemy**
  - **udemy:course**
@@ -433,6 +476,8 @@
  - **videoweed**: VideoWeed
  - **Vidme**
  - **Vidzi**
+ - **vier**
+ - **vier:videos**
  - **viki**
  - **vimeo**
  - **vimeo:album**
@@ -460,6 +505,7 @@
  - **WDR**
  - **wdr:mobile**
  - **WDRMaus**: Sendung mit der Maus
+ - **WebOfStories**
  - **Weibo**
  - **Wimp**
  - **Wistia**
@@ -472,7 +518,9 @@
  - **XNXX**
  - **XTube**
  - **XTubeUser**: XTube user profile
+ - **Xuite**
  - **XVideos**
+ - **XXXYMovies**
  - **Yahoo**: Yahoo screen and movies
  - **YesJapan**
  - **Ynet**
@@ -491,7 +539,6 @@
  - **youtube:search_url**: YouTube.com search URLs
  - **youtube:show**: YouTube.com (multi-season) shows
  - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
- - **youtube:toplist**: YouTube.com top lists, "yttoplist:{channel}:{list title}" (Example: "yttoplist:music:Top Tracks")
  - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
  - **youtube:watch_later**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
  - **ZDF**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 46af42171..a96e74629 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.02.5'
+__version__ = '2015.02.03'

From 23d9ded655f45c5a21c8d93ad167a072322ae88f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 10:15:05 +0100
Subject: [PATCH 1408/1937] [franceculture] Rewrite for new HTML scheme (Fixes
 #4853)

---
 test/helper.py                        | 10 ++++
 youtube_dl/extractor/franceculture.py | 72 ++++++++++++---------------
 2 files changed, 42 insertions(+), 40 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 805467c82..651ef99b9 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -103,6 +103,16 @@ def expect_info_dict(self, got_dict, expected_dict):
             self.assertTrue(
                 match_rex.match(got),
                 'field %s (value: %r) should match %r' % (info_field, got, match_str))
+        elif isinstance(expected, compat_str) and expected.startswith('startswith:'):
+            got = got_dict.get(info_field)
+            start_str = expected[len('startswith:'):]
+            self.assertTrue(
+                isinstance(got, compat_str),
+                'Expected a %s object, but got %s for field %s' % (
+                    compat_str.__name__, type(got).__name__, info_field))
+            self.assertTrue(
+                got.startswith(start_str),
+                'field %s (value: %r) should start with %r' % (info_field, got, start_str))
         elif isinstance(expected, type):
             got = got_dict.get(info_field)
             self.assertTrue(isinstance(got, expected),
diff --git a/youtube_dl/extractor/franceculture.py b/youtube_dl/extractor/franceculture.py
index 0c2972162..1e83a4e7e 100644
--- a/youtube_dl/extractor/franceculture.py
+++ b/youtube_dl/extractor/franceculture.py
@@ -1,77 +1,69 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
 from ..compat import (
-    compat_parse_qs,
     compat_urlparse,
 )
+from ..utils import (
+    determine_ext,
+    int_or_none,
+)
 
 
 class FranceCultureIE(InfoExtractor):
-    _VALID_URL = r'(?P<baseurl>http://(?:www\.)?franceculture\.fr/)player/reecouter\?play=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?franceculture\.fr/player/reecouter\?play=(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.franceculture.fr/player/reecouter?play=4795174',
         'info_dict': {
             'id': '4795174',
             'ext': 'mp3',
             'title': 'Rendez-vous au pays des geeks',
+            'alt_title': 'Carnet nomade | 13-14',
             'vcodec': 'none',
-            'uploader': 'Colette Fellous',
             'upload_date': '20140301',
-            'duration': 3601,
             'thumbnail': r're:^http://www\.franceculture\.fr/.*/images/player/Carnet-nomade\.jpg$',
-            'description': 'Avec :Jean-Baptiste Péretié pour son documentaire sur Arte "La revanche des « geeks », une enquête menée aux Etats-Unis dans la S ...',
+            'description': 'startswith:Avec :Jean-Baptiste Péretié pour son documentaire sur Arte "La revanche des « geeks », une enquête menée aux Etats',
+            'timestamp': 1393700400,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        baseurl = mobj.group('baseurl')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        params_code = self._search_regex(
-            r"<param name='movie' value='/sites/all/modules/rf/rf_player/swf/loader.swf\?([^']+)' />",
-            webpage, 'parameter code')
-        params = compat_parse_qs(params_code)
-        video_url = compat_urlparse.urljoin(baseurl, params['urlAOD'][0])
+
+        video_path = self._search_regex(
+            r'<a id="player".*?href="([^"]+)"', webpage, 'video path')
+        video_url = compat_urlparse.urljoin(url, video_path)
+        timestamp = int_or_none(self._search_regex(
+            r'<a id="player".*?data-date="([0-9]+)"',
+            webpage, 'upload date', fatal=False))
+        thumbnail = self._search_regex(
+            r'<a id="player".*?>\s+<img src="([^"]+)"',
+            webpage, 'thumbnail', fatal=False)
 
         title = self._html_search_regex(
-            r'<h1 class="title[^"]+">(.+?)</h1>', webpage, 'title')
+            r'<span class="title-diffusion">(.*?)</span>', webpage, 'title')
+        alt_title = self._html_search_regex(
+            r'<span class="title">(.*?)</span>',
+            webpage, 'alt_title', fatal=False)
+        description = self._html_search_regex(
+            r'<span class="description">(.*?)</span>',
+            webpage, 'description', fatal=False)
+
         uploader = self._html_search_regex(
             r'(?s)<div id="emission".*?<span class="author">(.*?)</span>',
-            webpage, 'uploader', fatal=False)
-        thumbnail_part = self._html_search_regex(
-            r'(?s)<div id="emission".*?<img src="([^"]+)"', webpage,
-            'thumbnail', fatal=False)
-        if thumbnail_part is None:
-            thumbnail = None
-        else:
-            thumbnail = compat_urlparse.urljoin(baseurl, thumbnail_part)
-        description = self._html_search_regex(
-            r'(?s)<p class="desc">(.*?)</p>', webpage, 'description')
-
-        info = json.loads(params['infoData'][0])[0]
-        duration = info.get('media_length')
-        upload_date_candidate = info.get('media_section5')
-        upload_date = (
-            upload_date_candidate
-            if (upload_date_candidate is not None and
-                re.match(r'[0-9]{8}$', upload_date_candidate))
-            else None)
+            webpage, 'uploader', default=None)
+        vcodec = 'none' if determine_ext(video_url.lower()) == 'mp3' else None
 
         return {
             'id': video_id,
             'url': video_url,
-            'vcodec': 'none' if video_url.lower().endswith('.mp3') else None,
-            'duration': duration,
+            'vcodec': vcodec,
             'uploader': uploader,
-            'upload_date': upload_date,
+            'timestamp': timestamp,
             'title': title,
+            'alt_title': alt_title,
             'thumbnail': thumbnail,
             'description': description,
         }

From d86204230157fc7a8a6800b7b814e8e3bbfbdf20 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 10:18:32 +0100
Subject: [PATCH 1409/1937] [aftonbladet] Modernize

---
 youtube_dl/extractor/aftonbladet.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/aftonbladet.py b/youtube_dl/extractor/aftonbladet.py
index cfc7370ae..8442019ea 100644
--- a/youtube_dl/extractor/aftonbladet.py
+++ b/youtube_dl/extractor/aftonbladet.py
@@ -1,8 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -21,9 +19,7 @@ class AftonbladetIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.search(self._VALID_URL, url)
-
-        video_id = mobj.group('video_id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
         # find internal video meta data

From f6c24009bea95104e186afd77a867abbc61b7f81 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 10:52:22 +0100
Subject: [PATCH 1410/1937] [YoutubeDL] Calculate thumbnail IDs automatically

---
 youtube_dl/YoutubeDL.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index c18ce9660..1730df4cd 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -964,9 +964,11 @@ class YoutubeDL(object):
             thumbnails.sort(key=lambda t: (
                 t.get('preference'), t.get('width'), t.get('height'),
                 t.get('id'), t.get('url')))
-            for t in thumbnails:
+            for i, t in enumerate(thumbnails):
                 if 'width' in t and 'height' in t:
                     t['resolution'] = '%dx%d' % (t['width'], t['height'])
+                if t.get('id') is None:
+                    t['id'] = '%d' % i
 
         if thumbnails and 'thumbnail' not in info_dict:
             info_dict['thumbnail'] = thumbnails[-1]['url']

From 1a6373ef3960add0117f797e0afe3322352c1c52 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 10:53:05 +0100
Subject: [PATCH 1411/1937] [sort_formats] Prefer bitrate over video size

720p @ 1000KB/s looks way better than 1080p @ 500KB/s
---
 youtube_dl/extractor/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9e517e3ac..653d793fc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -704,11 +704,11 @@ class InfoExtractor(object):
                 preference,
                 f.get('language_preference') if f.get('language_preference') is not None else -1,
                 f.get('quality') if f.get('quality') is not None else -1,
-                f.get('height') if f.get('height') is not None else -1,
-                f.get('width') if f.get('width') is not None else -1,
-                ext_preference,
                 f.get('tbr') if f.get('tbr') is not None else -1,
                 f.get('vbr') if f.get('vbr') is not None else -1,
+                ext_preference,
+                f.get('height') if f.get('height') is not None else -1,
+                f.get('width') if f.get('width') is not None else -1,
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,
                 f.get('fps') if f.get('fps') is not None else -1,

From 9bb8e0a3f9276f65de38cda431bf72f7bd266693 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 10:58:28 +0100
Subject: [PATCH 1412/1937] [wsj] Add new extractor (Fixes #4854)

---
 test/test_utils.py               |  3 ++
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/common.py   |  1 +
 youtube_dl/extractor/wsj.py      | 89 ++++++++++++++++++++++++++++++++
 youtube_dl/utils.py              |  2 +-
 5 files changed, 95 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/wsj.py

diff --git a/test/test_utils.py b/test/test_utils.py
index 0ffccd35f..80c765bc4 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -156,6 +156,9 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(
             unified_strdate('11/26/2014 11:30:00 AM PST', day_first=False),
             '20141126')
+        self.assertEqual(
+            unified_strdate('2/2/2015 6:47:40 PM', day_first=False),
+            '20150202')
 
     def test_find_xpath_attr(self):
         testxml = '''<root>
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5dcb14feb..5866a7617 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -554,6 +554,7 @@ from .wimp import WimpIE
 from .wistia import WistiaIE
 from .worldstarhiphop import WorldStarHipHopIE
 from .wrzuta import WrzutaIE
+from .wsj import WSJIE
 from .xbef import XBefIE
 from .xboxclips import XboxClipsIE
 from .xhamster import XHamsterIE
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 653d793fc..602601b24 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -145,6 +145,7 @@ class InfoExtractor(object):
     thumbnail:      Full URL to a video thumbnail image.
     description:    Full video description.
     uploader:       Full name of the video uploader.
+    creator:        The main artist who created the video.
     timestamp:      UNIX timestamp of the moment the video became available.
     upload_date:    Video upload date (YYYYMMDD).
                     If not explicitly set, calculated from timestamp.
diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py
new file mode 100644
index 000000000..cbe3dc7be
--- /dev/null
+++ b/youtube_dl/extractor/wsj.py
@@ -0,0 +1,89 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    unified_strdate,
+)
+
+
+class WSJIE(InfoExtractor):
+    _VALID_URL = r'https?://video-api\.wsj\.com/api-video/player/iframe\.html\?guid=(?P<id>[a-zA-Z0-9-]+)'
+    IE_DESC = 'Wall Street Journal'
+    _TEST = {
+        'url': 'http://video-api.wsj.com/api-video/player/iframe.html?guid=1BD01A4C-BFE8-40A5-A42F-8A8AF9898B1A',
+        'md5': '9747d7a6ebc2f4df64b981e1dde9efa9',
+        'info_dict': {
+            'id': '1BD01A4C-BFE8-40A5-A42F-8A8AF9898B1A',
+            'ext': 'mp4',
+            'upload_date': '20150202',
+            'uploader_id': 'bbright',
+            'creator': 'bbright',
+            'categories': list,  # a long list
+            'duration': 90,
+            'title': 'Bills Coach Rex Ryan Updates His Old Jets Tattoo',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        bitrates = [128, 174, 264, 320, 464, 664, 1264]
+        api_url = (
+            'http://video-api.wsj.com/api-video/find_all_videos.asp?'
+            'type=guid&count=1&query=%s&'
+            'fields=hls,adZone,thumbnailList,guid,state,secondsUntilStartTime,'
+            'author,description,name,linkURL,videoStillURL,duration,videoURL,'
+            'adCategory,catastrophic,linkShortURL,doctypeID,youtubeID,'
+            'titletag,rssURL,wsj-section,wsj-subsection,allthingsd-section,'
+            'allthingsd-subsection,sm-section,sm-subsection,provider,'
+            'formattedCreationDate,keywords,keywordsOmniture,column,editor,'
+            'emailURL,emailPartnerID,showName,omnitureProgramName,'
+            'omnitureVideoFormat,linkRelativeURL,touchCastID,'
+            'omniturePublishDate,%s') % (
+                video_id, ','.join('video%dkMP4Url' % br for br in bitrates))
+        info = self._download_json(api_url, video_id)['items'][0]
+
+        # Thumbnails are conveniently in the correct format already
+        thumbnails = info.get('thumbnailList')
+        creator = info.get('author')
+        uploader_id = info.get('editor')
+        categories = info.get('keywords')
+        duration = int_or_none(info.get('duration'))
+        upload_date = unified_strdate(
+            info.get('formattedCreationDate'), day_first=False)
+        title = info.get('name', info.get('titletag'))
+
+        formats = [{
+            'format_id': 'f4m',
+            'format_note': 'f4m (meta URL)',
+            'url': info['videoURL'],
+        }]
+        if info.get('hls'):
+            formats.extend(self._extract_m3u8_formats(
+                info['hls'], video_id, ext='mp4',
+                preference=0, entry_protocol='m3u8_native'))
+        for br in bitrates:
+            field = 'video%dkMP4Url' % br
+            if info.get(field):
+                formats.append({
+                    'format_id': 'mp4-%d' % br,
+                    'container': 'mp4',
+                    'tbr': br,
+                    'url': info[field],
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'creator': creator,
+            'uploader_id': uploader_id,
+            'duration': duration,
+            'upload_date': upload_date,
+            'title': title,
+            'formats': formats,
+            'categories': categories,
+        }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 251074bf5..8f5463f1c 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -701,7 +701,7 @@ def unified_strdate(date_str, day_first=True):
     # %z (UTC offset) is only supported in python>=3.2
     date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
     # Remove AM/PM + timezone
-    date_str = re.sub(r'(?i)\s*(?:AM|PM)\s+[A-Z]+', '', date_str)
+    date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
 
     format_expressions = [
         '%d %B %Y',

From cd7342755fb6928ca9f089bbd4bfdd17904f91b3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 10:59:27 +0100
Subject: [PATCH 1413/1937] release 2015.02.03.1

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 0cebe3981..b188be636 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -511,6 +511,7 @@
  - **Wistia**
  - **WorldStarHipHop**
  - **wrzuta.pl**
+ - **WSJ**: Wall Street Journal
  - **XBef**
  - **XboxClips**
  - **XHamster**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a96e74629..4975ff83e 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.03'
+__version__ = '2015.02.03.1'

From 12f0454cd60ffa67b57a822b2e30461cb761040d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Feb 2015 14:18:12 +0100
Subject: [PATCH 1414/1937] [README] Add an FAQ entry about anime sites

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index c1221417e..06dea400d 100644
--- a/README.md
+++ b/README.md
@@ -532,6 +532,14 @@ Either prepend `http://www.youtube.com/watch?v=` or separate the ID from the opt
     youtube-dl -- -wNyEUrxzFU
     youtube-dl "http://www.youtube.com/watch?v=-wNyEUrxzFU"
 
+### Can you add support for this anime video site, or site which shows current movies for free?
+
+As a matter of policy (as well as legality), youtube-dl does not include support for services that specialize in infringing copyright. As a rule of thumb, if you cannot easily find a video that the service is quite obviously allowed to distribute (i.e. that has been uploaded by the creator, the creator's distributor, or is published under a free license), the service is probably unfit for inclusion to youtube-dl.
+
+A note on the service that they don't host the infringing content, but just link to those who do, is evidence that the service should **not** be included into youtube-dl. The same goes for any DMCA note when the whole front page of the service is filled with videos they are not allowed to distribute. A "fair use" note is equally unconvincing if the service shows copyright-protected videos in full without authorization.
+
+Support requests for services that **do** purchase the rights to distribute their content are perfectly fine though. If in doubt, you can simply include a source that mentions the legitimate purchase of content.
+
 ### How can I detect whether a given URL is supported by youtube-dl?
 
 For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.

From f076b63821100af17379ed8b470e508846bb8432 Mon Sep 17 00:00:00 2001
From: naglis <njonaitis@gmail.com>
Date: Wed, 4 Feb 2015 15:33:37 +0200
Subject: [PATCH 1415/1937] [generic/ooyala] Add support for Ooyala embeds on
 SBN network websites (Fixes #4859)

---
 youtube_dl/extractor/generic.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a6f102a4e..fbbc79a57 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -140,6 +140,19 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Ooyala'],
         },
+        # multiple ooyala embeds on SBN network websites
+        {
+            'url': 'http://www.sbnation.com/college-football-recruiting/2015/2/3/7970291/national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
+            'info_dict': {
+                'id': 'national-signing-day-rationalizations-itll-be-ok-itll-be-ok',
+                'title': '25 lies you will tell yourself on National Signing Day - SBNation.com',
+            },
+            'playlist_mincount': 3,
+            'params': {
+                'skip_download': True,
+            },
+            'add_ie': ['Ooyala'],
+        },
         # google redirect
         {
             'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
@@ -882,10 +895,19 @@ class GenericIE(InfoExtractor):
 
         # Look for Ooyala videos
         mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
-                re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage))
+                re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or
+                re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage))
         if mobj is not None:
             return OoyalaIE._build_url_result(mobj.group('ec'))
 
+        # Look for multiple Ooyala embeds on SBN network websites
+        mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
+        if mobj is not None:
+            embeds = self._parse_json(mobj.group(1), video_id, fatal=False)
+            if embeds:
+                return _playlist_from_matches(
+                    embeds, getter=lambda v: OoyalaIE._url_for_embed_code(v['provider_video_id']), ie='Ooyala')
+
         # Look for Aparat videos
         mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
         if mobj is not None:

From 58ba6c016086f1f9f47778f7a64aca72beb55c23 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 4 Feb 2015 19:47:55 +0600
Subject: [PATCH 1416/1937] [mixcloud] Fix extraction (Closes #4862)

---
 youtube_dl/extractor/mixcloud.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 141bb463e..1831c6749 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -18,7 +18,7 @@ class MixcloudIE(InfoExtractor):
     _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([^/]+)/([^/]+)'
     IE_NAME = 'mixcloud'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.mixcloud.com/dholbach/cryptkeeper/',
         'info_dict': {
             'id': 'dholbach-cryptkeeper',
@@ -33,7 +33,20 @@ class MixcloudIE(InfoExtractor):
             'view_count': int,
             'like_count': int,
         },
-    }
+    }, {
+        'url': 'http://www.mixcloud.com/gillespeterson/caribou-7-inch-vinyl-mix-chat/',
+        'info_dict': {
+            'id': 'gillespeterson-caribou-7-inch-vinyl-mix-chat',
+            'ext': 'm4a',
+            'title': 'Electric Relaxation vol. 3',
+            'description': 'md5:2b8aec6adce69f9d41724647c65875e8',
+            'uploader': 'Daniel Drumz',
+            'uploader_id': 'gillespeterson',
+            'thumbnail': 're:https?://.*\.jpg',
+            'view_count': int,
+            'like_count': int,
+        },
+    }]
 
     def _get_url(self, track_id, template_url):
         server_count = 30
@@ -60,7 +73,7 @@ class MixcloudIE(InfoExtractor):
         webpage = self._download_webpage(url, track_id)
 
         preview_url = self._search_regex(
-            r'\s(?:data-preview-url|m-preview)="(.+?)"', webpage, 'preview url')
+            r'\s(?:data-preview-url|m-preview)="([^"]+)"', webpage, 'preview url')
         song_url = preview_url.replace('/previews/', '/c/originals/')
         template_url = re.sub(r'(stream\d*)', 'stream%d', song_url)
         final_song_url = self._get_url(track_id, template_url)

From 7dfc3566250894147910bd6b79530f97770a47f3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 4 Feb 2015 16:09:35 +0100
Subject: [PATCH 1417/1937] release 2015.02.04

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4975ff83e..36591a43d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.03.1'
+__version__ = '2015.02.04'

From 25e5ebf382b2fca9d4b6909ac1ac9b5af89bcdd9 Mon Sep 17 00:00:00 2001
From: Frans de Jonge <fransdejonge@gmail.com>
Date: Thu, 5 Feb 2015 12:51:33 +0100
Subject: [PATCH 1418/1937] Add NPO.nl subtitles

Implements #3638
---
 youtube_dl/extractor/npo.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 175b14583..0580461bb 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
+from .subtitles import SubtitlesInfoExtractor
 from ..utils import (
     fix_xml_ampersands,
     parse_duration,
@@ -11,7 +11,7 @@ from ..utils import (
 )
 
 
-class NPOBaseIE(InfoExtractor):
+class NPOBaseIE(SubtitlesInfoExtractor):
     def _get_token(self, video_id):
         token_page = self._download_webpage(
             'http://ida.omroep.nl/npoplayer/i.js',
@@ -159,6 +159,13 @@ class NPOIE(NPOBaseIE):
                     'quality': stream.get('kwaliteit'),
                 })
 
+        subtitles = {}
+
+        tt888 = metadata.get('tt888')
+        if self._have_to_download_any_subtitles and tt888 == 'ja':
+            subtitles['nl'] = 'http://e.omroep.nl/tt888/%s' % video_id
+            subtitles = self.extract_subtitles(video_id, subtitles)
+
         self._sort_formats(formats)
 
         return {
@@ -169,6 +176,7 @@ class NPOIE(NPOBaseIE):
             'upload_date': unified_strdate(metadata.get('gidsdatum')),
             'duration': parse_duration(metadata.get('tijdsduur')),
             'formats': formats,
+            'subtitles': subtitles,
         }
 
 

From 9e91449c8d1dae53eda6fb250384f0afa5122ce6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Feb 2015 20:13:28 +0600
Subject: [PATCH 1419/1937] [npo] Fix subtitles (Closes #3638)

---
 youtube_dl/extractor/npo.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 0580461bb..54be06a4e 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -159,15 +159,18 @@ class NPOIE(NPOBaseIE):
                     'quality': stream.get('kwaliteit'),
                 })
 
-        subtitles = {}
-
-        tt888 = metadata.get('tt888')
-        if self._have_to_download_any_subtitles and tt888 == 'ja':
-            subtitles['nl'] = 'http://e.omroep.nl/tt888/%s' % video_id
-            subtitles = self.extract_subtitles(video_id, subtitles)
-
         self._sort_formats(formats)
 
+        subtitles = {}
+        if metadata.get('tt888') == 'ja':
+            subtitles['nl'] = 'http://e.omroep.nl/tt888/%s' % video_id
+
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, subtitles)
+            return
+
+        subtitles = self.extract_subtitles(video_id, subtitles)
+
         return {
             'id': video_id,
             'title': metadata['titel'],

From 6a348cf7d507f0373e9a6f67358202b66a858cd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Feb 2015 20:14:56 +0600
Subject: [PATCH 1420/1937] Credit @Frenzie for npo subtitles (#4878)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 83d8161a9..85ca81a27 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -108,3 +108,4 @@ Enam Mijbah Noor
 David Luhmer
 Shaya Goldberg
 Paul Hartmann
+Frans de Jonge

From 60ca389c646ae03e90377261857bfe45333a8ff2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Feb 2015 22:16:27 +0600
Subject: [PATCH 1421/1937] [extractor/common] Prefix f4m/m3u8 entries with
 identifier

---
 youtube_dl/extractor/common.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 602601b24..8ec7cee06 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -765,7 +765,7 @@ class InfoExtractor(object):
         self.to_screen(msg)
         time.sleep(timeout)
 
-    def _extract_f4m_formats(self, manifest_url, video_id):
+    def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None):
         manifest = self._download_xml(
             manifest_url, video_id, 'Downloading f4m manifest',
             'Unable to download f4m manifest')
@@ -780,24 +780,25 @@ class InfoExtractor(object):
             if manifest_version == '2.0':
                 manifest_url = '/'.join(manifest_url.split('/')[:-1]) + '/' + media_el.attrib.get('href')
             tbr = int_or_none(media_el.attrib.get('bitrate'))
-            format_id = 'f4m-%d' % (i if tbr is None else tbr)
             formats.append({
-                'format_id': format_id,
+                'format_id': '-'.join(filter(None, [f4m_id, 'f4m-%d' % (i if tbr is None else tbr)])),
                 'url': manifest_url,
                 'ext': 'flv',
                 'tbr': tbr,
                 'width': int_or_none(media_el.attrib.get('width')),
                 'height': int_or_none(media_el.attrib.get('height')),
+                'preference': preference,
             })
         self._sort_formats(formats)
 
         return formats
 
     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
-                              entry_protocol='m3u8', preference=None):
+                              entry_protocol='m3u8', preference=None,
+                              m3u8_id=None):
 
         formats = [{
-            'format_id': 'm3u8-meta',
+            'format_id': '-'.join(filter(None, [m3u8_id, 'm3u8-meta'])),
             'url': m3u8_url,
             'ext': ext,
             'protocol': 'm3u8',
@@ -833,9 +834,8 @@ class InfoExtractor(object):
                     formats.append({'url': format_url(line)})
                     continue
                 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
-
                 f = {
-                    'format_id': 'm3u8-%d' % (tbr if tbr else len(formats)),
+                    'format_id': '-'.join(filter(None, [m3u8_id, 'm3u8-%d' % (tbr if tbr else len(formats))])),
                     'url': format_url(line.strip()),
                     'tbr': tbr,
                     'ext': ext,

From f13b88c61696eaeb96878b5c68b7ab949a3f4ca6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Feb 2015 22:17:50 +0600
Subject: [PATCH 1422/1937] [rts] Fix f4m and m3u8 extraction (Closes #4873)

---
 youtube_dl/extractor/rts.py | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index 5e84c1098..d0981115d 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -6,12 +6,14 @@ import re
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
+    compat_urllib_parse_urlparse,
 )
 from ..utils import (
     int_or_none,
     parse_duration,
     parse_iso8601,
     unescapeHTML,
+    xpath_text,
 )
 
 
@@ -159,11 +161,27 @@ class RTSIE(InfoExtractor):
             return int_or_none(self._search_regex(
                 r'-([0-9]+)k\.', url, 'bitrate', default=None))
 
-        formats = [{
-            'format_id': fid,
-            'url': furl,
-            'tbr': extract_bitrate(furl),
-        } for fid, furl in info['streams'].items()]
+        formats = []
+        for format_id, format_url in info['streams'].items():
+            if format_url.endswith('.f4m'):
+                token = self._download_xml(
+                    'http://tp.srgssr.ch/token/akahd.xml?stream=%s/*' % compat_urllib_parse_urlparse(format_url).path,
+                    video_id, 'Downloading %s token' % format_id)
+                auth_params = xpath_text(token, './/authparams', 'auth params')
+                if not auth_params:
+                    continue
+                formats.extend(self._extract_f4m_formats(
+                    '%s?%s&hdcore=3.4.0&plugin=aasp-3.4.0.132.66' % (format_url, auth_params),
+                    video_id, f4m_id=format_id))
+            elif format_url.endswith('.m3u8'):
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', m3u8_id=format_id))
+            else:
+                formats.append({
+                    'format_id': format_id,
+                    'url': format_url,
+                    'tbr': extract_bitrate(format_url),
+                })
 
         if 'media' in info:
             formats.extend([{

From 300753a0695c17979d6ac42147cb162ff0eedea5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Feb 2015 00:51:16 +0600
Subject: [PATCH 1423/1937] [YoutubeDL] Fix video+audio format field (Closes
 #4880)

---
 youtube_dl/YoutubeDL.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1730df4cd..9605f8f99 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1076,7 +1076,8 @@ class YoutubeDL(object):
                                 else self.params['merge_output_format'])
                             selected_format = {
                                 'requested_formats': formats_info,
-                                'format': rf,
+                                'format': '%s+%s' % (formats_info[0].get('format'),
+                                                     formats_info[1].get('format')),
                                 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
                                                         formats_info[1].get('format_id')),
                                 'width': formats_info[0].get('width'),

From e3aaace4005e7b767d28152e60d1cf6272c398ec Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 19:55:41 +0100
Subject: [PATCH 1424/1937] [tweakers] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tweakers.py | 41 ++++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 youtube_dl/extractor/tweakers.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5866a7617..047f7002a 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -475,6 +475,7 @@ from .tutv import TutvIE
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
+from .tweakers import TweakersIE
 from .twentyfourvideo import TwentyFourVideoIE
 from .twitch import (
     TwitchVideoIE,
diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
new file mode 100644
index 000000000..3080f9e46
--- /dev/null
+++ b/youtube_dl/extractor/tweakers.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class TweakersIE(InfoExtractor):
+    _VALID_URL = r'https?://tweakers\.net/video/(?P<id>[0-9]+).*'
+    _TEST = {
+        'url': 'https://tweakers.net/video/9926/new-nintendo-3ds-xl-op-alle-fronten-beter.html',
+        'md5': 'f7f7f3027166a7f32f024b4ae6571ced',
+        'info_dict': {
+            'id': '9926',
+            'ext': 'mp4',
+            'title': 'New-Nintendo-3Ds-Xl-Op-Alle-Fronten-Beter',
+            # TODO more properties, either as:
+            # * A value
+            # * MD5 checksum; start the string with md5:
+            # * A regular expression; start the string with re:
+            # * Any Python type (for example int or float)
+        }
+    }
+
+    def _real_extract(self, url):
+        splitted_url = re.split('.html|/', url)
+        del splitted_url[-1] # To remove extra '/' at the end
+        video_id = splitted_url[4]
+        title = splitted_url[5].title() # Retrieve title for URL and capitalize
+        splitted_url[3] = splitted_url[3] + '/player' # Add /player to get the player page
+        player_url = '/'.join(splitted_url) + '.html'
+        player_page = self._download_webpage(player_url, video_id)
+
+        return {
+            'id': video_id,
+            'ext': 'mp4',
+            'title': title,
+            'url':  re.findall('http.*mp4', player_page)[0],
+            'player_url': player_url
+        }

From 3ee6e02564f7f6c54480fd4eeb94ae08a747b9b6 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 19:59:36 +0100
Subject: [PATCH 1425/1937] Edit Tweakers extractor

Fixed code conventions (mainly adding two or more spaces before making
an inline comment)
---
 youtube_dl/extractor/tweakers.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index 3080f9e46..29cd04f00 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -25,10 +25,10 @@ class TweakersIE(InfoExtractor):
 
     def _real_extract(self, url):
         splitted_url = re.split('.html|/', url)
-        del splitted_url[-1] # To remove extra '/' at the end
+        del splitted_url[-1]  # To remove extra '/' at the end
         video_id = splitted_url[4]
-        title = splitted_url[5].title() # Retrieve title for URL and capitalize
-        splitted_url[3] = splitted_url[3] + '/player' # Add /player to get the player page
+        title = splitted_url[5].title()  # Retrieve title for URL and capitalize
+        splitted_url[3] = splitted_url[3] + '/player'  # Add /player to get the player page
         player_url = '/'.join(splitted_url) + '.html'
         player_page = self._download_webpage(player_url, video_id)
 
@@ -36,6 +36,6 @@ class TweakersIE(InfoExtractor):
             'id': video_id,
             'ext': 'mp4',
             'title': title,
-            'url':  re.findall('http.*mp4', player_page)[0],
+            'url': re.findall('http.*mp4', player_page)[0],
             'player_url': player_url
         }

From c893d70805be9b8f8a10c7d7dbe39216f8976ea5 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 21:38:35 +0100
Subject: [PATCH 1426/1937] Remove player-url in tweakers.py

Player-url only needed for rmftp, not for regular URLs
---
 youtube_dl/extractor/tweakers.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index 29cd04f00..14145e617 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -37,5 +37,4 @@ class TweakersIE(InfoExtractor):
             'ext': 'mp4',
             'title': title,
             'url': re.findall('http.*mp4', player_page)[0],
-            'player_url': player_url
         }

From bc63d56ccac8805ca8394e0b41890027e6d677d2 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 21:40:18 +0100
Subject: [PATCH 1427/1937] Remove unnecessary TODO-comments

---
 youtube_dl/extractor/tweakers.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index 14145e617..a4e749850 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -15,11 +15,6 @@ class TweakersIE(InfoExtractor):
             'id': '9926',
             'ext': 'mp4',
             'title': 'New-Nintendo-3Ds-Xl-Op-Alle-Fronten-Beter',
-            # TODO more properties, either as:
-            # * A value
-            # * MD5 checksum; start the string with md5:
-            # * A regular expression; start the string with re:
-            # * Any Python type (for example int or float)
         }
     }
 

From 67ce4f8820f94961c9b094f9c652f421c9d9f6fb Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 21:49:13 +0100
Subject: [PATCH 1428/1937] Use match_id method instead of splitted URL

---
 youtube_dl/extractor/tweakers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index a4e749850..e332d4694 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -21,7 +21,7 @@ class TweakersIE(InfoExtractor):
     def _real_extract(self, url):
         splitted_url = re.split('.html|/', url)
         del splitted_url[-1]  # To remove extra '/' at the end
-        video_id = splitted_url[4]
+        video_id = self._match_id(url)
         title = splitted_url[5].title()  # Retrieve title for URL and capitalize
         splitted_url[3] = splitted_url[3] + '/player'  # Add /player to get the player page
         player_url = '/'.join(splitted_url) + '.html'

From 9c91a8fa70e3c2f29dd23809e90b3357b6c1cdc4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Feb 2015 03:23:42 +0600
Subject: [PATCH 1429/1937] [tweakers] Switch extraction to xspf playlist,
 extract all formats and meta (#4881)

---
 youtube_dl/extractor/tweakers.py | 58 ++++++++++++++++++++++++--------
 1 file changed, 44 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index e332d4694..c80ec15cf 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -1,35 +1,65 @@
-# coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..utils import (
+    xpath_text,
+    xpath_with_ns,
+    int_or_none,
+    float_or_none,
+)
 
 
 class TweakersIE(InfoExtractor):
-    _VALID_URL = r'https?://tweakers\.net/video/(?P<id>[0-9]+).*'
+    _VALID_URL = r'https?://tweakers\.net/video/(?P<id>\d+)'
     _TEST = {
         'url': 'https://tweakers.net/video/9926/new-nintendo-3ds-xl-op-alle-fronten-beter.html',
-        'md5': 'f7f7f3027166a7f32f024b4ae6571ced',
+        'md5': '1b5afa817403bb5baa08359dca31e6df',
         'info_dict': {
             'id': '9926',
             'ext': 'mp4',
-            'title': 'New-Nintendo-3Ds-Xl-Op-Alle-Fronten-Beter',
+            'title': 'New Nintendo 3DS XL - Op alle fronten beter',
+            'description': 'md5:f97324cc71e86e11c853f0763820e3ba',
+            'thumbnail': 're:^https?://.*\.jpe?g$',
+            'duration': 386,
         }
     }
 
     def _real_extract(self, url):
-        splitted_url = re.split('.html|/', url)
-        del splitted_url[-1]  # To remove extra '/' at the end
         video_id = self._match_id(url)
-        title = splitted_url[5].title()  # Retrieve title for URL and capitalize
-        splitted_url[3] = splitted_url[3] + '/player'  # Add /player to get the player page
-        player_url = '/'.join(splitted_url) + '.html'
-        player_page = self._download_webpage(player_url, video_id)
+
+        playlist = self._download_xml(
+            'https://tweakers.net/video/s1playlist/%s/playlist.xspf' % video_id,
+            video_id)
+
+        NS_MAP = {
+            'xspf': 'http://xspf.org/ns/0/',
+            's1': 'http://static.streamone.nl/player/ns/0',
+        }
+
+        track = playlist.find(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP))
+
+        title = xpath_text(
+            track, xpath_with_ns('./xspf:title', NS_MAP), 'title')
+        description = xpath_text(
+            track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
+        thumbnail = xpath_text(
+            track, xpath_with_ns('./xspf:image', NS_MAP), 'thumbnail')
+        duration = float_or_none(
+            xpath_text(track, xpath_with_ns('./xspf:duration', NS_MAP), 'duration'),
+            1000)
+
+        formats = [{
+            'url': location.text,
+            'format_id': location.get(xpath_with_ns('s1:label', NS_MAP)),
+            'width': int_or_none(location.get(xpath_with_ns('s1:width', NS_MAP))),
+            'height': int_or_none(location.get(xpath_with_ns('s1:height', NS_MAP))),
+        } for location in track.findall(xpath_with_ns('./xspf:location', NS_MAP))]
 
         return {
             'id': video_id,
-            'ext': 'mp4',
             'title': title,
-            'url': re.findall('http.*mp4', player_page)[0],
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
         }

From 3900eec27c0a385d60f2b1dca50faa6f5e25b971 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Feb 2015 04:29:29 +0600
Subject: [PATCH 1430/1937] [extractor/common] Fix 2.0 manifest extraction
 (Closes #4830)

---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8ec7cee06..df1a4417b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -778,7 +778,8 @@ class InfoExtractor(object):
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
             if manifest_version == '2.0':
-                manifest_url = '/'.join(manifest_url.split('/')[:-1]) + '/' + media_el.attrib.get('href')
+                manifest_url = ('/'.join(manifest_url.split('/')[:-1]) + '/'
+                                + (media_el.attrib.get('href') or media_el.attrib.get('url')))
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
                 'format_id': '-'.join(filter(None, [f4m_id, 'f4m-%d' % (i if tbr is None else tbr)])),

From 1a2548d9e91b24f60d6948e658cfc0b68797b254 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 6 Feb 2015 13:41:58 +0200
Subject: [PATCH 1431/1937] [rtp] Pass --realtime to rtmpdump (Fixes #4882)

A workaround for video jumping back in time.
---
 youtube_dl/extractor/rtp.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/rtp.py b/youtube_dl/extractor/rtp.py
index 7736cabba..4511cba49 100644
--- a/youtube_dl/extractor/rtp.py
+++ b/youtube_dl/extractor/rtp.py
@@ -49,6 +49,7 @@ class RTPIE(InfoExtractor):
             'ext': ext,
             'vcodec': config.get('type') == 'audio' and 'none' or None,
             'player_url': 'http://programas.rtp.pt/play/player.swf?v3',
+            'rtmp_real_time': True,
         }]
 
         return {

From c8319733667ff5bc0e0cbe0a9c253c8cfc7d479e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 6 Feb 2015 14:38:30 +0100
Subject: [PATCH 1432/1937] release 2015.02.06

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index b188be636..2d8f9c316 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -441,6 +441,7 @@
  - **tvp.pl**
  - **tvp.pl:Series**
  - **TVPlay**: TV3Play and related services
+ - **Tweakers**
  - **twitch:bookmarks**
  - **twitch:chapter**
  - **twitch:past_broadcasts**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 36591a43d..1091ae61b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.04'
+__version__ = '2015.02.06'

From 1e1896f2de300bf1905a63adffe17e7ab68cc252 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 6 Feb 2015 15:16:43 +0100
Subject: [PATCH 1433/1937] [extractor/common] Correct sort order.

We should look at height and width before ext_preference.
---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index df1a4417b..980b63680 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -707,9 +707,9 @@ class InfoExtractor(object):
                 f.get('quality') if f.get('quality') is not None else -1,
                 f.get('tbr') if f.get('tbr') is not None else -1,
                 f.get('vbr') if f.get('vbr') is not None else -1,
-                ext_preference,
                 f.get('height') if f.get('height') is not None else -1,
                 f.get('width') if f.get('width') is not None else -1,
+                ext_preference,
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,
                 f.get('fps') if f.get('fps') is not None else -1,

From 7f2a9f1b4993693659820693c1b6432f3b01d21b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Feb 2015 21:15:01 +0600
Subject: [PATCH 1434/1937] [tvigle] Add support for cloud URLs (Closes #4887)

---
 youtube_dl/extractor/tvigle.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/tvigle.py b/youtube_dl/extractor/tvigle.py
index ba65996dc..102362b29 100644
--- a/youtube_dl/extractor/tvigle.py
+++ b/youtube_dl/extractor/tvigle.py
@@ -1,6 +1,8 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     float_or_none,
@@ -11,7 +13,7 @@ from ..utils import (
 class TvigleIE(InfoExtractor):
     IE_NAME = 'tvigle'
     IE_DESC = 'Интернет-телевидение Tvigle.ru'
-    _VALID_URL = r'http://(?:www\.)?tvigle\.ru/(?:[^/]+/)+(?P<id>[^/]+)/$'
+    _VALID_URL = r'https?://(?:www\.)?(?:tvigle\.ru/(?:[^/]+/)+(?P<display_id>[^/]+)/$|cloud\.tvigle\.ru/video/(?P<id>\d+))'
 
     _TESTS = [
         {
@@ -38,16 +40,22 @@ class TvigleIE(InfoExtractor):
                 'duration': 186.080,
                 'age_limit': 0,
             },
-        },
+        }, {
+            'url': 'https://cloud.tvigle.ru/video/5267604/',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id')
 
-        webpage = self._download_webpage(url, display_id)
-
-        video_id = self._html_search_regex(
-            r'<li class="video-preview current_playing" id="(\d+)">', webpage, 'video id')
+        if not video_id:
+            webpage = self._download_webpage(url, display_id)
+            video_id = self._html_search_regex(
+                r'<li class="video-preview current_playing" id="(\d+)">',
+                webpage, 'video id')
 
         video_data = self._download_json(
             'http://cloud.tvigle.ru/api/play/video/%s/' % video_id, display_id)

From d6eb66ed3c0dc1362c1fec46b2d9a60076812f50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Feb 2015 01:46:54 +0600
Subject: [PATCH 1435/1937] [aftenposten] Add extractor (Closes #4863)

---
 youtube_dl/extractor/__init__.py    |   1 +
 youtube_dl/extractor/aftenposten.py | 103 ++++++++++++++++++++++++++++
 2 files changed, 104 insertions(+)
 create mode 100644 youtube_dl/extractor/aftenposten.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 047f7002a..0d7a120bc 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -6,6 +6,7 @@ from .academicearth import AcademicEarthCourseIE
 from .addanime import AddAnimeIE
 from .adobetv import AdobeTVIE
 from .adultswim import AdultSwimIE
+from .aftenposten import AftenpostenIE
 from .aftonbladet import AftonbladetIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE
diff --git a/youtube_dl/extractor/aftenposten.py b/youtube_dl/extractor/aftenposten.py
new file mode 100644
index 000000000..2b257ede7
--- /dev/null
+++ b/youtube_dl/extractor/aftenposten.py
@@ -0,0 +1,103 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    xpath_with_ns,
+    xpath_text,
+    find_xpath_attr,
+)
+
+
+class AftenpostenIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?aftenposten\.no/webtv/([^/]+/)*(?P<id>[^/]+)-\d+\.html'
+
+    _TEST = {
+        'url': 'http://www.aftenposten.no/webtv/serier-og-programmer/sweatshopenglish/TRAILER-SWEATSHOP---I-cant-take-any-more-7800835.html?paging=&section=webtv_serierogprogrammer_sweatshop_sweatshopenglish',
+        'md5': 'fd828cd29774a729bf4d4425fe192972',
+        'info_dict': {
+            'id': '21039',
+            'ext': 'mov',
+            'title': 'TRAILER: "Sweatshop" - I can´t take any more',
+            'description': 'md5:21891f2b0dd7ec2f78d84a50e54f8238',
+            'timestamp': 1416927969,
+            'upload_date': '20141125',
+        }
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, display_id)
+
+        video_id = self._html_search_regex(
+            r'data-xs-id="(\d+)"', webpage, 'video id')
+
+        data = self._download_xml(
+            'http://frontend.xstream.dk/ap/feed/video/?platform=web&id=%s' % video_id, video_id)
+
+        NS_MAP = {
+            'atom': 'http://www.w3.org/2005/Atom',
+            'xt': 'http://xstream.dk/',
+            'media': 'http://search.yahoo.com/mrss/',
+        }
+
+        entry = data.find(xpath_with_ns('./atom:entry', NS_MAP))
+
+        title = xpath_text(
+            entry, xpath_with_ns('./atom:title', NS_MAP), 'title')
+        description = xpath_text(
+            entry, xpath_with_ns('./atom:summary', NS_MAP), 'description')
+        timestamp = parse_iso8601(xpath_text(
+            entry, xpath_with_ns('./atom:published', NS_MAP), 'upload date'))
+
+        formats = []
+        media_group = entry.find(xpath_with_ns('./media:group', NS_MAP))
+        for media_content in media_group.findall(xpath_with_ns('./media:content', NS_MAP)):
+            media_url = media_content.get('url')
+            if not media_url:
+                continue
+            tbr = int_or_none(media_content.get('bitrate'))
+            mobj = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', media_url)
+            if mobj:
+                formats.append({
+                    'url': mobj.group('url'),
+                    'play_path': 'mp4:%s' % mobj.group('playpath'),
+                    'app': mobj.group('app'),
+                    'ext': 'flv',
+                    'tbr': tbr,
+                    'format_id': 'rtmp-%d' % tbr,
+                })
+            else:
+                formats.append({
+                    'url': media_url,
+                    'tbr': tbr,
+                })
+        self._sort_formats(formats)
+
+        link = find_xpath_attr(
+            entry, xpath_with_ns('./atom:link', NS_MAP), 'rel', 'original')
+        if link is not None:
+            formats.append({
+                'url': link.get('href'),
+                'format_id': link.get('rel'),
+            })
+
+        thumbnails = [{
+            'url': splash.get('url'),
+            'width': int_or_none(splash.get('width')),
+            'height': int_or_none(splash.get('height')),
+        } for splash in media_group.findall(xpath_with_ns('./xt:splash', NS_MAP))]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'timestamp': timestamp,
+            'formats': formats,
+            'thumbnails': thumbnails,
+        }

From ad5747bad146c76f545870911d5f2fb586c0f7b1 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 6 Feb 2015 22:59:17 +0200
Subject: [PATCH 1436/1937] [rtp] Construct regular HTTP download URLs (#4882)

---
 youtube_dl/extractor/rtp.py | 42 +++++++++++++++++++++++++++++--------
 1 file changed, 33 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/rtp.py b/youtube_dl/extractor/rtp.py
index 4511cba49..ecf4939cd 100644
--- a/youtube_dl/extractor/rtp.py
+++ b/youtube_dl/extractor/rtp.py
@@ -1,16 +1,16 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import json
+import re
 
 from .common import InfoExtractor
-from ..utils import js_to_json
 
 
 class RTPIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?rtp\.pt/play/p(?P<program_id>[0-9]+)/(?P<id>[^/?#]+)/?'
     _TESTS = [{
         'url': 'http://www.rtp.pt/play/p405/e174042/paixoes-cruzadas',
+        'md5': 'e736ce0c665e459ddb818546220b4ef8',
         'info_dict': {
             'id': 'e174042',
             'ext': 'mp3',
@@ -18,9 +18,6 @@ class RTPIE(InfoExtractor):
             'description': 'As paixões musicais de António Cartaxo e António Macedo',
             'thumbnail': 're:^https?://.*\.jpg',
         },
-        'params': {
-            'skip_download': True,  # RTMP download
-        },
     }, {
         'url': 'http://www.rtp.pt/play/p831/a-quimica-das-coisas',
         'only_matching': True,
@@ -37,21 +34,48 @@ class RTPIE(InfoExtractor):
 
         player_config = self._search_regex(
             r'(?s)RTPPLAY\.player\.newPlayer\(\s*(\{.*?\})\s*\)', webpage, 'player config')
-        config = json.loads(js_to_json(player_config))
+        config = self._parse_json(player_config, video_id)
 
         path, ext = config.get('file').rsplit('.', 1)
         formats = [{
+            'format_id': 'rtmp',
+            'ext': ext,
+            'vcodec': config.get('type') == 'audio' and 'none' or None,
+            'preference': -2,
+            'url': 'rtmp://{streamer:s}/{application:s}'.format(**config),
             'app': config.get('application'),
             'play_path': '{ext:s}:{path:s}'.format(ext=ext, path=path),
             'page_url': url,
-            'url': 'rtmp://{streamer:s}/{application:s}'.format(**config),
             'rtmp_live': config.get('live', False),
-            'ext': ext,
-            'vcodec': config.get('type') == 'audio' and 'none' or None,
             'player_url': 'http://programas.rtp.pt/play/player.swf?v3',
             'rtmp_real_time': True,
         }]
 
+        # Construct regular HTTP download URLs
+        replacements = {
+            'audio': {
+                'format_id': 'mp3',
+                'pattern': r'^nas2\.share/wavrss/',
+                'repl': 'http://rsspod.rtp.pt/podcasts/',
+                'vcodec': 'none',
+            },
+            'video': {
+                'format_id': 'mp4_h264',
+                'pattern': r'^nas2\.share/h264/',
+                'repl': 'http://rsspod.rtp.pt/videocasts/',
+                'vcodec': 'h264',
+            },
+        }
+        r = replacements[config['type']]
+        if re.match(r['pattern'], config['file']) is not None:
+            formats.append({
+                'format_id': r['format_id'],
+                'url': re.sub(r['pattern'], r['repl'], config['file']),
+                'vcodec': r['vcodec'],
+            })
+
+        self._sort_formats(formats)
+
         return {
             'id': video_id,
             'title': title,

From 467d3c9a0c364fe4e24c6efd1d20317a80eee547 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Feb 2015 22:05:11 +0100
Subject: [PATCH 1437/1937] [ffmpeg] --extrac-audio: Use the same options for
 avconv and ffmpeg

They have been available in ffmpeg since version 0.9, and we require 1.0 or higher.
---
 youtube_dl/postprocessor/ffmpeg.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 4a4422c5a..01d25f760 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -166,14 +166,13 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
         if filecodec is None:
             raise PostProcessingError('WARNING: unable to obtain file audio codec with ffprobe')
 
-        uses_avconv = self._uses_avconv()
         more_opts = []
         if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'):
             if filecodec == 'aac' and self._preferredcodec in ['m4a', 'best']:
                 # Lossless, but in another container
                 acodec = 'copy'
                 extension = 'm4a'
-                more_opts = ['-bsf:a' if uses_avconv else '-absf', 'aac_adtstoasc']
+                more_opts = ['-bsf:a', 'aac_adtstoasc']
             elif filecodec in ['aac', 'mp3', 'vorbis', 'opus']:
                 # Lossless if possible
                 acodec = 'copy'
@@ -189,9 +188,9 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 more_opts = []
                 if self._preferredquality is not None:
                     if int(self._preferredquality) < 10:
-                        more_opts += ['-q:a' if uses_avconv else '-aq', self._preferredquality]
+                        more_opts += ['-q:a', self._preferredquality]
                     else:
-                        more_opts += ['-b:a' if uses_avconv else '-ab', self._preferredquality + 'k']
+                        more_opts += ['-b:a', self._preferredquality + 'k']
         else:
             # We convert the audio (lossy)
             acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'opus': 'opus', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec]
@@ -200,13 +199,13 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
             if self._preferredquality is not None:
                 # The opus codec doesn't support the -aq option
                 if int(self._preferredquality) < 10 and extension != 'opus':
-                    more_opts += ['-q:a' if uses_avconv else '-aq', self._preferredquality]
+                    more_opts += ['-q:a', self._preferredquality]
                 else:
-                    more_opts += ['-b:a' if uses_avconv else '-ab', self._preferredquality + 'k']
+                    more_opts += ['-b:a', self._preferredquality + 'k']
             if self._preferredcodec == 'aac':
                 more_opts += ['-f', 'adts']
             if self._preferredcodec == 'm4a':
-                more_opts += ['-bsf:a' if uses_avconv else '-absf', 'aac_adtstoasc']
+                more_opts += ['-bsf:a', 'aac_adtstoasc']
             if self._preferredcodec == 'vorbis':
                 extension = 'ogg'
             if self._preferredcodec == 'wav':

From b3a286d69d2bf28ba32a77b0fe66932eb6d1bc65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Feb 2015 22:23:06 +0100
Subject: [PATCH 1438/1937] [YoutubeDL] _calc_cookies: add get_header method to
 _PseudoRequest (#4861)

---
 youtube_dl/YoutubeDL.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 9605f8f99..633e3d8a1 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -938,6 +938,9 @@ class YoutubeDL(object):
             def has_header(self, h):
                 return h in self.headers
 
+            def get_header(self, h, default=None):
+                return self.headers.get(h, default)
+
         pr = _PseudoRequest(info_dict['url'])
         self.cookiejar.add_cookie_header(pr)
         return pr.headers.get('Cookie')

From 2b4ecde2c8e7d405758a3a80190c646940fe7e9a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Feb 2015 23:54:25 +0100
Subject: [PATCH 1439/1937] [test/YoutubeDL] Add a simple test for
 postprocesors

Just checks that the 'keepvideo' option works as intended.
---
 test/test_YoutubeDL.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 678b9f7d1..47ab102a9 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -13,6 +13,7 @@ import copy
 from test.helper import FakeYDL, assertRegexpMatches
 from youtube_dl import YoutubeDL
 from youtube_dl.extractor import YoutubeIE
+from youtube_dl.postprocessor.common import PostProcessor
 
 
 class YDL(FakeYDL):
@@ -370,5 +371,36 @@ class TestFormatSelection(unittest.TestCase):
             'vbr': 10,
         }), '^\s*10k$')
 
+    def test_postprocessors(self):
+        filename = 'post-processor-testfile.mp4'
+        audiofile = filename + '.mp3'
+
+        class SimplePP(PostProcessor):
+            def run(self, info):
+                print(audiofile)
+                with open(audiofile, 'wt') as f:
+                    f.write('EXAMPLE')
+                info['filepath']
+                return False, info
+
+        def run_pp(params):
+            with open(filename, 'wt') as f:
+                f.write('EXAMPLE')
+            ydl = YoutubeDL(params)
+            ydl.add_post_processor(SimplePP())
+            ydl.post_process(filename, {'filepath': filename})
+
+        run_pp({'keepvideo': True})
+        self.assertTrue(os.path.exists(filename), '%s doesn\'t exist' % filename)
+        self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
+        os.unlink(filename)
+        os.unlink(audiofile)
+
+        run_pp({'keepvideo': False})
+        self.assertFalse(os.path.exists(filename), '%s exists' % filename)
+        self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
+        os.unlink(audiofile)
+
+
 if __name__ == '__main__':
     unittest.main()

From 0161353d7d2d0f0fce1ecaa2f77471d787ee9bdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Feb 2015 23:58:01 +0100
Subject: [PATCH 1440/1937] [test/test_YoutubeDL] Remove debug print call

---
 test/test_YoutubeDL.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 47ab102a9..b1cd6a69f 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -377,7 +377,6 @@ class TestFormatSelection(unittest.TestCase):
 
         class SimplePP(PostProcessor):
             def run(self, info):
-                print(audiofile)
                 with open(audiofile, 'wt') as f:
                     f.write('EXAMPLE')
                 info['filepath']

From c0e46412e99ef87f5730326ab8fa4a56038c43d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Feb 2015 17:30:29 +0600
Subject: [PATCH 1441/1937] [aparat] Fix extraction (Closes #4897)

---
 youtube_dl/extractor/aparat.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/aparat.py b/youtube_dl/extractor/aparat.py
index 15006336f..5d70758a3 100644
--- a/youtube_dl/extractor/aparat.py
+++ b/youtube_dl/extractor/aparat.py
@@ -34,7 +34,8 @@ class AparatIE(InfoExtractor):
                      video_id + '/vt/frame')
         webpage = self._download_webpage(embed_url, video_id)
 
-        video_urls = re.findall(r'fileList\[[0-9]+\]\s*=\s*"([^"]+)"', webpage)
+        video_urls = [video_url.replace('\\/', '/') for video_url in re.findall(
+            r'(?:fileList\[[0-9]+\]\s*=|"file"\s*:)\s*"([^"]+)"', webpage)]
         for i, video_url in enumerate(video_urls):
             req = HEADRequest(video_url)
             res = self._request_webpage(
@@ -46,7 +47,7 @@ class AparatIE(InfoExtractor):
 
         title = self._search_regex(r'\s+title:\s*"([^"]+)"', webpage, 'title')
         thumbnail = self._search_regex(
-            r'\s+image:\s*"([^"]+)"', webpage, 'thumbnail', fatal=False)
+            r'image:\s*"([^"]+)"', webpage, 'thumbnail', fatal=False)
 
         return {
             'id': video_id,

From 77d2b106ccf81e50cf41a7aa0bb320433a84e110 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 13:42:41 +0100
Subject: [PATCH 1442/1937] [Gamekings] Fix 404 when large isn't available

When trying to download some GameKings videos, not all worked. This was
because not all videos had a "/large"-URL available. The extractor
checks now if the /large URL is available, if it isn't, it tries to get
the normal URL.
---
 youtube_dl/extractor/gamekings.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index cf8e90d7d..bac325bd8 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -31,6 +31,8 @@ class GamekingsIE(InfoExtractor):
 
         # Todo: add medium format
         video_url = video_url.replace(video_id, 'large/' + video_id)
+        if not (self._is_valid_url(video_url, video_id)):
+            video_url = video_url.replace(video_id + '/large', video_id)
 
         return {
             'id': video_id,

From f23a3ca69975346d5fc30ee09b0cdceb4a384879 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 13:47:27 +0100
Subject: [PATCH 1443/1937] [Gamekings] Fixed typo in URL replacement

---
 youtube_dl/extractor/gamekings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index bac325bd8..1697f5e44 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -32,7 +32,7 @@ class GamekingsIE(InfoExtractor):
         # Todo: add medium format
         video_url = video_url.replace(video_id, 'large/' + video_id)
         if not (self._is_valid_url(video_url, video_id)):
-            video_url = video_url.replace(video_id + '/large', video_id)
+            video_url = video_url.replace(video_id + 'large/', video_id)
 
         return {
             'id': video_id,

From 2f38289b79ed6c265f2e7cc91e417d0f2178371a Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 13:49:32 +0100
Subject: [PATCH 1444/1937] [Gamekings] Fix order of replacement string

Oops.
---
 youtube_dl/extractor/gamekings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 1697f5e44..a5d854ade 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -32,7 +32,7 @@ class GamekingsIE(InfoExtractor):
         # Todo: add medium format
         video_url = video_url.replace(video_id, 'large/' + video_id)
         if not (self._is_valid_url(video_url, video_id)):
-            video_url = video_url.replace(video_id + 'large/', video_id)
+            video_url = video_url.replace('large/' + video_id, video_id)
 
         return {
             'id': video_id,

From ba322d82090bd1126774e772b699283121ffa4b8 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 14:23:37 +0100
Subject: [PATCH 1445/1937] [Gamekings] Added test and replaced video_url

Quick and dirty fix for the Gamekings extractor. It gives an error about
the video_url, but it downloads it now instead of giving a 404 error on
newer Gamekings videos
---
 youtube_dl/extractor/gamekings.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index a5d854ade..aa589390a 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -7,7 +7,8 @@ from .common import InfoExtractor
 
 class GamekingsIE(InfoExtractor):
     _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-]+)'
-    _TEST = {
+    _TESTS = [
+        {
         'url': 'http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
         # MD5 is flaky, seems to change regularly
         # 'md5': '2f32b1f7b80fdc5cb616efb4f387f8a3',
@@ -16,8 +17,17 @@ class GamekingsIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Phoenix Wright: Ace Attorney \u2013 Dual Destinies Review',
             'description': 'md5:36fd701e57e8c15ac8682a2374c99731',
+            }
+        },
+        {
+        'url': 'http://www.gamekings.tv/videos/the-legend-of-zelda-majoras-mask/',
+        'info_dict': {
+            'id': '118933752',
+            'ext': 'mp4',
+            'title': 'The Legend of Zelda: Majora’s Mask'
+            }
         }
-    }
+    ]
 
     def _real_extract(self, url):
 
@@ -33,6 +43,7 @@ class GamekingsIE(InfoExtractor):
         video_url = video_url.replace(video_id, 'large/' + video_id)
         if not (self._is_valid_url(video_url, video_id)):
             video_url = video_url.replace('large/' + video_id, video_id)
+            video_url = video_url.replace('http://stream.gamekings.tv/', '')
 
         return {
             'id': video_id,

From 8ca8cbe2bd262676cbe5a63aaee6559dfa795634 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 14:41:14 +0100
Subject: [PATCH 1446/1937] [Gamekings] Check string for vimeo, fix test

The test now doesn't fail anymore. It just checks the string for having
"vimeo" in it, instead of using the method for URL-checking, since it's
returns an error.

The tests don't fail, and the extractor works fine now.
---
 youtube_dl/extractor/gamekings.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index aa589390a..ba4d2e082 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -24,7 +24,8 @@ class GamekingsIE(InfoExtractor):
         'info_dict': {
             'id': '118933752',
             'ext': 'mp4',
-            'title': 'The Legend of Zelda: Majora’s Mask'
+            'title': 'The Legend of Zelda: Majora’s Mask',
+            'description': 'md5:9917825fe0e9f4057601fe1e38860de3'
             }
         }
     ]
@@ -41,7 +42,7 @@ class GamekingsIE(InfoExtractor):
 
         # Todo: add medium format
         video_url = video_url.replace(video_id, 'large/' + video_id)
-        if not (self._is_valid_url(video_url, video_id)):
+        if "vimeo" in video_url:
             video_url = video_url.replace('large/' + video_id, video_id)
             video_url = video_url.replace('http://stream.gamekings.tv/', '')
 

From 5d678df64a7a4a695da769f7691c86f39fca26bb Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 15:34:37 +0100
Subject: [PATCH 1447/1937] [Gamekings] Download playlist

Todo: URL and Thumbnail should be extracted with XPath
---
 youtube_dl/extractor/gamekings.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index ba4d2e082..f2760d444 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-
+from ..utils import xpath_text
 
 class GamekingsIE(InfoExtractor):
     _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-]+)'
@@ -35,6 +35,17 @@ class GamekingsIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         name = mobj.group('name')
         webpage = self._download_webpage(url, name)
+
+        playlist_id = re.search(r'(?:gogoVideo)\(\d+,"?(?P<playlist_id>.*)"', webpage, re.MULTILINE).group('playlist_id')
+        playlist_url = 'http://www.gamekings.tv/wp-content/themes/gk2010/rss_playlist.php?id=' + playlist_id
+        playlist_rss = self._download_xml(playlist_url, playlist_id)
+
+        NS_MAP {
+            'rss': 'http://rss.jwpcdn.com/'
+         }
+
+        # Todo: Implement Xpath for searching the video link
+        
         video_url = self._og_search_video_url(webpage)
 
         video = re.search(r'[0-9]+', video_url)

From a14292e8480f339bdbf9e52e93460aec5c31ef13 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 8 Feb 2015 16:57:07 +0200
Subject: [PATCH 1448/1937] [soulanime] Remove extractor (#4554)

Was supposed to be deleted by 67c2bcd
---
 youtube_dl/extractor/soulanime.py | 80 -------------------------------
 1 file changed, 80 deletions(-)
 delete mode 100644 youtube_dl/extractor/soulanime.py

diff --git a/youtube_dl/extractor/soulanime.py b/youtube_dl/extractor/soulanime.py
deleted file mode 100644
index feef33e27..000000000
--- a/youtube_dl/extractor/soulanime.py
+++ /dev/null
@@ -1,80 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    HEADRequest,
-    urlhandle_detect_ext,
-)
-
-
-class SoulAnimeWatchingIE(InfoExtractor):
-    IE_NAME = "soulanime:watching"
-    IE_DESC = "SoulAnime video"
-    _TEST = {
-        'url': 'http://www.soul-anime.net/watching/seirei-tsukai-no-blade-dance-episode-9/',
-        'md5': '05fae04abf72298098b528e98abf4298',
-        'info_dict': {
-            'id': 'seirei-tsukai-no-blade-dance-episode-9',
-            'ext': 'mp4',
-            'title': 'seirei-tsukai-no-blade-dance-episode-9',
-            'description': 'seirei-tsukai-no-blade-dance-episode-9'
-        }
-    }
-    _VALID_URL = r'http://[w.]*soul-anime\.(?P<domain>[^/]+)/watch[^/]*/(?P<id>[^/]+)'
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        domain = mobj.group('domain')
-
-        page = self._download_webpage(url, video_id)
-
-        video_url_encoded = self._html_search_regex(
-            r'<div id="download">[^<]*<a href="(?P<url>[^"]+)"', page, 'url')
-        video_url = "http://www.soul-anime." + domain + video_url_encoded
-
-        ext_req = HEADRequest(video_url)
-        ext_handle = self._request_webpage(
-            ext_req, video_id, note='Determining extension')
-        ext = urlhandle_detect_ext(ext_handle)
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'ext': ext,
-            'title': video_id,
-            'description': video_id
-        }
-
-
-class SoulAnimeSeriesIE(InfoExtractor):
-    IE_NAME = "soulanime:series"
-    IE_DESC = "SoulAnime Series"
-
-    _VALID_URL = r'http://[w.]*soul-anime\.(?P<domain>[^/]+)/anime./(?P<id>[^/]+)'
-
-    _EPISODE_REGEX = r'<option value="(/watch[^/]*/[^"]+)">[^<]*</option>'
-
-    _TEST = {
-        'url': 'http://www.soul-anime.net/anime1/black-rock-shooter-tv/',
-        'info_dict': {
-            'id': 'black-rock-shooter-tv'
-        },
-        'playlist_count': 8
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        series_id = mobj.group('id')
-        domain = mobj.group('domain')
-
-        pattern = re.compile(self._EPISODE_REGEX)
-
-        page = self._download_webpage(url, series_id, "Downloading series page")
-        mobj = pattern.findall(page)
-
-        entries = [self.url_result("http://www.soul-anime." + domain + obj) for obj in mobj]
-
-        return self.playlist_result(entries, series_id)

From 69319969de40f62ccf1c471427d309b6555483f9 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 8 Feb 2015 17:39:00 +0200
Subject: [PATCH 1449/1937] [extractor/common] Add new helper method
 _family_friendly_search

---
 youtube_dl/extractor/common.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 980b63680..2f5ba7aee 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -656,6 +656,21 @@ class InfoExtractor(object):
         }
         return RATING_TABLE.get(rating.lower(), None)
 
+    def _family_friendly_search(self, html):
+        # See http://schema.org/VideoObj
+        family_friendly = self._html_search_meta('isFamilyFriendly', html)
+
+        if not family_friendly:
+            return None
+
+        RATING_TABLE = {
+            '1': 0,
+            'true': 0,
+            '0': 18,
+            'false': 18,
+        }
+        return RATING_TABLE.get(family_friendly.lower(), None)
+
     def _twitter_search_player(self, html):
         return self._html_search_meta('twitter:player', html,
                                       'twitter card player')

From 955c5505e7df737b452d54d869958a8f5425edf5 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 16:44:25 +0100
Subject: [PATCH 1450/1937] [Gamekings] Use xpath

XPath is used for extracting the video url and the thumbnail
---
 youtube_dl/extractor/gamekings.py | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index f2760d444..3c1f7f195 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -3,7 +3,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import xpath_text
+from ..utils import (
+    xpath_text,
+    xpath_with_ns
+ )
+
 
 class GamekingsIE(InfoExtractor):
     _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-]+)'
@@ -39,23 +43,22 @@ class GamekingsIE(InfoExtractor):
         playlist_id = re.search(r'(?:gogoVideo)\(\d+,"?(?P<playlist_id>.*)"', webpage, re.MULTILINE).group('playlist_id')
         playlist_url = 'http://www.gamekings.tv/wp-content/themes/gk2010/rss_playlist.php?id=' + playlist_id
         playlist_rss = self._download_xml(playlist_url, playlist_id)
+        
 
-        NS_MAP {
-            'rss': 'http://rss.jwpcdn.com/'
+        NS_MAP = {
+            'jwplayer': 'http://rss.jwpcdn.com/'
          }
 
-        # Todo: Implement Xpath for searching the video link
+        item = playlist_rss.find('./channel/item')
         
-        video_url = self._og_search_video_url(webpage)
-
+        image = xpath_text(item, xpath_with_ns('./jwplayer:image', NS_MAP), 'image')
+        file_node = item.find(xpath_with_ns('./jwplayer:source', NS_MAP))
+        
+        video_url = file_node.get('file')
         video = re.search(r'[0-9]+', video_url)
         video_id = video.group(0)
-
-        # Todo: add medium format
-        video_url = video_url.replace(video_id, 'large/' + video_id)
-        if "vimeo" in video_url:
-            video_url = video_url.replace('large/' + video_id, video_id)
-            video_url = video_url.replace('http://stream.gamekings.tv/', '')
+        
+        # Todo: Add medium format
 
         return {
             'id': video_id,

From 641eb10d34629c177445dd38e9d3f1f73c4cb92f Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 8 Feb 2015 17:45:38 +0200
Subject: [PATCH 1451/1937] Use _family_friendly_search for determining
 age_limit

---
 youtube_dl/extractor/aparat.py   | 2 ++
 youtube_dl/extractor/goshgay.py  | 4 +---
 youtube_dl/extractor/izlesene.py | 5 +----
 youtube_dl/extractor/teamcoco.py | 7 +++++--
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/aparat.py b/youtube_dl/extractor/aparat.py
index 5d70758a3..63429780e 100644
--- a/youtube_dl/extractor/aparat.py
+++ b/youtube_dl/extractor/aparat.py
@@ -20,6 +20,7 @@ class AparatIE(InfoExtractor):
             'id': 'wP8On',
             'ext': 'mp4',
             'title': 'تیم گلکسی 11 - زومیت',
+            'age_limit': 0,
         },
         # 'skip': 'Extremely unreliable',
     }
@@ -55,4 +56,5 @@ class AparatIE(InfoExtractor):
             'url': video_url,
             'ext': 'mp4',
             'thumbnail': thumbnail,
+            'age_limit': self._family_friendly_search(webpage),
         }
diff --git a/youtube_dl/extractor/goshgay.py b/youtube_dl/extractor/goshgay.py
index b116d251d..1d9166455 100644
--- a/youtube_dl/extractor/goshgay.py
+++ b/youtube_dl/extractor/goshgay.py
@@ -34,8 +34,6 @@ class GoshgayIE(InfoExtractor):
         duration = parse_duration(self._html_search_regex(
             r'<span class="duration">\s*-?\s*(.*?)</span>',
             webpage, 'duration', fatal=False))
-        family_friendly = self._html_search_meta(
-            'isFamilyFriendly', webpage, default='false')
 
         flashvars = compat_parse_qs(self._html_search_regex(
             r'<embed.+?id="flash-player-embed".+?flashvars="([^"]+)"',
@@ -49,5 +47,5 @@ class GoshgayIE(InfoExtractor):
             'title': title,
             'thumbnail': thumbnail,
             'duration': duration,
-            'age_limit': 0 if family_friendly == 'true' else 18,
+            'age_limit': self._family_friendly_search(webpage),
         }
diff --git a/youtube_dl/extractor/izlesene.py b/youtube_dl/extractor/izlesene.py
index d16d483ee..99a1361f8 100644
--- a/youtube_dl/extractor/izlesene.py
+++ b/youtube_dl/extractor/izlesene.py
@@ -80,9 +80,6 @@ class IzleseneIE(InfoExtractor):
             r'comment_count\s*=\s*\'([^\']+)\';',
             webpage, 'comment_count', fatal=False)
 
-        family_friendly = self._html_search_meta(
-            'isFamilyFriendly', webpage, 'age limit', fatal=False)
-
         content_url = self._html_search_meta(
             'contentURL', webpage, 'content URL', fatal=False)
         ext = determine_ext(content_url, 'mp4')
@@ -120,6 +117,6 @@ class IzleseneIE(InfoExtractor):
             'duration': duration,
             'view_count': int_or_none(view_count),
             'comment_count': int_or_none(comment_count),
-            'age_limit': 18 if family_friendly == 'False' else 0,
+            'age_limit': self._family_friendly_search(webpage),
             'formats': formats,
         }
diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 18a823719..e85d452a3 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -15,7 +15,8 @@ class TeamcocoIE(InfoExtractor):
                 'id': '80187',
                 'ext': 'mp4',
                 'title': 'Conan Becomes A Mary Kay Beauty Consultant',
-                'description': 'Mary Kay is perhaps the most trusted name in female beauty, so of course Conan is a natural choice to sell their products.'
+                'description': 'Mary Kay is perhaps the most trusted name in female beauty, so of course Conan is a natural choice to sell their products.',
+                'age_limit': 0,
             }
         }, {
             'url': 'http://teamcoco.com/video/louis-ck-interview-george-w-bush',
@@ -24,7 +25,8 @@ class TeamcocoIE(InfoExtractor):
                 'id': '19705',
                 'ext': 'mp4',
                 "description": "Louis C.K. got starstruck by George W. Bush, so what? Part one.",
-                "title": "Louis C.K. Interview Pt. 1 11/3/11"
+                "title": "Louis C.K. Interview Pt. 1 11/3/11",
+                'age_limit': 0,
             }
         }
     ]
@@ -83,4 +85,5 @@ class TeamcocoIE(InfoExtractor):
             'title': self._og_search_title(webpage),
             'thumbnail': self._og_search_thumbnail(webpage),
             'description': self._og_search_description(webpage),
+            'age_limit': self._family_friendly_search(webpage),
         }

From c36b09a5026172c1ca452038fffccd68b14c528c Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Sun, 8 Feb 2015 16:46:13 +0100
Subject: [PATCH 1452/1937] [Gamekings] Use thumbnail in return statement

---
 youtube_dl/extractor/gamekings.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 3c1f7f195..1821f26d8 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -66,4 +66,5 @@ class GamekingsIE(InfoExtractor):
             'url': video_url,
             'title': self._og_search_title(webpage),
             'description': self._og_search_description(webpage),
+            'thumbnail': image
         }

From d87ec897e99d503e30fb677b0b4acf7256fa6143 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Feb 2015 23:03:12 +0600
Subject: [PATCH 1453/1937] [gamekings] Improve extraction

---
 youtube_dl/extractor/gamekings.py | 64 ++++++++++++++-----------------
 1 file changed, 29 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 1821f26d8..929779f60 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -1,70 +1,64 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     xpath_text,
-    xpath_with_ns
- )
+    xpath_with_ns,
+)
 
 
 class GamekingsIE(InfoExtractor):
-    _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-]+)'
-    _TESTS = [
-        {
+    _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<id>[^/]+)'
+    _TESTS = [{
         'url': 'http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
         # MD5 is flaky, seems to change regularly
         # 'md5': '2f32b1f7b80fdc5cb616efb4f387f8a3',
         'info_dict': {
-            'id': '20130811',
+            'id': 'phoenix-wright-ace-attorney-dual-destinies-review',
             'ext': 'mp4',
             'title': 'Phoenix Wright: Ace Attorney \u2013 Dual Destinies Review',
             'description': 'md5:36fd701e57e8c15ac8682a2374c99731',
-            }
+            'thumbnail': 're:^https?://.*\.jpg$',
         },
-        {
+    }, {
+        # vimeo video
         'url': 'http://www.gamekings.tv/videos/the-legend-of-zelda-majoras-mask/',
+        'md5': '12bf04dfd238e70058046937657ea68d',
         'info_dict': {
-            'id': '118933752',
+            'id': 'the-legend-of-zelda-majoras-mask',
             'ext': 'mp4',
             'title': 'The Legend of Zelda: Majora’s Mask',
-            'description': 'md5:9917825fe0e9f4057601fe1e38860de3'
-            }
-        }
-    ]
+            'description': 'md5:9917825fe0e9f4057601fe1e38860de3',
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+    }]
 
     def _real_extract(self, url):
+        video_id = self._match_id(url)
 
-        mobj = re.match(self._VALID_URL, url)
-        name = mobj.group('name')
-        webpage = self._download_webpage(url, name)
+        webpage = self._download_webpage(url, video_id)
 
-        playlist_id = re.search(r'(?:gogoVideo)\(\d+,"?(?P<playlist_id>.*)"', webpage, re.MULTILINE).group('playlist_id')
-        playlist_url = 'http://www.gamekings.tv/wp-content/themes/gk2010/rss_playlist.php?id=' + playlist_id
-        playlist_rss = self._download_xml(playlist_url, playlist_id)
-        
+        playlist_id = self._search_regex(
+            r'gogoVideo\(\s*\d+\s*,\s*"([^"]+)', webpage, 'playlist id')
+
+        playlist = self._download_xml(
+            'http://www.gamekings.tv/wp-content/themes/gk2010/rss_playlist.php?id=%s' % playlist_id,
+            video_id)
 
         NS_MAP = {
             'jwplayer': 'http://rss.jwpcdn.com/'
-         }
+        }
 
-        item = playlist_rss.find('./channel/item')
-        
-        image = xpath_text(item, xpath_with_ns('./jwplayer:image', NS_MAP), 'image')
-        file_node = item.find(xpath_with_ns('./jwplayer:source', NS_MAP))
-        
-        video_url = file_node.get('file')
-        video = re.search(r'[0-9]+', video_url)
-        video_id = video.group(0)
-        
-        # Todo: Add medium format
+        item = playlist.find('./channel/item')
+
+        thumbnail = xpath_text(item, xpath_with_ns('./jwplayer:image', NS_MAP), 'thumbnail')
+        video_url = item.find(xpath_with_ns('./jwplayer:source', NS_MAP)).get('file')
 
         return {
             'id': video_id,
-            'ext': 'mp4',
             'url': video_url,
             'title': self._og_search_title(webpage),
             'description': self._og_search_description(webpage),
-            'thumbnail': image
+            'thumbnail': thumbnail,
         }

From 41c23b0da58ba36805a67f4736890475ab7abedf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Feb 2015 23:12:59 +0600
Subject: [PATCH 1454/1937] [gamekings] Support videos from news pages

---
 youtube_dl/extractor/gamekings.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 929779f60..027f55eb2 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class GamekingsIE(InfoExtractor):
-    _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<id>[^/]+)'
+    _VALID_URL = r'http://www\.gamekings\.tv/(?:videos|nieuws)/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/',
         # MD5 is flaky, seems to change regularly
@@ -32,6 +32,9 @@ class GamekingsIE(InfoExtractor):
             'description': 'md5:9917825fe0e9f4057601fe1e38860de3',
             'thumbnail': 're:^https?://.*\.jpg$',
         },
+    }, {
+        'url': 'http://www.gamekings.tv/nieuws/gamekings-extra-shelly-en-david-bereiden-zich-voor-op-de-livestream/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 12931e1c6eb85adeffa19dcbd3b911335a5ab032 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Feb 2015 23:33:29 +0600
Subject: [PATCH 1455/1937] Credit @robin007bond for tweakers (#4881) and
 gamekings fixes (#4901)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 85ca81a27..3d6985ab6 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -109,3 +109,4 @@ David Luhmer
 Shaya Goldberg
 Paul Hartmann
 Frans de Jonge
+Robin de Rooij

From 9ddb6925bfc53e0f31f7d114490259e293e91edd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Feb 2015 01:07:43 +0600
Subject: [PATCH 1456/1937] [YoutubeDL] Allow filtering by string properties
 (#4906)

---
 youtube_dl/YoutubeDL.py | 41 +++++++++++++++++++++++++++++------------
 1 file changed, 29 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 633e3d8a1..0934911ec 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -832,21 +832,38 @@ class YoutubeDL(object):
             \]$
             ''' % '|'.join(map(re.escape, OPERATORS.keys())))
         m = operator_rex.search(format_spec)
+        if m:
+            try:
+                comparison_value = int(m.group('value'))
+            except ValueError:
+                comparison_value = parse_filesize(m.group('value'))
+                if comparison_value is None:
+                    comparison_value = parse_filesize(m.group('value') + 'B')
+                if comparison_value is None:
+                    raise ValueError(
+                        'Invalid value %r in format specification %r' % (
+                            m.group('value'), format_spec))
+            op = OPERATORS[m.group('op')]
+
+        if not m:
+            STR_OPERATORS = {
+                '=': operator.eq,
+                '!=': operator.ne,
+            }
+            str_operator_rex = re.compile(r'''(?x)\s*\[
+                \s*(?P<key>ext|acodec|vcodec|container|protocol)
+                \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
+                \s*(?P<value>[a-zA-Z0-9_-]+)
+                \s*\]$
+                ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
+            m = str_operator_rex.search(format_spec)
+            if m:
+                comparison_value = m.group('value')
+                op = STR_OPERATORS[m.group('op')]
+
         if not m:
             raise ValueError('Invalid format specification %r' % format_spec)
 
-        try:
-            comparison_value = int(m.group('value'))
-        except ValueError:
-            comparison_value = parse_filesize(m.group('value'))
-            if comparison_value is None:
-                comparison_value = parse_filesize(m.group('value') + 'B')
-            if comparison_value is None:
-                raise ValueError(
-                    'Invalid value %r in format specification %r' % (
-                        m.group('value'), format_spec))
-        op = OPERATORS[m.group('op')]
-
         def _filter(f):
             actual_value = f.get(m.group('key'))
             if actual_value is None:

From 2ec19e955870e82bd91949b0996dfd9acdde0232 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Feb 2015 01:09:45 +0600
Subject: [PATCH 1457/1937] [YoutubeDL] Allow filtering by audio sampling rate

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0934911ec..c7108dbc1 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -826,7 +826,7 @@ class YoutubeDL(object):
             '!=': operator.ne,
         }
         operator_rex = re.compile(r'''(?x)\s*\[
-            (?P<key>width|height|tbr|abr|vbr|filesize|fps)
+            (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
             \]$

From 6128bf07a91b941b9241c5ea0d611df9e69d48b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Feb 2015 01:27:27 +0600
Subject: [PATCH 1458/1937] [options] Update help on string comparisons

---
 youtube_dl/options.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index d5e575964..6a795e750 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -298,7 +298,9 @@ def parseOpts(overrideArguments=None):
             ' brackets, as in -f "best[height=720]"'
             ' (or -f "[filesize>10M]"). '
             ' This works for filesize, height, width, tbr, abr, vbr, and fps'
-            ' and the comparisons <, <=, >, >=, =, != .'
+            ' and the comparisons <, <=, >, >=, =, !='
+            ' and for ext, acodec, vcodec, container and protocol'
+            ' and the comparisons =, != .'
             ' Formats for which the value is not known are excluded unless you'
             ' put a question mark (?) after the operator.'
             ' You can combine format filters, so  '

From b29231c0401b7b70337bb52b0c20bc92aa3b704e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 8 Feb 2015 20:28:38 +0100
Subject: [PATCH 1459/1937] release 2015.02.08

---
 README.md              | 4 +++-
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 06dea400d..3eafad7c7 100644
--- a/README.md
+++ b/README.md
@@ -294,7 +294,9 @@ which means you can modify it, redistribute it or use it however you like.
                                      -f "[filesize>10M]").  This works for
                                      filesize, height, width, tbr, abr, vbr, and
                                      fps and the comparisons <, <=, >, >=, =, !=
-                                     . Formats for which the value is not known
+                                     and for ext, acodec, vcodec, container and
+                                     protocol and the comparisons =, != .
+                                     Formats for which the value is not known
                                      are excluded unless you put a question mark
                                      (?) after the operator. You can combine
                                      format filters, so  -f "[height <=?
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2d8f9c316..84a132684 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -14,6 +14,7 @@
  - **AddAnime**
  - **AdobeTV**
  - **AdultSwim**
+ - **Aftenposten**
  - **Aftonbladet**
  - **AlJazeera**
  - **Allocine**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1091ae61b..b0ddff6e2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.06'
+__version__ = '2015.02.08'

From 64f9baa084c60b8bfd8e33179fd6caa50084a642 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Feb 2015 01:35:16 +0600
Subject: [PATCH 1460/1937] [options] Mention asr as possible filter

---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 6a795e750..048d9992c 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -297,9 +297,9 @@ def parseOpts(overrideArguments=None):
             ' You can filter the video results by putting a condition in'
             ' brackets, as in -f "best[height=720]"'
             ' (or -f "[filesize>10M]"). '
-            ' This works for filesize, height, width, tbr, abr, vbr, and fps'
+            ' This works for filesize, height, width, tbr, abr, vbr, asr, and fps'
             ' and the comparisons <, <=, >, >=, =, !='
-            ' and for ext, acodec, vcodec, container and protocol'
+            ' and for ext, acodec, vcodec, container, and protocol'
             ' and the comparisons =, != .'
             ' Formats for which the value is not known are excluded unless you'
             ' put a question mark (?) after the operator.'

From bdb186f3b074f1f7ac0f154be419ca04241add6f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Markus=20M=C3=BCller?= <mmu@grummel.net>
Date: Sun, 8 Feb 2015 21:55:39 +0100
Subject: [PATCH 1461/1937] fix rtlnow for newer series like "Der Bachelor"
 season 5

---
 youtube_dl/extractor/rtlnow.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py
index 285c3c4be..e235f3c57 100644
--- a/youtube_dl/extractor/rtlnow.py
+++ b/youtube_dl/extractor/rtlnow.py
@@ -134,9 +134,18 @@ class RTLnowIE(InfoExtractor):
                     'player_url': video_page_url + 'includes/vodplayer.swf',
                 }
             else:
-                fmt = {
-                    'url': filename.text,
-                }
+                mobj = re.search(r'.*/(?P<hoster>[^/]+)/videos/(?P<play_path>.+)\.f4m', filename.text)
+                if mobj:
+                    fmt = {
+                        'url': 'rtmpe://fmspay-fra2.rtl.de/' + mobj.group('hoster'),
+                        'play_path': 'mp4:' + mobj.group('play_path'),
+                        'page_url': url,
+                        'player_url': video_page_url + 'includes/vodplayer.swf',
+                    }
+                else:
+                    fmt = {
+                        'url': filename.text,
+                    }
             fmt.update({
                 'width': int_or_none(filename.get('width')),
                 'height': int_or_none(filename.get('height')),

From dd8930684ea4069a1c4e28b3298b0d328145d00f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 10:28:16 +0100
Subject: [PATCH 1462/1937] release 2015.02.09

---
 README.md             | 28 ++++++++++++++--------------
 youtube_dl/version.py |  2 +-
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index 3eafad7c7..23295df4b 100644
--- a/README.md
+++ b/README.md
@@ -292,20 +292,20 @@ which means you can modify it, redistribute it or use it however you like.
                                      video results by putting a condition in
                                      brackets, as in -f "best[height=720]" (or
                                      -f "[filesize>10M]").  This works for
-                                     filesize, height, width, tbr, abr, vbr, and
-                                     fps and the comparisons <, <=, >, >=, =, !=
-                                     and for ext, acodec, vcodec, container and
-                                     protocol and the comparisons =, != .
-                                     Formats for which the value is not known
-                                     are excluded unless you put a question mark
-                                     (?) after the operator. You can combine
-                                     format filters, so  -f "[height <=?
-                                     720][tbr>500]" selects up to 720p videos
-                                     (or videos where the height is not known)
-                                     with a bitrate of at least 500 KBit/s. By
-                                     default, youtube-dl will pick the best
-                                     quality. Use commas to download multiple
-                                     audio formats, such as -f
+                                     filesize, height, width, tbr, abr, vbr,
+                                     asr, and fps and the comparisons <, <=, >,
+                                     >=, =, != and for ext, acodec, vcodec,
+                                     container, and protocol and the comparisons
+                                     =, != . Formats for which the value is not
+                                     known are excluded unless you put a
+                                     question mark (?) after the operator. You
+                                     can combine format filters, so  -f "[height
+                                     <=? 720][tbr>500]" selects up to 720p
+                                     videos (or videos where the height is not
+                                     known) with a bitrate of at least 500
+                                     KBit/s. By default, youtube-dl will pick
+                                     the best quality. Use commas to download
+                                     multiple audio formats, such as -f
                                      136/137/mp4/bestvideo,140/m4a/bestaudio.
                                      You can merge the video and audio of two
                                      formats into a single file using -f <video-
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b0ddff6e2..b62fcd63a 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.08'
+__version__ = '2015.02.09'

From a0f719854463c6f4226e4042dfa80c1b17154e1d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 10:42:25 +0100
Subject: [PATCH 1463/1937] [generic] Add support for jwPlayer YouTube videos

This makes nationalarchives.gov.uk work (Fixes #4907, fixes #4876)
---
 youtube_dl/extractor/generic.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index fbbc79a57..8a0f10803 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -524,6 +524,19 @@ class GenericIE(InfoExtractor):
                 'upload_date': '20150126',
             },
             'add_ie': ['Viddler'],
+        },
+        # jwplayer YouTube
+        {
+            'url': 'http://media.nationalarchives.gov.uk/index.php/webinar-using-discovery-national-archives-online-catalogue/',
+            'info_dict': {
+                'id': 'Mrj4DVp2zeA',
+                'ext': 'mp4',
+                'upload_date': '20150204',
+                'uploader': 'The National Archives UK',
+                'description': 'md5:a236581cd2449dd2df4f93412f3f01c6',
+                'uploader_id': 'NationalArchives08',
+                'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
+            },
         }
     ]
 
@@ -1065,6 +1078,8 @@ class GenericIE(InfoExtractor):
             return self.url_result(mobj.group('url'), 'Livestream')
 
         def check_video(vurl):
+            if YoutubeIE.suitable(vurl):
+                return True
             vpath = compat_urlparse.urlparse(vurl).path
             vext = determine_ext(vpath)
             return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml')
@@ -1082,7 +1097,8 @@ class GenericIE(InfoExtractor):
                     JWPlayerOptions|
                     jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
                 )
-                .*?file\s*:\s*["\'](.*?)["\']''', webpage))
+                .*?
+                ['"]?file['"]?\s*:\s*["\'](.*?)["\']''', webpage))
         if not found:
             # Broaden the search a little bit
             found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))

From 5269028951d5abd6f20d46c267b0ef610b2becef Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 10:47:19 +0100
Subject: [PATCH 1464/1937] [rtlnow] Add test for @mmue's extension (#4908)

---
 youtube_dl/extractor/rtlnow.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py
index e235f3c57..fd93cc66f 100644
--- a/youtube_dl/extractor/rtlnow.py
+++ b/youtube_dl/extractor/rtlnow.py
@@ -91,6 +91,15 @@ class RTLnowIE(InfoExtractor):
             },
         },
         {
+            'url': 'http://rtl-now.rtl.de/der-bachelor/folge-4.php?film_id=188729&player=1&season=5',
+            'info_dict': {
+                'id': '188729',
+                'ext': 'flv',
+                'upload_date': '20150204',
+                'description': 'md5:5e1ce23095e61a79c166d134b683cecc',
+                'title': 'Der Bachelor - Folge 4',
+            }
+        }, {
             'url': 'http://www.n-tvnow.de/deluxe-alles-was-spass-macht/thema-ua-luxushotel-fuer-vierbeiner.php?container_id=153819&player=1&season=0',
             'only_matching': True,
         },

From 196121c51baf2e7b7bb9c657f2f832b2840432de Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 10:49:10 +0100
Subject: [PATCH 1465/1937] release 2015.02.09.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b62fcd63a..e3f6d1e90 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.09'
+__version__ = '2015.02.09.1'

From e98b8e79ead03f33705356b32a08664ef2df2628 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 14:46:10 +0100
Subject: [PATCH 1466/1937] [generic] Improve SBS detection (Fixes #4899)

---
 youtube_dl/extractor/generic.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8a0f10803..f4500e931 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1047,7 +1047,12 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded sbs.com.au player
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:www\.)sbs\.com\.au/ondemand/video/single/.+?)\1',
+            r'''(?x)
+            (?:
+                <meta\s+property="og:video"\s+content=|
+                <iframe[^>]+?src=
+            )
+            (["\'])(?P<url>https?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1''',
             webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'SBS')

From 3255fe7141a1e11d65997da3de436ba3007a8fc3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 14:46:30 +0100
Subject: [PATCH 1467/1937] release 2015.02.09.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e3f6d1e90..d8911cae6 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.09.1'
+__version__ = '2015.02.09.2'

From ba617964585a118e30932461f24a6c3627047738 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 9 Feb 2015 15:04:22 +0100
Subject: [PATCH 1468/1937] [youtube] Don't override format info from the dash
 manifest (fixes #4911)

---
 youtube_dl/extractor/youtube.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e4b26b84f..8b91fff18 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -780,8 +780,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                     fo for fo in formats
                     if fo['format_id'] == format_id)
             except StopIteration:
-                f.update(self._formats.get(format_id, {}).items())
-                formats.append(f)
+                full_info = self._formats.get(format_id, {}).copy()
+                full_info.update(f)
+                formats.append(full_info)
             else:
                 existing_format.update(f)
         return formats

From 1309b396d0ff87f0e762f8893b6d0bd2aa96432a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 15:56:46 +0100
Subject: [PATCH 1469/1937] [svtplay] Add new extractor (Fixes #4914)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/svtplay.py  | 55 ++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 youtube_dl/extractor/svtplay.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0d7a120bc..55ca0d6e4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -428,6 +428,7 @@ from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
 from .streetvoice import StreetVoiceIE
 from .sunporno import SunPornoIE
+from .svtplay import SVTPlayIE
 from .swrmediathek import SWRMediathekIE
 from .syfy import SyfyIE
 from .sztvhu import SztvHuIE
diff --git a/youtube_dl/extractor/svtplay.py b/youtube_dl/extractor/svtplay.py
new file mode 100644
index 000000000..727b3d1fc
--- /dev/null
+++ b/youtube_dl/extractor/svtplay.py
@@ -0,0 +1,55 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+)
+
+
+class SVTPlayIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?svtplay\.se/video/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.svtplay.se/video/2609989/sm-veckan/sm-veckan-rally-final-sasong-1-sm-veckan-rally-final',
+        'md5': '2521cd644e862936cf2e698206e47385',
+        'info_dict': {
+            'id': '3966754',
+            'ext': 'mp4',
+            'title': 'FIFA 14 - E3 2013 Trailer',
+            'duration': 4500,
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        info = self._download_json(
+            'http://www.svtplay.se/video/%s?output=json' % video_id, video_id)
+
+        title = info['context']['title']
+        thumbnail = info['context'].get('thumbnailImage')
+
+        video_info = info['video']
+        formats = []
+        for vr in video_info['videoReferences']:
+            vurl = vr['url']
+            if determine_ext(vurl) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    vurl, video_id,
+                    ext='mp4', entry_protocol='m3u8_native',
+                    m3u8_id=vr.get('playerType')))
+            else:
+                formats.append({
+                    'format_id': vr.get('playerType'),
+                    'url': vurl,
+                })
+        self._sort_formats(formats)
+
+        duration = video_info.get('materialLength')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'duration': duration,
+        }

From 43f244b6d51aefac4683d130d70adf94641bf6d8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 15:57:40 +0100
Subject: [PATCH 1470/1937] [YoutubeDL] Do not show worst in --list-formats
 output

Nobody wants to know what the worst possible format is. And if they do, they can still provide -f worst.
---
 youtube_dl/YoutubeDL.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index c7108dbc1..3975ae0bc 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1546,7 +1546,6 @@ class YoutubeDL(object):
             line(f, idlen) for f in formats
             if f.get('preference') is None or f['preference'] >= -1000]
         if len(formats) > 1:
-            formats_s[0] += (' ' if self._format_note(formats[0]) else '') + '(worst)'
             formats_s[-1] += (' ' if self._format_note(formats[-1]) else '') + '(best)'
 
         header_line = line({

From 2caf182f37562a8dc3857f88a19f6902f6d829e7 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 9 Feb 2015 16:55:31 +0200
Subject: [PATCH 1471/1937] [trilulilu] Add support for videos without category
 in the URL (Closes #4067)

Also, update the testcase, detect private/country restricted videos and modernize a bit.
---
 youtube_dl/extractor/trilulilu.py | 51 ++++++++++++++++++++-----------
 1 file changed, 33 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/trilulilu.py b/youtube_dl/extractor/trilulilu.py
index 220a05b7b..185accc4b 100644
--- a/youtube_dl/extractor/trilulilu.py
+++ b/youtube_dl/extractor/trilulilu.py
@@ -1,40 +1,55 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
-import json
+import re
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
 
 class TriluliluIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?trilulilu\.ro/video-[^/]+/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?trilulilu\.ro/(?:video-[^/]+/)?(?P<id>[^/#\?]+)'
     _TEST = {
         'url': 'http://www.trilulilu.ro/video-animatie/big-buck-bunny-1',
+        'md5': 'c1450a00da251e2769b74b9005601cac',
         'info_dict': {
-            'id': 'big-buck-bunny-1',
+            'id': 'ae2899e124140b',
             'ext': 'mp4',
             'title': 'Big Buck Bunny',
             'description': ':) pentru copilul din noi',
         },
-        # Server ignores Range headers (--test)
-        'params': {
-            'skip_download': True
-        }
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
 
+        if re.search(r'Fişierul nu este disponibil pentru vizionare în ţara dumneavoastră', webpage):
+            raise ExtractorError(
+                'This video is not available in your country.', expected=True)
+        elif re.search('Fişierul poate fi accesat doar de către prietenii lui', webpage):
+            raise ExtractorError('This video is private.', expected=True)
+
+        flashvars_str = self._search_regex(
+            r'block_flash_vars\s*=\s*(\{[^\}]+\})', webpage, 'flashvars', fatal=False, default=None)
+
+        if flashvars_str:
+            flashvars = self._parse_json(flashvars_str, display_id)
+        else:
+            raise ExtractorError(
+                'This page does not contain videos', expected=True)
+
+        if flashvars['isMP3'] == 'true':
+            raise ExtractorError(
+                'Audio downloads are currently not supported', expected=True)
+
+        video_id = flashvars['hash']
         title = self._og_search_title(webpage)
         thumbnail = self._og_search_thumbnail(webpage)
-        description = self._og_search_description(webpage)
-
-        log_str = self._search_regex(
-            r'block_flash_vars[ ]=[ ]({[^}]+})', webpage, 'log info')
-        log = json.loads(log_str)
+        description = self._og_search_description(webpage, default=None)
 
         format_url = ('http://fs%(server)s.trilulilu.ro/%(hash)s/'
-                      'video-formats2' % log)
+                      'video-formats2' % flashvars)
         format_doc = self._download_xml(
             format_url, video_id,
             note='Downloading formats',
@@ -44,10 +59,10 @@ class TriluliluIE(InfoExtractor):
             'http://fs%(server)s.trilulilu.ro/stream.php?type=video'
             '&source=site&hash=%(hash)s&username=%(userid)s&'
             'key=ministhebest&format=%%s&sig=&exp=' %
-            log)
+            flashvars)
         formats = [
             {
-                'format': fnode.text,
+                'format_id': fnode.text.partition('-')[2],
                 'url': video_url_template % fnode.text,
                 'ext': fnode.text.partition('-')[0]
             }
@@ -56,8 +71,8 @@ class TriluliluIE(InfoExtractor):
         ]
 
         return {
-            '_type': 'video',
             'id': video_id,
+            'display_id': display_id,
             'formats': formats,
             'title': title,
             'description': description,

From f18e3a2fc0b73011c98c39d54576fb835cee4561 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 15:59:19 +0100
Subject: [PATCH 1472/1937] release 2015.02.09.3

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 84a132684..93572f6a8 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -392,6 +392,7 @@
  - **StreamCZ**
  - **StreetVoice**
  - **SunPorno**
+ - **SVTPlay**
  - **SWRMediathek**
  - **Syfy**
  - **SztvHu**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d8911cae6..30447a368 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.09.2'
+__version__ = '2015.02.09.3'

From 28f1272870c43700a90af4671b6eae46d0fab6c6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 16:05:01 +0100
Subject: [PATCH 1473/1937] [svtplay] Correct test case

---
 youtube_dl/extractor/svtplay.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/svtplay.py b/youtube_dl/extractor/svtplay.py
index 727b3d1fc..eadb9ccb4 100644
--- a/youtube_dl/extractor/svtplay.py
+++ b/youtube_dl/extractor/svtplay.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
@@ -10,13 +11,13 @@ class SVTPlayIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?svtplay\.se/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.svtplay.se/video/2609989/sm-veckan/sm-veckan-rally-final-sasong-1-sm-veckan-rally-final',
-        'md5': '2521cd644e862936cf2e698206e47385',
+        'md5': 'f4a184968bc9c802a9b41316657aaa80',
         'info_dict': {
-            'id': '3966754',
+            'id': '2609989',
             'ext': 'mp4',
-            'title': 'FIFA 14 - E3 2013 Trailer',
+            'title': 'SM veckan vinter, Örebro - Rally, final',
             'duration': 4500,
-            'thumbnail': 're:^https?://.*\.jpg$',
+            'thumbnail': 're:^https?://.*[\.-]jpg$',
         },
     }
 

From 99c2398bc641c478dbea85c013081d883479827e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Feb 2015 19:08:51 +0100
Subject: [PATCH 1474/1937] [bandcamp] Use our API to get more stable error
 messages (#1194)

---
 youtube_dl/extractor/bandcamp.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index aea0263d6..27d8652f6 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -72,26 +72,29 @@ class BandcampIE(InfoExtractor):
 
         download_link = m_download.group(1)
         video_id = self._search_regex(
-            r'var TralbumData = {.*?id: (?P<id>\d+),?$',
-            webpage, 'video id', flags=re.MULTILINE | re.DOTALL)
+            r'(?ms)var TralbumData = {.*?id: (?P<id>\d+),?$',
+            webpage, 'video id')
 
         download_webpage = self._download_webpage(download_link, video_id, 'Downloading free downloads page')
         # We get the dictionary of the track from some javascript code
-        info = re.search(r'items: (.*?),$', download_webpage, re.MULTILINE).group(1)
-        info = json.loads(info)[0]
+        all_info = self._parse_json(self._search_regex(
+            r'(?sm)items: (.*?),$', download_webpage, 'items'), video_id)
+        info = All_info[0]
         # We pick mp3-320 for now, until format selection can be easily implemented.
         mp3_info = info['downloads']['mp3-320']
         # If we try to use this url it says the link has expired
         initial_url = mp3_info['url']
-        re_url = r'(?P<server>http://(.*?)\.bandcamp\.com)/download/track\?enc=mp3-320&fsig=(?P<fsig>.*?)&id=(?P<id>.*?)&ts=(?P<ts>.*)$'
-        m_url = re.match(re_url, initial_url)
+        m_url = re.match(
+            r'(?P<server>http://(.*?)\.bandcamp\.com)/download/track\?enc=mp3-320&fsig=(?P<fsig>.*?)&id=(?P<id>.*?)&ts=(?P<ts>.*)$',
+            initial_url)
         # We build the url we will use to get the final track url
         # This url is build in Bandcamp in the script download_bunde_*.js
         request_url = '%s/statdownload/track?enc=mp3-320&fsig=%s&id=%s&ts=%s&.rand=665028774616&.vrs=1' % (m_url.group('server'), m_url.group('fsig'), video_id, m_url.group('ts'))
         final_url_webpage = self._download_webpage(request_url, video_id, 'Requesting download url')
         # If we could correctly generate the .rand field the url would be
         # in the "download_url" key
-        final_url = re.search(r'"retry_url":"(.*?)"', final_url_webpage).group(1)
+        final_url = self._search_regex(
+            r'"retry_url":"(.*?)"', final_url_webpage, 'final video URL')
 
         return {
             'id': video_id,

From 3a5bcd03263059bd7b905d4586c5f23f7f120974 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:13:57 +0100
Subject: [PATCH 1475/1937] [extractor/common] Wrap extractor errors (Fixes
 #1194)

For now, we just wrap some common errors. More may follow. We do not want to catch actual programming errors in the extractors, such as 1 // 0.
---
 youtube_dl/extractor/common.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 2f5ba7aee..eee936a6f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -264,8 +264,15 @@ class InfoExtractor(object):
 
     def extract(self, url):
         """Extracts URL information and returns it in list of dicts."""
-        self.initialize()
-        return self._real_extract(url)
+        try:
+            self.initialize()
+            return self._real_extract(url)
+        except ExtractorError:
+            raise
+        except compat_http_client.IncompleteRead as e:
+            raise ExtractorError('A network error has occured.', cause=e, expected=True)
+        except (KeyError,) as e:
+            raise ExtractorError('An extractor error has occured.', cause=e)
 
     def set_downloader(self, downloader):
         """Sets the downloader for this IE."""

From 34814eb66e948b5d73420e435f3aea7112b243d5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:19:52 +0100
Subject: [PATCH 1476/1937] release 2015.02.10

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 30447a368..2414627da 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.09.3'
+__version__ = '2015.02.10'

From 753fad4adc32b57a0d18518fddd06d2411d09635 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:34:01 +0100
Subject: [PATCH 1477/1937] [commonmistakes] Correct logic error

---
 youtube_dl/extractor/commonmistakes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/commonmistakes.py b/youtube_dl/extractor/commonmistakes.py
index 75c06903f..dbbf27a74 100644
--- a/youtube_dl/extractor/commonmistakes.py
+++ b/youtube_dl/extractor/commonmistakes.py
@@ -24,6 +24,6 @@ class CommonMistakesIE(InfoExtractor):
             'That doesn\'t make any sense. '
             'Simply remove the parameter in your command or configuration.'
         ) % url
-        if self._downloader.params.get('verbose'):
+        if not self._downloader.params.get('verbose'):
             msg += ' Add -v to the command line to see what arguments and configuration youtube-dl got.'
         raise ExtractorError(msg, expected=True)

From c06a9fa34f41edf090f4beb385212e44a6e1bb08 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:36:38 +0100
Subject: [PATCH 1478/1937] Use snake_case instead of camelCase

---
 youtube_dl/options.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 048d9992c..9d9195b60 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -734,22 +734,22 @@ def parseOpts(overrideArguments=None):
         if opts.verbose:
             write_string('[debug] Override config: ' + repr(overrideArguments) + '\n')
     else:
-        commandLineConf = sys.argv[1:]
-        if '--ignore-config' in commandLineConf:
-            systemConf = []
-            userConf = []
+        command_line_conf = sys.argv[1:]
+        if '--ignore-config' in command_line_conf:
+            system_conf = []
+            user_conf = []
         else:
-            systemConf = _readOptions('/etc/youtube-dl.conf')
-            if '--ignore-config' in systemConf:
-                userConf = []
+            system_conf = _readOptions('/etc/youtube-dl.conf')
+            if '--ignore-config' in system_conf:
+                user_conf = []
             else:
-                userConf = _readUserConf()
-        argv = systemConf + userConf + commandLineConf
+                user_conf = _readUserConf()
+        argv = system_conf + user_conf + command_line_conf
 
         opts, args = parser.parse_args(argv)
         if opts.verbose:
-            write_string('[debug] System config: ' + repr(_hide_login_info(systemConf)) + '\n')
-            write_string('[debug] User config: ' + repr(_hide_login_info(userConf)) + '\n')
-            write_string('[debug] Command-line args: ' + repr(_hide_login_info(commandLineConf)) + '\n')
+            write_string('[debug] System config: ' + repr(_hide_login_info(system_conf)) + '\n')
+            write_string('[debug] User config: ' + repr(_hide_login_info(user_conf)) + '\n')
+            write_string('[debug] Command-line args: ' + repr(_hide_login_info(command_line_conf)) + '\n')
 
     return parser, opts, args

From 834bf069d20745263586284c98196b5b0605b916 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:37:14 +0100
Subject: [PATCH 1479/1937] [bandcamp] Correct variable name

---
 youtube_dl/extractor/bandcamp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 27d8652f6..490cc961a 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -79,7 +79,7 @@ class BandcampIE(InfoExtractor):
         # We get the dictionary of the track from some javascript code
         all_info = self._parse_json(self._search_regex(
             r'(?sm)items: (.*?),$', download_webpage, 'items'), video_id)
-        info = All_info[0]
+        info = all_info[0]
         # We pick mp3-320 for now, until format selection can be easily implemented.
         mp3_info = info['downloads']['mp3-320']
         # If we try to use this url it says the link has expired

From c73fae1e2e4421df664aefd1d14a72596caf9e2f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:39:43 +0100
Subject: [PATCH 1480/1937] [commonmistakes] Detect BOMs at the beginning of
 URLs

Reported at https://bugzilla.redhat.com/show_bug.cgi?id=1093517 .
---
 youtube_dl/extractor/__init__.py       |  2 +-
 youtube_dl/extractor/commonmistakes.py | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 55ca0d6e4..fb1e7f325 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -74,7 +74,7 @@ from .collegehumor import CollegeHumorIE
 from .collegerama import CollegeRamaIE
 from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
 from .comcarcoff import ComCarCoffIE
-from .commonmistakes import CommonMistakesIE
+from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
 from .condenast import CondeNastIE
 from .cracked import CrackedIE
 from .criterion import CriterionIE
diff --git a/youtube_dl/extractor/commonmistakes.py b/youtube_dl/extractor/commonmistakes.py
index dbbf27a74..2f86e2381 100644
--- a/youtube_dl/extractor/commonmistakes.py
+++ b/youtube_dl/extractor/commonmistakes.py
@@ -27,3 +27,20 @@ class CommonMistakesIE(InfoExtractor):
         if not self._downloader.params.get('verbose'):
             msg += ' Add -v to the command line to see what arguments and configuration youtube-dl got.'
         raise ExtractorError(msg, expected=True)
+
+
+class UnicodeBOMIE(InfoExtractor):
+        IE_DESC = False
+        _VALID_URL = r'(?P<bom>\ufeff)(?P<id>.*)$'
+
+        _TESTS = [{
+            'url': '\ufeffhttp://www.youtube.com/watch?v=BaW_jenozKc',
+            'only_matching': True,
+        }]
+
+        def _real_extract(self, url):
+            real_url = self._match_id(url)
+            self.report_warning(
+                'Your URL starts with a Byte Order Mark (BOM). '
+                'Removing the BOM and looking for "%s" ...' % real_url)
+            return self.url_result(real_url)

From 88296505131f4b91ff91eaa0af34318664d892c9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 01:46:09 +0100
Subject: [PATCH 1481/1937] release 2015.02.10.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2414627da..7d144dadc 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10'
+__version__ = '2015.02.10.1'

From 347de4931cb3e496fb7b1dfb0314c213f44cce6b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 03:32:21 +0100
Subject: [PATCH 1482/1937] [YoutubeDL] Add generic video filtering (Fixes
 #4916)

This functionality is intended to eventually encompass the current format filtering.
---
 test/test_utils.py      | 32 +++++++++++++++++
 youtube_dl/YoutubeDL.py | 14 +++++++-
 youtube_dl/__init__.py  |  7 +++-
 youtube_dl/options.py   | 19 ++++++++++
 youtube_dl/utils.py     | 77 +++++++++++++++++++++++++++++++++++++++++
 5 files changed, 147 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 80c765bc4..1c29d0889 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -53,6 +53,7 @@ from youtube_dl.utils import (
     version_tuple,
     xpath_with_ns,
     render_table,
+    match_str,
 )
 
 
@@ -459,6 +460,37 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
             '123  4\n'
             '9999 51')
 
+    def test_match_str(self):
+        self.assertRaises(ValueError, match_str, 'xy>foobar', {})
+        self.assertFalse(match_str('xy', {'x': 1200}))
+        self.assertTrue(match_str('!xy', {'x': 1200}))
+        self.assertTrue(match_str('x', {'x': 1200}))
+        self.assertFalse(match_str('!x', {'x': 1200}))
+        self.assertTrue(match_str('x', {'x': 0}))
+        self.assertFalse(match_str('x>0', {'x': 0}))
+        self.assertFalse(match_str('x>0', {}))
+        self.assertTrue(match_str('x>?0', {}))
+        self.assertTrue(match_str('x>1K', {'x': 1200}))
+        self.assertFalse(match_str('x>2K', {'x': 1200}))
+        self.assertTrue(match_str('x>=1200 & x < 1300', {'x': 1200}))
+        self.assertFalse(match_str('x>=1100 & x < 1200', {'x': 1200}))
+        self.assertFalse(match_str('y=a212', {'y': 'foobar42'}))
+        self.assertTrue(match_str('y=foobar42', {'y': 'foobar42'}))
+        self.assertFalse(match_str('y!=foobar42', {'y': 'foobar42'}))
+        self.assertTrue(match_str('y!=foobar2', {'y': 'foobar42'}))
+        self.assertFalse(match_str(
+            'like_count > 100 & dislike_count <? 50 & description',
+            {'like_count': 90, 'description': 'foo'}))
+        self.assertTrue(match_str(
+            'like_count > 100 & dislike_count <? 50 & description',
+            {'like_count': 190, 'description': 'foo'}))
+        self.assertFalse(match_str(
+            'like_count > 100 & dislike_count <? 50 & description',
+            {'like_count': 190, 'dislike_count': 60, 'description': 'foo'}))
+        self.assertFalse(match_str(
+            'like_count > 100 & dislike_count <? 50 & description',
+            {'like_count': 190, 'dislike_count': 10}))
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3975ae0bc..dda222fee 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -228,6 +228,11 @@ class YoutubeDL(object):
     external_downloader:  Executable of the external downloader to call.
     listformats:       Print an overview of available video formats and exit.
     list_thumbnails:   Print a table of all thumbnails and exit.
+    match_filter:      A function that gets called with the info_dict of
+                       every video.
+                       If it returns a message, the video is ignored.
+                       If it returns None, the video is downloaded.
+                       match_filter_func in utils.py is one example for this.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
@@ -583,9 +588,16 @@ class YoutubeDL(object):
             if max_views is not None and view_count > max_views:
                 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
         if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
-            return 'Skipping "%s" because it is age restricted' % title
+            return 'Skipping "%s" because it is age restricted' % video_title
         if self.in_download_archive(info_dict):
             return '%s has already been recorded in archive' % video_title
+
+        match_filter = self.params.get('match_filter')
+        if match_filter is not None:
+            ret = match_filter(info_dict)
+            if ret is not None:
+                return ret
+
         return None
 
     @staticmethod
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index e90679ff9..eefca0fe4 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -23,9 +23,10 @@ from .compat import (
 )
 from .utils import (
     DateRange,
-    DEFAULT_OUTTMPL,
     decodeOption,
+    DEFAULT_OUTTMPL,
     DownloadError,
+    match_filter_func,
     MaxDownloadsReached,
     preferredencoding,
     read_batch_urls,
@@ -247,6 +248,9 @@ def _real_main(argv=None):
             xattr  # Confuse flake8
         except ImportError:
             parser.error('setting filesize xattr requested but python-xattr is not available')
+    match_filter = (
+        None if opts.match_filter is None
+        else match_filter_func(opts.match_filter))
 
     ydl_opts = {
         'usenetrc': opts.usenetrc,
@@ -344,6 +348,7 @@ def _real_main(argv=None):
         'list_thumbnails': opts.list_thumbnails,
         'playlist_items': opts.playlist_items,
         'xattr_set_filesize': opts.xattr_set_filesize,
+        'match_filter': match_filter,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 9d9195b60..f64aa5b85 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -244,6 +244,25 @@ def parseOpts(overrideArguments=None):
         '--max-views',
         metavar='COUNT', dest='max_views', default=None, type=int,
         help='Do not download any videos with more than COUNT views')
+    selection.add_option(
+        '--match-filter',
+        metavar='FILTER', dest='match_filter', default=None,
+        help=(
+            '(Experimental) Generic video filter. '
+            'Specify any key (see help for -o for a list of available keys) to'
+            ' match if the key is present, '
+            '!key to check if the key is not present,'
+            'key > NUMBER (like "comment_count > 12", also works with '
+            '>=, <, <=, !=, =) to compare against a number, and '
+            '& to require multiple matches. '
+            'Values which are not known are excluded unless you'
+            ' put a question mark (?) after the operator.'
+            'For example, to only match videos that have been liked more than '
+            '100 times and disliked less than 50 times (or the dislike '
+            'functionality is not available at the given service), but who '
+            'also have a description, use  --match-filter '
+            '"like_count > 100 & dislike_count <? 50 & description" .'
+        ))
     selection.add_option(
         '--no-playlist',
         action='store_true', dest='noplaylist', default=False,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 8f5463f1c..03566d223 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -17,6 +17,7 @@ import io
 import json
 import locale
 import math
+import operator
 import os
 import pipes
 import platform
@@ -1678,3 +1679,79 @@ def render_table(header_row, data):
     max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
     format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
     return '\n'.join(format_str % tuple(row) for row in table)
+
+
+def _match_one(filter_part, dct):
+    COMPARISON_OPERATORS = {
+        '<': operator.lt,
+        '<=': operator.le,
+        '>': operator.gt,
+        '>=': operator.ge,
+        '=': operator.eq,
+        '!=': operator.ne,
+    }
+    operator_rex = re.compile(r'''(?x)\s*
+        (?P<key>[a-z_]+)
+        \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
+        (?:
+            (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
+            (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
+        )
+        \s*$
+        ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
+    m = operator_rex.search(filter_part)
+    if m:
+        op = COMPARISON_OPERATORS[m.group('op')]
+        if m.group('strval') is not None:
+            if m.group('op') not in ('=', '!='):
+                raise ValueError(
+                    'Operator %s does not support string values!' % m.group('op'))
+            comparison_value = m.group('strval')
+        else:
+            try:
+                comparison_value = int(m.group('intval'))
+            except ValueError:
+                comparison_value = parse_filesize(m.group('intval'))
+                if comparison_value is None:
+                    comparison_value = parse_filesize(m.group('intval') + 'B')
+                if comparison_value is None:
+                    raise ValueError(
+                        'Invalid integer value %r in filter part %r' % (
+                            m.group('intval'), filter_part))
+        actual_value = dct.get(m.group('key'))
+        if actual_value is None:
+            return m.group('none_inclusive')
+        return op(actual_value, comparison_value)
+
+    UNARY_OPERATORS = {
+        '': lambda v: v is not None,
+        '!': lambda v: v is None,
+    }
+    operator_rex = re.compile(r'''(?x)\s*
+        (?P<op>%s)\s*(?P<key>[a-z_]+)
+        \s*$
+        ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
+    m = operator_rex.search(filter_part)
+    if m:
+        op = UNARY_OPERATORS[m.group('op')]
+        actual_value = dct.get(m.group('key'))
+        return op(actual_value)
+
+    raise ValueError('Invalid filter part %r' % filter_part)
+
+
+def match_str(filter_str, dct):
+    """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
+
+    return all(
+        _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
+
+
+def match_filter_func(filter_str):
+    def _match_func(info_dict):
+        if match_str(filter_str, info_dict):
+            return None
+        else:
+            video_title = info_dict.get('title', info_dict.get('id', 'video'))
+            return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
+    return _match_func

From 845734773d334b1a338129dad7c656c12c8fbf6f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 03:32:55 +0100
Subject: [PATCH 1483/1937] release 2015.02.10.2

---
 README.md             | 17 +++++++++++++++++
 youtube_dl/version.py |  2 +-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 23295df4b..7aad3007f 100644
--- a/README.md
+++ b/README.md
@@ -119,6 +119,23 @@ which means you can modify it, redistribute it or use it however you like.
                                      COUNT views
     --max-views COUNT                Do not download any videos with more than
                                      COUNT views
+    --match-filter FILTER            (Experimental) Generic video filter.
+                                     Specify any key (see help for -o for a list
+                                     of available keys) to match if the key is
+                                     present, !key to check if the key is not
+                                     present,key > NUMBER (like "comment_count >
+                                     12", also works with >=, <, <=, !=, =) to
+                                     compare against a number, and & to require
+                                     multiple matches. Values which are not
+                                     known are excluded unless you put a
+                                     question mark (?) after the operator.For
+                                     example, to only match videos that have
+                                     been liked more than 100 times and disliked
+                                     less than 50 times (or the dislike
+                                     functionality is not available at the given
+                                     service), but who also have a description,
+                                     use  --match-filter "like_count > 100 &
+                                     dislike_count <? 50 & description" .
     --no-playlist                    If the URL refers to a video and a
                                      playlist, download only the video.
     --age-limit YEARS                download only videos suitable for the given
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7d144dadc..9ab5b49b8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10.1'
+__version__ = '2015.02.10.2'

From f7a211dcc8d86fe5a4d25f4d209ffb3efc05bdfc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 03:41:31 +0100
Subject: [PATCH 1484/1937] [pornhd] Fix extraction (fixes #4915)

---
 youtube_dl/extractor/pornhd.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index 954dfccb7..dbb2c3bd9 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -46,16 +46,17 @@ class PornHdIE(InfoExtractor):
 
         quality = qualities(['sd', 'hd'])
         sources = json.loads(js_to_json(self._search_regex(
-            r"(?s)'sources'\s*:\s*(\{.+?\})\s*\}\);", webpage, 'sources')))
+            r"(?s)'sources'\s*:\s*(\{.+?\})\s*\}[;,)]",
+            webpage, 'sources')))
         formats = []
-        for container, s in sources.items():
-            for qname, video_url in s.items():
-                formats.append({
-                    'url': video_url,
-                    'container': container,
-                    'format_id': '%s-%s' % (container, qname),
-                    'quality': quality(qname),
-                })
+        for qname, video_url in sources.items():
+            if not video_url:
+                continue
+            formats.append({
+                'url': video_url,
+                'format_id': qname,
+                'quality': quality(qname),
+            })
         self._sort_formats(formats)
 
         return {

From 7e5db8c930e5f154287f98c6c0da90fffd2ede4f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 04:22:10 +0100
Subject: [PATCH 1485/1937] [options] Add --no-color

---
 youtube_dl/YoutubeDL.py        | 5 +++--
 youtube_dl/__init__.py         | 1 +
 youtube_dl/extractor/common.py | 2 +-
 youtube_dl/options.py          | 5 +++++
 4 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index dda222fee..e2ec55c90 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -233,6 +233,7 @@ class YoutubeDL(object):
                        If it returns a message, the video is ignored.
                        If it returns None, the video is downloaded.
                        match_filter_func in utils.py is one example for this.
+    no_color:          Do not emit color codes in output.
 
 
     The following parameters are not used by YoutubeDL itself, they are used by
@@ -490,7 +491,7 @@ class YoutubeDL(object):
         else:
             if self.params.get('no_warnings'):
                 return
-            if self._err_file.isatty() and os.name != 'nt':
+            if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
                 _msg_header = '\033[0;33mWARNING:\033[0m'
             else:
                 _msg_header = 'WARNING:'
@@ -502,7 +503,7 @@ class YoutubeDL(object):
         Do the same as trouble, but prefixes the message with 'ERROR:', colored
         in red if stderr is a tty file.
         '''
-        if self._err_file.isatty() and os.name != 'nt':
+        if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
             _msg_header = '\033[0;31mERROR:\033[0m'
         else:
             _msg_header = 'ERROR:'
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index eefca0fe4..ed22f169f 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -349,6 +349,7 @@ def _real_main(argv=None):
         'playlist_items': opts.playlist_items,
         'xattr_set_filesize': opts.xattr_set_filesize,
         'match_filter': match_filter,
+        'no_color': opts.no_color,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index eee936a6f..382e846c5 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -514,7 +514,7 @@ class InfoExtractor(object):
                 if mobj:
                     break
 
-        if os.name != 'nt' and sys.stderr.isatty():
+        if not self._downloader.params.get('no_color') and os.name != 'nt' and sys.stderr.isatty():
             _name = '\033[0;34m%s\033[0m' % name
         else:
             _name = name
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index f64aa5b85..0219e6ba4 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -165,6 +165,11 @@ def parseOpts(overrideArguments=None):
         action='store_const', dest='extract_flat', const='in_playlist',
         default=False,
         help='Do not extract the videos of a playlist, only list them.')
+    general.add_option(
+        '--no-color', '--no-colors',
+        action='store_true', dest='no_color',
+        default=False,
+        help='Do not emit color codes in output.')
 
     network = optparse.OptionGroup(parser, 'Network Options')
     network.add_option(

From 2f543a2142e05bac988ccd6566e0c9fc5cadca90 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 04:52:33 +0100
Subject: [PATCH 1486/1937] [options] Add alias --dump-header for
 --print-traffic

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 0219e6ba4..873432bee 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -557,7 +557,7 @@ def parseOpts(overrideArguments=None):
         action='store_true', dest='youtube_print_sig_code', default=False,
         help=optparse.SUPPRESS_HELP)
     verbosity.add_option(
-        '--print-traffic',
+        '--print-traffic', '--dump-headers',
         dest='debug_printtraffic', action='store_true', default=False,
         help='Display sent and read HTTP traffic')
     verbosity.add_option(

From bf8f082a90f4396a0a70464b55bf5c11fdb36c47 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 04:53:21 +0100
Subject: [PATCH 1487/1937] [vimeo:album] Add support for album passwords
 (Fixes #4917)

---
 youtube_dl/extractor/vimeo.py | 59 ++++++++++++++++++++++++++++++-----
 1 file changed, 52 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 1bb473514..303e81447 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -188,9 +188,9 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
         password_request = compat_urllib_request.Request(pass_url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         password_request.add_header('Cookie', 'xsrft=%s' % token)
-        self._download_webpage(password_request, video_id,
-                               'Verifying the password',
-                               'Wrong password')
+        return self._download_webpage(
+            password_request, video_id,
+            'Verifying the password', 'Wrong password')
 
     def _verify_player_video_password(self, url, video_id):
         password = self._downloader.params.get('videopassword', None)
@@ -266,7 +266,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
             if re.search('The creator of this video has not given you permission to embed it on this domain.', webpage):
                 raise ExtractorError('The author has restricted the access to this video, try with the "--referer" option')
 
-            if re.search('<form[^>]+?id="pw_form"', webpage) is not None:
+            if re.search(r'<form[^>]+?id="pw_form"', webpage) is not None:
                 self._verify_video_password(url, video_id, webpage)
                 return self._real_extract(url)
             else:
@@ -412,12 +412,47 @@ class VimeoChannelIE(InfoExtractor):
     def _extract_list_title(self, webpage):
         return self._html_search_regex(self._TITLE_RE, webpage, 'list title')
 
+    def _login_list_password(self, page_url, list_id, webpage):
+        login_form = self._search_regex(
+            r'(?s)<form[^>]+?id="pw_form"(.*?)</form>',
+            webpage, 'login form', default=None)
+        if not login_form:
+            return webpage
+
+        password = self._downloader.params.get('videopassword', None)
+        if password is None:
+            raise ExtractorError('This album is protected by a password, use the --video-password option', expected=True)
+        fields = dict(re.findall(r'''(?x)<input\s+
+            type="hidden"\s+
+            name="([^"]+)"\s+
+            value="([^"]*)"
+            ''', login_form))
+        token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
+        fields['token'] = token
+        fields['password'] = password
+        post = compat_urllib_parse.urlencode(fields)
+        password_path = self._search_regex(
+            r'action="([^"]+)"', login_form, 'password URL')
+        password_url = compat_urlparse.urljoin(page_url, password_path)
+        password_request = compat_urllib_request.Request(password_url, post)
+        password_request.add_header('Content-type', 'application/x-www-form-urlencoded')
+        self._set_cookie('vimeo.com', 'xsrft', token)
+
+        return self._download_webpage(
+            password_request, list_id,
+            'Verifying the password', 'Wrong password')
+
     def _extract_videos(self, list_id, base_url):
         video_ids = []
         for pagenum in itertools.count(1):
+            page_url = self._page_url(base_url, pagenum)
             webpage = self._download_webpage(
-                self._page_url(base_url, pagenum), list_id,
+                page_url, list_id,
                 'Downloading page %s' % pagenum)
+
+            if pagenum == 1:
+                webpage = self._login_list_password(page_url, list_id, webpage)
+
             video_ids.extend(re.findall(r'id="clip_(\d+?)"', webpage))
             if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
                 break
@@ -464,14 +499,24 @@ class VimeoAlbumIE(VimeoChannelIE):
             'title': 'Staff Favorites: November 2013',
         },
         'playlist_mincount': 13,
+    }, {
+        'note': 'Password-protected album',
+        'url': 'https://vimeo.com/album/3253534',
+        'info_dict': {
+            'title': 'test',
+            'id': '3253534',
+        },
+        'playlist_count': 1,
+        'params': {
+            'videopassword': 'youtube-dl',
+        }
     }]
 
     def _page_url(self, base_url, pagenum):
         return '%s/page:%d/' % (base_url, pagenum)
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        album_id = mobj.group('id')
+        album_id = self._match_id(url)
         return self._extract_videos(album_id, 'http://vimeo.com/album/%s' % album_id)
 
 

From c0e1a415fdcd8d36d98a9fb6ff5bf9146004ddd3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 05:28:48 +0100
Subject: [PATCH 1488/1937] [firstpost] Modernize

---
 youtube_dl/extractor/firstpost.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/firstpost.py b/youtube_dl/extractor/firstpost.py
index 0993af1c9..298227d57 100644
--- a/youtube_dl/extractor/firstpost.py
+++ b/youtube_dl/extractor/firstpost.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -20,11 +18,10 @@ class FirstpostIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         page = self._download_webpage(url, video_id)
-        title = self._html_search_meta('twitter:title', page, 'title')
+
+        title = self._html_search_meta('twitter:title', page, 'title', fatal=True)
         description = self._html_search_meta('twitter:description', page, 'title')
 
         data = self._download_xml(
@@ -42,6 +39,7 @@ class FirstpostIE(InfoExtractor):
                 'height': int(details.find('./height').text.strip()),
             } for details in item.findall('./source/file_details') if details.find('./file').text
         ]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From 6a78740211b37cab5d20e707db9f73dca6c60049 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 05:28:59 +0100
Subject: [PATCH 1489/1937] [test/test_youtube_signature] Use fake YDL

---
 test/test_youtube_signature.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 13d228cd8..09696e19a 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -8,11 +8,11 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-
 import io
 import re
 import string
 
+from test.helper import FakeYDL
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.compat import compat_str, compat_urlretrieve
 
@@ -88,7 +88,8 @@ def make_tfunc(url, stype, sig_input, expected_sig):
         if not os.path.exists(fn):
             compat_urlretrieve(url, fn)
 
-        ie = YoutubeIE()
+        ydl = FakeYDL()
+        ie = YoutubeIE(ydl)
         if stype == 'js':
             with io.open(fn, encoding='utf-8') as testf:
                 jscode = testf.read()

From 8f84f5718319293485eef7174702340f109c7227 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 05:42:41 +0100
Subject: [PATCH 1490/1937] [ccc] Add new extractor (Fixes #4890)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ccc.py      | 99 ++++++++++++++++++++++++++++++++
 2 files changed, 100 insertions(+)
 create mode 100644 youtube_dl/extractor/ccc.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index fb1e7f325..da3b2b2d0 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -54,6 +54,7 @@ from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .cbs import CBSIE
 from .cbsnews import CBSNewsIE
+from .ccc import CCCIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chilloutzone import ChilloutzoneIE
diff --git a/youtube_dl/extractor/ccc.py b/youtube_dl/extractor/ccc.py
new file mode 100644
index 000000000..2a5d4be18
--- /dev/null
+++ b/youtube_dl/extractor/ccc.py
@@ -0,0 +1,99 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    qualities,
+    unified_strdate,
+)
+
+
+class CCCIE(InfoExtractor):
+    IE_NAME = 'media.ccc.de'
+    _VALID_URL = r'https?://(?:www\.)?media\.ccc\.de/[^?#]+/[^?#/]*?_(?P<id>[0-9]{8,})._[^?#/]*\.html'
+
+    _TEST = {
+        'url': 'http://media.ccc.de/browse/congress/2013/30C3_-_5443_-_en_-_saal_g_-_201312281830_-_introduction_to_processor_design_-_byterazor.html#video',
+        'md5': '205a365d0d57c0b1e43a12c9ffe8f9be',
+        'info_dict': {
+            'id': '20131228183',
+            'ext': 'mp4',
+            'title': 'Introduction to Processor Design',
+            'description': 'md5:5ddbf8c734800267f2cee4eab187bc1b',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'view_count': int,
+            'upload_date': '20131229',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        if self._downloader.params.get('prefer_free_formats'):
+            preference = qualities(['mp3', 'opus', 'mp4-lq', 'webm-lq', 'h264-sd', 'mp4-sd', 'webm-sd', 'mp4', 'webm', 'mp4-hd', 'h264-hd', 'webm-hd'])
+        else:
+            preference = qualities(['opus', 'mp3', 'webm-lq', 'mp4-lq', 'webm-sd', 'h264-sd', 'mp4-sd', 'webm', 'mp4', 'webm-hd', 'mp4-hd', 'h264-hd'])
+
+        title = self._html_search_regex(
+            r'(?s)<h1>(.*?)</h1>', webpage, 'title')
+        description = self._html_search_regex(
+            r"(?s)<p class='description'>(.*?)</p>",
+            webpage, 'description', fatal=False)
+        upload_date = unified_strdate(self._html_search_regex(
+            r"(?s)<span class='[^']*fa-calendar-o'></span>(.*?)</li>",
+            webpage, 'upload date', fatal=False))
+        view_count = int_or_none(self._html_search_regex(
+            r"(?s)<span class='[^']*fa-eye'></span>(.*?)</li>",
+            webpage, 'view count', fatal=False))
+
+        matches = re.finditer(r'''(?xs)
+            <(?:span|div)\s+class='label\s+filetype'>(?P<format>.*?)</(?:span|div)>\s*
+            <a\s+href='(?P<http_url>[^']+)'>\s*
+            (?:
+                .*?
+                <a\s+href='(?P<torrent_url>[^']+\.torrent)'
+            )?''', webpage)
+        formats = []
+        for m in matches:
+            format = m.group('format')
+            format_id = self._search_regex(
+                r'.*/([a-z0-9_-]+)/[^/]*$',
+                m.group('http_url'), 'format id', default=None)
+            vcodec = 'h264' if 'h264' in format_id else (
+                'none' if format_id in ('mp3', 'opus') else None
+            )
+            formats.append({
+                'format_id': format_id,
+                'format': format,
+                'url': m.group('http_url'),
+                'vcodec': vcodec,
+                'preference': preference(format_id),
+            })
+
+            if m.group('torrent_url'):
+                formats.append({
+                    'format_id': 'torrent-%s' % (format if format_id is None else format_id),
+                    'format': '%s (torrent)' % format,
+                    'proto': 'torrent',
+                    'format_note': '(unsupported; will just download the .torrent file)',
+                    'vcodec': vcodec,
+                    'preference': -100 + preference(format_id),
+                    'url': m.group('torrent_url'),
+                })
+        self._sort_formats(formats)
+
+        thumbnail = self._html_search_regex(
+            r"<video.*?poster='([^']+)'", webpage, 'thumbnail', fatal=False)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'view_count': view_count,
+            'upload_date': upload_date,
+            'formats': formats,
+        }

From 04bbe41330a4c8619e6fa3228d47c4e7b0396899 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 05:42:47 +0100
Subject: [PATCH 1491/1937] release 2015.02.10.3

---
 README.md              | 1 +
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7aad3007f..731cea1e1 100644
--- a/README.md
+++ b/README.md
@@ -77,6 +77,7 @@ which means you can modify it, redistribute it or use it however you like.
                                      on Windows)
     --flat-playlist                  Do not extract the videos of a playlist,
                                      only list them.
+    --no-color                       Do not emit color codes in output.
 
 ## Network Options:
     --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 93572f6a8..8bce8fede 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -225,6 +225,7 @@
  - **mailru**: Видео@Mail.Ru
  - **Malemotion**
  - **MDR**
+ - **media.ccc.de**
  - **metacafe**
  - **Metacritic**
  - **Mgoon**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9ab5b49b8..13c436219 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10.2'
+__version__ = '2015.02.10.3'

From 442c37b7a9a912cf3b0eb972335b7d24407f4537 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 11:28:28 +0100
Subject: [PATCH 1492/1937] [YoutubeDL] Do not perform filter matching on
 partial results (Fixes #4921)

---
 youtube_dl/YoutubeDL.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e2ec55c90..13d18e25e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -560,7 +560,7 @@ class YoutubeDL(object):
             self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
             return None
 
-    def _match_entry(self, info_dict):
+    def _match_entry(self, info_dict, incomplete):
         """ Returns None iff the file should be downloaded """
 
         video_title = info_dict.get('title', info_dict.get('id', 'video'))
@@ -593,11 +593,12 @@ class YoutubeDL(object):
         if self.in_download_archive(info_dict):
             return '%s has already been recorded in archive' % video_title
 
-        match_filter = self.params.get('match_filter')
-        if match_filter is not None:
-            ret = match_filter(info_dict)
-            if ret is not None:
-                return ret
+        if not incomplete:
+            match_filter = self.params.get('match_filter')
+            if match_filter is not None:
+                ret = match_filter(info_dict)
+                if ret is not None:
+                    return ret
 
         return None
 
@@ -792,7 +793,7 @@ class YoutubeDL(object):
                     'extractor_key': ie_result['extractor_key'],
                 }
 
-                reason = self._match_entry(entry)
+                reason = self._match_entry(entry, incomplete=True)
                 if reason is not None:
                     self.to_screen('[download] ' + reason)
                     continue
@@ -1166,7 +1167,7 @@ class YoutubeDL(object):
         if 'format' not in info_dict:
             info_dict['format'] = info_dict['ext']
 
-        reason = self._match_entry(info_dict)
+        reason = self._match_entry(info_dict, incomplete=False)
         if reason is not None:
             self.to_screen('[download] ' + reason)
             return

From 612ee37365cf0e0e5c6a59fcce2986fa0754605c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 11:28:34 +0100
Subject: [PATCH 1493/1937] release 2015.02.10.4

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 13c436219..895d2da0d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10.3'
+__version__ = '2015.02.10.4'

From 596ac6e31f6baaadf8280e2c54ca90a7cf2fbc2a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 15:45:36 +0100
Subject: [PATCH 1494/1937] [escapist] Modernize

---
 youtube_dl/extractor/escapist.py | 51 ++++++++++++++++----------------
 1 file changed, 25 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index e240cb859..98b698277 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -1,18 +1,17 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
 )
 from ..utils import (
     ExtractorError,
+    js_to_json,
 )
 
 
 class EscapistIE(InfoExtractor):
-    _VALID_URL = r'^https?://?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<id>[0-9]+)-'
+    _VALID_URL = r'https?://?(www\.)?escapistmagazine\.com/videos/view/[^/?#]+/(?P<id>[0-9]+)-[^/?#]*(?:$|[?#])'
     _TEST = {
         'url': 'http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
         'md5': 'ab3a706c681efca53f0a35f1415cf0d1',
@@ -20,31 +19,30 @@ class EscapistIE(InfoExtractor):
             'id': '6618',
             'ext': 'mp4',
             'description': "Baldur's Gate: Original, Modded or Enhanced Edition? I'll break down what you can expect from the new Baldur's Gate: Enhanced Edition.",
-            'uploader': 'the-escapist-presents',
+            'uploader_id': 'the-escapist-presents',
+            'uploader': 'The Escapist Presents',
             'title': "Breaking Down Baldur's Gate",
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        showName = mobj.group('showname')
-        video_id = mobj.group('id')
-
-        self.report_extraction(video_id)
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        videoDesc = self._html_search_regex(
-            r'<meta name="description" content="([^"]*)"',
-            webpage, 'description', fatal=False)
+        uploader_id = self._html_search_regex(
+            r"<h1 class='headline'><a href='/videos/view/(.*?)'",
+            webpage, 'uploader ID', fatal=False)
+        uploader = self._html_search_regex(
+            r"<h1 class='headline'>(.*?)</a>",
+            webpage, 'uploader', fatal=False)
+        description = self._html_search_meta('description', webpage)
 
-        playerUrl = self._og_search_video_url(webpage, name='player URL')
+        raw_title = self._html_search_meta('title', webpage, fatal=True)
+        title = raw_title.partition(' : ')[2]
 
-        title = self._html_search_regex(
-            r'<meta name="title" content="([^"]*)"',
-            webpage, 'title').split(' : ')[-1]
-
-        configUrl = self._search_regex('config=(.*)$', playerUrl, 'config URL')
-        configUrl = compat_urllib_parse.unquote(configUrl)
+        player_url = self._og_search_video_url(webpage, name='player URL')
+        config_url = compat_urllib_parse.unquote(self._search_regex(
+            r'config=(.*)$', player_url, 'config URL'))
 
         formats = []
 
@@ -53,7 +51,7 @@ class EscapistIE(InfoExtractor):
                 cfgurl, video_id,
                 'Downloading ' + name + ' configuration',
                 'Unable to download ' + name + ' configuration',
-                transform_source=lambda s: s.replace("'", '"'))
+                transform_source=js_to_json)
 
             playlist = config['playlist']
             formats.append({
@@ -62,9 +60,9 @@ class EscapistIE(InfoExtractor):
                 'quality': quality,
             })
 
-        _add_format('normal', configUrl, quality=0)
-        hq_url = (configUrl +
-                  ('&hq=1' if '?' in configUrl else configUrl + '?hq=1'))
+        _add_format('normal', config_url, quality=0)
+        hq_url = (config_url +
+                  ('&hq=1' if '?' in config_url else config_url + '?hq=1'))
         try:
             _add_format('hq', hq_url, quality=1)
         except ExtractorError:
@@ -75,9 +73,10 @@ class EscapistIE(InfoExtractor):
         return {
             'id': video_id,
             'formats': formats,
-            'uploader': showName,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
             'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
-            'description': videoDesc,
-            'player_url': playerUrl,
+            'description': description,
+            'player_url': player_url,
         }

From 9650885be954b3cc626205aa67f24c31f21da7b6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 15:55:51 +0100
Subject: [PATCH 1495/1937] [escapist] Filter video differently (Fixes #4919)

---
 youtube_dl/extractor/common.py   | 2 +-
 youtube_dl/extractor/escapist.py | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 382e846c5..8d71494b4 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -271,7 +271,7 @@ class InfoExtractor(object):
             raise
         except compat_http_client.IncompleteRead as e:
             raise ExtractorError('A network error has occured.', cause=e, expected=True)
-        except (KeyError,) as e:
+        except (KeyError, StopIteration) as e:
             raise ExtractorError('An extractor error has occured.', cause=e)
 
     def set_downloader(self, downloader):
diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 98b698277..4303feccd 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -54,8 +54,11 @@ class EscapistIE(InfoExtractor):
                 transform_source=js_to_json)
 
             playlist = config['playlist']
+            video_url = next(
+                p['url'] for p in playlist
+                if p.get('eventCategory') == 'Video')
             formats.append({
-                'url': playlist[1]['url'],
+                'url': video_url,
                 'format_id': name,
                 'quality': quality,
             })

From af0d11f2441728ce12ba8a424de29aa873b12d86 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 10 Feb 2015 15:56:04 +0100
Subject: [PATCH 1496/1937] release 2015.02.10.5

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 895d2da0d..225b30f8c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10.4'
+__version__ = '2015.02.10.5'

From 054fe3cc40cc0feaa69e21b31946c568a749eb76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 10 Feb 2015 21:35:34 +0600
Subject: [PATCH 1497/1937] [ntvru] Adapt to new direct delivery and modernize
 (Closes #4918)

---
 youtube_dl/extractor/ntvru.py | 97 +++++++++++++++--------------------
 1 file changed, 40 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/ntvru.py b/youtube_dl/extractor/ntvru.py
index 0ab8d5100..2cd924d05 100644
--- a/youtube_dl/extractor/ntvru.py
+++ b/youtube_dl/extractor/ntvru.py
@@ -3,7 +3,9 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    unescapeHTML
+    clean_html,
+    xpath_text,
+    int_or_none,
 )
 
 
@@ -14,73 +16,63 @@ class NTVRuIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://www.ntv.ru/novosti/863142/',
+            'md5': 'ba7ea172a91cb83eb734cad18c10e723',
             'info_dict': {
                 'id': '746000',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Командующий Черноморским флотом провел переговоры в штабе ВМС Украины',
                 'description': 'Командующий Черноморским флотом провел переговоры в штабе ВМС Украины',
+                'thumbnail': 're:^http://.*\.jpg',
                 'duration': 136,
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         },
         {
             'url': 'http://www.ntv.ru/video/novosti/750370/',
+            'md5': 'adecff79691b4d71e25220a191477124',
             'info_dict': {
                 'id': '750370',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
                 'description': 'Родные пассажиров пропавшего Boeing не верят в трагический исход',
+                'thumbnail': 're:^http://.*\.jpg',
                 'duration': 172,
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         },
         {
             'url': 'http://www.ntv.ru/peredacha/segodnya/m23700/o232416',
+            'md5': '82dbd49b38e3af1d00df16acbeab260c',
             'info_dict': {
                 'id': '747480',
-                'ext': 'flv',
-                'title': '«Сегодня». 21 марта 2014 года. 16:00 ',
-                'description': '«Сегодня». 21 марта 2014 года. 16:00 ',
+                'ext': 'mp4',
+                'title': '«Сегодня». 21 марта 2014 года. 16:00',
+                'description': '«Сегодня». 21 марта 2014 года. 16:00',
+                'thumbnail': 're:^http://.*\.jpg',
                 'duration': 1496,
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         },
         {
             'url': 'http://www.ntv.ru/kino/Koma_film',
+            'md5': 'f825770930937aa7e5aca0dc0d29319a',
             'info_dict': {
-                'id': '758100',
-                'ext': 'flv',
+                'id': '1007609',
+                'ext': 'mp4',
                 'title': 'Остросюжетный фильм «Кома»',
                 'description': 'Остросюжетный фильм «Кома»',
+                'thumbnail': 're:^http://.*\.jpg',
                 'duration': 5592,
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         },
         {
             'url': 'http://www.ntv.ru/serial/Delo_vrachey/m31760/o233916/',
+            'md5': '9320cd0e23f3ea59c330dc744e06ff3b',
             'info_dict': {
                 'id': '751482',
-                'ext': 'flv',
+                'ext': 'mp4',
                 'title': '«Дело врачей»: «Деревце жизни»',
                 'description': '«Дело врачей»: «Деревце жизни»',
+                'thumbnail': 're:^http://.*\.jpg',
                 'duration': 2590,
             },
-            'params': {
-                # rtmp download
-                'skip_download': True,
-            },
         },
     ]
 
@@ -92,45 +84,36 @@ class NTVRuIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        page = self._download_webpage(url, video_id)
 
-        video_id = self._html_search_regex(self._VIDEO_ID_REGEXES, page, 'video id')
+        webpage = self._download_webpage(url, video_id)
 
-        player = self._download_xml('http://www.ntv.ru/vi%s/' % video_id, video_id, 'Downloading video XML')
-        title = unescapeHTML(player.find('./data/title').text)
-        description = unescapeHTML(player.find('./data/description').text)
+        video_id = self._html_search_regex(self._VIDEO_ID_REGEXES, webpage, 'video id')
+
+        player = self._download_xml(
+            'http://www.ntv.ru/vi%s/' % video_id,
+            video_id, 'Downloading video XML')
+        title = clean_html(xpath_text(player, './data/title', 'title', fatal=True))
+        description = clean_html(xpath_text(player, './data/description', 'description'))
 
         video = player.find('./data/video')
-        video_id = video.find('./id').text
-        thumbnail = video.find('./splash').text
-        duration = int(video.find('./totaltime').text)
-        view_count = int(video.find('./views').text)
-        puid22 = video.find('./puid22').text
+        video_id = xpath_text(video, './id', 'video id')
+        thumbnail = xpath_text(video, './splash', 'thumbnail')
+        duration = int_or_none(xpath_text(video, './totaltime', 'duration'))
+        view_count = int_or_none(xpath_text(video, './views', 'view count'))
 
-        apps = {
-            '4': 'video1',
-            '7': 'video2',
-        }
-
-        app = apps.get(puid22, apps['4'])
+        token = self._download_webpage(
+            'http://stat.ntv.ru/services/access/token',
+            video_id, 'Downloading access token')
 
         formats = []
         for format_id in ['', 'hi', 'webm']:
-            file = video.find('./%sfile' % format_id)
-            if file is None:
+            file_ = video.find('./%sfile' % format_id)
+            if file_ is None:
                 continue
             size = video.find('./%ssize' % format_id)
             formats.append({
-                'url': 'rtmp://media.ntv.ru/%s' % app,
-                'app': app,
-                'play_path': file.text,
-                'rtmp_conn': 'B:1',
-                'player_url': 'http://www.ntv.ru/swf/vps1.swf?update=20131128',
-                'page_url': 'http://www.ntv.ru',
-                'flash_version': 'LNX 11,2,202,341',
-                'rtmp_live': True,
-                'ext': 'flv',
-                'filesize': int(size.text),
+                'url': 'http://media2.ntv.ru/vod/%s&tok=%s' % (file_.text, token),
+                'filesize': int_or_none(size.text if size is not None else None),
             })
         self._sort_formats(formats)
 

From 8708d764252a83e650f36fe47f89ace4f2a91a5b Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 11 Feb 2015 16:39:15 +0800
Subject: [PATCH 1498/1937] [camdemy] Add new extractor

Single file download done, while folder extaction in plan
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/camdemy.py  | 78 ++++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 youtube_dl/extractor/camdemy.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index da3b2b2d0..496e4d0b7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -49,6 +49,7 @@ from .brightcove import BrightcoveIE
 from .buzzfeed import BuzzFeedIE
 from .byutv import BYUtvIE
 from .c56 import C56IE
+from .camdemy import CamdemyIE
 from .canal13cl import Canal13clIE
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
new file mode 100644
index 000000000..54959f2cf
--- /dev/null
+++ b/youtube_dl/extractor/camdemy.py
@@ -0,0 +1,78 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import parse_iso8601
+
+
+class CamdemyIE(InfoExtractor):
+    _VALID_URL = r'http://www.camdemy.com/media/(?P<id>\d+).*'
+    _TESTS = [{
+        # single file
+        'url': 'http://www.camdemy.com/media/5181/',
+        'md5': '5a5562b6a98b37873119102e052e311b',
+        'info_dict': {
+            'id': '5181',
+            'ext': 'mp4',
+            'title': 'Ch1-1 Introduction, Signals (02-23-2012)',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': '',
+            'creator': 'ss11spring',
+            'upload_date': '20130114',
+            'timestamp': 1358154556,
+        }
+    }, {
+        # With non-empty description
+        'url': 'http://www.camdemy.com/media/13885',
+        'md5': '4576a3bb2581f86c61044822adbd1249',
+        'info_dict': {
+            'id': '13885',
+            'ext': 'mp4',
+            'title': 'EverCam + Camdemy QuickStart',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'description': 'md5:050b62f71ed62928f8a35f1a41e186c9',
+            'creator': 'evercam',
+            'upload_date': '20140620',
+            'timestamp': 1403271569,
+        }
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        page = self._download_webpage(url, video_id)
+
+        oembed_obj = self._download_json(
+            'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)
+
+        thumb_url = oembed_obj['thumbnail_url']
+        video_folder = compat_urlparse.urljoin(thumb_url, 'video/')
+        fileListXML = self._download_xml(
+            compat_urlparse.urljoin(video_folder, 'fileList.xml'),
+            video_id, 'Filelist XML')
+        fileName = fileListXML.find('./video/item/fileName').text
+
+        creation_time = self._html_search_regex(
+            r"<div class='title'>Posted :</div>.*<div class='value'>([0-9:\- ]+)<",
+            page, 'creation time', flags=re.MULTILINE | re.DOTALL) + '+08:00'
+        creation_timestamp = parse_iso8601(creation_time, delimiter=' ')
+
+        view_count_str = self._html_search_regex(
+            r"<div class='title'>Views :</div>.*<div class='value'>([0-9,]+)<",
+            page, 'view count', flags=re.MULTILINE | re.DOTALL)
+        views = int(view_count_str.replace(',', ''))
+
+        return {
+            'id': video_id,
+            'url': compat_urlparse.urljoin(video_folder, fileName),
+            'title': oembed_obj['title'],
+            'thumbnail': thumb_url,
+            'description': self._html_search_meta('description', page),
+            'creator': oembed_obj['author_name'],
+            'duration': oembed_obj['duration'],
+            'timestamp': creation_timestamp,
+            'view_count': views,
+        }

From 3811c567e734f38f830e8c2e0a45afa7c48cbf12 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 11 Feb 2015 15:47:19 +0200
Subject: [PATCH 1499/1937] [teamcoco] Fix video id extraction

---
 youtube_dl/extractor/teamcoco.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index e85d452a3..a73da1c9c 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -30,6 +30,11 @@ class TeamcocoIE(InfoExtractor):
             }
         }
     ]
+    _VIDEO_ID_REGEXES = (
+        r'"eVar42"\s*:\s*(\d+)',
+        r'Ginger\.TeamCoco\.openInApp\("video",\s*"([^"]+)"',
+        r'"id_not"\s*:\s*(\d+)'
+    )
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -40,8 +45,7 @@ class TeamcocoIE(InfoExtractor):
         video_id = mobj.group("video_id")
         if not video_id:
             video_id = self._html_search_regex(
-                r'<div\s+class="player".*?data-id="(\d+?)"',
-                webpage, 'video id')
+                self._VIDEO_ID_REGEXES, webpage, 'video id')
 
         data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id
         data = self._download_xml(

From ea5db8469ee3e5f1bb4ba3deb88f28b4d9838e09 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Wed, 11 Feb 2015 16:21:52 +0200
Subject: [PATCH 1500/1937] [canalplus] Add support for itele.fr URLs (Closes
 #4931)

---
 youtube_dl/extractor/canalplus.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 11d18d74a..1b14471e5 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -15,12 +15,13 @@ from ..utils import (
 
 class CanalplusIE(InfoExtractor):
     IE_DESC = 'canalplus.fr, piwiplus.fr and d8.tv'
-    _VALID_URL = r'https?://(?:www\.(?P<site>canalplus\.fr|piwiplus\.fr|d8\.tv)/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
+    _VALID_URL = r'https?://(?:www\.(?P<site>canalplus\.fr|piwiplus\.fr|d8\.tv|itele\.fr)/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>[0-9]+))'
     _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/%s/%s'
     _SITE_ID_MAP = {
         'canalplus.fr': 'cplus',
         'piwiplus.fr': 'teletoon',
         'd8.tv': 'd8',
+        'itele.fr': 'itele',
     }
 
     _TESTS = [{
@@ -53,6 +54,16 @@ class CanalplusIE(InfoExtractor):
             'upload_date': '20131108',
         },
         'skip': 'videos get deleted after a while',
+    }, {
+        'url': 'http://www.itele.fr/france/video/aubervilliers-un-lycee-en-colere-111559',
+        'md5': '65aa83ad62fe107ce29e564bb8712580',
+        'info_dict': {
+            'id': '1213714',
+            'ext': 'flv',
+            'title': 'Aubervilliers : un lycée en colère - Le 11/02/2015 à 06h45',
+            'description': 'md5:8216206ec53426ea6321321f3b3c16db',
+            'upload_date': '20150211',
+        },
     }]
 
     def _real_extract(self, url):

From c56d7d899d5587cd73cb017b776ceb302494b6fd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Feb 2015 22:10:33 +0600
Subject: [PATCH 1501/1937] [dctptv] Skip rtmp download

---
 youtube_dl/extractor/dctp.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/dctp.py b/youtube_dl/extractor/dctp.py
index 6ed354379..aa2c09eb6 100644
--- a/youtube_dl/extractor/dctp.py
+++ b/youtube_dl/extractor/dctp.py
@@ -14,6 +14,10 @@ class DctpTvIE(InfoExtractor):
             'display_id': 'videoinstallation-fuer-eine-kaufhausfassade',
             'ext': 'flv',
             'title': 'Videoinstallation für eine Kaufhausfassade'
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
         }
     }
 

From 8367d3f3cbad7ddae5116c3cd5d39ebfed9711c5 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 Feb 2015 00:11:33 +0800
Subject: [PATCH 1502/1937] [camdemy] Detection of external sources

---
 youtube_dl/extractor/camdemy.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 54959f2cf..9a6341f0f 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -38,6 +38,19 @@ class CamdemyIE(InfoExtractor):
             'upload_date': '20140620',
             'timestamp': 1403271569,
         }
+    }, {
+        # External source
+        'url': 'http://www.camdemy.com/media/14842',
+        'md5': '50e1c3c3aa233d3d7b7daa2fa10b1cf7',
+        'info_dict': {
+            'id': '2vsYQzNIsJo',
+            'ext': 'mp4',
+            'upload_date': '20130211',
+            'uploader': 'Hun Kim',
+            'description': 'Excel 2013 Tutorial for Beginners - How to add Password Protection',
+            'uploader_id': 'hunkimtutorials',
+            'title': 'Excel 2013 Tutorial - How to add Password Protection',
+        }
     }]
 
     def _real_extract(self, url):
@@ -45,6 +58,13 @@ class CamdemyIE(InfoExtractor):
 
         page = self._download_webpage(url, video_id)
 
+        srcFrom = self._html_search_regex(
+            r"<div class='srcFrom'>Source: <a title='([^']+)'", page,
+            'external source', default=None)
+
+        if srcFrom:
+            return self.url_result(srcFrom)
+
         oembed_obj = self._download_json(
             'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)
 

From d304209a850793cd121820307be9bee7e9b9ae1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 11 Feb 2015 17:25:04 +0100
Subject: [PATCH 1503/1937] [test/parameters.json] Set 'fixup' to 'never'

The fixed audio files for Youtube have a size lower that the minimum required.
---
 test/parameters.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/parameters.json b/test/parameters.json
index 098cd0cd0..af77b89b4 100644
--- a/test/parameters.json
+++ b/test/parameters.json
@@ -39,5 +39,6 @@
     "writesubtitles": false,
     "allsubtitles": false,
     "listssubtitles": false,
-    "socket_timeout": 20
+    "socket_timeout": 20,
+    "fixup": "never"
 }

From 124f3bc67d61adeb4dc257eb8a678df821c27314 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Feb 2015 22:33:03 +0600
Subject: [PATCH 1504/1937] [dotsub] Fix extraction and modernize

---
 youtube_dl/extractor/dotsub.py | 44 +++++++++++++++++++++-------------
 1 file changed, 27 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/dotsub.py b/youtube_dl/extractor/dotsub.py
index 638bb33cd..f51d88a98 100644
--- a/youtube_dl/extractor/dotsub.py
+++ b/youtube_dl/extractor/dotsub.py
@@ -1,13 +1,14 @@
 from __future__ import unicode_literals
 
-import re
-import time
-
 from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    int_or_none,
+)
 
 
 class DotsubIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?dotsub\.com/view/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?dotsub\.com/view/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://dotsub.com/view/aed3b8b2-1889-4df5-ae63-ad85f5572f27',
         'md5': '0914d4d69605090f623b7ac329fea66e',
@@ -15,28 +16,37 @@ class DotsubIE(InfoExtractor):
             'id': 'aed3b8b2-1889-4df5-ae63-ad85f5572f27',
             'ext': 'flv',
             'title': 'Pyramids of Waste (2010), AKA The Lightbulb Conspiracy - Planned obsolescence documentary',
+            'description': 'md5:699a0f7f50aeec6042cb3b1db2d0d074',
+            'thumbnail': 're:^https?://dotsub.com/media/aed3b8b2-1889-4df5-ae63-ad85f5572f27/p',
+            'duration': 3169,
             'uploader': '4v4l0n42',
-            'description': 'Pyramids of Waste (2010) also known as "The lightbulb conspiracy" is a documentary about how our economic system based on consumerism  and planned obsolescence is breaking our planet down.\r\n\r\nSolutions to this can be found at:\r\nhttp://robotswillstealyourjob.com\r\nhttp://www.federicopistono.org\r\n\r\nhttp://opensourceecology.org\r\nhttp://thezeitgeistmovement.com',
-            'thumbnail': 'http://dotsub.com/media/aed3b8b2-1889-4df5-ae63-ad85f5572f27/p',
+            'timestamp': 1292248482.625,
             'upload_date': '20101213',
+            'view_count': int,
         }
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        info_url = "https://dotsub.com/api/media/%s/metadata" % video_id
-        info = self._download_json(info_url, video_id)
-        date = time.gmtime(info['dateCreated'] / 1000)  # The timestamp is in miliseconds
+        video_id = self._match_id(url)
+
+        info = self._download_json(
+            'https://dotsub.com/api/media/%s/metadata' % video_id, video_id)
+        video_url = info.get('mediaURI')
+
+        if not video_url:
+            webpage = self._download_webpage(url, video_id)
+            video_url = self._search_regex(
+                r'"file"\s*:\s*\'([^\']+)', webpage, 'video url')
 
         return {
             'id': video_id,
-            'url': info['mediaURI'],
+            'url': video_url,
             'ext': 'flv',
             'title': info['title'],
-            'thumbnail': info['screenshotURI'],
-            'description': info['description'],
-            'uploader': info['user'],
-            'view_count': info['numberOfViews'],
-            'upload_date': '%04i%02i%02i' % (date.tm_year, date.tm_mon, date.tm_mday),
+            'description': info.get('description'),
+            'thumbnail': info.get('screenshotURI'),
+            'duration': int_or_none(info.get('duration'), 1000),
+            'uploader': info.get('user'),
+            'timestamp': float_or_none(info.get('dateCreated'), 1000),
+            'view_count': int_or_none(info.get('numberOfViews')),
         }

From f36f92f4da3664f7286b8b4327ac5791fb98e5b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 11 Feb 2015 18:07:12 +0100
Subject: [PATCH 1505/1937] [aes] style: Put __all__ variable at the end of the
 file

---
 youtube_dl/aes.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py
index 5efd0f836..07224d508 100644
--- a/youtube_dl/aes.py
+++ b/youtube_dl/aes.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-__all__ = ['aes_encrypt', 'key_expansion', 'aes_ctr_decrypt', 'aes_cbc_decrypt', 'aes_decrypt_text']
-
 import base64
 from math import ceil
 
@@ -329,3 +327,5 @@ def inc(data):
             data[i] = data[i] + 1
             break
     return data
+
+__all__ = ['aes_encrypt', 'key_expansion', 'aes_ctr_decrypt', 'aes_cbc_decrypt', 'aes_decrypt_text']

From 050fa4356122926d673635f7ba343293126e6d5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 11 Feb 2015 18:08:42 +0100
Subject: [PATCH 1506/1937] flake8: Ignore some error added in pep8 1.6

* E402: we exectute code between imports, like modifying 'sys.path' in the tests
* E731: we assign to lambdas in a lot of places, we may want to consider defining functions in a single line instead (what pep8 recommends)
---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 02aa56257..26857750c 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -3,4 +3,4 @@ universal = True
 
 [flake8]
 exclude = youtube_dl/extractor/__init__.py,devscripts/buildserver.py,setup.py,build,.git
-ignore = E501
+ignore = E402,E501,E731

From 2d30521ab9f535559dd208293982138516b88e8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 11 Feb 2015 18:39:31 +0100
Subject: [PATCH 1507/1937] [youtube] Extract average rating (closes #2362)

---
 youtube_dl/extractor/common.py  | 1 +
 youtube_dl/extractor/youtube.py | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8d71494b4..48742189a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -157,6 +157,7 @@ class InfoExtractor(object):
     view_count:     How many users have watched the video on the platform.
     like_count:     Number of positive ratings of the video
     dislike_count:  Number of negative ratings of the video
+    average_rating: Average rating give by users, the scale used depends on the webpage
     comment_count:  Number of comments on the video
     comments:       A list of comments, each with one or more of the following
                     properties (all but one of text or html optional):
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8b91fff18..35ef4c303 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -25,6 +25,7 @@ from ..compat import (
 from ..utils import (
     clean_html,
     ExtractorError,
+    float_or_none,
     get_element_by_attribute,
     get_element_by_id,
     int_or_none,
@@ -1124,6 +1125,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             'view_count': view_count,
             'like_count': like_count,
             'dislike_count': dislike_count,
+            'average_rating': float_or_none(video_info.get('avg_rating', [None])[0]),
             'formats': formats,
         }
 

From 5277f09dfce6ba1a6f191871df1515ddd68ddd42 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 11 Feb 2015 19:02:39 +0100
Subject: [PATCH 1508/1937] release 2015.02.11

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 225b30f8c..492ddf2ea 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.10.5'
+__version__ = '2015.02.11'

From c40feaba77d8717243467a65fcce4dcab6dfe7bf Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 Feb 2015 14:13:19 +0800
Subject: [PATCH 1509/1937] [camdemy] Add support for folders

---
 youtube_dl/extractor/__init__.py |  5 ++-
 youtube_dl/extractor/camdemy.py  | 68 +++++++++++++++++++++++++++-----
 2 files changed, 63 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 496e4d0b7..095feabf9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -49,7 +49,10 @@ from .brightcove import BrightcoveIE
 from .buzzfeed import BuzzFeedIE
 from .byutv import BYUtvIE
 from .c56 import C56IE
-from .camdemy import CamdemyIE
+from .camdemy import (
+    CamdemyIE,
+    CamdemyFolderIE
+)
 from .canal13cl import Canal13clIE
 from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 9a6341f0f..0d64b5a16 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -4,12 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urlparse
+from ..compat import compat_urllib_parse
 from ..utils import parse_iso8601
 
 
 class CamdemyIE(InfoExtractor):
-    _VALID_URL = r'http://www.camdemy.com/media/(?P<id>\d+).*'
+    _VALID_URL = r'http://www.camdemy.com/media/(?P<id>\d+)'
     _TESTS = [{
         # single file
         'url': 'http://www.camdemy.com/media/5181/',
@@ -69,25 +69,25 @@ class CamdemyIE(InfoExtractor):
             'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)
 
         thumb_url = oembed_obj['thumbnail_url']
-        video_folder = compat_urlparse.urljoin(thumb_url, 'video/')
+        video_folder = compat_urllib_parse.urljoin(thumb_url, 'video/')
         fileListXML = self._download_xml(
-            compat_urlparse.urljoin(video_folder, 'fileList.xml'),
+            compat_urllib_parse.urljoin(video_folder, 'fileList.xml'),
             video_id, 'Filelist XML')
         fileName = fileListXML.find('./video/item/fileName').text
 
         creation_time = self._html_search_regex(
-            r"<div class='title'>Posted :</div>.*<div class='value'>([0-9:\- ]+)<",
-            page, 'creation time', flags=re.MULTILINE | re.DOTALL) + '+08:00'
+            r"<div class='title'>Posted :</div>[\r\n ]*<div class='value'>([^<>]+)<",
+            page, 'creation time', flags=re.MULTILINE) + '+08:00'
         creation_timestamp = parse_iso8601(creation_time, delimiter=' ')
 
         view_count_str = self._html_search_regex(
-            r"<div class='title'>Views :</div>.*<div class='value'>([0-9,]+)<",
-            page, 'view count', flags=re.MULTILINE | re.DOTALL)
+            r"<div class='title'>Views :</div>[\r\n ]*<div class='value'>([^<>]+)<",
+            page, 'view count', flags=re.MULTILINE)
         views = int(view_count_str.replace(',', ''))
 
         return {
             'id': video_id,
-            'url': compat_urlparse.urljoin(video_folder, fileName),
+            'url': compat_urllib_parse.urljoin(video_folder, fileName),
             'title': oembed_obj['title'],
             'thumbnail': thumb_url,
             'description': self._html_search_meta('description', page),
@@ -96,3 +96,53 @@ class CamdemyIE(InfoExtractor):
             'timestamp': creation_timestamp,
             'view_count': views,
         }
+
+
+class CamdemyFolderIE(InfoExtractor):
+    _VALID_URL = r'http://www.camdemy.com/folder/(?P<id>\d+)'
+    _TESTS = [{
+        # links with trailing slash
+        'url': 'http://www.camdemy.com/folder/450',
+        'info_dict': {
+            'id': '450',
+            'title': '信號與系統 2012 & 2011 (Signals and Systems)',
+        },
+        'playlist_mincount': 145
+    }, {
+        # links without trailing slash
+        # and multi-page
+        'url': 'http://www.camdemy.com/folder/853',
+        'info_dict': {
+            'id': '853',
+            'title': '科學計算 - 使用 Matlab'
+        },
+        'playlist_mincount': 20
+    }, {
+        # with displayMode parameter. For testing the codes to add parameters
+        'url': 'http://www.camdemy.com/folder/853/?displayMode=defaultOrderByOrg',
+        'info_dict': {
+            'id': '853',
+            'title': '科學計算 - 使用 Matlab'
+        },
+        'playlist_mincount': 20
+    }]
+
+    def _real_extract(self, url):
+        folder_id = self._match_id(url)
+
+        # Add displayMode=list so that all links are displayed in a single page
+        parsed_url = list(compat_urllib_parse.urlparse(url))
+        query = dict(compat_urllib_parse.parse_qsl(parsed_url[4]))
+        query.update({'displayMode': 'list'})
+        parsed_url[4] = compat_urllib_parse.urlencode(query)
+        final_url = compat_urllib_parse.urlunparse(parsed_url)
+
+        page = self._download_webpage(final_url, folder_id)
+        matches = re.findall(r"href='(/media/\d+/?)'", page)
+
+        entries = [self.url_result('http://www.camdemy.com' + media_path)
+                   for media_path in matches]
+
+        folder_title = self._html_search_meta('keywords', page)
+
+        return self.playlist_result(entries, folder_id, folder_title)

From 7e6011101f00e4f7216467e80091cc89c2846f01 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Thu, 12 Feb 2015 14:23:25 +0800
Subject: [PATCH 1510/1937] [camdemy] Python2 compatibility

---
 youtube_dl/extractor/camdemy.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 0d64b5a16..1bc602c31 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse
+from ..compat import (compat_urllib_parse, compat_urlparse)
 from ..utils import parse_iso8601
 
 
@@ -69,9 +69,9 @@ class CamdemyIE(InfoExtractor):
             'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)
 
         thumb_url = oembed_obj['thumbnail_url']
-        video_folder = compat_urllib_parse.urljoin(thumb_url, 'video/')
+        video_folder = compat_urlparse.urljoin(thumb_url, 'video/')
         fileListXML = self._download_xml(
-            compat_urllib_parse.urljoin(video_folder, 'fileList.xml'),
+            compat_urlparse.urljoin(video_folder, 'fileList.xml'),
             video_id, 'Filelist XML')
         fileName = fileListXML.find('./video/item/fileName').text
 
@@ -87,7 +87,7 @@ class CamdemyIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'url': compat_urllib_parse.urljoin(video_folder, fileName),
+            'url': compat_urlparse.urljoin(video_folder, fileName),
             'title': oembed_obj['title'],
             'thumbnail': thumb_url,
             'description': self._html_search_meta('description', page),
@@ -131,11 +131,11 @@ class CamdemyFolderIE(InfoExtractor):
         folder_id = self._match_id(url)
 
         # Add displayMode=list so that all links are displayed in a single page
-        parsed_url = list(compat_urllib_parse.urlparse(url))
-        query = dict(compat_urllib_parse.parse_qsl(parsed_url[4]))
+        parsed_url = list(compat_urlparse.urlparse(url))
+        query = dict(compat_urlparse.parse_qsl(parsed_url[4]))
         query.update({'displayMode': 'list'})
         parsed_url[4] = compat_urllib_parse.urlencode(query)
-        final_url = compat_urllib_parse.urlunparse(parsed_url)
+        final_url = compat_urlparse.urlunparse(parsed_url)
 
         page = self._download_webpage(final_url, folder_id)
         matches = re.findall(r"href='(/media/\d+/?)'", page)

From 08b38d54015706d90d05371455ba86b7887cabcc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 12 Feb 2015 08:55:06 +0100
Subject: [PATCH 1511/1937] [camdemy] Simplify and make more robust (#4938)

Do not throw errors if view count or upload date extraction fails.
Dispose of re.MULTILINE, which had absolutely no effect without any ^ or $ in sight.
Follow PEP8 naming conventions.
---
 youtube_dl/extractor/camdemy.py | 47 ++++++++++++++++++---------------
 youtube_dl/utils.py             | 27 ++++++++++---------
 2 files changed, 40 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 1bc602c31..5de5879b4 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -1,11 +1,18 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import datetime
 import re
 
 from .common import InfoExtractor
-from ..compat import (compat_urllib_parse, compat_urlparse)
-from ..utils import parse_iso8601
+from ..compat import (
+    compat_urllib_parse,
+    compat_urlparse,
+)
+from ..utils import (
+    parse_iso8601,
+    str_to_int,
+)
 
 
 class CamdemyIE(InfoExtractor):
@@ -23,6 +30,7 @@ class CamdemyIE(InfoExtractor):
             'creator': 'ss11spring',
             'upload_date': '20130114',
             'timestamp': 1358154556,
+            'view_count': int,
         }
     }, {
         # With non-empty description
@@ -55,46 +63,43 @@ class CamdemyIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         page = self._download_webpage(url, video_id)
 
-        srcFrom = self._html_search_regex(
+        src_from = self._html_search_regex(
             r"<div class='srcFrom'>Source: <a title='([^']+)'", page,
             'external source', default=None)
-
-        if srcFrom:
-            return self.url_result(srcFrom)
+        if src_from:
+            return self.url_result(src_from)
 
         oembed_obj = self._download_json(
             'http://www.camdemy.com/oembed/?format=json&url=' + url, video_id)
 
         thumb_url = oembed_obj['thumbnail_url']
         video_folder = compat_urlparse.urljoin(thumb_url, 'video/')
-        fileListXML = self._download_xml(
+        file_list_doc = self._download_xml(
             compat_urlparse.urljoin(video_folder, 'fileList.xml'),
             video_id, 'Filelist XML')
-        fileName = fileListXML.find('./video/item/fileName').text
+        file_name = file_list_doc.find('./video/item/fileName').text
+        video_url = compat_urlparse.urljoin(video_folder, file_name)
 
-        creation_time = self._html_search_regex(
-            r"<div class='title'>Posted :</div>[\r\n ]*<div class='value'>([^<>]+)<",
-            page, 'creation time', flags=re.MULTILINE) + '+08:00'
-        creation_timestamp = parse_iso8601(creation_time, delimiter=' ')
-
-        view_count_str = self._html_search_regex(
-            r"<div class='title'>Views :</div>[\r\n ]*<div class='value'>([^<>]+)<",
-            page, 'view count', flags=re.MULTILINE)
-        views = int(view_count_str.replace(',', ''))
+        timestamp = parse_iso8601(self._html_search_regex(
+            r"<div class='title'>Posted\s*:</div>\s*<div class='value'>([^<>]+)<",
+            page, 'creation time', fatal=False),
+            delimiter=' ', timezone=datetime.timedelta(hours=8))
+        view_count = str_to_int(self._html_search_regex(
+            r"<div class='title'>Views\s*:</div>\s*<div class='value'>([^<>]+)<",
+            page, 'view count', fatal=False))
 
         return {
             'id': video_id,
-            'url': compat_urlparse.urljoin(video_folder, fileName),
+            'url': video_url,
             'title': oembed_obj['title'],
             'thumbnail': thumb_url,
             'description': self._html_search_meta('description', page),
             'creator': oembed_obj['author_name'],
             'duration': oembed_obj['duration'],
-            'timestamp': creation_timestamp,
-            'view_count': views,
+            'timestamp': timestamp,
+            'view_count': view_count,
         }
 
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 03566d223..54fa17c38 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -666,26 +666,27 @@ class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
             req, **kwargs)
 
 
-def parse_iso8601(date_str, delimiter='T'):
+def parse_iso8601(date_str, delimiter='T', timezone=None):
     """ Return a UNIX timestamp from the given date """
 
     if date_str is None:
         return None
 
-    m = re.search(
-        r'(\.[0-9]+)?(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
-        date_str)
-    if not m:
-        timezone = datetime.timedelta()
-    else:
-        date_str = date_str[:-len(m.group(0))]
-        if not m.group('sign'):
+    if timezone is None:
+        m = re.search(
+            r'(\.[0-9]+)?(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
+            date_str)
+        if not m:
             timezone = datetime.timedelta()
         else:
-            sign = 1 if m.group('sign') == '+' else -1
-            timezone = datetime.timedelta(
-                hours=sign * int(m.group('hours')),
-                minutes=sign * int(m.group('minutes')))
+            date_str = date_str[:-len(m.group(0))]
+            if not m.group('sign'):
+                timezone = datetime.timedelta()
+            else:
+                sign = 1 if m.group('sign') == '+' else -1
+                timezone = datetime.timedelta(
+                    hours=sign * int(m.group('hours')),
+                    minutes=sign * int(m.group('minutes')))
     date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
     dt = datetime.datetime.strptime(date_str, date_format) - timezone
     return calendar.timegm(dt.timetuple())

From 7c24ce225d1ddb350f91b0fd78fefa40f6aa4481 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 12 Feb 2015 19:19:55 +0100
Subject: [PATCH 1512/1937] [NPORadio] Added extractor for live radio

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/nporadio.py | 40 ++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100644 youtube_dl/extractor/nporadio.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 095feabf9..b118c3d1d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -320,6 +320,7 @@ from .npo import (
     NPOLiveIE,
     TegenlichtVproIE,
 )
+from .nporadio import NPORadioIE
 from .nrk import (
     NRKIE,
     NRKTVIE,
diff --git a/youtube_dl/extractor/nporadio.py b/youtube_dl/extractor/nporadio.py
new file mode 100644
index 000000000..dd8d97cb7
--- /dev/null
+++ b/youtube_dl/extractor/nporadio.py
@@ -0,0 +1,40 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+
+
+class NPORadioIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/radio/(?P<id>.*)'
+    _TEST = {
+        'url': 'http://www.npo.nl/radio/radio-1',
+        'info_dict': {
+            'id': 'radio-1',
+            'ext': 'mp3',
+            'title': 'NPO Radio 1',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(
+                self._html_get_attribute_regex('data-channel'), webpage, 'title')
+       
+        json_data = json.loads(
+                     self._html_search_regex(
+                     self._html_get_attribute_regex('data-streams'), webpage, 'data-streams'))
+        
+        return {
+            'id': video_id,
+            'title': title,
+            'ext': json_data['codec'],
+            'url': json_data['url']
+        }
+
+    def _html_get_attribute_regex(self, attribute):
+        return r'{0}\s*=\s*\'([^\']+)\''.format(attribute)
+

From c3d64fc1b30b46cc4f58aaa55c02918248bb1bee Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 12 Feb 2015 19:27:59 +0100
Subject: [PATCH 1513/1937] [nporadio] Edit to confirm to flake8 standards

---
 youtube_dl/extractor/nporadio.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/nporadio.py b/youtube_dl/extractor/nporadio.py
index dd8d97cb7..d1c4ec167 100644
--- a/youtube_dl/extractor/nporadio.py
+++ b/youtube_dl/extractor/nporadio.py
@@ -22,12 +22,12 @@ class NPORadioIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         title = self._html_search_regex(
-                self._html_get_attribute_regex('data-channel'), webpage, 'title')
-       
+            self._html_get_attribute_regex('data-channel'), webpage, 'title')
+
         json_data = json.loads(
-                     self._html_search_regex(
-                     self._html_get_attribute_regex('data-streams'), webpage, 'data-streams'))
-        
+            self._html_search_regex(
+                self._html_get_attribute_regex('data-streams'), webpage, 'data-streams'))
+
         return {
             'id': video_id,
             'title': title,
@@ -37,4 +37,3 @@ class NPORadioIE(InfoExtractor):
 
     def _html_get_attribute_regex(self, attribute):
         return r'{0}\s*=\s*\'([^\']+)\''.format(attribute)
-

From 171ca612af948b2966e56054ce3a25a9f5ab2a76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Feb 2015 01:36:54 +0600
Subject: [PATCH 1514/1937] [npo:radio] Move to extractor to common npo place
 and add extractor for fragments

---
 youtube_dl/extractor/__init__.py |  3 +-
 youtube_dl/extractor/npo.py      | 83 +++++++++++++++++++++++++++++++-
 youtube_dl/extractor/nporadio.py | 39 ---------------
 3 files changed, 83 insertions(+), 42 deletions(-)
 delete mode 100644 youtube_dl/extractor/nporadio.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b118c3d1d..a4fab540b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -318,9 +318,10 @@ from .nowvideo import NowVideoIE
 from .npo import (
     NPOIE,
     NPOLiveIE,
+    NPORadioIE,
+    NPORadioFragmentIE,
     TegenlichtVproIE,
 )
-from .nporadio import NPORadioIE
 from .nrk import (
     NRKIE,
     NRKTVIE,
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 54be06a4e..c075618e8 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..utils import (
     fix_xml_ampersands,
     parse_duration,
@@ -22,7 +23,7 @@ class NPOBaseIE(SubtitlesInfoExtractor):
 
 class NPOIE(NPOBaseIE):
     IE_NAME = 'npo.nl'
-    _VALID_URL = r'https?://www\.npo\.nl/[^/]+/[^/]+/(?P<id>[^/?]+)'
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/(?!live|radio)[^/]+/[^/]+/(?P<id>[^/?]+)'
 
     _TESTS = [
         {
@@ -185,7 +186,7 @@ class NPOIE(NPOBaseIE):
 
 class NPOLiveIE(NPOBaseIE):
     IE_NAME = 'npo.nl:live'
-    _VALID_URL = r'https?://www\.npo\.nl/live/(?P<id>.+)'
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/live/(?P<id>.+)'
 
     _TEST = {
         'url': 'http://www.npo.nl/live/npo-1',
@@ -260,6 +261,84 @@ class NPOLiveIE(NPOBaseIE):
         }
 
 
+class NPORadioIE(InfoExtractor):
+    IE_NAME = 'npo.nl:radio'
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/radio/(?P<id>[^/]+)/?$'
+
+    _TEST = {
+        'url': 'http://www.npo.nl/radio/radio-1',
+        'info_dict': {
+            'id': 'radio-1',
+            'ext': 'mp3',
+            'title': 're:^NPO Radio 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        }
+    }
+
+    @staticmethod
+    def _html_get_attribute_regex(attribute):
+        return r'{0}\s*=\s*\'([^\']+)\''.format(attribute)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_regex(
+            self._html_get_attribute_regex('data-channel'), webpage, 'title')
+
+        stream = self._parse_json(
+            self._html_search_regex(self._html_get_attribute_regex('data-streams'), webpage, 'data-streams'),
+            video_id)
+
+        codec = stream.get('codec')
+
+        return {
+            'id': video_id,
+            'url': stream['url'],
+            'title': self._live_title(title),
+            'acodec': codec,
+            'ext': codec,
+            'is_live': True,
+        }
+
+
+class NPORadioFragmentIE(InfoExtractor):
+    IE_NAME = 'npo.nl:radio:fragment'
+    _VALID_URL = r'https?://(?:www\.)?npo\.nl/radio/[^/]+/fragment/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://www.npo.nl/radio/radio-5/fragment/174356',
+        'md5': 'dd8cc470dad764d0fdc70a9a1e2d18c2',
+        'info_dict': {
+            'id': '174356',
+            'ext': 'mp3',
+            'title': 'Jubileumconcert Willeke Alberti',
+        },
+    }
+
+    def _real_extract(self, url):
+        audio_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, audio_id)
+
+        title = self._html_search_regex(
+            r'href="/radio/[^/]+/fragment/%s" title="([^"]+)"' % audio_id,
+            webpage, 'title')
+
+        audio_url = self._search_regex(
+            r"data-streams='([^']+)'", webpage, 'audio url')
+
+        return {
+            'id': audio_id,
+            'url': audio_url,
+            'title': title,
+        }
+
+
 class TegenlichtVproIE(NPOIE):
     IE_NAME = 'tegenlicht.vpro.nl'
     _VALID_URL = r'https?://tegenlicht\.vpro\.nl/afleveringen/.*?'
diff --git a/youtube_dl/extractor/nporadio.py b/youtube_dl/extractor/nporadio.py
deleted file mode 100644
index d1c4ec167..000000000
--- a/youtube_dl/extractor/nporadio.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import json
-
-from .common import InfoExtractor
-
-
-class NPORadioIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?npo\.nl/radio/(?P<id>.*)'
-    _TEST = {
-        'url': 'http://www.npo.nl/radio/radio-1',
-        'info_dict': {
-            'id': 'radio-1',
-            'ext': 'mp3',
-            'title': 'NPO Radio 1',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._html_search_regex(
-            self._html_get_attribute_regex('data-channel'), webpage, 'title')
-
-        json_data = json.loads(
-            self._html_search_regex(
-                self._html_get_attribute_regex('data-streams'), webpage, 'data-streams'))
-
-        return {
-            'id': video_id,
-            'title': title,
-            'ext': json_data['codec'],
-            'url': json_data['url']
-        }
-
-    def _html_get_attribute_regex(self, attribute):
-        return r'{0}\s*=\s*\'([^\']+)\''.format(attribute)

From 7105440cecf82aff295df4f32575f6c8b64b3c2d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 13 Feb 2015 15:14:23 +0800
Subject: [PATCH 1515/1937] [Yam] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/yam.py      | 70 ++++++++++++++++++++++++++++++++
 youtube_dl/utils.py              | 20 +++++++--
 3 files changed, 87 insertions(+), 4 deletions(-)
 create mode 100644 youtube_dl/extractor/yam.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a4fab540b..f749ec333 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -578,6 +578,7 @@ from .yahoo import (
     YahooIE,
     YahooSearchIE,
 )
+from .yam import YamIE
 from .yesjapan import YesJapanIE
 from .ynet import YnetIE
 from .youjizz import YouJizzIE
diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
new file mode 100644
index 000000000..511c8b67c
--- /dev/null
+++ b/youtube_dl/extractor/yam.py
@@ -0,0 +1,70 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import month_by_abbreviation
+import re
+
+
+class YamIE(InfoExtractor):
+    _VALID_URL = r'http://mymedia.yam.com/m/(?P<id>\d+)'
+
+    _TESTS = [{
+        # An audio hosted on Yam
+        'url': 'http://mymedia.yam.com/m/2283921',
+        'md5': 'c011b8e262a52d5473d9c2e3c9963b9c',
+        'info_dict': {
+            'id': '2283921',
+            'ext': 'mp3',
+            'title': '發現 - 趙薇 京華煙雲主題曲',
+            'uploader_id': 'princekt',
+            'upload_date': '20080807',
+            'duration': 313.0,
+        }
+    }, {
+        # An external video hosted on YouTube
+        'url': 'http://mymedia.yam.com/m/3598173',
+        'md5': '0238ceec479c654e8c2f1223755bf3e9',
+        'info_dict': {
+            'id': 'pJ2Deys283c',
+            'ext': 'mp4',
+            'upload_date': '20150202',
+            'uploader': '新莊社大瑜伽社',
+            'description': 'md5:f5cc72f0baf259a70fb731654b0d2eff',
+            'uploader_id': '2323agoy',
+            'title': '外婆的澎湖灣KTV-潘安邦',
+        }
+    }]
+
+    def _real_extract(self, url):
+        media_id = self._match_id(url)
+        page = self._download_webpage(url, media_id)
+
+        # Is it hosted externally on YouTube?
+        youtube_url = self._html_search_regex(
+            r'<embed src="(http://www.youtube.com/[^"]+)"',
+            page, 'YouTube url', default=None)
+        if youtube_url:
+            return self.url_result(youtube_url, 'Youtube')
+
+        api_page = self._download_webpage(
+            'http://mymedia.yam.com/api/a/?pID=' + media_id, media_id)
+        api_result_obj = compat_urlparse.parse_qs(api_page)
+
+        author = self._html_search_regex(
+            r'<!-- 發表作者 -->：[\n ]+<a href="/([a-z]+)"', page, 'author')
+        mobj = re.search(r'<!-- 發表於 -->(?P<mon>[A-Z][a-z]{2})  ' +
+                         r'(?P<day>\d{1,2}), (?P<year>\d{4})', page)
+        upload_date = '%s%02d%02d' % (mobj.group('year'),
+                                      month_by_abbreviation(mobj.group('mon')),
+                                      int(mobj.group('day')))
+
+        return {
+            'id': media_id,
+            'url': api_result_obj['mp3file'][0],
+            'title': self._html_search_meta('description', page),
+            'duration': float(api_result_obj['totaltime'][0]) / 1000.0,
+            'uploader_id': author,
+            'upload_date': upload_date,
+        }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 54fa17c38..3eb6bc6d4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -62,6 +62,11 @@ std_headers = {
 }
 
 
+ENGLISH_MONTH_NAMES = [
+    'January', 'February', 'March', 'April', 'May', 'June',
+    'July', 'August', 'September', 'October', 'November', 'December']
+
+
 def preferredencoding():
     """Get preferred encoding.
 
@@ -1185,11 +1190,18 @@ def get_term_width():
 def month_by_name(name):
     """ Return the number of a month by (locale-independently) English name """
 
-    ENGLISH_NAMES = [
-        'January', 'February', 'March', 'April', 'May', 'June',
-        'July', 'August', 'September', 'October', 'November', 'December']
     try:
-        return ENGLISH_NAMES.index(name) + 1
+        return ENGLISH_MONTH_NAMES.index(name) + 1
+    except ValueError:
+        return None
+
+
+def month_by_abbreviation(abbrev):
+    """ Return the number of a month by (locale-independently) English
+        abbreviations """
+
+    try:
+        return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
     except ValueError:
         return None
 

From ae6423d7042de863c303fcbdc1646ea370233741 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 13 Feb 2015 11:36:33 +0100
Subject: [PATCH 1516/1937] [bambuser] Fix 'uploader_id' extraction (fixes
 #4944)

---
 youtube_dl/extractor/bambuser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index 98e1443ab..c193e66ca 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -50,7 +50,7 @@ class BambuserIE(InfoExtractor):
             'duration': int(info['length']),
             'view_count': int(info['views_total']),
             'uploader': info['username'],
-            'uploader_id': info['uid'],
+            'uploader_id': info['owner']['uid'],
         }
 
 

From 36e7a4ca2e85be8dc16955e28f9c563e01aa9eca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 13 Feb 2015 14:43:50 +0100
Subject: [PATCH 1517/1937] [test/subtitles] Update checksums

---
 test/test_subtitles.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 6336dd317..3e329438f 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -247,7 +247,7 @@ class TestVimeoSubtitles(BaseTestSubtitles):
     def test_subtitles(self):
         self.DL.params['writesubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '26399116d23ae3cf2c087cea94bc43b4')
+        self.assertEqual(md5(subtitles['en']), '8062383cf4dec168fc40a088aa6d5888')
 
     def test_subtitles_lang(self):
         self.DL.params['writesubtitles'] = True
@@ -334,7 +334,7 @@ class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['cs']))
-        self.assertEqual(md5(subtitles['cs']), '9bf52d9549533c32c427e264bf0847d4')
+        self.assertEqual(md5(subtitles['cs']), 'cc3957b2c6dff1db71e5f2e83d467480')
 
     def test_nosubtitles(self):
         self.DL.expect_warning('video doesn\'t have subtitles')

From 037e9437e435f32d287354c77f6586d7dc34544b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Feb 2015 20:10:42 +0600
Subject: [PATCH 1518/1937] [camdemy] Fix _VALID_URL

---
 youtube_dl/extractor/camdemy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/camdemy.py b/youtube_dl/extractor/camdemy.py
index 5de5879b4..897f3a104 100644
--- a/youtube_dl/extractor/camdemy.py
+++ b/youtube_dl/extractor/camdemy.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class CamdemyIE(InfoExtractor):
-    _VALID_URL = r'http://www.camdemy.com/media/(?P<id>\d+)'
+    _VALID_URL = r'http://(?:www\.)?camdemy\.com/media/(?P<id>\d+)'
     _TESTS = [{
         # single file
         'url': 'http://www.camdemy.com/media/5181/',

From 9347fddbfc81114f85fecc72e8452813ef130fdb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Feb 2015 02:04:28 +0600
Subject: [PATCH 1519/1937] [1tv] Cover arbitraty URLs

---
 youtube_dl/extractor/firsttv.py | 57 ++++++++++++++++++++++-----------
 1 file changed, 39 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/firsttv.py b/youtube_dl/extractor/firsttv.py
index 08ceee4ed..6e015ca16 100644
--- a/youtube_dl/extractor/firsttv.py
+++ b/youtube_dl/extractor/firsttv.py
@@ -8,45 +8,66 @@ from ..utils import int_or_none
 
 
 class FirstTVIE(InfoExtractor):
-    IE_NAME = 'firsttv'
-    IE_DESC = 'Видеоархив - Первый канал'
-    _VALID_URL = r'http://(?:www\.)?1tv\.ru/videoarchive/(?P<id>\d+)'
+    IE_NAME = '1tv'
+    IE_DESC = 'Первый канал'
+    _VALID_URL = r'http://(?:www\.)?1tv\.ru/(?:[^/]+/)+(?P<id>.+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.1tv.ru/videoarchive/73390',
-        'md5': '3de6390cf0cca4a5eae1d1d83895e5ad',
+        'md5': '777f525feeec4806130f4f764bc18a4f',
         'info_dict': {
             'id': '73390',
             'ext': 'mp4',
             'title': 'Олимпийские канатные дороги',
-            'description': 'md5:cc730d2bf4215463e37fff6a1e277b13',
-            'thumbnail': 'http://img1.1tv.ru/imgsize640x360/PR20140210114657.JPG',
+            'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
+            'thumbnail': 're:^https?://.*\.(?:jpg|JPG)$',
             'duration': 149,
+            'like_count': int,
+            'dislike_count': int,
         },
         'skip': 'Only works from Russia',
-    }
+    }, {
+        'url': 'http://www.1tv.ru/prj/inprivate/vypusk/35930',
+        'md5': 'a1b6b60d530ebcf8daacf4565762bbaf',
+        'info_dict': {
+            'id': '35930',
+            'ext': 'mp4',
+            'title': 'Наедине со всеми. Людмила Сенчина',
+            'description': 'md5:89553aed1d641416001fe8d450f06cb9',
+            'thumbnail': 're:^https?://.*\.(?:jpg|JPG)$',
+            'duration': 2694,
+        },
+        'skip': 'Only works from Russia',
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id, 'Downloading page')
 
         video_url = self._html_search_regex(
-            r'''(?s)jwplayer\('flashvideoportal_1'\)\.setup\({.*?'file': '([^']+)'.*?}\);''', webpage, 'video URL')
+            r'''(?s)(?:jwplayer\('flashvideoportal_1'\)\.setup\({|var\s+playlistObj\s*=).*?'file'\s*:\s*'([^']+)'.*?}\);''',
+            webpage, 'video URL')
 
         title = self._html_search_regex(
-            r'<div class="tv_translation">\s*<h1><a href="[^"]+">([^<]*)</a>', webpage, 'title')
+            [r'<div class="tv_translation">\s*<h1><a href="[^"]+">([^<]*)</a>',
+             r"'title'\s*:\s*'([^']+)'"], webpage, 'title')
         description = self._html_search_regex(
-            r'<div class="descr">\s*<div>&nbsp;</div>\s*<p>([^<]*)</p></div>', webpage, 'description', fatal=False)
+            r'<div class="descr">\s*<div>&nbsp;</div>\s*<p>([^<]*)</p></div>',
+            webpage, 'description', default=None) or self._html_search_meta(
+                'description', webpage, 'description')
 
         thumbnail = self._og_search_thumbnail(webpage)
-        duration = self._og_search_property('video:duration', webpage, 'video duration', fatal=False)
+        duration = self._og_search_property(
+            'video:duration', webpage,
+            'video duration', fatal=False)
 
-        like_count = self._html_search_regex(r'title="Понравилось".*?/></label> \[(\d+)\]',
-                                             webpage, 'like count', fatal=False)
-        dislike_count = self._html_search_regex(r'title="Не понравилось".*?/></label> \[(\d+)\]',
-                                                webpage, 'dislike count', fatal=False)
+        like_count = self._html_search_regex(
+            r'title="Понравилось".*?/></label> \[(\d+)\]',
+            webpage, 'like count', default=None)
+        dislike_count = self._html_search_regex(
+            r'title="Не понравилось".*?/></label> \[(\d+)\]',
+            webpage, 'dislike count', default=None)
 
         return {
             'id': video_id,

From cffcbc02de504d84e1c2677bb525c00b03e04f53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 13 Feb 2015 22:25:34 +0100
Subject: [PATCH 1520/1937] [postprocessor/ffmpeg] Don't let ffmpeg read from
 stdin (fixes #4945)

If you run 'while read aurl ; do youtube-dl --extract-audio "${aurl}"; done < path_to_batch_file'  (batch_file contains one url per line) each call to youtube-dl consumed some characters and 'read' would assing to 'aurl' a non valid url, something like 'tube.com/watch?v=<id>'.
---
 youtube_dl/postprocessor/ffmpeg.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 01d25f760..5238ce534 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -95,7 +95,7 @@ class FFmpegPostProcessor(PostProcessor):
 
         if self._downloader.params.get('verbose', False):
             self._downloader.to_screen('[debug] ffmpeg command line: %s' % shell_quote(cmd))
-        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
         stdout, stderr = p.communicate()
         if p.returncode != 0:
             stderr = stderr.decode('utf-8', 'replace')
@@ -134,7 +134,7 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
                 encodeFilename(self._probe_executable, True),
                 encodeArgument('-show_streams'),
                 encodeFilename(self._ffmpeg_filename_argument(path), True)]
-            handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE)
+            handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE, stdin=subprocess.PIPE)
             output = handle.communicate()[0]
             if handle.wait() != 0:
                 return None

From d9aa2b784d914ae99c7d9bbaf83d06f1b9dbd04e Mon Sep 17 00:00:00 2001
From: Ryan Schmidt <github@ryandesign.com>
Date: Sat, 14 Feb 2015 04:10:23 -0600
Subject: [PATCH 1521/1937] Support NBC Nightly News broadcasts

---
 youtube_dl/extractor/nbc.py | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index f840f6532..3e3de9e2d 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -54,7 +54,7 @@ class NBCIE(InfoExtractor):
 class NBCNewsIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://www\.nbcnews\.com/
         ((video/.+?/(?P<id>\d+))|
-        (feature/[^/]+/(?P<title>.+)))
+        ((?P<program>feature|nightly-news)/[^/]+/(?P<title>.+)))
         '''
 
     _TESTS = [
@@ -89,6 +89,16 @@ class NBCNewsIE(InfoExtractor):
                 'description': 'md5:757988edbaae9d7be1d585eb5d55cc04',
             },
         },
+        {
+            'url': 'http://www.nbcnews.com/nightly-news/video/nightly-news-with-brian-williams-full-broadcast-february-4-394064451844',
+            'md5': 'b5dda8cddd8650baa0dcb616dd2cf60d',
+            'info_dict': {
+                'id': 'sekXqyTVnmN3',
+                'ext': 'mp4',
+                'title': 'Nightly News with Brian Williams Full Broadcast (February 4)',
+                'description': 'md5:1c10c1eccbe84a26e5debb4381e2d3c5',
+            },
+        },
     ]
 
     def _real_extract(self, url):
@@ -107,12 +117,19 @@ class NBCNewsIE(InfoExtractor):
                 'thumbnail': find_xpath_attr(info, 'media', 'type', 'thumbnail').text,
             }
         else:
-            # "feature" pages use theplatform.com
+            # "feature" and "nightly-news" pages use theplatform.com
             title = mobj.group('title')
             webpage = self._download_webpage(url, title)
-            bootstrap_json = self._search_regex(
-                r'var bootstrapJson = ({.+})\s*$', webpage, 'bootstrap json',
-                flags=re.MULTILINE)
+            program = mobj.group('program')
+            if program == 'feature':
+                bootstrap_json = self._search_regex(
+                    r'var bootstrapJson = ({.+})\s*$', webpage, 'bootstrap json',
+                    flags=re.MULTILINE)
+            else:
+                # nightly-news
+                bootstrap_json = self._search_regex(
+                    r'var playlistData = ({.+});\s*$', webpage, 'playlist data',
+                    flags=re.MULTILINE)
             bootstrap = json.loads(bootstrap_json)
             info = bootstrap['results'][0]['video']
             mpxid = info['mpxId']

From a4f3d779db13ec3c8bda67e897de8cd849a7f811 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Feb 2015 17:42:12 +0600
Subject: [PATCH 1522/1937] [nbcnews] Simplify

---
 youtube_dl/extractor/nbc.py | 21 +++++++--------------
 1 file changed, 7 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 3e3de9e2d..52e8595ea 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -52,9 +52,9 @@ class NBCIE(InfoExtractor):
 
 
 class NBCNewsIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://www\.nbcnews\.com/
-        ((video/.+?/(?P<id>\d+))|
-        ((?P<program>feature|nightly-news)/[^/]+/(?P<title>.+)))
+    _VALID_URL = r'''(?x)https?://(?:www\.)?nbcnews\.com/
+        (?:video/.+?/(?P<id>\d+)|
+        (?:feature|nightly-news)/[^/]+/(?P<title>.+))
         '''
 
     _TESTS = [
@@ -120,17 +120,10 @@ class NBCNewsIE(InfoExtractor):
             # "feature" and "nightly-news" pages use theplatform.com
             title = mobj.group('title')
             webpage = self._download_webpage(url, title)
-            program = mobj.group('program')
-            if program == 'feature':
-                bootstrap_json = self._search_regex(
-                    r'var bootstrapJson = ({.+})\s*$', webpage, 'bootstrap json',
-                    flags=re.MULTILINE)
-            else:
-                # nightly-news
-                bootstrap_json = self._search_regex(
-                    r'var playlistData = ({.+});\s*$', webpage, 'playlist data',
-                    flags=re.MULTILINE)
-            bootstrap = json.loads(bootstrap_json)
+            bootstrap_json = self._search_regex(
+                r'var\s+(?:bootstrapJson|playlistData)\s*=\s*({.+});?\s*$',
+                webpage, 'bootstrap json', flags=re.MULTILINE)
+            bootstrap = self._parse_json(bootstrap_json, video_id)
             info = bootstrap['results'][0]['video']
             mpxid = info['mpxId']
 

From 3215c50f25bcf01a037e066747d7d60bd274ae71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Feb 2015 17:44:24 +0600
Subject: [PATCH 1523/1937] Credit @ryandesign for nbcnews nightly news (#4948)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 3d6985ab6..47f12a9ee 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -110,3 +110,4 @@ Shaya Goldberg
 Paul Hartmann
 Frans de Jonge
 Robin de Rooij
+Ryan Schmidt

From ae1580d790ecca2b6ad132da26bc834e4f9873f3 Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Sat, 14 Feb 2015 13:29:44 +0100
Subject: [PATCH 1524/1937] [sunporno] fix extraction

---
 youtube_dl/extractor/sunporno.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dl/extractor/sunporno.py
index 8a333f1d2..f2a1d6550 100644
--- a/youtube_dl/extractor/sunporno.py
+++ b/youtube_dl/extractor/sunporno.py
@@ -52,7 +52,7 @@ class SunPornoIE(InfoExtractor):
 
         formats = []
         quality = qualities(['mp4', 'flv'])
-        for video_url in re.findall(r'<source src="([^"]+)"', webpage):
+        for video_url in re.findall(r'<video src="([^"]+)"', webpage):
             video_ext = determine_ext(video_url)
             formats.append({
                 'url': video_url,

From 68f2d273bfeb7aa10eba68b6c62cc0502a948c72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Feb 2015 18:33:52 +0600
Subject: [PATCH 1525/1937] [sunporno] Keep old video regex just in case

---
 youtube_dl/extractor/sunporno.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dl/extractor/sunporno.py
index f2a1d6550..854d01bee 100644
--- a/youtube_dl/extractor/sunporno.py
+++ b/youtube_dl/extractor/sunporno.py
@@ -52,7 +52,7 @@ class SunPornoIE(InfoExtractor):
 
         formats = []
         quality = qualities(['mp4', 'flv'])
-        for video_url in re.findall(r'<video src="([^"]+)"', webpage):
+        for video_url in re.findall(r'<(?:source|video) src="([^"]+)"', webpage):
             video_ext = determine_ext(video_url)
             formats.append({
                 'url': video_url,

From 10e3c4c2215a7d65391d23efd19aab96197cdcf8 Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Sat, 14 Feb 2015 13:40:35 +0100
Subject: [PATCH 1526/1937] [drtuber] fix extraction

---
 youtube_dl/extractor/drtuber.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/drtuber.py b/youtube_dl/extractor/drtuber.py
index ca274dff6..2f06e64cc 100644
--- a/youtube_dl/extractor/drtuber.py
+++ b/youtube_dl/extractor/drtuber.py
@@ -36,7 +36,7 @@ class DrTuberIE(InfoExtractor):
             r'<source src="([^"]+)"', webpage, 'video URL')
 
         title = self._html_search_regex(
-            r'<title>([^<]+)\s*-\s*Free', webpage, 'title')
+            r'<title>([^<]+) - \d+', webpage, 'title')
 
         thumbnail = self._html_search_regex(
             r'poster="([^"]+)"',

From 52e1d0ccc415377bd9f7d862f645c87909705ace Mon Sep 17 00:00:00 2001
From: peugeot <peugeot.com>
Date: Sat, 14 Feb 2015 13:42:42 +0100
Subject: [PATCH 1527/1937] [beeg] fix test

---
 youtube_dl/extractor/beeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beeg.py b/youtube_dl/extractor/beeg.py
index 4e79fea8f..b38057f2f 100644
--- a/youtube_dl/extractor/beeg.py
+++ b/youtube_dl/extractor/beeg.py
@@ -9,7 +9,7 @@ class BeegIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?beeg\.com/(?P<id>\d+)'
     _TEST = {
         'url': 'http://beeg.com/5416503',
-        'md5': '634526ae978711f6b748fe0dd6c11f57',
+        'md5': '1bff67111adb785c51d1b42959ec10e5',
         'info_dict': {
             'id': '5416503',
             'ext': 'mp4',

From 76d1466b08d4e3cebc40a7fb7dae957c3b34b1eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Feb 2015 18:50:13 +0600
Subject: [PATCH 1528/1937] [drtuber] Add one more title regex

---
 youtube_dl/extractor/drtuber.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/drtuber.py b/youtube_dl/extractor/drtuber.py
index 2f06e64cc..37c5c181f 100644
--- a/youtube_dl/extractor/drtuber.py
+++ b/youtube_dl/extractor/drtuber.py
@@ -15,7 +15,7 @@ class DrTuberIE(InfoExtractor):
             'id': '1740434',
             'display_id': 'hot-perky-blonde-naked-golf',
             'ext': 'mp4',
-            'title': 'Hot Perky Blonde Naked Golf',
+            'title': 'hot perky blonde naked golf',
             'like_count': int,
             'dislike_count': int,
             'comment_count': int,
@@ -36,7 +36,8 @@ class DrTuberIE(InfoExtractor):
             r'<source src="([^"]+)"', webpage, 'video URL')
 
         title = self._html_search_regex(
-            r'<title>([^<]+) - \d+', webpage, 'title')
+            [r'class="hd_title" style="[^"]+">([^<]+)</h1>', r'<title>([^<]+) - \d+'],
+            webpage, 'title')
 
         thumbnail = self._html_search_regex(
             r'poster="([^"]+)"',

From a294bce82f84482091fa6f44ce913a7a86fe8be8 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 14 Feb 2015 17:48:04 +0200
Subject: [PATCH 1529/1937] [streamcz] Fix extraction (Closes #4940)

---
 youtube_dl/extractor/streamcz.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index c3ceb5f76..e92b93285 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -1,14 +1,30 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import hashlib
+import time
+
 from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+)
 from ..utils import (
     int_or_none,
 )
 
 
+def _get_api_key(api_path):
+    if api_path.endswith('?'):
+        api_path = api_path[:-1]
+
+    api_key = 'fb5f58a820353bd7095de526253c14fd'
+    a = '{0:}{1:}{2:}'.format(api_key, api_path, int(round(time.time() / 24 / 3600)))
+    return hashlib.md5(a.encode('ascii')).hexdigest()
+
+
 class StreamCZIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?stream\.cz/.+/(?P<id>[0-9]+)'
+    _API_URL = 'http://www.stream.cz/API'
 
     _TESTS = [{
         'url': 'http://www.stream.cz/peklonataliri/765767-ecka-pro-deti',
@@ -36,8 +52,11 @@ class StreamCZIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        data = self._download_json(
-            'http://www.stream.cz/API/episode/%s' % video_id, video_id)
+        api_path = '/episode/%s' % video_id
+
+        req = compat_urllib_request.Request(self._API_URL + api_path)
+        req.add_header('Api-Password', _get_api_key(api_path))
+        data = self._download_json(req, video_id)
 
         formats = []
         for quality, video in enumerate(data['video_qualities']):

From b0ab0fac49057ea0df3db75fb407af92deaaa453 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Feb 2015 22:18:09 +0100
Subject: [PATCH 1530/1937] Remove unused imports

---
 youtube_dl/extractor/firsttv.py | 2 --
 youtube_dl/extractor/nbc.py     | 1 -
 2 files changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/firsttv.py b/youtube_dl/extractor/firsttv.py
index 6e015ca16..510d4b108 100644
--- a/youtube_dl/extractor/firsttv.py
+++ b/youtube_dl/extractor/firsttv.py
@@ -1,8 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import int_or_none
 
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 52e8595ea..89a2845fe 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..compat import (

From 6ca7732d5e792f02be030d4a9c9d101fff2f4079 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Feb 2015 22:20:24 +0100
Subject: [PATCH 1531/1937] [extractor/common] Fix link to external
 documentation

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 48742189a..c784eedb9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -665,7 +665,7 @@ class InfoExtractor(object):
         return RATING_TABLE.get(rating.lower(), None)
 
     def _family_friendly_search(self, html):
-        # See http://schema.org/VideoObj
+        # See http://schema.org/VideoObject
         family_friendly = self._html_search_meta('isFamilyFriendly', html)
 
         if not family_friendly:

From 9fb2f1cd6d14dafca7a2d6cd74b0394b0f24afae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Feb 2015 04:56:12 +0600
Subject: [PATCH 1532/1937] [theplatform] Add URL sign capability

---
 youtube_dl/extractor/theplatform.py | 42 ++++++++++++++++++++++++++---
 1 file changed, 38 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 110ed976d..1579822f2 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -2,6 +2,11 @@ from __future__ import unicode_literals
 
 import re
 import json
+import time
+import hmac
+import binascii
+import hashlib
+
 
 from .subtitles import SubtitlesInfoExtractor
 from ..compat import (
@@ -11,6 +16,7 @@ from ..utils import (
     determine_ext,
     ExtractorError,
     xpath_with_ns,
+    unsmuggle_url,
 )
 
 _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language'})
@@ -18,7 +24,7 @@ _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language
 
 class ThePlatformIE(SubtitlesInfoExtractor):
     _VALID_URL = r'''(?x)
-        (?:https?://(?:link|player)\.theplatform\.com/[sp]/[^/]+/
+        (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
            (?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)?
          |theplatform:)(?P<id>[^/\?&]+)'''
 
@@ -38,9 +44,33 @@ class ThePlatformIE(SubtitlesInfoExtractor):
         },
     }
 
+    @staticmethod
+    def _sign_url(url, sig_key, sig_secret, life=600, include_qs=False):
+        flags = '10' if include_qs else '00'
+        expiration_date = '%x' % (int(time.time()) + life)
+
+        def str_to_hex(str):
+            return binascii.b2a_hex(str.encode('ascii')).decode('ascii')
+
+        def hex_to_str(hex):
+            return binascii.a2b_hex(hex)
+
+        relative_path = url.split('http://link.theplatform.com/s/')[1].split('?')[0]
+        clear_text = hex_to_str(flags + expiration_date + str_to_hex(relative_path))
+        checksum = hmac.new(sig_key.encode('ascii'), clear_text, hashlib.sha1).hexdigest()
+        sig = flags + expiration_date + checksum + str_to_hex(sig_secret)
+        return '%s&sig=%s' % (url, sig)
+
     def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+
         mobj = re.match(self._VALID_URL, url)
+        provider_id = mobj.group('provider_id')
         video_id = mobj.group('id')
+
+        if not provider_id:
+            provider_id = 'dJ5BDC'
+
         if mobj.group('config'):
             config_url = url + '&form=json'
             config_url = config_url.replace('swf/', 'config/')
@@ -48,8 +78,12 @@ class ThePlatformIE(SubtitlesInfoExtractor):
             config = self._download_json(config_url, video_id, 'Downloading config')
             smil_url = config['releaseUrl'] + '&format=SMIL&formats=MPEG4&manifest=f4m'
         else:
-            smil_url = ('http://link.theplatform.com/s/dJ5BDC/{0}/meta.smil?'
-                        'format=smil&mbr=true'.format(video_id))
+            smil_url = ('http://link.theplatform.com/s/{0}/{1}/meta.smil?'
+                        'format=smil&mbr=true'.format(provider_id, video_id))
+
+        sig = smuggled_data.get('sig')
+        if sig:
+            smil_url = self._sign_url(smil_url, sig['key'], sig['secret'])
 
         meta = self._download_xml(smil_url, video_id)
         try:
@@ -62,7 +96,7 @@ class ThePlatformIE(SubtitlesInfoExtractor):
         else:
             raise ExtractorError(error_msg, expected=True)
 
-        info_url = 'http://link.theplatform.com/s/dJ5BDC/{0}?format=preview'.format(video_id)
+        info_url = 'http://link.theplatform.com/s/{0}/{1}?format=preview'.format(provider_id, video_id)
         info_json = self._download_webpage(info_url, video_id)
         info = json.loads(info_json)
 

From b9c7a97318f4ca68cacf0ff5395c0d4a6f74b525 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Feb 2015 04:57:52 +0600
Subject: [PATCH 1533/1937] [history] Add extractor (Closes #4934)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/history.py  | 31 +++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 youtube_dl/extractor/history.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a4fab540b..13292073c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -189,6 +189,7 @@ from .hellporno import HellPornoIE
 from .helsinki import HelsinkiIE
 from .hentaistigma import HentaiStigmaIE
 from .historicfilms import HistoricFilmsIE
+from .history import HistoryIE
 from .hitbox import HitboxIE, HitboxLiveIE
 from .hornbunny import HornBunnyIE
 from .hostingbulk import HostingBulkIE
diff --git a/youtube_dl/extractor/history.py b/youtube_dl/extractor/history.py
new file mode 100644
index 000000000..f86164afe
--- /dev/null
+++ b/youtube_dl/extractor/history.py
@@ -0,0 +1,31 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import smuggle_url
+
+
+class HistoryIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?history\.com/(?:[^/]+/)+(?P<id>[^/]+?)(?:$|[?#])'
+
+    _TESTS = [{
+        'url': 'http://www.history.com/topics/valentines-day/history-of-valentines-day/videos/bet-you-didnt-know-valentines-day?m=528e394da93ae&s=undefined&f=1&free=false',
+        'md5': '6fe632d033c92aa10b8d4a9be047a7c5',
+        'info_dict': {
+            'id': 'bLx5Dv5Aka1G',
+            'ext': 'mp4',
+            'title': "Bet You Didn't Know: Valentine's Day",
+            'description': 'md5:7b57ea4829b391995b405fa60bd7b5f7',
+        },
+        'add_ie': ['ThePlatform'],
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        video_url = self._search_regex(
+            r'data-href="[^"]*/%s"[^>]+data-release-url="([^"]+)"' % video_id,
+            webpage, 'video url')
+
+        return self.url_result(smuggle_url(video_url, {'sig': {'key': 'crazyjava', 'secret': 's3cr3t'}}))

From f813928e4b669627007b772c9b150eed135b18be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Feb 2015 16:32:38 +0600
Subject: [PATCH 1534/1937] [bbccouk] Fix fallback to legacy playlist

---
 youtube_dl/extractor/bbccouk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index 126c8824c..f23e39545 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -273,7 +273,7 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
                     formats, subtitles = self._download_media_selector(programme_id)
                 return programme_id, title, description, duration, formats, subtitles
         except ExtractorError as ee:
-            if not isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404:
+            if not (isinstance(ee.cause, compat_HTTPError) and ee.cause.code == 404):
                 raise
 
         # fallback to legacy playlist

From 8fb474fb17a64ff2aa9f6315ebbc99ae7938c4e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 15 Feb 2015 14:59:00 +0100
Subject: [PATCH 1535/1937] [test/subtitles] Fix some tests

The checksym for the CeskaTelevize subtitles has changed again, so we just test that it has a reasonable length.
---
 test/test_subtitles.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 3e329438f..bcc69a778 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -138,7 +138,7 @@ class TestDailymotionSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(len(subtitles.keys()), 5)
+        self.assertTrue(len(subtitles.keys()) >= 6)
 
     def test_list_subtitles(self):
         self.DL.expect_warning('Automatic Captions not supported by this server')
@@ -334,7 +334,7 @@ class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['cs']))
-        self.assertEqual(md5(subtitles['cs']), 'cc3957b2c6dff1db71e5f2e83d467480')
+        self.assertTrue(len(subtitles['cs']) > 20000)
 
     def test_nosubtitles(self):
         self.DL.expect_warning('video doesn\'t have subtitles')

From 73fac4e9119bc53d49e646c38529f1b6d8de734d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 13 Feb 2015 11:14:01 +0100
Subject: [PATCH 1536/1937] [ffmpeg] Add --ffmpeg-location

---
 youtube_dl/YoutubeDL.py            |  4 +-
 youtube_dl/__init__.py             |  1 +
 youtube_dl/downloader/hls.py       |  7 +--
 youtube_dl/options.py              |  4 ++
 youtube_dl/postprocessor/ffmpeg.py | 95 ++++++++++++++++++++++--------
 5 files changed, 80 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 13d18e25e..dbb26272d 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1298,7 +1298,7 @@ class YoutubeDL(object):
                     downloaded = []
                     success = True
                     merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
-                    if not merger._executable:
+                    if not merger.available():
                         postprocessors = []
                         self.report_warning('You have requested multiple '
                                             'formats but ffmpeg or avconv are not installed.'
@@ -1647,7 +1647,7 @@ class YoutubeDL(object):
         self._write_string('[debug] Python version %s - %s\n' % (
             platform.python_version(), platform_name()))
 
-        exe_versions = FFmpegPostProcessor.get_versions()
+        exe_versions = FFmpegPostProcessor.get_versions(self)
         exe_versions['rtmpdump'] = rtmpdump_version()
         exe_str = ', '.join(
             '%s %s' % (exe, v)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ed22f169f..108fb3c7a 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -350,6 +350,7 @@ def _real_main(argv=None):
         'xattr_set_filesize': opts.xattr_set_filesize,
         'match_filter': match_filter,
         'no_color': opts.no_color,
+        'ffmpeg_location': opts.ffmpeg_location,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index e527ee425..8be4f4249 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -23,15 +23,14 @@ class HlsFD(FileDownloader):
         tmpfilename = self.temp_name(filename)
 
         ffpp = FFmpegPostProcessor(downloader=self)
-        program = ffpp._executable
-        if program is None:
+        if not ffpp.available:
             self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
             return False
         ffpp.check_version()
 
         args = [
             encodeArgument(opt)
-            for opt in (program, '-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
+            for opt in (ffpp.executable, '-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc')]
         args.append(encodeFilename(tmpfilename, True))
 
         retval = subprocess.call(args)
@@ -48,7 +47,7 @@ class HlsFD(FileDownloader):
             return True
         else:
             self.to_stderr('\n')
-            self.report_error('%s exited with code %d' % (program, retval))
+            self.report_error('%s exited with code %d' % (ffpp.basename, retval))
             return False
 
 
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 873432bee..ba35399cf 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -735,6 +735,10 @@ def parseOpts(overrideArguments=None):
         '--prefer-ffmpeg',
         action='store_true', dest='prefer_ffmpeg',
         help='Prefer ffmpeg over avconv for running the postprocessors')
+    postproc.add_option(
+        '--ffmpeg-location', '--avconv-location', metavar='PATH',
+        dest='ffmpeg_location',
+        help='Location of the ffmpeg/avconv binary; either the path to the binary or its containing directory.')
     postproc.add_option(
         '--exec',
         metavar='CMD', dest='exec_cmd',
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 01d25f760..504a71193 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -30,54 +30,97 @@ class FFmpegPostProcessorError(PostProcessingError):
 class FFmpegPostProcessor(PostProcessor):
     def __init__(self, downloader=None, deletetempfiles=False):
         PostProcessor.__init__(self, downloader)
-        self._versions = self.get_versions()
         self._deletetempfiles = deletetempfiles
+        self._determine_executables()
 
     def check_version(self):
-        if not self._executable:
+        if not self.available():
             raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')
 
         required_version = '10-0' if self._uses_avconv() else '1.0'
         if is_outdated_version(
-                self._versions[self._executable], required_version):
+                self._versions[self.basename], required_version):
             warning = 'Your copy of %s is outdated, update %s to version %s or newer if you encounter any errors.' % (
-                self._executable, self._executable, required_version)
+                self.basename, self.basename, required_version)
             if self._downloader:
                 self._downloader.report_warning(warning)
 
     @staticmethod
-    def get_versions():
+    def get_versions(downloader=None):
+        return FFmpegPostProcessor(downloader)._versions
+
+    def _determine_executables(self):
         programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
-        return dict((p, get_exe_version(p, args=['-version'])) for p in programs)
+        prefer_ffmpeg = self._downloader.params.get('prefer_ffmpeg', False)
 
-    @property
-    def available(self):
-        return self._executable is not None
+        self.basename = None
+        self.probe_basename = None
 
-    @property
-    def _executable(self):
-        if self._downloader.params.get('prefer_ffmpeg', False):
+        self._paths = None
+        self._versions = None
+        if self._downloader:
+            location = self._downloader.params.get('ffmpeg_location')
+            if location is not None:
+                if not os.path.exists(location):
+                    self._downloader.report_warning(
+                        'ffmpeg-location %s does not exist! '
+                        'Continuing without avconv/ffmpeg.' % (location))
+                    self._versions = {}
+                    return
+                elif not os.path.isdir(location):
+                    basename = os.path.splitext(os.path.basename(location))[0]
+                    if basename not in programs:
+                        self._downloader.report_warning(
+                            'Cannot identify executable %s, its basename should be one of %s. '
+                            'Continuing without avconv/ffmpeg.' %
+                            (location, ', '.join(programs)))
+                        self._versions = {}
+                        return None
+                    location = os.path.dirname(os.path.abspath(location))
+                    if basename in ('ffmpeg', 'ffprobe'):
+                        prefer_ffmpeg = True
+
+                self._paths = dict(
+                    (p, os.path.join(location, p)) for p in programs)
+                self._versions = dict(
+                    (p, get_exe_version(self._paths[p], args=['-version']))
+                    for p in programs)
+        if self._versions is None:
+            self._versions = dict(
+                (p, get_exe_version(p, args=['-version'])) for p in programs)
+            self._paths = dict((p, p) for p in programs)
+
+        if prefer_ffmpeg:
             prefs = ('ffmpeg', 'avconv')
         else:
             prefs = ('avconv', 'ffmpeg')
         for p in prefs:
             if self._versions[p]:
-                return p
-        return None
+                self.basename = p
+                break
 
-    @property
-    def _probe_executable(self):
-        if self._downloader.params.get('prefer_ffmpeg', False):
+        if prefer_ffmpeg:
             prefs = ('ffprobe', 'avprobe')
         else:
             prefs = ('avprobe', 'ffprobe')
         for p in prefs:
             if self._versions[p]:
-                return p
-        return None
+                self.probe_basename = p
+                break
+
+    def available(self):
+        return self.basename is not None
 
     def _uses_avconv(self):
-        return self._executable == 'avconv'
+        return self.basename == 'avconv'
+
+    @property
+    def executable(self):
+        return self._paths[self.basename]
+
+    @property
+    def probe_executable(self):
+        return self._paths[self.probe_basename]
 
     def run_ffmpeg_multiple_files(self, input_paths, out_path, opts):
         self.check_version()
@@ -88,7 +131,7 @@ class FFmpegPostProcessor(PostProcessor):
         files_cmd = []
         for path in input_paths:
             files_cmd.extend([encodeArgument('-i'), encodeFilename(path, True)])
-        cmd = ([encodeFilename(self._executable, True), encodeArgument('-y')] +
+        cmd = ([encodeFilename(self.executable, True), encodeArgument('-y')] +
                files_cmd +
                [encodeArgument(o) for o in opts] +
                [encodeFilename(self._ffmpeg_filename_argument(out_path), True)])
@@ -127,13 +170,15 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
 
     def get_audio_codec(self, path):
 
-        if not self._probe_executable:
+        if not self.probe_executable:
             raise PostProcessingError('ffprobe or avprobe not found. Please install one.')
         try:
             cmd = [
-                encodeFilename(self._probe_executable, True),
+                encodeFilename(self.probe_executable, True),
                 encodeArgument('-show_streams'),
                 encodeFilename(self._ffmpeg_filename_argument(path), True)]
+            if self._downloader.params.get('verbose', False):
+                self._downloader.to_screen('[debug] ffprobe command line: %s' % shell_quote(cmd))
             handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE)
             output = handle.communicate()[0]
             if handle.wait() != 0:
@@ -223,14 +268,14 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor):
             if self._nopostoverwrites and os.path.exists(encodeFilename(new_path)):
                 self._downloader.to_screen('[youtube] Post-process file %s exists, skipping' % new_path)
             else:
-                self._downloader.to_screen('[' + self._executable + '] Destination: ' + new_path)
+                self._downloader.to_screen('[' + self.basename + '] Destination: ' + new_path)
                 self.run_ffmpeg(path, new_path, acodec, more_opts)
         except:
             etype, e, tb = sys.exc_info()
             if isinstance(e, AudioConversionError):
                 msg = 'audio conversion failed: ' + e.msg
             else:
-                msg = 'error running ' + self._executable
+                msg = 'error running ' + self.basename
             raise PostProcessingError(msg)
 
         # Try to update the date time for extracted audio file.

From 03cd72b0070c430e95b7e9d612afcad026690153 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 16 Feb 2015 04:37:55 +0100
Subject: [PATCH 1537/1937] [extractor/common] Move up filesize

filesize and tbr should correlate, so it doesn't make sense to treat them differently.
---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c784eedb9..e74b7bf25 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -729,6 +729,7 @@ class InfoExtractor(object):
                 f.get('language_preference') if f.get('language_preference') is not None else -1,
                 f.get('quality') if f.get('quality') is not None else -1,
                 f.get('tbr') if f.get('tbr') is not None else -1,
+                f.get('filesize') if f.get('filesize') is not None else -1,
                 f.get('vbr') if f.get('vbr') is not None else -1,
                 f.get('height') if f.get('height') is not None else -1,
                 f.get('width') if f.get('width') is not None else -1,
@@ -736,7 +737,6 @@ class InfoExtractor(object):
                 f.get('abr') if f.get('abr') is not None else -1,
                 audio_ext_preference,
                 f.get('fps') if f.get('fps') is not None else -1,
-                f.get('filesize') if f.get('filesize') is not None else -1,
                 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
                 f.get('source_preference') if f.get('source_preference') is not None else -1,
                 f.get('format_id'),

From 85e80f71cddafe956886968d0eb60eff0aa7432c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 16 Feb 2015 04:50:57 +0100
Subject: [PATCH 1538/1937] [yam] Allow faults in optional fields (#4943)

---
 youtube_dl/extractor/yam.py | 37 ++++++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
index 511c8b67c..b294767c5 100644
--- a/youtube_dl/extractor/yam.py
+++ b/youtube_dl/extractor/yam.py
@@ -1,10 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..compat import compat_urlparse
-from ..utils import month_by_abbreviation
-import re
+from ..utils import (
+    float_or_none,
+    month_by_abbreviation,
+)
 
 
 class YamIE(InfoExtractor):
@@ -38,8 +42,8 @@ class YamIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        media_id = self._match_id(url)
-        page = self._download_webpage(url, media_id)
+        video_id = self._match_id(url)
+        page = self._download_webpage(url, video_id)
 
         # Is it hosted externally on YouTube?
         youtube_url = self._html_search_regex(
@@ -49,22 +53,29 @@ class YamIE(InfoExtractor):
             return self.url_result(youtube_url, 'Youtube')
 
         api_page = self._download_webpage(
-            'http://mymedia.yam.com/api/a/?pID=' + media_id, media_id)
+            'http://mymedia.yam.com/api/a/?pID=' + video_id, video_id,
+            note='Downloading API page')
         api_result_obj = compat_urlparse.parse_qs(api_page)
 
-        author = self._html_search_regex(
-            r'<!-- 發表作者 -->：[\n ]+<a href="/([a-z]+)"', page, 'author')
+        uploader_id = self._html_search_regex(
+            r'<!-- 發表作者 -->：[\n ]+<a href="/([a-z]+)"',
+            page, 'uploader id', fatal=False)
         mobj = re.search(r'<!-- 發表於 -->(?P<mon>[A-Z][a-z]{2})  ' +
                          r'(?P<day>\d{1,2}), (?P<year>\d{4})', page)
-        upload_date = '%s%02d%02d' % (mobj.group('year'),
-                                      month_by_abbreviation(mobj.group('mon')),
-                                      int(mobj.group('day')))
+        if mobj:
+            upload_date = '%s%02d%02d' % (
+                mobj.group('year'),
+                month_by_abbreviation(mobj.group('mon')),
+                int(mobj.group('day')))
+        else:
+            upload_date = None
+        duration = float_or_none(api_result_obj['totaltime'][0], scale=1000)
 
         return {
-            'id': media_id,
+            'id': video_id,
             'url': api_result_obj['mp3file'][0],
             'title': self._html_search_meta('description', page),
-            'duration': float(api_result_obj['totaltime'][0]) / 1000.0,
-            'uploader_id': author,
+            'duration': duration,
+            'uploader_id': uploader_id,
             'upload_date': upload_date,
         }

From a5681804415a9b8d011cd238b7d61e6eb4ec7c4b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 16 Feb 2015 04:51:20 +0100
Subject: [PATCH 1539/1937] release 2015.02.16

---
 README.md              | 3 +++
 docs/supportedsites.md | 8 +++++++-
 youtube_dl/version.py  | 2 +-
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 731cea1e1..d68896202 100644
--- a/README.md
+++ b/README.md
@@ -397,6 +397,9 @@ which means you can modify it, redistribute it or use it however you like.
                                      postprocessors (default)
     --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the
                                      postprocessors
+    --ffmpeg-location PATH           Location of the ffmpeg/avconv binary;
+                                     either the path to the binary or its
+                                     containing directory.
     --exec CMD                       Execute a command on the file after
                                      downloading, similar to find's -exec
                                      syntax. Example: --exec 'adb push {}
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 8bce8fede..894fa4323 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -1,4 +1,5 @@
 # Supported sites
+ - **1tv**: Первый канал
  - **1up.com**
  - **220.ro**
  - **24video**
@@ -60,6 +61,8 @@
  - **Brightcove**
  - **BuzzFeed**
  - **BYUtv**
+ - **Camdemy**
+ - **CamdemyFolder**
  - **Canal13cl**
  - **canalc2.tv**
  - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
@@ -134,7 +137,6 @@
  - **fernsehkritik.tv:postecke**
  - **Firedrive**
  - **Firstpost**
- - **firsttv**: Видеоархив - Первый канал
  - **Flickr**
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
  - **Foxgay**
@@ -174,6 +176,7 @@
  - **Helsinki**: helsinki.fi
  - **HentaiStigma**
  - **HistoricFilms**
+ - **History**
  - **hitbox**
  - **hitbox:live**
  - **HornBunny**
@@ -287,6 +290,8 @@
  - **nowvideo**: NowVideo
  - **npo.nl**
  - **npo.nl:live**
+ - **npo.nl:radio**
+ - **npo.nl:radio:fragment**
  - **NRK**
  - **NRKTV**
  - **ntv.ru**
@@ -527,6 +532,7 @@
  - **XVideos**
  - **XXXYMovies**
  - **Yahoo**: Yahoo screen and movies
+ - **Yam**
  - **YesJapan**
  - **Ynet**
  - **YouJizz**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 492ddf2ea..e58c59969 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.11'
+__version__ = '2015.02.16'

From 59b8ab583491410f4f365b1cb268a4360d7be877 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 16 Feb 2015 15:45:01 +0100
Subject: [PATCH 1540/1937] [rtlnl|generic] Add support for rtl.nl embeds
 (Fixes #4959)

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/generic.py  | 17 ++++++++-
 youtube_dl/extractor/rtlnl.py    | 63 ++++++++++++++++++++++++--------
 3 files changed, 65 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5b776a7a1..7d0909b5d 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -371,7 +371,7 @@ from .rottentomatoes import RottenTomatoesIE
 from .roxwel import RoxwelIE
 from .rtbf import RTBFIE
 from .rte import RteIE
-from .rtlnl import RtlXlIE
+from .rtlnl import RtlNlIE
 from .rtlnow import RTLnowIE
 from .rtl2 import RTL2IE
 from .rtp import RTPIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f4500e931..c4ba25a96 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -537,6 +537,15 @@ class GenericIE(InfoExtractor):
                 'uploader_id': 'NationalArchives08',
                 'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
             },
+        },
+        # rtl.nl embed
+        {
+            'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
+            'playlist_mincount': 5,
+            'info_dict': {
+                'id': 'aanslagen-kopenhagen',
+                'title': 'Aanslagen Kopenhagen | RTL Nieuws',
+            }
         }
     ]
 
@@ -782,6 +791,13 @@ class GenericIE(InfoExtractor):
                 'entries': entries,
             }
 
+        # Look for embedded rtl.nl player
+        matches = re.findall(
+            r'<iframe\s+(?:[a-zA-Z-]+="[^"]+"\s+)*?src="((?:https?:)?//(?:www\.)?rtl\.nl/system/videoplayer/[^"]+video_embed[^"]+)"',
+            webpage)
+        if matches:
+            return _playlist_from_matches(matches, ie='RtlNl')
+
         # Look for embedded (iframe) Vimeo player
         mobj = re.search(
             r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//player\.vimeo\.com/video/.+?)\1', webpage)
@@ -789,7 +805,6 @@ class GenericIE(InfoExtractor):
             player_url = unescapeHTML(mobj.group('url'))
             surl = smuggle_url(player_url, {'Referer': url})
             return self.url_result(surl)
-
         # Look for embedded (swf embed) Vimeo player
         mobj = re.search(
             r'<embed[^>]+?src="((?:https?:)?//(?:www\.)?vimeo\.com/moogaloop\.swf.+?)"', webpage)
diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py
index a3ca79f2c..cfce4550a 100644
--- a/youtube_dl/extractor/rtlnl.py
+++ b/youtube_dl/extractor/rtlnl.py
@@ -1,16 +1,25 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import parse_duration
+from ..utils import (
+    int_or_none,
+    parse_duration,
+)
 
 
-class RtlXlIE(InfoExtractor):
-    IE_NAME = 'rtlxl.nl'
-    _VALID_URL = r'https?://(www\.)?rtlxl\.nl/#!/[^/]+/(?P<uuid>[^/?]+)'
+class RtlNlIE(InfoExtractor):
+    IE_NAME = 'rtl.nl'
+    IE_DESC = 'rtl.nl and rtlxl.nl'
+    _VALID_URL = r'''(?x)
+        https?://(www\.)?
+        (?:
+            rtlxl\.nl/\#!/[^/]+/|
+            rtl\.nl/system/videoplayer/[^?#]+?/video_embed\.html\#uuid=
+        )
+        (?P<id>[0-9a-f-]+)'''
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.rtlxl.nl/#!/rtl-nieuws-132237/6e4203a6-0a5e-3596-8424-c599a59e0677',
         'md5': 'cc16baa36a6c169391f0764fa6b16654',
         'info_dict': {
@@ -22,21 +31,30 @@ class RtlXlIE(InfoExtractor):
             'upload_date': '20140814',
             'duration': 576.880,
         },
-    }
+    }, {
+        'url': 'http://www.rtl.nl/system/videoplayer/derden/rtlnieuws/video_embed.html#uuid=84ae5571-ac25-4225-ae0c-ef8d9efb2aed/autoplay=false',
+        'md5': 'dea7474214af1271d91ef332fb8be7ea',
+        'info_dict': {
+            'id': '84ae5571-ac25-4225-ae0c-ef8d9efb2aed',
+            'ext': 'mp4',
+            'timestamp': 1424039400,
+            'title': 'RTL Nieuws - Nieuwe beelden Kopenhagen: chaos direct na aanslag',
+            'thumbnail': 're:^https?://screenshots\.rtl\.nl/system/thumb/sz=[0-9]+x[0-9]+/uuid=84ae5571-ac25-4225-ae0c-ef8d9efb2aed$',
+            'upload_date': '20150215',
+            'description': 'Er zijn nieuwe beelden vrijgegeven die vlak na de aanslag in Kopenhagen zijn gemaakt. Op de video is goed te zien hoe omstanders zich bekommeren om één van de slachtoffers, terwijl de eerste agenten ter plaatse komen.',
+        }
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        uuid = mobj.group('uuid')
-
+        uuid = self._match_id(url)
         info = self._download_json(
             'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=flash/' % uuid,
             uuid)
 
         material = info['material'][0]
-        episode_info = info['episodes'][0]
-
         progname = info['abstracts'][0]['name']
         subtitle = material['title'] or info['episodes'][0]['name']
+        description = material.get('synopsis') or info['episodes'][0]['synopsis']
 
         # Use unencrypted m3u8 streams (See https://github.com/rg3/youtube-dl/issues/4118)
         videopath = material['videopath'].replace('.f4m', '.m3u8')
@@ -58,14 +76,29 @@ class RtlXlIE(InfoExtractor):
                 'quality': 0,
             }
         ])
-
         self._sort_formats(formats)
 
+        thumbnails = []
+        meta = info.get('meta', {})
+        for p in ('poster_base_url', '"thumb_base_url"'):
+            if not meta.get(p):
+                continue
+
+            thumbnails.append({
+                'url': self._proto_relative_url(meta[p] + uuid),
+                'width': int_or_none(self._search_regex(
+                    r'/sz=([0-9]+)', meta[p], 'thumbnail width', fatal=False)),
+                'height': int_or_none(self._search_regex(
+                    r'/sz=[0-9]+x([0-9]+)',
+                    meta[p], 'thumbnail height', fatal=False))
+            })
+
         return {
             'id': uuid,
             'title': '%s - %s' % (progname, subtitle),
             'formats': formats,
             'timestamp': material['original_date'],
-            'description': episode_info['synopsis'],
+            'description': description,
             'duration': parse_duration(material.get('duration')),
+            'thumbnails': thumbnails,
         }

From 1db5fbcfe3ef95654aa642ebe483fb070852cb49 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 16 Feb 2015 15:47:13 +0100
Subject: [PATCH 1541/1937] release 2015.02.16.1

---
 docs/supportedsites.md | 2 +-
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 894fa4323..4bb68fdc5 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -338,9 +338,9 @@
  - **Roxwel**
  - **RTBF**
  - **Rte**
+ - **rtl.nl**: rtl.nl and rtlxl.nl
  - **RTL2**
  - **RTLnow**
- - **rtlxl.nl**
  - **RTP**
  - **RTS**: RTS.ch
  - **rtve.es:alacarta**: RTVE a la carta
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e58c59969..bbe0e53b5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.16'
+__version__ = '2015.02.16.1'

From a504ced097e703a9bc6c18b6e31bcafb4783ed80 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 15 Feb 2015 18:03:41 +0100
Subject: [PATCH 1542/1937] Improve subtitles support

For each language the extractor builds a list with the available formats sorted (like for video formats), then YoutubeDL selects one of them using the '--sub-format' option which now allows giving the format preferences (for example 'ass/srt/best').
For each format the 'url' field can be set so that we only download the contents if needed, or if the contents needs to be processed (like in crunchyroll) the 'data' field can be used.

The reasons for this change are:
* We weren't checking that the format given with '--sub-format' was available, checking it in each extractor would be repetitive.
* It allows to easily support giving a format preference.
* The subtitles were automatically downloaded in the extractor, but I think that if you use for example the '--dump-json' option you want to finish as fast as possible.

Currently only the ted extractor has been updated, but the old system still works.
---
 test/test_subtitles.py             | 24 ++++-----
 youtube_dl/YoutubeDL.py            | 85 +++++++++++++++++++++++++++---
 youtube_dl/__init__.py             |  1 -
 youtube_dl/extractor/common.py     | 20 ++++++-
 youtube_dl/extractor/ted.py        | 18 ++++---
 youtube_dl/options.py              |  4 +-
 youtube_dl/postprocessor/ffmpeg.py |  6 +--
 7 files changed, 121 insertions(+), 37 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index bcc69a778..fbc9eaf4d 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -27,15 +27,23 @@ class BaseTestSubtitles(unittest.TestCase):
 
     def setUp(self):
         self.DL = FakeYDL()
-        self.ie = self.IE(self.DL)
+        self.ie = self.IE()
+        self.DL.add_info_extractor(self.ie)
 
     def getInfoDict(self):
-        info_dict = self.ie.extract(self.url)
+        info_dict = self.DL.extract_info(self.url, download=False)
         return info_dict
 
     def getSubtitles(self):
         info_dict = self.getInfoDict()
-        return info_dict['subtitles']
+        subtitles = info_dict['subtitles']
+        if not subtitles:
+            return subtitles
+        for sub_info in subtitles.values():
+            if sub_info.get('data') is None:
+                uf = self.DL.urlopen(sub_info['url'])
+                sub_info['data'] = uf.read().decode('utf-8')
+        return dict((l, sub_info['data']) for l, sub_info in subtitles.items())
 
 
 class TestYoutubeSubtitles(BaseTestSubtitles):
@@ -176,7 +184,7 @@ class TestTedSubtitles(BaseTestSubtitles):
 
     def test_no_writesubtitles(self):
         subtitles = self.getSubtitles()
-        self.assertEqual(subtitles, None)
+        self.assertFalse(subtitles)
 
     def test_subtitles(self):
         self.DL.params['writesubtitles'] = True
@@ -196,18 +204,10 @@ class TestTedSubtitles(BaseTestSubtitles):
         self.assertTrue(len(subtitles.keys()) >= 28)
 
     def test_list_subtitles(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
-    def test_automatic_captions(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
-        self.DL.params['writeautomaticsub'] = True
-        self.DL.params['subtitleslang'] = ['en']
-        subtitles = self.getSubtitles()
-        self.assertTrue(len(subtitles.keys()) == 0)
-
     def test_multiple_langs(self):
         self.DL.params['writesubtitles'] = True
         langs = ['es', 'fr', 'de']
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 13d18e25e..e665e3d53 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -154,7 +154,7 @@ class YoutubeDL(object):
     allsubtitles:      Downloads all the subtitles of the video
                        (requires writesubtitles or writeautomaticsub)
     listsubtitles:     Lists all available subtitles for the video
-    subtitlesformat:   Subtitle format [srt/sbv/vtt] (default=srt)
+    subtitlesformat:   The format code for subtitles
     subtitleslangs:    List of languages of the subtitles to download
     keepvideo:         Keep the video file after post-processing
     daterange:         A DateRange object, download only if the upload_date is in the range.
@@ -1019,6 +1019,11 @@ class YoutubeDL(object):
                 info_dict['timestamp'])
             info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
 
+        if self.params.get('listsubtitles', False):
+            self.list_subtitles(info_dict['id'], info_dict.get('subtitles'))
+            return
+        info_dict['subtitles'] = self.process_subtitles(info_dict['id'], info_dict.get('subtitles'))
+
         # This extractors handle format selection themselves
         if info_dict['extractor'] in ['Youku']:
             if download:
@@ -1147,6 +1152,53 @@ class YoutubeDL(object):
         info_dict.update(formats_to_download[-1])
         return info_dict
 
+    def process_subtitles(self, video_id, available_subs):
+        """Select the requested subtitles and their format"""
+        if not available_subs:
+            return available_subs
+
+        if self.params.get('allsubtitles', False):
+            requested_langs = available_subs.keys()
+        else:
+            if self.params.get('subtitleslangs', False):
+                requested_langs = self.params.get('subtitleslangs')
+            elif 'en' in available_subs:
+                requested_langs = ['en']
+            else:
+                requested_langs = [list(available_subs.keys())[0]]
+
+        formats_query = self.params.get('subtitlesformat', 'best')
+        formats_preference = formats_query.split('/') if formats_query else []
+        subs = {}
+        for lang in requested_langs:
+            formats = available_subs.get(lang)
+            if formats is None:
+                self.report_warning('%s subtitles not available for %s' % (lang, video_id))
+                continue
+            if isinstance(formats, compat_str):
+                # TODO: convert all IE with subtitles support to the new format
+                # and remove this
+                subs[lang] = {
+                    'ext': formats_preference[0],
+                    'data': formats,
+                }
+                continue
+            for ext in formats_preference:
+                if ext == 'best':
+                    f = formats[-1]
+                    break
+                matches = list(filter(lambda f: f['ext'] == ext, formats))
+                if matches:
+                    f = matches[-1]
+                    break
+            else:
+                f = formats[-1]
+                self.report_warning(
+                    'No subtitle format found matching "%s" for language %s, '
+                    'using %s' % (formats_query, lang, f['ext']))
+            subs[lang] = f
+        return subs
+
     def process_info(self, info_dict):
         """Process a single resolved IE result."""
 
@@ -1253,11 +1305,18 @@ class YoutubeDL(object):
             # subtitles download errors are already managed as troubles in relevant IE
             # that way it will silently go on when used with unsupporting IE
             subtitles = info_dict['subtitles']
-            sub_format = self.params.get('subtitlesformat', 'srt')
-            for sub_lang in subtitles.keys():
-                sub = subtitles[sub_lang]
-                if sub is None:
-                    continue
+            for sub_lang, sub_info in subtitles.items():
+                sub_format = sub_info['ext']
+                if sub_info.get('data') is not None:
+                    sub_data = sub_info['data']
+                else:
+                    try:
+                        uf = self.urlopen(sub_info['url'])
+                        sub_data = uf.read().decode('utf-8')
+                    except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+                        self.report_warning('Unable to download subtitle for "%s": %s' %
+                                            (sub_lang, compat_str(err)))
+                        continue
                 try:
                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)
                     if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
@@ -1265,7 +1324,7 @@ class YoutubeDL(object):
                     else:
                         self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
                         with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
-                            subfile.write(sub)
+                            subfile.write(sub_data)
                 except (OSError, IOError):
                     self.report_error('Cannot write subtitles file ' + sub_filename)
                     return
@@ -1586,6 +1645,18 @@ class YoutubeDL(object):
             ['ID', 'width', 'height', 'URL'],
             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
 
+    def list_subtitles(self, video_id, subtitles):
+        if not subtitles:
+            self.to_screen('%s has no subtitles' % video_id)
+            return
+        header_line = 'Language    formats'
+        sub_lines = [
+            '%-12s%s' % (lang, ', '.join(f['ext'] for f in reversed(formats)))
+            for lang, formats in subtitles.items()]
+        self.to_screen(
+            'Available subtitles for %s:\n%s\n%s' %
+            (video_id, header_line, '\n'.join(sub_lines)))
+
     def urlopen(self, req):
         """ Start an HTTP download """
 
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ed22f169f..5f2585003 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -226,7 +226,6 @@ def _real_main(argv=None):
     if opts.embedsubtitles:
         postprocessors.append({
             'key': 'FFmpegEmbedSubtitle',
-            'subtitlesformat': opts.subtitlesformat,
         })
     if opts.xattrs:
         postprocessors.append({'key': 'XAttrMetadata'})
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index c784eedb9..161c623eb 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -151,8 +151,14 @@ class InfoExtractor(object):
                     If not explicitly set, calculated from timestamp.
     uploader_id:    Nickname or id of the video uploader.
     location:       Physical location where the video was filmed.
-    subtitles:      The subtitle file contents as a dictionary in the format
-                    {language: subtitles}.
+    subtitles:      The available subtitles as a dictionary in the format
+                    {language: subformats}. "subformats" is a list sorted from
+                    lower to higher preference, each element is a dictionary
+                    with the "ext" entry and one of:
+                        * "data": The subtitles file contents
+                        * "url": A url pointing to the subtitles file
+                    Note: YoutubeDL.extract_info will get the requested
+                    format and replace the "subformats" list with it.
     duration:       Length of the video in seconds, as an integer.
     view_count:     How many users have watched the video on the platform.
     like_count:     Number of positive ratings of the video
@@ -993,6 +999,16 @@ class InfoExtractor(object):
             any_restricted = any_restricted or is_restricted
         return not any_restricted
 
+    def extract_subtitles(self, *args, **kwargs):
+        subtitles = {}
+        list_subtitles = self._downloader.params.get('listsubtitles')
+        if self._downloader.params.get('writesubtitles', False) or list_subtitles:
+            subtitles.update(self._get_subtitles(*args, **kwargs))
+        return subtitles
+
+    def _get_subtitles(self, *args, **kwargs):
+        raise NotImplementedError("This method must be implemented by subclasses")
+
 
 class SearchInfoExtractor(InfoExtractor):
     """
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 10b3b706a..1809eaae4 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -3,14 +3,14 @@ from __future__ import unicode_literals
 import json
 import re
 
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 
 from ..compat import (
     compat_str,
 )
 
 
-class TEDIE(SubtitlesInfoExtractor):
+class TEDIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         (?P<proto>https?://)
         (?P<type>www|embed(?:-ssl)?)(?P<urlmain>\.ted\.com/
@@ -165,9 +165,6 @@ class TEDIE(SubtitlesInfoExtractor):
         video_id = compat_str(talk_info['id'])
         # subtitles
         video_subtitles = self.extract_subtitles(video_id, talk_info)
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, talk_info)
-            return
 
         thumbnail = talk_info['thumb']
         if not thumbnail.startswith('http'):
@@ -183,13 +180,18 @@ class TEDIE(SubtitlesInfoExtractor):
             'duration': talk_info.get('duration'),
         }
 
-    def _get_available_subtitles(self, video_id, talk_info):
+    def _get_subtitles(self, video_id, talk_info):
         languages = [lang['languageCode'] for lang in talk_info.get('languages', [])]
         if languages:
             sub_lang_list = {}
             for l in languages:
-                url = 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/srt' % (video_id, l)
-                sub_lang_list[l] = url
+                sub_lang_list[l] = [
+                    {
+                        'url': 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/%s' % (video_id, l, ext),
+                        'ext': ext,
+                    }
+                    for ext in ['ted', 'srt']
+                ]
             return sub_lang_list
         else:
             self._downloader.report_warning('video doesn\'t have subtitles')
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 873432bee..4fcf8c83d 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -387,8 +387,8 @@ def parseOpts(overrideArguments=None):
         help='lists all available subtitles for the video')
     subtitles.add_option(
         '--sub-format',
-        action='store', dest='subtitlesformat', metavar='FORMAT', default='srt',
-        help='subtitle format (default=srt) ([sbv/vtt] youtube only)')
+        action='store', dest='subtitlesformat', metavar='FORMAT', default='best',
+        help='subtitle format, accepts formats preference, for example: "ass/srt/best"')
     subtitles.add_option(
         '--sub-lang', '--sub-langs', '--srt-lang',
         action='callback', dest='subtitleslangs', metavar='LANGS', type='str',
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 5238ce534..d1bbfbfe3 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -453,10 +453,6 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
         'zu': 'zul',
     }
 
-    def __init__(self, downloader=None, subtitlesformat='srt'):
-        super(FFmpegEmbedSubtitlePP, self).__init__(downloader)
-        self._subformat = subtitlesformat
-
     @classmethod
     def _conver_lang_code(cls, code):
         """Convert language code from ISO 639-1 to ISO 639-2/T"""
@@ -472,7 +468,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
 
         sub_langs = [key for key in information['subtitles']]
         filename = information['filepath']
-        input_files = [filename] + [subtitles_filename(filename, lang, self._subformat) for lang in sub_langs]
+        input_files = [filename] + [subtitles_filename(filename, lang, sub_info['ext']) for lang, sub_info in information['subtitles'].items()]
 
         opts = [
             '-map', '0',

From b5857f62e2c2ca70316e041212aa9e89d54cc253 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 15 Feb 2015 18:21:42 +0100
Subject: [PATCH 1543/1937] [crunchyroll] Convert to new subtitles system

---
 youtube_dl/extractor/crunchyroll.py | 66 +++++++++++++++--------------
 1 file changed, 35 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 1680f532f..f1da7d09b 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -9,7 +9,7 @@ import xml.etree.ElementTree
 
 from hashlib import sha1
 from math import pow, sqrt, floor
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
@@ -25,10 +25,9 @@ from ..aes import (
     aes_cbc_decrypt,
     inc,
 )
-from .common import InfoExtractor
 
 
-class CrunchyrollIE(SubtitlesInfoExtractor):
+class CrunchyrollIE(InfoExtractor):
     _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
     _TESTS = [{
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
@@ -187,6 +186,38 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
 
         return output
 
+    def _get_subtitles(self, video_id, webpage):
+        subtitles = {}
+        for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
+            sub_page = self._download_webpage(
+                'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id,
+                video_id, note='Downloading subtitles for ' + sub_name)
+            id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False)
+            iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False)
+            data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False)
+            if not id or not iv or not data:
+                continue
+            id = int(id)
+            iv = base64.b64decode(iv)
+            data = base64.b64decode(data)
+
+            subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8')
+            lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
+            if not lang_code:
+                continue
+            sub_root = xml.etree.ElementTree.fromstring(subtitle)
+            subtitles[lang_code] = [
+                {
+                    'ext': 'srt',
+                    'data': self._convert_subtitles_to_srt(sub_root),
+                },
+                {
+                    'ext': 'ass',
+                    'data': self._convert_subtitles_to_ass(sub_root),
+                },
+            ]
+        return subtitles
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('video_id')
@@ -249,34 +280,7 @@ Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
                 'format_id': video_format,
             })
 
-        subtitles = {}
-        sub_format = self._downloader.params.get('subtitlesformat', 'srt')
-        for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
-            sub_page = self._download_webpage(
-                'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id=' + sub_id,
-                video_id, note='Downloading subtitles for ' + sub_name)
-            id = self._search_regex(r'id=\'([0-9]+)', sub_page, 'subtitle_id', fatal=False)
-            iv = self._search_regex(r'<iv>([^<]+)', sub_page, 'subtitle_iv', fatal=False)
-            data = self._search_regex(r'<data>([^<]+)', sub_page, 'subtitle_data', fatal=False)
-            if not id or not iv or not data:
-                continue
-            id = int(id)
-            iv = base64.b64decode(iv)
-            data = base64.b64decode(data)
-
-            subtitle = self._decrypt_subtitles(data, iv, id).decode('utf-8')
-            lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
-            if not lang_code:
-                continue
-            sub_root = xml.etree.ElementTree.fromstring(subtitle)
-            if sub_format == 'ass':
-                subtitles[lang_code] = self._convert_subtitles_to_ass(sub_root)
-            else:
-                subtitles[lang_code] = self._convert_subtitles_to_srt(sub_root)
-
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
+        subtitles = self.extract_subtitles(video_id, webpage)
 
         return {
             'id': video_id,

From 6b597516c12c7fd81e832f3ec05dd0dca6089823 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 15 Feb 2015 18:32:40 +0100
Subject: [PATCH 1544/1937] [atresplayer] Convert to new subtitles system

---
 youtube_dl/extractor/atresplayer.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index f016368fa..7669e0e3d 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import time
 import hmac
 
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..compat import (
     compat_str,
     compat_urllib_parse,
@@ -17,7 +17,7 @@ from ..utils import (
 )
 
 
-class AtresPlayerIE(SubtitlesInfoExtractor):
+class AtresPlayerIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?atresplayer\.com/television/[^/]+/[^/]+/[^/]+/(?P<id>.+?)_\d+\.html'
     _TESTS = [
         {
@@ -144,13 +144,12 @@ class AtresPlayerIE(SubtitlesInfoExtractor):
         thumbnail = xpath_text(episode, './media/asset/files/background', 'thumbnail')
 
         subtitles = {}
-        subtitle = xpath_text(episode, './media/asset/files/subtitle', 'subtitle')
-        if subtitle:
-            subtitles['es'] = subtitle
-
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
+        subtitle_url = xpath_text(episode, './media/asset/files/subtitle', 'subtitle')
+        if subtitle_url:
+            subtitles['es'] = [{
+                'ext': 'srt',
+                'url': subtitle_url,
+            }]
 
         return {
             'id': video_id,
@@ -159,5 +158,5 @@ class AtresPlayerIE(SubtitlesInfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
-            'subtitles': self.extract_subtitles(video_id, subtitles),
+            'subtitles': subtitles,
         }

From 65469a7f8b0ba50bd3c8918707e35125962aa2cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 15 Feb 2015 18:52:07 +0100
Subject: [PATCH 1545/1937] [vimeo] Convert to new subtitles system

Removed some tests, the behaviour should be checked in a test for the YoutubeDL class
---
 test/parameters.json          |  2 +-
 test/test_subtitles.py        | 17 -----------------
 youtube_dl/extractor/vimeo.py | 15 ++++++---------
 3 files changed, 7 insertions(+), 27 deletions(-)

diff --git a/test/parameters.json b/test/parameters.json
index af77b89b4..cbff9bd16 100644
--- a/test/parameters.json
+++ b/test/parameters.json
@@ -28,7 +28,7 @@
     "retries": 10, 
     "simulate": false, 
     "subtitleslang": null, 
-    "subtitlesformat": "srt",
+    "subtitlesformat": "best",
     "test": true, 
     "updatetime": true, 
     "usenetrc": false, 
diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index fbc9eaf4d..3f2d61d36 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -240,10 +240,6 @@ class TestVimeoSubtitles(BaseTestSubtitles):
     url = 'http://vimeo.com/76979871'
     IE = VimeoIE
 
-    def test_no_writesubtitles(self):
-        subtitles = self.getSubtitles()
-        self.assertEqual(subtitles, None)
-
     def test_subtitles(self):
         self.DL.params['writesubtitles'] = True
         subtitles = self.getSubtitles()
@@ -261,19 +257,6 @@ class TestVimeoSubtitles(BaseTestSubtitles):
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['de', 'en', 'es', 'fr']))
 
-    def test_list_subtitles(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
-        self.DL.params['listsubtitles'] = True
-        info_dict = self.getInfoDict()
-        self.assertEqual(info_dict, None)
-
-    def test_automatic_captions(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
-        self.DL.params['writeautomaticsub'] = True
-        self.DL.params['subtitleslang'] = ['en']
-        subtitles = self.getSubtitles()
-        self.assertTrue(len(subtitles.keys()) == 0)
-
     def test_nosubtitles(self):
         self.DL.expect_warning('video doesn\'t have subtitles')
         self.url = 'http://vimeo.com/56015672'
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 303e81447..5930d5984 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -6,7 +6,6 @@ import re
 import itertools
 
 from .common import InfoExtractor
-from .subtitles import SubtitlesInfoExtractor
 from ..compat import (
     compat_HTTPError,
     compat_urllib_parse,
@@ -51,7 +50,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         self._download_webpage(login_request, None, False, 'Wrong login info')
 
 
-class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
+class VimeoIE(VimeoBaseInfoExtractor):
     """Information extractor for vimeo.com."""
 
     # _VALID_URL matches Vimeo URLs
@@ -368,12 +367,10 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
         text_tracks = config['request'].get('text_tracks')
         if text_tracks:
             for tt in text_tracks:
-                subtitles[tt['lang']] = 'http://vimeo.com' + tt['url']
-
-        video_subtitles = self.extract_subtitles(video_id, subtitles)
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
+                subtitles[tt['lang']] = [{
+                    'ext': 'vtt',
+                    'url': 'http://vimeo.com' + tt['url'],
+                }]
 
         return {
             'id': video_id,
@@ -389,7 +386,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
             'view_count': view_count,
             'like_count': like_count,
             'comment_count': comment_count,
-            'subtitles': video_subtitles,
+            'subtitles': subtitles,
         }
 
 

From c84dd8a90dcc75547b343449b921b644a2119c4f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 16 Feb 2015 21:12:31 +0100
Subject: [PATCH 1546/1937] [YoutubeDL] store the subtitles to download in the
 'requested_subtitles' field

We need to keep the orginal subtitles information, so that the '--load-info' option can be used to list or select the subtitles again.
We'll also be able to have a separate field for storing the automatic captions info.
---
 test/test_subtitles.py             | 2 +-
 youtube_dl/YoutubeDL.py            | 6 +++---
 youtube_dl/extractor/common.py     | 2 --
 youtube_dl/postprocessor/ffmpeg.py | 7 ++++---
 4 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 3f2d61d36..b3c615c4f 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -36,7 +36,7 @@ class BaseTestSubtitles(unittest.TestCase):
 
     def getSubtitles(self):
         info_dict = self.getInfoDict()
-        subtitles = info_dict['subtitles']
+        subtitles = info_dict['requested_subtitles']
         if not subtitles:
             return subtitles
         for sub_info in subtitles.values():
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e665e3d53..8545dc9e9 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1022,7 +1022,7 @@ class YoutubeDL(object):
         if self.params.get('listsubtitles', False):
             self.list_subtitles(info_dict['id'], info_dict.get('subtitles'))
             return
-        info_dict['subtitles'] = self.process_subtitles(info_dict['id'], info_dict.get('subtitles'))
+        info_dict['requested_subtitles'] = self.process_subtitles(info_dict['id'], info_dict.get('subtitles'))
 
         # This extractors handle format selection themselves
         if info_dict['extractor'] in ['Youku']:
@@ -1301,10 +1301,10 @@ class YoutubeDL(object):
         subtitles_are_requested = any([self.params.get('writesubtitles', False),
                                        self.params.get('writeautomaticsub')])
 
-        if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']:
+        if subtitles_are_requested and info_dict.get('requested_subtitles'):
             # subtitles download errors are already managed as troubles in relevant IE
             # that way it will silently go on when used with unsupporting IE
-            subtitles = info_dict['subtitles']
+            subtitles = info_dict['requested_subtitles']
             for sub_lang, sub_info in subtitles.items():
                 sub_format = sub_info['ext']
                 if sub_info.get('data') is not None:
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 161c623eb..d149e0f92 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -157,8 +157,6 @@ class InfoExtractor(object):
                     with the "ext" entry and one of:
                         * "data": The subtitles file contents
                         * "url": A url pointing to the subtitles file
-                    Note: YoutubeDL.extract_info will get the requested
-                    format and replace the "subformats" list with it.
     duration:       Length of the video in seconds, as an integer.
     view_count:     How many users have watched the video on the platform.
     like_count:     Number of positive ratings of the video
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index d1bbfbfe3..e42298f0e 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -462,13 +462,14 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
         if information['ext'] != 'mp4':
             self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4 files')
             return True, information
-        if not information.get('subtitles'):
+        subtitles = information.get('requested_subtitles')
+        if not subtitles:
             self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to embed')
             return True, information
 
-        sub_langs = [key for key in information['subtitles']]
+        sub_langs = list(subtitles.keys())
         filename = information['filepath']
-        input_files = [filename] + [subtitles_filename(filename, lang, sub_info['ext']) for lang, sub_info in information['subtitles'].items()]
+        input_files = [filename] + [subtitles_filename(filename, lang, sub_info['ext']) for lang, sub_info in subtitles.items()]
 
         opts = [
             '-map', '0',

From a1f2a06b34807a2e1b5eb5176fa418da2405392d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 16 Feb 2015 21:28:06 +0100
Subject: [PATCH 1547/1937] [dailymotion] Convert to new subtitles system

---
 test/test_subtitles.py              | 17 -----------------
 youtube_dl/extractor/dailymotion.py | 10 +++-------
 2 files changed, 3 insertions(+), 24 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index b3c615c4f..84ae0e714 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -127,10 +127,6 @@ class TestDailymotionSubtitles(BaseTestSubtitles):
     url = 'http://www.dailymotion.com/video/xczg00'
     IE = DailymotionIE
 
-    def test_no_writesubtitles(self):
-        subtitles = self.getSubtitles()
-        self.assertEqual(subtitles, None)
-
     def test_subtitles(self):
         self.DL.params['writesubtitles'] = True
         subtitles = self.getSubtitles()
@@ -148,19 +144,6 @@ class TestDailymotionSubtitles(BaseTestSubtitles):
         subtitles = self.getSubtitles()
         self.assertTrue(len(subtitles.keys()) >= 6)
 
-    def test_list_subtitles(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
-        self.DL.params['listsubtitles'] = True
-        info_dict = self.getInfoDict()
-        self.assertEqual(info_dict, None)
-
-    def test_automatic_captions(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
-        self.DL.params['writeautomaticsub'] = True
-        self.DL.params['subtitleslang'] = ['en']
-        subtitles = self.getSubtitles()
-        self.assertTrue(len(subtitles.keys()) == 0)
-
     def test_nosubtitles(self):
         self.DL.expect_warning('video doesn\'t have subtitles')
         self.url = 'http://www.dailymotion.com/video/x12u166_le-zapping-tele-star-du-08-aout-2013_tv'
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index cf5841a7c..4ca892926 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -6,7 +6,6 @@ import json
 import itertools
 
 from .common import InfoExtractor
-from .subtitles import SubtitlesInfoExtractor
 
 from ..compat import (
     compat_str,
@@ -31,7 +30,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor):
         return request
 
 
-class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
+class DailymotionIE(DailymotionBaseInfoExtractor):
     """Information Extractor for Dailymotion"""
 
     _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(embed|#)/)?video/(?P<id>[^/?_]+)'
@@ -143,9 +142,6 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
 
         # subtitles
         video_subtitles = self.extract_subtitles(video_id, webpage)
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, webpage)
-            return
 
         view_count = str_to_int(self._search_regex(
             r'video_views_count[^>]+>\s+([\d\.,]+)',
@@ -169,7 +165,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
             'view_count': view_count,
         }
 
-    def _get_available_subtitles(self, video_id, webpage):
+    def _get_subtitles(self, video_id, webpage):
         try:
             sub_list = self._download_webpage(
                 'https://api.dailymotion.com/video/%s/subtitles?fields=id,language,url' % video_id,
@@ -179,7 +175,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
             return {}
         info = json.loads(sub_list)
         if (info['total'] > 0):
-            sub_lang_list = dict((l['language'], l['url']) for l in info['list'])
+            sub_lang_list = dict((l['language'], [{'url': l['url'], 'ext': 'srt'}]) for l in info['list'])
             return sub_lang_list
         self._downloader.report_warning('video doesn\'t have subtitles')
         return {}

From 360e1ca5ccabcb5d48228d9472b09f1bce68bbc4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 16 Feb 2015 21:44:17 +0100
Subject: [PATCH 1548/1937] [youtube] Convert to new subtitles system

The automatic captions are stored in the 'automactic_captions' field, which is used if no normal subtitles are found for an specific language.
---
 test/test_subtitles.py          |  5 ---
 youtube_dl/YoutubeDL.py         | 24 ++++++++++----
 youtube_dl/extractor/common.py  | 12 +++++++
 youtube_dl/extractor/youtube.py | 57 ++++++++++++++++++---------------
 4 files changed, 61 insertions(+), 37 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 84ae0e714..91cebce28 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -50,11 +50,6 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
     url = 'QRS8MkLhQmM'
     IE = YoutubeIE
 
-    def test_youtube_no_writesubtitles(self):
-        self.DL.params['writesubtitles'] = False
-        subtitles = self.getSubtitles()
-        self.assertEqual(subtitles, None)
-
     def test_youtube_subtitles(self):
         self.DL.params['writesubtitles'] = True
         subtitles = self.getSubtitles()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8545dc9e9..a47f8f5de 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1020,9 +1020,13 @@ class YoutubeDL(object):
             info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
 
         if self.params.get('listsubtitles', False):
-            self.list_subtitles(info_dict['id'], info_dict.get('subtitles'))
+            if 'automatic_captions' in info_dict:
+                self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
+            self.list_subtitles(info_dict['id'], info_dict.get('subtitles'), 'subtitles')
             return
-        info_dict['requested_subtitles'] = self.process_subtitles(info_dict['id'], info_dict.get('subtitles'))
+        info_dict['requested_subtitles'] = self.process_subtitles(
+            info_dict['id'], info_dict.get('subtitles'),
+            info_dict.get('automatic_captions'))
 
         # This extractors handle format selection themselves
         if info_dict['extractor'] in ['Youku']:
@@ -1152,8 +1156,14 @@ class YoutubeDL(object):
         info_dict.update(formats_to_download[-1])
         return info_dict
 
-    def process_subtitles(self, video_id, available_subs):
+    def process_subtitles(self, video_id, available_subs, available_autocaps):
         """Select the requested subtitles and their format"""
+        if available_autocaps and self.params.get('writeautomaticsub'):
+            available_subs = available_subs.copy()
+            for lang, cap_info in available_autocaps.items():
+                if lang not in available_subs:
+                    available_subs[lang] = cap_info
+
         if not available_subs:
             return available_subs
 
@@ -1645,17 +1655,17 @@ class YoutubeDL(object):
             ['ID', 'width', 'height', 'URL'],
             [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
 
-    def list_subtitles(self, video_id, subtitles):
+    def list_subtitles(self, video_id, subtitles, name='subtitles'):
         if not subtitles:
-            self.to_screen('%s has no subtitles' % video_id)
+            self.to_screen('%s has no %s' % (video_id, name))
             return
         header_line = 'Language    formats'
         sub_lines = [
             '%-12s%s' % (lang, ', '.join(f['ext'] for f in reversed(formats)))
             for lang, formats in subtitles.items()]
         self.to_screen(
-            'Available subtitles for %s:\n%s\n%s' %
-            (video_id, header_line, '\n'.join(sub_lines)))
+            'Available %s for %s:\n%s\n%s' %
+            (name, video_id, header_line, '\n'.join(sub_lines)))
 
     def urlopen(self, req):
         """ Start an HTTP download """
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d149e0f92..fe7d8dbc9 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -157,6 +157,8 @@ class InfoExtractor(object):
                     with the "ext" entry and one of:
                         * "data": The subtitles file contents
                         * "url": A url pointing to the subtitles file
+    automatic_captions: Like 'subtitles', used by the YoutubeIE for
+                    automatically generated captions
     duration:       Length of the video in seconds, as an integer.
     view_count:     How many users have watched the video on the platform.
     like_count:     Number of positive ratings of the video
@@ -1007,6 +1009,16 @@ class InfoExtractor(object):
     def _get_subtitles(self, *args, **kwargs):
         raise NotImplementedError("This method must be implemented by subclasses")
 
+    def extract_automatic_captions(self, *args, **kwargs):
+        automatic_captions = {}
+        list_subtitles = self._downloader.params.get('listsubtitles')
+        if self._downloader.params.get('writeautomaticsub', False) or list_subtitles:
+            automatic_captions.update(self._get_automatic_captions(*args, **kwargs))
+        return automatic_captions
+
+    def _get_automatic_captions(self, *args, **kwargs):
+        raise NotImplementedError("This method must be implemented by subclasses")
+
 
 class SearchInfoExtractor(InfoExtractor):
     """
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 35ef4c303..1b2dbf276 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -11,7 +11,6 @@ import time
 import traceback
 
 from .common import InfoExtractor, SearchInfoExtractor
-from .subtitles import SubtitlesInfoExtractor
 from ..jsinterp import JSInterpreter
 from ..swfinterp import SWFInterpreter
 from ..compat import (
@@ -185,7 +184,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             return
 
 
-class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
+class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_DESC = 'YouTube.com'
     _VALID_URL = r"""(?x)^
                      (
@@ -644,7 +643,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             raise ExtractorError(
                 'Signature extraction failed: ' + tb, cause=e)
 
-    def _get_available_subtitles(self, video_id, webpage):
+    def _get_subtitles(self, video_id, webpage):
         try:
             subs_doc = self._download_xml(
                 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
@@ -658,23 +657,27 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             lang = track.attrib['lang_code']
             if lang in sub_lang_list:
                 continue
-            params = compat_urllib_parse.urlencode({
-                'lang': lang,
-                'v': video_id,
-                'fmt': self._downloader.params.get('subtitlesformat', 'srt'),
-                'name': track.attrib['name'].encode('utf-8'),
-            })
-            url = 'https://www.youtube.com/api/timedtext?' + params
-            sub_lang_list[lang] = url
+            sub_formats = []
+            for ext in ['sbv', 'vtt', 'srt']:
+                params = compat_urllib_parse.urlencode({
+                    'lang': lang,
+                    'v': video_id,
+                    'fmt': ext,
+                    'name': track.attrib['name'].encode('utf-8'),
+                })
+                sub_formats.append({
+                    'url': 'https://www.youtube.com/api/timedtext?' + params,
+                    'ext': ext,
+                })
+            sub_lang_list[lang] = sub_formats
         if not sub_lang_list:
             self._downloader.report_warning('video doesn\'t have subtitles')
             return {}
         return sub_lang_list
 
-    def _get_available_automatic_caption(self, video_id, webpage):
+    def _get_automatic_captions(self, video_id, webpage):
         """We need the webpage for getting the captions url, pass it as an
            argument to speed up the process."""
-        sub_format = self._downloader.params.get('subtitlesformat', 'srt')
         self.to_screen('%s: Looking for automatic captions' % video_id)
         mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
         err_msg = 'Couldn\'t find automatic captions for %s' % video_id
@@ -704,14 +707,20 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             sub_lang_list = {}
             for lang_node in caption_list.findall('target'):
                 sub_lang = lang_node.attrib['lang_code']
-                params = compat_urllib_parse.urlencode({
-                    'lang': original_lang,
-                    'tlang': sub_lang,
-                    'fmt': sub_format,
-                    'ts': timestamp,
-                    'kind': caption_kind,
-                })
-                sub_lang_list[sub_lang] = caption_url + '&' + params
+                sub_formats = []
+                for ext in ['sbv', 'vtt', 'srt']:
+                    params = compat_urllib_parse.urlencode({
+                        'lang': original_lang,
+                        'tlang': sub_lang,
+                        'fmt': ext,
+                        'ts': timestamp,
+                        'kind': caption_kind,
+                    })
+                    sub_formats.append({
+                        'url': caption_url + '&' + params,
+                        'ext': ext,
+                    })
+                sub_lang_list[sub_lang] = sub_formats
             return sub_lang_list
         # An extractor error can be raise by the download process if there are
         # no automatic captions but there are subtitles
@@ -966,10 +975,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
 
         # subtitles
         video_subtitles = self.extract_subtitles(video_id, video_webpage)
-
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, video_webpage)
-            return
+        automatic_captions = self.extract_automatic_captions(video_id, video_webpage)
 
         if 'length_seconds' not in video_info:
             self._downloader.report_warning('unable to extract video duration')
@@ -1118,6 +1124,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             'description': video_description,
             'categories': video_categories,
             'subtitles': video_subtitles,
+            'automatic_captions': automatic_captions,
             'duration': video_duration,
             'age_limit': 18 if age_gate else 0,
             'annotations': video_annotations,

From 85729c51afad484ef784faf5d82bad8acab77d5e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 12:09:12 +0100
Subject: [PATCH 1549/1937] [downloader] Add --hls-prefer-native to use the
 native HLS downloader (#4966)

---
 youtube_dl/YoutubeDL.py           | 5 ++++-
 youtube_dl/__init__.py            | 1 +
 youtube_dl/downloader/__init__.py | 3 +++
 youtube_dl/options.py             | 4 ++++
 4 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index dbb26272d..ea2435e0a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -225,7 +225,6 @@ class YoutubeDL(object):
     call_home:         Boolean, true iff we are allowed to contact the
                        youtube-dl servers for debugging.
     sleep_interval:    Number of seconds to sleep before each download.
-    external_downloader:  Executable of the external downloader to call.
     listformats:       Print an overview of available video formats and exit.
     list_thumbnails:   Print a table of all thumbnails and exit.
     match_filter:      A function that gets called with the info_dict of
@@ -235,6 +234,10 @@ class YoutubeDL(object):
                        match_filter_func in utils.py is one example for this.
     no_color:          Do not emit color codes in output.
 
+    The following options determine which downloader is picked:
+    external_downloader: Executable of the external downloader to call.
+                       None or unset for standard (built-in) downloader.
+    hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
 
     The following parameters are not used by YoutubeDL itself, they are used by
     the FileDownloader:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 108fb3c7a..eac2a26ec 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -351,6 +351,7 @@ def _real_main(argv=None):
         'match_filter': match_filter,
         'no_color': opts.no_color,
         'ffmpeg_location': opts.ffmpeg_location,
+        'hls_prefer_native': opts.hls_prefer_native,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index eff1122c5..9fb66e2f7 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -34,6 +34,9 @@ def get_suitable_downloader(info_dict, params={}):
         if ed.supports(info_dict):
             return ed
 
+    if protocol == 'm3u8' and params.get('hls_prefer_native'):
+        return NativeHlsFD
+
     return PROTOCOL_MAP.get(protocol, HttpFD)
 
 
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index ba35399cf..5f678f76b 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -424,6 +424,10 @@ def parseOpts(overrideArguments=None):
         '--xattr-set-filesize',
         dest='xattr_set_filesize', action='store_true',
         help='(experimental) set file xattribute ytdl.filesize with expected filesize')
+    downloader.add_option(
+        '--hls-prefer-native',
+        dest='hls_prefer_native', action='store_true',
+        help='(experimental) Use the native HLS downloader instead of ffmpeg.')
     downloader.add_option(
         '--external-downloader',
         dest='external_downloader', metavar='COMMAND',

From 163d966707a7e49bcdad4ebd9189922b58223395 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 16:21:02 +0100
Subject: [PATCH 1550/1937] [downloader/external] curl: Add the '--location'
 flag

curl doesn't follow redirections by default
---
 youtube_dl/downloader/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index ff031d2e0..51c41c704 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -75,7 +75,7 @@ class ExternalFD(FileDownloader):
 
 class CurlFD(ExternalFD):
     def _make_cmd(self, tmpfilename, info_dict):
-        cmd = [self.exe, '-o', tmpfilename]
+        cmd = [self.exe, '--location', '-o', tmpfilename]
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._source_address('--interface')

From 662435f7281da418612131512725fa5e367605be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 16:29:24 +0100
Subject: [PATCH 1551/1937] [YoutubeDL] Use a Request object for getting the
 cookies (fixes #4970)

So that we don't have to implement all the methods used by the cookiejar.
---
 youtube_dl/YoutubeDL.py | 25 ++-----------------------
 1 file changed, 2 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ea2435e0a..0a41152d6 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -954,30 +954,9 @@ class YoutubeDL(object):
         return res
 
     def _calc_cookies(self, info_dict):
-        class _PseudoRequest(object):
-            def __init__(self, url):
-                self.url = url
-                self.headers = {}
-                self.unverifiable = False
-
-            def add_unredirected_header(self, k, v):
-                self.headers[k] = v
-
-            def get_full_url(self):
-                return self.url
-
-            def is_unverifiable(self):
-                return self.unverifiable
-
-            def has_header(self, h):
-                return h in self.headers
-
-            def get_header(self, h, default=None):
-                return self.headers.get(h, default)
-
-        pr = _PseudoRequest(info_dict['url'])
+        pr = compat_urllib_request.Request(info_dict['url'])
         self.cookiejar.add_cookie_header(pr)
-        return pr.headers.get('Cookie')
+        return pr.get_header('Cookie')
 
     def process_video_result(self, info_dict, download=True):
         assert info_dict.get('_type', 'video') == 'video'

From fbc503d69698fca33525707b859b41d491d681ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 16:40:42 +0100
Subject: [PATCH 1552/1937] [downloader/hls] Fix detection of ffmpeg/avconv
 (reported in #4966)

---
 youtube_dl/downloader/hls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 8be4f4249..3650d715e 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -23,7 +23,7 @@ class HlsFD(FileDownloader):
         tmpfilename = self.temp_name(filename)
 
         ffpp = FFmpegPostProcessor(downloader=self)
-        if not ffpp.available:
+        if not ffpp.available():
             self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
             return False
         ffpp.check_version()

From f740fae2a418ce6c1d229890b48730d257a80dee Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 17:26:41 +0100
Subject: [PATCH 1553/1937] [ffmpeg] Make available a property

---
 youtube_dl/YoutubeDL.py            | 2 +-
 youtube_dl/postprocessor/ffmpeg.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ea2435e0a..a74e73b0c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1301,7 +1301,7 @@ class YoutubeDL(object):
                     downloaded = []
                     success = True
                     merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
-                    if not merger.available():
+                    if not merger.available:
                         postprocessors = []
                         self.report_warning('You have requested multiple '
                                             'formats but ffmpeg or avconv are not installed.'
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 16babf6a5..e98fe9834 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -34,7 +34,7 @@ class FFmpegPostProcessor(PostProcessor):
         self._determine_executables()
 
     def check_version(self):
-        if not self.available():
+        if not self.available:
             raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')
 
         required_version = '10-0' if self._uses_avconv() else '1.0'
@@ -108,6 +108,7 @@ class FFmpegPostProcessor(PostProcessor):
                 self.probe_basename = p
                 break
 
+    @property
     def available(self):
         return self.basename is not None
 

From 65bf37ef83b67bd0e2c594283852d120734b2a3a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 17:27:29 +0100
Subject: [PATCH 1554/1937] [ffmpeg] Remove trivial helper method

---
 youtube_dl/postprocessor/ffmpeg.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index e98fe9834..3f2e6cf1d 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -37,7 +37,7 @@ class FFmpegPostProcessor(PostProcessor):
         if not self.available:
             raise FFmpegPostProcessorError('ffmpeg or avconv not found. Please install one.')
 
-        required_version = '10-0' if self._uses_avconv() else '1.0'
+        required_version = '10-0' if self.basename == 'avconv' else '1.0'
         if is_outdated_version(
                 self._versions[self.basename], required_version):
             warning = 'Your copy of %s is outdated, update %s to version %s or newer if you encounter any errors.' % (
@@ -112,9 +112,6 @@ class FFmpegPostProcessor(PostProcessor):
     def available(self):
         return self.basename is not None
 
-    def _uses_avconv(self):
-        return self.basename == 'avconv'
-
     @property
     def executable(self):
         return self._paths[self.basename]

From 46312e0b46147ff26008eee62b2f9ba47b30fe73 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 17:29:32 +0100
Subject: [PATCH 1555/1937] release 2015.02.17

---
 README.md             | 2 ++
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d68896202..acd0ef41b 100644
--- a/README.md
+++ b/README.md
@@ -161,6 +161,8 @@ which means you can modify it, redistribute it or use it however you like.
     --playlist-reverse               Download playlist videos in reverse order
     --xattr-set-filesize             (experimental) set file xattribute
                                      ytdl.filesize with expected filesize
+    --hls-prefer-native              (experimental) Use the native HLS
+                                     downloader instead of ffmpeg.
     --external-downloader COMMAND    (experimental) Use the specified external
                                      downloader. Currently supports
                                      aria2c,curl,wget
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index bbe0e53b5..88834e058 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.16.1'
+__version__ = '2015.02.17'

From 8ac27a68e618ecc789bbef4dd0eb408324498fdd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 17:35:03 +0100
Subject: [PATCH 1556/1937] [hls] Switch to available as a property

---
 youtube_dl/downloader/hls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 3650d715e..8be4f4249 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -23,7 +23,7 @@ class HlsFD(FileDownloader):
         tmpfilename = self.temp_name(filename)
 
         ffpp = FFmpegPostProcessor(downloader=self)
-        if not ffpp.available():
+        if not ffpp.available:
             self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
             return False
         ffpp.check_version()

From 7010577720bbea309566c38adbd912c15400b9fc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 17:35:08 +0100
Subject: [PATCH 1557/1937] release 2015.02.17.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 88834e058..4ffbd3ae4 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.17'
+__version__ = '2015.02.17.1'

From 4069766c527d10b8e25b9262a3882101367deb3e Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Tue, 17 Feb 2015 22:31:35 +0600
Subject: [PATCH 1558/1937] [extractor/common] Test URLs with GET

---
 youtube_dl/extractor/common.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e74b7bf25..4b0ee882c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -753,9 +753,7 @@ class InfoExtractor(object):
 
     def _is_valid_url(self, url, video_id, item='video'):
         try:
-            self._request_webpage(
-                HEADRequest(url), video_id,
-                'Checking %s URL' % item)
+            self._request_webpage(url, video_id, 'Checking %s URL' % item)
             return True
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError):

From 0d93378887bd527b1df04e6138b4bc41382dd48f Mon Sep 17 00:00:00 2001
From: Sergey M? <dstftw@gmail.com>
Date: Tue, 17 Feb 2015 22:34:29 +0600
Subject: [PATCH 1559/1937] [videolecturesnet] Check http format URLs (Closes
 #4968)

---
 youtube_dl/extractor/videolecturesnet.py | 33 +++++++++++++++++-------
 1 file changed, 24 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index ebd2a3dca..1ec5298e9 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -49,15 +49,30 @@ class VideoLecturesNetIE(InfoExtractor):
         thumbnail = (
             None if thumbnail_el is None else thumbnail_el.attrib.get('src'))
 
-        formats = [{
-            'url': v.attrib['src'],
-            'width': int_or_none(v.attrib.get('width')),
-            'height': int_or_none(v.attrib.get('height')),
-            'filesize': int_or_none(v.attrib.get('size')),
-            'tbr': int_or_none(v.attrib.get('systemBitrate')) / 1000.0,
-            'ext': v.attrib.get('ext'),
-        } for v in switch.findall('./video')
-            if v.attrib.get('proto') == 'http']
+        formats = []
+        for v in switch.findall('./video'):
+            proto = v.attrib.get('proto')
+            if not proto in ['http', 'rtmp']:
+                continue
+            f = {
+                'width': int_or_none(v.attrib.get('width')),
+                'height': int_or_none(v.attrib.get('height')),
+                'filesize': int_or_none(v.attrib.get('size')),
+                'tbr': int_or_none(v.attrib.get('systemBitrate')) / 1000.0,
+                'ext': v.attrib.get('ext'),
+            }
+            src = v.attrib['src']
+            if proto == 'http':
+                if self._is_valid_url(src, video_id):
+                    f['url'] = src
+                    formats.append(f)
+            elif proto == 'rtmp':
+                f.update({
+                    'url': v.attrib['streamer'],
+                    'play_path': src,
+                })
+                formats.append(f)
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From be24c8697f51fbc68e2cd99668e675a07a3735de Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 17:38:31 +0100
Subject: [PATCH 1560/1937] release 2015.02.17.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 4ffbd3ae4..34a8d1184 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.17.1'
+__version__ = '2015.02.17.2'

From 4cd95bcbc34d74e506e6158251d50d83697da5fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 18:55:53 +0100
Subject: [PATCH 1561/1937] [twitch:stream] Prefer the 'source' format (fixes
 #4972)

---
 youtube_dl/extractor/common.py | 11 +++++++++++
 youtube_dl/extractor/twitch.py |  6 ++++++
 2 files changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4b0ee882c..31599bf52 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -839,6 +839,7 @@ class InfoExtractor(object):
             note='Downloading m3u8 information',
             errnote='Failed to download m3u8 information')
         last_info = None
+        last_media= None
         kv_rex = re.compile(
             r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
         for line in m3u8_doc.splitlines():
@@ -849,6 +850,13 @@ class InfoExtractor(object):
                     if v.startswith('"'):
                         v = v[1:-1]
                     last_info[m.group('key')] = v
+            elif line.startswith('#EXT-X-MEDIA:'):
+                last_media = {}
+                for m in kv_rex.finditer(line):
+                    v = m.group('val')
+                    if v.startswith('"'):
+                        v = v[1:-1]
+                    last_media[m.group('key')] = v
             elif line.startswith('#') or not line.strip():
                 continue
             else:
@@ -877,6 +885,9 @@ class InfoExtractor(object):
                     width_str, height_str = resolution.split('x')
                     f['width'] = int(width_str)
                     f['height'] = int(height_str)
+                if last_media is not None:
+                    f['m3u8_media'] = last_media
+                    last_media = None
                 formats.append(f)
                 last_info = {}
         self._sort_formats(formats)
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 87290d002..8e296698e 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -348,6 +348,12 @@ class TwitchStreamIE(TwitchBaseIE):
             '%s/api/channel/hls/%s.m3u8?%s'
             % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query).encode('utf-8')),
             channel_id, 'mp4')
+        # prefer the 'source' stream, the others are limited to 30 fps
+        def _sort_source(f):
+            if f.get('m3u8_media') is not None and f['m3u8_media'].get('NAME') == 'Source':
+                return 1
+            return 0
+        formats = sorted(formats, key=_sort_source)
 
         view_count = stream.get('viewers')
         timestamp = parse_iso8601(stream.get('created_at'))

From 4a8d4a53b107dde6e78f37fd63dee22b3d29a877 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Feb 2015 01:16:49 +0600
Subject: [PATCH 1562/1937] [videolecturesnet] Fix rtmp stream glitches (Closes
 #4968)

---
 youtube_dl/extractor/videolecturesnet.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index 1ec5298e9..d46234856 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -70,6 +70,7 @@ class VideoLecturesNetIE(InfoExtractor):
                 f.update({
                     'url': v.attrib['streamer'],
                     'play_path': src,
+                    'rtmp_real_time': True,
                 })
                 formats.append(f)
         self._sort_formats(formats)

From 48246541da66a12486505804f9519391a298ff54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 21:17:47 +0100
Subject: [PATCH 1563/1937] [ceskatelevize] Convert to new subtitles system

---
 youtube_dl/extractor/ceskatelevize.py | 30 +++++++++++++--------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index f70e090bb..65f6be623 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 
 import re
 
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..compat import (
     compat_urllib_request,
     compat_urllib_parse,
@@ -15,7 +15,7 @@ from ..utils import (
 )
 
 
-class CeskaTelevizeIE(SubtitlesInfoExtractor):
+class CeskaTelevizeIE(InfoExtractor):
     _VALID_URL = r'https?://www\.ceskatelevize\.cz/(porady|ivysilani)/(.+/)?(?P<id>[^?#]+)'
 
     _TESTS = [
@@ -107,13 +107,7 @@ class CeskaTelevizeIE(SubtitlesInfoExtractor):
         subtitles = {}
         subs = item.get('subtitles')
         if subs:
-            subtitles['cs'] = subs[0]['url']
-
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
-
-        subtitles = self._fix_subtitles(self.extract_subtitles(video_id, subtitles))
+            subtitles = self.extract_subtitles(episode_id, subs)
 
         return {
             'id': episode_id,
@@ -125,11 +119,20 @@ class CeskaTelevizeIE(SubtitlesInfoExtractor):
             'subtitles': subtitles,
         }
 
+    def _get_subtitles(self, episode_id, subs):
+        original_subtitles = self._download_webpage(
+            subs[0]['url'], episode_id, 'Downloading subtitles')
+        srt_subs = self._fix_subtitles(original_subtitles)
+        return {
+            'cs': [{
+                'ext': 'srt',
+                'data': srt_subs,
+            }]
+        }
+
     @staticmethod
     def _fix_subtitles(subtitles):
         """ Convert millisecond-based subtitles to SRT """
-        if subtitles is None:
-            return subtitles  # subtitles not requested
 
         def _msectotimecode(msec):
             """ Helper utility to convert milliseconds to timecode """
@@ -149,7 +152,4 @@ class CeskaTelevizeIE(SubtitlesInfoExtractor):
                 else:
                     yield line
 
-        fixed_subtitles = {}
-        for k, v in subtitles.items():
-            fixed_subtitles[k] = "\r\n".join(_fix_subtitle(v))
-        return fixed_subtitles
+        return "\r\n".join(_fix_subtitle(subtitles))

From bd7fe0cf6668c9ea9272dbe25774072b383e67d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 21:23:09 +0100
Subject: [PATCH 1564/1937] [walla] Convert to new subtitles system

---
 youtube_dl/extractor/walla.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/walla.py b/youtube_dl/extractor/walla.py
index 672bda7a7..24efbd6e6 100644
--- a/youtube_dl/extractor/walla.py
+++ b/youtube_dl/extractor/walla.py
@@ -3,14 +3,14 @@ from __future__ import unicode_literals
 
 import re
 
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..utils import (
     xpath_text,
     int_or_none,
 )
 
 
-class WallaIE(SubtitlesInfoExtractor):
+class WallaIE(InfoExtractor):
     _VALID_URL = r'http://vod\.walla\.co\.il/[^/]+/(?P<id>\d+)/(?P<display_id>.+)'
     _TEST = {
         'url': 'http://vod.walla.co.il/movie/2642630/one-direction-all-for-one',
@@ -52,13 +52,10 @@ class WallaIE(SubtitlesInfoExtractor):
         subtitles = {}
         for subtitle in item.findall('./subtitles/subtitle'):
             lang = xpath_text(subtitle, './title')
-            subtitles[self._SUBTITLE_LANGS.get(lang, lang)] = xpath_text(subtitle, './src')
-
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
-
-        subtitles = self.extract_subtitles(video_id, subtitles)
+            subtitles[self._SUBTITLE_LANGS.get(lang, lang)] = [{
+                'ext': 'srt',
+                'url': xpath_text(subtitle, './src'),
+            }]
 
         formats = []
         for quality in item.findall('./qualities/quality'):

From 98f000409f072e544d6c3b07809022f703b8b23d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 19:54:04 +0100
Subject: [PATCH 1565/1937] [radio.de] Fix extraction

---
 youtube_dl/extractor/radiode.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/radiode.py b/youtube_dl/extractor/radiode.py
index f95bc9454..aa5f6f8ad 100644
--- a/youtube_dl/extractor/radiode.py
+++ b/youtube_dl/extractor/radiode.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import json
-
 from .common import InfoExtractor
 
 
@@ -10,13 +8,13 @@ class RadioDeIE(InfoExtractor):
     _VALID_URL = r'https?://(?P<id>.+?)\.(?:radio\.(?:de|at|fr|pt|es|pl|it)|rad\.io)'
     _TEST = {
         'url': 'http://ndr2.radio.de/',
-        'md5': '3b4cdd011bc59174596b6145cda474a4',
         'info_dict': {
             'id': 'ndr2',
             'ext': 'mp3',
             'title': 're:^NDR 2 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'description': 'md5:591c49c702db1a33751625ebfb67f273',
             'thumbnail': 're:^https?://.*\.png',
+            'is_live': True,
         },
         'params': {
             'skip_download': True,
@@ -25,16 +23,15 @@ class RadioDeIE(InfoExtractor):
 
     def _real_extract(self, url):
         radio_id = self._match_id(url)
-
         webpage = self._download_webpage(url, radio_id)
+        jscode = self._search_regex(
+            r"'components/station/stationService':\s*\{\s*'?station'?:\s*(\{.*?\s*\}),\n",
+            webpage, 'broadcast')
 
-        broadcast = json.loads(self._search_regex(
-            r'_getBroadcast\s*=\s*function\(\s*\)\s*{\s*return\s+({.+?})\s*;\s*}',
-            webpage, 'broadcast'))
-
+        broadcast = self._parse_json(jscode, radio_id)
         title = self._live_title(broadcast['name'])
         description = broadcast.get('description') or broadcast.get('shortDescription')
-        thumbnail = broadcast.get('picture4Url') or broadcast.get('picture4TransUrl')
+        thumbnail = broadcast.get('picture4Url') or broadcast.get('picture4TransUrl') or broadcast.get('logo100x100')
 
         formats = [{
             'url': stream['streamUrl'],

From 5cda4eda7253d766611363a880af46895c11ad17 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 21:37:48 +0100
Subject: [PATCH 1566/1937] [YoutubeDL] Use a progress hook for progress
 reporting

Instead of every downloader calling two helper functions, let our progress report be an ordinary progress hook like everyone else's.
Closes #4875.
---
 youtube_dl/YoutubeDL.py         | 13 +++--
 youtube_dl/downloader/common.py | 87 +++++++++++++++++++++------------
 youtube_dl/downloader/f4m.py    | 70 ++++++++++++++++----------
 youtube_dl/downloader/http.py   | 13 +++--
 youtube_dl/downloader/rtmp.py   | 18 +++----
 5 files changed, 128 insertions(+), 73 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3658332ec..b087d356f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -199,18 +199,25 @@ class YoutubeDL(object):
                        postprocessor.
     progress_hooks:    A list of functions that get called on download
                        progress, with a dictionary with the entries
-                       * status: One of "downloading" and "finished".
+                       * status: One of "downloading", "error", or "finished".
                                  Check this first and ignore unknown values.
 
-                       If status is one of "downloading" or "finished", the
+                       If status is one of "downloading", or "finished", the
                        following properties may also be present:
                        * filename: The final filename (always present)
+                       * tmpfilename: The filename we're currently writing to
                        * downloaded_bytes: Bytes on disk
                        * total_bytes: Size of the whole file, None if unknown
-                       * tmpfilename: The filename we're currently writing to
+                       * total_bytes_estimate: Guess of the eventual file size,
+                                               None if unavailable.
+                       * elapsed: The number of seconds since download started.
                        * eta: The estimated time in seconds, None if unknown
                        * speed: The download speed in bytes/second, None if
                                 unknown
+                       * fragment_index: The counter of the currently
+                                         downloaded video fragment.
+                       * fragment_count: The number of fragments (= individual
+                                         files that will be merged)
 
                        Progress hooks are guaranteed to be called at least once
                        (with status "finished") if the download is successful.
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 7bb3a948d..45e55b99c 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+from __future__ import division, unicode_literals
 
 import os
 import re
@@ -54,6 +54,7 @@ class FileDownloader(object):
         self.ydl = ydl
         self._progress_hooks = []
         self.params = params
+        self.add_progress_hook(self.report_progress)
 
     @staticmethod
     def format_seconds(seconds):
@@ -226,42 +227,64 @@ class FileDownloader(object):
             self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
         self.to_console_title('youtube-dl ' + msg)
 
-    def report_progress(self, percent, data_len_str, speed, eta):
-        """Report download progress."""
-        if self.params.get('noprogress', False):
+    def report_progress(self, s):
+        if s['status'] == 'finished':
+            if self.params.get('noprogress', False):
+                self.to_screen('[download] Download completed')
+            else:
+                s['_total_bytes_str'] = format_bytes(s['total_bytes'])
+                if s.get('elapsed') is not None:
+                    s['_elapsed_str'] = self.format_seconds(s['elapsed'])
+                    msg_template = '100%% of %(_total_bytes_str)s in %(_elapsed_str)s'
+                else:
+                    msg_template = '100%% of %(_total_bytes_str)s'
+                self._report_progress_status(
+                    msg_template % s, is_last_line=True)
+
+        if self.params.get('noprogress'):
             return
-        if eta is not None:
-            eta_str = self.format_eta(eta)
-        else:
-            eta_str = 'Unknown ETA'
-        if percent is not None:
-            percent_str = self.format_percent(percent)
-        else:
-            percent_str = 'Unknown %'
-        speed_str = self.format_speed(speed)
 
-        msg = ('%s of %s at %s ETA %s' %
-               (percent_str, data_len_str, speed_str, eta_str))
-        self._report_progress_status(msg)
-
-    def report_progress_live_stream(self, downloaded_data_len, speed, elapsed):
-        if self.params.get('noprogress', False):
+        if s['status'] != 'downloading':
             return
-        downloaded_str = format_bytes(downloaded_data_len)
-        speed_str = self.format_speed(speed)
-        elapsed_str = FileDownloader.format_seconds(elapsed)
-        msg = '%s at %s (%s)' % (downloaded_str, speed_str, elapsed_str)
-        self._report_progress_status(msg)
 
-    def report_finish(self, data_len_str, tot_time):
-        """Report download finished."""
-        if self.params.get('noprogress', False):
-            self.to_screen('[download] Download completed')
+        if s.get('eta') is not None:
+            s['_eta_str'] = self.format_eta(s['eta'])
         else:
-            self._report_progress_status(
-                ('100%% of %s in %s' %
-                 (data_len_str, self.format_seconds(tot_time))),
-                is_last_line=True)
+            s['_eta_str'] = 'Unknown ETA'
+
+        if s.get('total_bytes') and s.get('downloaded_bytes') is not None:
+            s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes'])
+        elif s.get('total_bytes_estimate') and s.get('downloaded_bytes') is not None:
+            s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes_estimate'])
+        else:
+            if s.get('downloaded_bytes') == 0:
+                s['_percent_str'] = self.format_percent(0)
+            else:
+                s['_percent_str'] = 'Unknown %'
+
+        if s.get('speed') is not None:
+            s['_speed_str'] = self.format_speed(s['speed'])
+        else:
+            s['_speed_str'] = 'Unknown speed'
+
+        if s.get('total_bytes') is not None:
+            s['_total_bytes_str'] = format_bytes(s['total_bytes'])
+            msg_template = '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'
+        elif s.get('total_bytes_estimate') is not None:
+            s['_total_bytes_estimate_str'] = format_bytes(s['total_bytes_estimate'])
+            msg_template = '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'
+        else:
+            if s.get('downloaded_bytes') is not None:
+                s['_downloaded_bytes_str'] = format_bytes(s['downloaded_bytes'])
+                if s.get('elapsed'):
+                    s['_elapsed_str'] = self.format_seconds(s['elapsed'])
+                    msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'
+                else:
+                    msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
+            else:
+                msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'
+
+        self._report_progress_status(msg_template % s)
 
     def report_resuming_byte(self, resume_len):
         """Report attempt to resume at given byte."""
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 0e7a1c200..5a1f8e680 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+from __future__ import division, unicode_literals
 
 import base64
 import io
@@ -252,17 +252,6 @@ class F4mFD(FileDownloader):
         requested_bitrate = info_dict.get('tbr')
         self.to_screen('[download] Downloading f4m manifest')
         manifest = self.ydl.urlopen(man_url).read()
-        self.report_destination(filename)
-        http_dl = HttpQuietDownloader(
-            self.ydl,
-            {
-                'continuedl': True,
-                'quiet': True,
-                'noprogress': True,
-                'ratelimit': self.params.get('ratelimit', None),
-                'test': self.params.get('test', False),
-            }
-        )
 
         doc = etree.fromstring(manifest)
         formats = [(int(f.attrib.get('bitrate', -1)), f)
@@ -298,39 +287,67 @@ class F4mFD(FileDownloader):
         # For some akamai manifests we'll need to add a query to the fragment url
         akamai_pv = xpath_text(doc, _add_ns('pv-2.0'))
 
+        self.report_destination(filename)
+        http_dl = HttpQuietDownloader(
+            self.ydl,
+            {
+                'continuedl': True,
+                'quiet': True,
+                'noprogress': True,
+                'ratelimit': self.params.get('ratelimit', None),
+                'test': self.params.get('test', False),
+            }
+        )
         tmpfilename = self.temp_name(filename)
         (dest_stream, tmpfilename) = sanitize_open(tmpfilename, 'wb')
+
         write_flv_header(dest_stream)
         write_metadata_tag(dest_stream, metadata)
 
         # This dict stores the download progress, it's updated by the progress
         # hook
         state = {
+            'status': 'downloading',
             'downloaded_bytes': 0,
-            'frag_counter': 0,
+            'frag_index': 0,
+            'frag_count': total_frags,
+            'filename': filename,
+            'tmpfilename': tmpfilename,
         }
         start = time.time()
 
-        def frag_progress_hook(status):
-            frag_total_bytes = status.get('total_bytes', 0)
-            estimated_size = (state['downloaded_bytes'] +
-                              (total_frags - state['frag_counter']) * frag_total_bytes)
-            if status['status'] == 'finished':
+        def frag_progress_hook(s):
+            if s['status'] not in ('downloading', 'finished'):
+                return
+
+            frag_total_bytes = s.get('total_bytes', 0)
+            if s['status'] == 'finished':
                 state['downloaded_bytes'] += frag_total_bytes
-                state['frag_counter'] += 1
-                progress = self.calc_percent(state['frag_counter'], total_frags)
+                state['frag_index'] += 1
+
+            estimated_size = (
+                (state['downloaded_bytes'] + frag_total_bytes)
+                / (state['frag_index'] + 1) * total_frags)
+            time_now = time.time()
+            state['total_bytes_estimate'] = estimated_size
+            state['elapsed'] = time_now - start
+
+            if s['status'] == 'finished':
+                progress = self.calc_percent(state['frag_index'], total_frags)
                 byte_counter = state['downloaded_bytes']
             else:
-                frag_downloaded_bytes = status['downloaded_bytes']
+                frag_downloaded_bytes = s['downloaded_bytes']
                 byte_counter = state['downloaded_bytes'] + frag_downloaded_bytes
                 frag_progress = self.calc_percent(frag_downloaded_bytes,
                                                   frag_total_bytes)
-                progress = self.calc_percent(state['frag_counter'], total_frags)
+                progress = self.calc_percent(state['frag_index'], total_frags)
                 progress += frag_progress / float(total_frags)
 
-            eta = self.calc_eta(start, time.time(), estimated_size, byte_counter)
-            self.report_progress(progress, format_bytes(estimated_size),
-                                 status.get('speed'), eta)
+                state['eta'] = self.calc_eta(
+                    start, time_now, estimated_size, state['downloaded_bytes'] + frag_downloaded_bytes)
+                state['speed'] = s.get('speed')
+            self._hook_progress(state)
+
         http_dl.add_progress_hook(frag_progress_hook)
 
         frags_filenames = []
@@ -354,8 +371,8 @@ class F4mFD(FileDownloader):
             frags_filenames.append(frag_filename)
 
         dest_stream.close()
-        self.report_finish(format_bytes(state['downloaded_bytes']), time.time() - start)
 
+        elapsed = time.time() - start
         self.try_rename(tmpfilename, filename)
         for frag_file in frags_filenames:
             os.remove(frag_file)
@@ -366,6 +383,7 @@ class F4mFD(FileDownloader):
             'total_bytes': fsize,
             'filename': filename,
             'status': 'finished',
+            'elapsed': elapsed,
         })
 
         return True
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 49170cf9d..d37522aea 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -200,16 +200,16 @@ class HttpFD(FileDownloader):
             else:
                 percent = self.calc_percent(byte_counter, data_len)
                 eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
-            self.report_progress(percent, data_len_str, speed, eta)
 
             self._hook_progress({
+                'status': 'downloading',
                 'downloaded_bytes': byte_counter,
                 'total_bytes': data_len,
                 'tmpfilename': tmpfilename,
                 'filename': filename,
-                'status': 'downloading',
                 'eta': eta,
                 'speed': speed,
+                'elapsed': now - start,
             })
 
             if is_test and byte_counter == data_len:
@@ -221,7 +221,13 @@ class HttpFD(FileDownloader):
             return False
         if tmpfilename != '-':
             stream.close()
-        self.report_finish(data_len_str, (time.time() - start))
+
+        self._hook_progress({
+            'downloaded_bytes': byte_counter,
+            'total_bytes': data_len,
+            'tmpfilename': tmpfilename,
+            'status': 'error',
+        })
         if data_len is not None and byte_counter != data_len:
             raise ContentTooShortError(byte_counter, int(data_len))
         self.try_rename(tmpfilename, filename)
@@ -235,6 +241,7 @@ class HttpFD(FileDownloader):
             'total_bytes': byte_counter,
             'filename': filename,
             'status': 'finished',
+            'elapsed': time.time() - start,
         })
 
         return True
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index f7eeb6f43..79836fe99 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -51,23 +51,23 @@ class RtmpFD(FileDownloader):
                     if not resume_percent:
                         resume_percent = percent
                         resume_downloaded_data_len = downloaded_data_len
-                    eta = self.calc_eta(start, time.time(), 100 - resume_percent, percent - resume_percent)
-                    speed = self.calc_speed(start, time.time(), downloaded_data_len - resume_downloaded_data_len)
+                    time_now = time.time()
+                    eta = self.calc_eta(start, time_now, 100 - resume_percent, percent - resume_percent)
+                    speed = self.calc_speed(start, time_now, downloaded_data_len - resume_downloaded_data_len)
                     data_len = None
                     if percent > 0:
                         data_len = int(downloaded_data_len * 100 / percent)
-                    data_len_str = '~' + format_bytes(data_len)
-                    self.report_progress(percent, data_len_str, speed, eta)
-                    cursor_in_new_line = False
                     self._hook_progress({
+                        'status': 'downloading',
                         'downloaded_bytes': downloaded_data_len,
-                        'total_bytes': data_len,
+                        'total_bytes_estimate': data_len,
                         'tmpfilename': tmpfilename,
                         'filename': filename,
-                        'status': 'downloading',
                         'eta': eta,
+                        'elapsed': time_now - start,
                         'speed': speed,
                     })
+                    cursor_in_new_line = False
                 else:
                     # no percent for live streams
                     mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec', line)
@@ -75,15 +75,15 @@ class RtmpFD(FileDownloader):
                         downloaded_data_len = int(float(mobj.group(1)) * 1024)
                         time_now = time.time()
                         speed = self.calc_speed(start, time_now, downloaded_data_len)
-                        self.report_progress_live_stream(downloaded_data_len, speed, time_now - start)
-                        cursor_in_new_line = False
                         self._hook_progress({
                             'downloaded_bytes': downloaded_data_len,
                             'tmpfilename': tmpfilename,
                             'filename': filename,
                             'status': 'downloading',
+                            'elapsed': time_now - start,
                             'speed': speed,
                         })
+                        cursor_in_new_line = False
                     elif self.params.get('verbose', False):
                         if not cursor_in_new_line:
                             self.to_screen('')

From 16e7711e22648027739096560914a976b8eea786 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 21:42:31 +0100
Subject: [PATCH 1567/1937] [downloader/http] Remove gruesome import

---
 youtube_dl/downloader/http.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 49170cf9d..25032ad4e 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -1,10 +1,9 @@
 from __future__ import unicode_literals
 
-import os
-import time
-
-from socket import error as SocketError
 import errno
+import os
+import socket
+import time
 
 from .common import FileDownloader
 from ..compat import (
@@ -102,7 +101,7 @@ class HttpFD(FileDownloader):
                             resume_len = 0
                             open_mode = 'wb'
                             break
-            except SocketError as e:
+            except socket.error as e:
                 if e.errno != errno.ECONNRESET:
                     # Connection reset is no problem, just retry
                     raise

From a91a2c1a83fdd195e850d4ad9c298c01a145ebf0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 21:44:41 +0100
Subject: [PATCH 1568/1937] [downloader] Remove various unneeded assignments
 and imports

---
 youtube_dl/downloader/f4m.py  | 3 ---
 youtube_dl/downloader/http.py | 5 +----
 youtube_dl/downloader/rtmp.py | 1 -
 3 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 5a1f8e680..b40ebfa50 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -15,7 +15,6 @@ from ..compat import (
 from ..utils import (
     struct_pack,
     struct_unpack,
-    format_bytes,
     encodeFilename,
     sanitize_open,
     xpath_text,
@@ -334,10 +333,8 @@ class F4mFD(FileDownloader):
 
             if s['status'] == 'finished':
                 progress = self.calc_percent(state['frag_index'], total_frags)
-                byte_counter = state['downloaded_bytes']
             else:
                 frag_downloaded_bytes = s['downloaded_bytes']
-                byte_counter = state['downloaded_bytes'] + frag_downloaded_bytes
                 frag_progress = self.calc_percent(frag_downloaded_bytes,
                                                   frag_total_bytes)
                 progress = self.calc_percent(state['frag_index'], total_frags)
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index d37522aea..6dec528f2 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -15,7 +15,6 @@ from ..utils import (
     ContentTooShortError,
     encodeFilename,
     sanitize_open,
-    format_bytes,
 )
 
 
@@ -137,7 +136,6 @@ class HttpFD(FileDownloader):
                 self.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len))
                 return False
 
-        data_len_str = format_bytes(data_len)
         byte_counter = 0 + resume_len
         block_size = self.params.get('buffersize', 1024)
         start = time.time()
@@ -196,9 +194,8 @@ class HttpFD(FileDownloader):
             # Progress message
             speed = self.calc_speed(start, now, byte_counter - resume_len)
             if data_len is None:
-                eta = percent = None
+                eta = None
             else:
-                percent = self.calc_percent(byte_counter, data_len)
                 eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
 
             self._hook_progress({
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 79836fe99..0a52c34c7 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -11,7 +11,6 @@ from ..compat import compat_str
 from ..utils import (
     check_executable,
     encodeFilename,
-    format_bytes,
     get_exe_version,
 )
 

From fa156077737d5162795a221fe2995a276d31c6d3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 17 Feb 2015 21:46:20 +0100
Subject: [PATCH 1569/1937] PEP8 fixes

---
 youtube_dl/extractor/common.py           | 3 +--
 youtube_dl/extractor/twitch.py           | 1 +
 youtube_dl/extractor/videolecturesnet.py | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 31599bf52..08b8ad37c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -27,7 +27,6 @@ from ..utils import (
     compiled_regex_type,
     ExtractorError,
     float_or_none,
-    HEADRequest,
     int_or_none,
     RegexNotFoundError,
     sanitize_filename,
@@ -839,7 +838,7 @@ class InfoExtractor(object):
             note='Downloading m3u8 information',
             errnote='Failed to download m3u8 information')
         last_info = None
-        last_media= None
+        last_media = None
         kv_rex = re.compile(
             r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
         for line in m3u8_doc.splitlines():
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 8e296698e..4b0d8988d 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -348,6 +348,7 @@ class TwitchStreamIE(TwitchBaseIE):
             '%s/api/channel/hls/%s.m3u8?%s'
             % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query).encode('utf-8')),
             channel_id, 'mp4')
+
         # prefer the 'source' stream, the others are limited to 30 fps
         def _sort_source(f):
             if f.get('m3u8_media') is not None and f['m3u8_media'].get('NAME') == 'Source':
diff --git a/youtube_dl/extractor/videolecturesnet.py b/youtube_dl/extractor/videolecturesnet.py
index d46234856..d6a7eb203 100644
--- a/youtube_dl/extractor/videolecturesnet.py
+++ b/youtube_dl/extractor/videolecturesnet.py
@@ -52,7 +52,7 @@ class VideoLecturesNetIE(InfoExtractor):
         formats = []
         for v in switch.findall('./video'):
             proto = v.attrib.get('proto')
-            if not proto in ['http', 'rtmp']:
+            if proto not in ['http', 'rtmp']:
                 continue
             f = {
                 'width': int_or_none(v.attrib.get('width')),

From 85920dd01d98cf74ea7d3ab7834a3b50cd6f1fde Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 21:56:25 +0100
Subject: [PATCH 1570/1937] [bliptv] Convert to new subtitles system

---
 test/test_subtitles.py         |  2 --
 youtube_dl/extractor/bliptv.py | 34 ++++++++++++++++++----------------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 91cebce28..0ca510310 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -200,13 +200,11 @@ class TestBlipTVSubtitles(BaseTestSubtitles):
     IE = BlipTVIE
 
     def test_list_subtitles(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
 
     def test_allsubtitles(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py
index 436cc5155..8c7ba4b91 100644
--- a/youtube_dl/extractor/bliptv.py
+++ b/youtube_dl/extractor/bliptv.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .subtitles import SubtitlesInfoExtractor
 
 from ..compat import (
     compat_str,
@@ -18,7 +17,7 @@ from ..utils import (
 )
 
 
-class BlipTVIE(SubtitlesInfoExtractor):
+class BlipTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:\w+\.)?blip\.tv/(?:(?:.+-|rss/flash/)(?P<id>\d+)|((?:play/|api\.swf#)(?P<lookup_id>[\da-zA-Z+_]+)))'
 
     _TESTS = [
@@ -143,7 +142,7 @@ class BlipTVIE(SubtitlesInfoExtractor):
         categories = [category.text for category in item.findall('category')]
 
         formats = []
-        subtitles = {}
+        subtitles_urls = {}
 
         media_group = item.find(media('group'))
         for media_content in media_group.findall(media('content')):
@@ -161,7 +160,7 @@ class BlipTVIE(SubtitlesInfoExtractor):
                 }
                 lang = role.rpartition('-')[-1].strip().lower()
                 langcode = LANGS.get(lang, lang)
-                subtitles[langcode] = url
+                subtitles_urls[langcode] = url
             elif media_type.startswith('video/'):
                 formats.append({
                     'url': real_url,
@@ -175,11 +174,7 @@ class BlipTVIE(SubtitlesInfoExtractor):
                 })
         self._sort_formats(formats)
 
-        # subtitles
-        video_subtitles = self.extract_subtitles(video_id, subtitles)
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
+        subtitles = self.extract_subtitles(video_id, subtitles_urls)
 
         return {
             'id': video_id,
@@ -192,15 +187,22 @@ class BlipTVIE(SubtitlesInfoExtractor):
             'thumbnail': thumbnail,
             'categories': categories,
             'formats': formats,
-            'subtitles': video_subtitles,
+            'subtitles': subtitles,
         }
 
-    def _download_subtitle_url(self, sub_lang, url):
-        # For some weird reason, blip.tv serves a video instead of subtitles
-        # when we request with a common UA
-        req = compat_urllib_request.Request(url)
-        req.add_header('User-Agent', 'youtube-dl')
-        return self._download_webpage(req, None, note=False)
+    def _get_subtitles(self, video_id, subtitles_urls):
+        subtitles = {}
+        for lang, url in subtitles_urls.items():
+            # For some weird reason, blip.tv serves a video instead of subtitles
+            # when we request with a common UA
+            req = compat_urllib_request.Request(url)
+            req.add_header('User-Agent', 'youtube-dl')
+            subtitles[lang] = [{
+                # The extension is 'srt' but it's actually an 'ass' file
+                'ext': 'ass',
+                'data': self._download_webpage(req, None, note=False),
+            }]
+        return subtitles
 
 
 class BlipTVUserIE(InfoExtractor):

From 9868ea493626a3a81d30d084fd00d22982a0f86a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 22:16:29 +0100
Subject: [PATCH 1571/1937] [extractor/common] Simplify subtitles handling
 methods

Initially I was going to use a single method for handling both subtitles and automatic captions, that's why I used the 'list_subtitles' and the 'subtitles' variables.
---
 youtube_dl/extractor/common.py | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index fe7d8dbc9..7d8ce1808 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1000,21 +1000,19 @@ class InfoExtractor(object):
         return not any_restricted
 
     def extract_subtitles(self, *args, **kwargs):
-        subtitles = {}
-        list_subtitles = self._downloader.params.get('listsubtitles')
-        if self._downloader.params.get('writesubtitles', False) or list_subtitles:
-            subtitles.update(self._get_subtitles(*args, **kwargs))
-        return subtitles
+        if (self._downloader.params.get('writesubtitles', False) or
+                self._downloader.params.get('listsubtitles')):
+            return self._get_subtitles(*args, **kwargs)
+        return {}
 
     def _get_subtitles(self, *args, **kwargs):
         raise NotImplementedError("This method must be implemented by subclasses")
 
     def extract_automatic_captions(self, *args, **kwargs):
-        automatic_captions = {}
-        list_subtitles = self._downloader.params.get('listsubtitles')
-        if self._downloader.params.get('writeautomaticsub', False) or list_subtitles:
-            automatic_captions.update(self._get_automatic_captions(*args, **kwargs))
-        return automatic_captions
+        if (self._downloader.params.get('writeautomaticsub', False) or
+                self._downloader.params.get('listsubtitles')):
+            return self._get_automatic_captions(*args, **kwargs)
+        return {}
 
     def _get_automatic_captions(self, *args, **kwargs):
         raise NotImplementedError("This method must be implemented by subclasses")

From edab9dbf4d00a7f76fbfd2df9ef4b205c88e47a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 17 Feb 2015 22:59:19 +0100
Subject: [PATCH 1572/1937] [YoutubeDL] use the 'render_table' function for
 listing the subtitles

---
 youtube_dl/YoutubeDL.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index a47f8f5de..f8b8fb0c1 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1659,13 +1659,12 @@ class YoutubeDL(object):
         if not subtitles:
             self.to_screen('%s has no %s' % (video_id, name))
             return
-        header_line = 'Language    formats'
-        sub_lines = [
-            '%-12s%s' % (lang, ', '.join(f['ext'] for f in reversed(formats)))
-            for lang, formats in subtitles.items()]
         self.to_screen(
-            'Available %s for %s:\n%s\n%s' %
-            (name, video_id, header_line, '\n'.join(sub_lines)))
+            'Available %s for %s:' % (name, video_id))
+        self.to_screen(render_table(
+            ['Language', 'formats'],
+            [[lang, ', '.join(f['ext'] for f in reversed(formats))]
+                for lang, formats in subtitles.items()]))
 
     def urlopen(self, req):
         """ Start an HTTP download """

From 30965ac66a4e33b8e70d6c0cf5da1a746054d6cd Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:27:57 +0100
Subject: [PATCH 1573/1937] [vimeo] Prevent infinite loops if video password
 verification fails
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We're seeing this in the tests¹ right now, which do not terminate.

¹  https://travis-ci.org/jaimeMF/youtube-dl/jobs/51135858
---
 youtube_dl/extractor/vimeo.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 303e81447..1959d5e36 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -18,6 +18,7 @@ from ..utils import (
     InAdvancePagedList,
     int_or_none,
     RegexNotFoundError,
+    smuggle_url,
     std_headers,
     unsmuggle_url,
     urlencode_postdata,
@@ -267,8 +268,11 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
                 raise ExtractorError('The author has restricted the access to this video, try with the "--referer" option')
 
             if re.search(r'<form[^>]+?id="pw_form"', webpage) is not None:
+                if data and '_video_password_verified' in data:
+                    raise ExtractorError('video password verification failed!')
                 self._verify_video_password(url, video_id, webpage)
-                return self._real_extract(url)
+                return self._real_extract(
+                    smuggle_url(url, {'_video_password_verified': 'verified'}))
             else:
                 raise ExtractorError('Unable to extract info section',
                                      cause=e)

From a3fa5da4967bf2699bfd982cf56a8f627a13bda4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:33:31 +0100
Subject: [PATCH 1574/1937] [vimeo] Amend playlist tests

---
 youtube_dl/extractor/vimeo.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 1959d5e36..e7284049d 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -405,6 +405,7 @@ class VimeoChannelIE(InfoExtractor):
     _TESTS = [{
         'url': 'http://vimeo.com/channels/tributes',
         'info_dict': {
+            'id': 'tributes',
             'title': 'Vimeo Tributes',
         },
         'playlist_mincount': 25,
@@ -483,6 +484,7 @@ class VimeoUserIE(VimeoChannelIE):
         'url': 'http://vimeo.com/nkistudio/videos',
         'info_dict': {
             'title': 'Nki',
+            'id': 'nkistudio',
         },
         'playlist_mincount': 66,
     }]
@@ -500,6 +502,7 @@ class VimeoAlbumIE(VimeoChannelIE):
     _TESTS = [{
         'url': 'http://vimeo.com/album/2632481',
         'info_dict': {
+            'id': '2632481',
             'title': 'Staff Favorites: November 2013',
         },
         'playlist_mincount': 13,
@@ -530,6 +533,7 @@ class VimeoGroupsIE(VimeoAlbumIE):
     _TESTS = [{
         'url': 'http://vimeo.com/groups/rolexawards',
         'info_dict': {
+            'id': 'rolexawards',
             'title': 'Rolex Awards for Enterprise',
         },
         'playlist_mincount': 73,
@@ -612,6 +616,7 @@ class VimeoLikesIE(InfoExtractor):
         'url': 'https://vimeo.com/user755559/likes/',
         'playlist_mincount': 293,
         "info_dict": {
+            'id': 'user755559_likes',
             "description": "See all the videos urza likes",
             "title": 'Videos urza likes',
         },

From 15ec669374727c9bdc3aad0df141fdf305cb80be Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:33:41 +0100
Subject: [PATCH 1575/1937] [vk] Amend playlist test

---
 youtube_dl/extractor/vk.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 81e02a624..7dea8c59d 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -217,6 +217,9 @@ class VKUserVideosIE(InfoExtractor):
     _TEMPLATE_URL = 'https://vk.com/videos'
     _TEST = {
         'url': 'http://vk.com/videos205387401',
+        'info_dict': {
+            'id': '205387401',
+        },
         'playlist_mincount': 4,
     }
 

From 1def5f359ef6e197e4cc5d610ed86e109e240080 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:34:45 +0100
Subject: [PATCH 1576/1937] [livestream] Correct playlist ID and add a test for
 it

---
 youtube_dl/extractor/livestream.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 5247c6f58..3642089f7 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -37,6 +37,7 @@ class LivestreamIE(InfoExtractor):
         'url': 'http://new.livestream.com/tedx/cityenglish',
         'info_dict': {
             'title': 'TEDCity2.0 (English)',
+            'id': '2245590',
         },
         'playlist_mincount': 4,
     }, {
@@ -148,7 +149,8 @@ class LivestreamIE(InfoExtractor):
                   if is_relevant(video_data, video_id)]
         if video_id is None:
             # This is an event page:
-            return self.playlist_result(videos, info['id'], info['full_name'])
+            return self.playlist_result(
+                videos, '%s' % info['id'], info['full_name'])
         else:
             if not videos:
                 raise ExtractorError('Cannot find video %s' % video_id)

From 6f53c63df66d95ea684f84ef2f7f0e697acc1d20 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:37:48 +0100
Subject: [PATCH 1577/1937] [test/helper] Only output a newline for forgotten
 keys if keys are really missing

---
 test/helper.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index 651ef99b9..fef2db284 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -163,7 +163,9 @@ def expect_info_dict(self, got_dict, expected_dict):
             info_dict_str += ''.join(
                 '    %s: %s,\n' % (_repr(k), _repr(v))
                 for k, v in test_info_dict.items() if k not in missing_keys)
-            info_dict_str += '\n'
+
+            if info_dict_str:
+                info_dict_str += '\n'
         info_dict_str += ''.join(
             '    %s: %s,\n' % (_repr(k), _repr(test_info_dict[k]))
             for k in missing_keys)

From f21e915fb9ad53a62df65e5c75c984243aaf0aea Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:38:42 +0100
Subject: [PATCH 1578/1937] [test/helper] Render info_dict with a final comma

---
 test/helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index fef2db284..570c91440 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -170,7 +170,7 @@ def expect_info_dict(self, got_dict, expected_dict):
             '    %s: %s,\n' % (_repr(k), _repr(test_info_dict[k]))
             for k in missing_keys)
         write_string(
-            '\n\'info_dict\': {\n' + info_dict_str + '}\n', out=sys.stderr)
+            '\n\'info_dict\': {\n' + info_dict_str + '},\n', out=sys.stderr)
         self.assertFalse(
             missing_keys,
             'Missing keys in test definition: %s' % (

From 4d278fde645a5ab70b046293a26c12bba146acb0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:38:55 +0100
Subject: [PATCH 1579/1937] [ign] Amend playlist test

---
 youtube_dl/extractor/ign.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index 3db668cd0..3aade9e74 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -34,6 +34,9 @@ class IGNIE(InfoExtractor):
         },
         {
             'url': 'http://me.ign.com/en/feature/15775/100-little-things-in-gta-5-that-will-blow-your-mind',
+            'info_dict': {
+                'id': '100-little-things-in-gta-5-that-will-blow-your-mind',
+            },
             'playlist': [
                 {
                     'info_dict': {

From 6e99868e4c73468734f3552d430ba6375c4e0880 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:41:45 +0100
Subject: [PATCH 1580/1937] [buzzfeed] Fix playlist test case

---
 youtube_dl/extractor/buzzfeed.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/buzzfeed.py b/youtube_dl/extractor/buzzfeed.py
index a5d2af174..df503ecc0 100644
--- a/youtube_dl/extractor/buzzfeed.py
+++ b/youtube_dl/extractor/buzzfeed.py
@@ -33,6 +33,7 @@ class BuzzFeedIE(InfoExtractor):
             'skip_download': True,  # Got enough YouTube download tests
         },
         'info_dict': {
+            'id': 'look-at-this-cute-dog-omg',
             'description': 're:Munchkin the Teddy Bear is back ?!',
             'title': 'You Need To Stop What You\'re Doing And Watching This Dog Walk On A Treadmill',
         },
@@ -42,8 +43,8 @@ class BuzzFeedIE(InfoExtractor):
                 'ext': 'mp4',
                 'upload_date': '20141124',
                 'uploader_id': 'CindysMunchkin',
-                'description': 're:© 2014 Munchkin the Shih Tzu',
-                'uploader': 'Munchkin the Shih Tzu',
+                'description': 're:© 2014 Munchkin the',
+                'uploader': 're:^Munchkin the',
                 'title': 're:Munchkin the Teddy Bear gets her exercise',
             },
         }]

From 72c1f8de06f11aea670039a17ca63146a1b85af2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:48:52 +0100
Subject: [PATCH 1581/1937] [bandcamp:album] Fix extractor results and
 associated test

---
 youtube_dl/extractor/bandcamp.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 490cc961a..869294967 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -109,7 +109,7 @@ class BandcampIE(InfoExtractor):
 
 class BandcampAlbumIE(InfoExtractor):
     IE_NAME = 'Bandcamp:album'
-    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<title>[^?#]+)|/?(?:$|[?#]))'
+    _VALID_URL = r'https?://(?:(?P<subdomain>[^.]+)\.)?bandcamp\.com(?:/album/(?P<album_id>[^?#]+)|/?(?:$|[?#]))'
 
     _TESTS = [{
         'url': 'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1',
@@ -133,31 +133,37 @@ class BandcampAlbumIE(InfoExtractor):
         ],
         'info_dict': {
             'title': 'Jazz Format Mixtape vol.1',
+            'id': 'jazz-format-mixtape-vol-1',
+            'uploader_id': 'blazo',
         },
         'params': {
             'playlistend': 2
         },
-        'skip': 'Bandcamp imposes download limits. See test_playlists:test_bandcamp_album for the playlist test'
+        'skip': 'Bandcamp imposes download limits.'
     }, {
         'url': 'http://nightbringer.bandcamp.com/album/hierophany-of-the-open-grave',
         'info_dict': {
             'title': 'Hierophany of the Open Grave',
+            'uploader_id': 'nightbringer',
+            'id': 'hierophany-of-the-open-grave',
         },
         'playlist_mincount': 9,
     }, {
         'url': 'http://dotscale.bandcamp.com',
         'info_dict': {
             'title': 'Loom',
+            'id': 'dotscale',
+            'uploader_id': 'dotscale',
         },
         'playlist_mincount': 7,
     }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        playlist_id = mobj.group('subdomain')
-        title = mobj.group('title')
-        display_id = title or playlist_id
-        webpage = self._download_webpage(url, display_id)
+        uploader_id = mobj.group('subdomain')
+        album_id = mobj.group('album_id')
+        playlist_id = album_id or uploader_id
+        webpage = self._download_webpage(url, playlist_id)
         tracks_paths = re.findall(r'<a href="(.*?)" itemprop="url">', webpage)
         if not tracks_paths:
             raise ExtractorError('The page doesn\'t contain any tracks')
@@ -168,8 +174,8 @@ class BandcampAlbumIE(InfoExtractor):
             r'album_title\s*:\s*"(.*?)"', webpage, 'title', fatal=False)
         return {
             '_type': 'playlist',
+            'uploader_id': uploader_id,
             'id': playlist_id,
-            'display_id': display_id,
             'title': title,
             'entries': entries,
         }

From 11e611a7fa809a447939e8a98c8549f2827c862f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:49:10 +0100
Subject: [PATCH 1582/1937] Extend various playlist tests

---
 youtube_dl/extractor/adultswim.py     | 2 ++
 youtube_dl/extractor/appletrailers.py | 3 +++
 youtube_dl/extractor/dailymotion.py   | 1 +
 youtube_dl/extractor/generic.py       | 1 +
 4 files changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 502a9c25a..34b8b0115 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -38,6 +38,7 @@ class AdultSwimIE(InfoExtractor):
             },
         ],
         'info_dict': {
+            'id': 'rQxZvXQ4ROaSOqq-or2Mow',
             'title': 'Rick and Morty - Pilot',
             'description': "Rick moves in with his daughter's family and establishes himself as a bad influence on his grandson, Morty. "
         }
@@ -55,6 +56,7 @@ class AdultSwimIE(InfoExtractor):
             }
         ],
         'info_dict': {
+            'id': '-t8CamQlQ2aYZ49ItZCFog',
             'title': 'American Dad - Putting Francine Out of Business',
             'description': 'Stan hatches a plan to get Francine out of the real estate business.Watch more American Dad on [adult swim].'
         },
diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 287f71e07..43e82847f 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -14,6 +14,9 @@ class AppleTrailersIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/trailers/(?P<company>[^/]+)/(?P<movie>[^/]+)'
     _TEST = {
         "url": "http://trailers.apple.com/trailers/wb/manofsteel/",
+        'info_dict': {
+            'id': 'manofsteel',
+        },
         "playlist": [
             {
                 "md5": "d97a8e575432dbcb81b7c3acb741f8a8",
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index cf5841a7c..b2dbf4a92 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -194,6 +194,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
         'url': 'http://www.dailymotion.com/playlist/xv4bw_nqtv_sport/1#video=xl8v3q',
         'info_dict': {
             'title': 'SPORT',
+            'id': 'xv4bw_nqtv_sport',
         },
         'playlist_mincount': 20,
     }]
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c4ba25a96..36a1f65bf 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -473,6 +473,7 @@ class GenericIE(InfoExtractor):
         {
             'url': 'http://discourse.ubuntu.com/t/unity-8-desktop-mode-windows-on-mir/1986',
             'info_dict': {
+                'id': '1986',
                 'title': 'Unity 8 desktop-mode windows on Mir! - Ubuntu Discourse',
             },
             'playlist_mincount': 2,

From 9cad27008b88b5ba3dfb1d747d7476b05582f385 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 00:49:34 +0100
Subject: [PATCH 1583/1937] release 2015.02.18

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 34a8d1184..635b6bdb8 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.17.2'
+__version__ = '2015.02.18'

From 6d1a55a521b524dd0c562e3855f81434afb6f807 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 10:39:14 +0100
Subject: [PATCH 1584/1937] [youtube] Show entire player URL when -v is given

---
 youtube_dl/extractor/youtube.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 35ef4c303..2c02da2b7 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -541,16 +541,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         if cache_spec is not None:
             return lambda s: ''.join(s[i] for i in cache_spec)
 
+        download_note = (
+            'Downloading player %s' % player_url
+            if self._downloader.params.get('verbose') else
+            'Downloading %s player %s' % (player_type, player_id)
+        )
         if player_type == 'js':
             code = self._download_webpage(
                 player_url, video_id,
-                note='Downloading %s player %s' % (player_type, player_id),
+                note=download_note,
                 errnote='Download of %s failed' % player_url)
             res = self._parse_sig_js(code)
         elif player_type == 'swf':
             urlh = self._request_webpage(
                 player_url, video_id,
-                note='Downloading %s player %s' % (player_type, player_id),
+                note=download_note,
                 errnote='Download of %s failed' % player_url)
             code = urlh.read()
             res = self._parse_sig_swf(code)

From 785521bf4fbd99b2916bdab5d847d84424196c1d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 10:42:23 +0100
Subject: [PATCH 1585/1937] [youtube] Remove useless if

---
 youtube_dl/extractor/youtube.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2c02da2b7..3d3d43491 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -562,10 +562,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         else:
             assert False, 'Invalid player type %r' % player_type
 
-        if cache_spec is None:
-            test_string = ''.join(map(compat_chr, range(len(example_sig))))
-            cache_res = res(test_string)
-            cache_spec = [ord(c) for c in cache_res]
+        test_string = ''.join(map(compat_chr, range(len(example_sig))))
+        cache_res = res(test_string)
+        cache_spec = [ord(c) for c in cache_res]
 
         self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
         return res

From 3eff81fbf72939abf885afcca2f1155cc976999b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 10:47:40 +0100
Subject: [PATCH 1586/1937] [jsinterp] Disable comment support

We need a proper lexer to be able to understand YouTube's code, which contains /* inside of strings.
For now it's sufficient to just disable comment support altogether.

Fixes #4976, fixes #4979, fixes #4980, fixes #4981, fixes #4982.
Closes #4977.
---
 test/test_jsinterp.py          | 11 +++++++++++
 test/test_youtube_signature.py |  6 ++++++
 youtube_dl/jsinterp.py         |  5 +----
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index b91b8c492..fc73e5dc2 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -70,6 +70,8 @@ class TestJSInterpreter(unittest.TestCase):
         self.assertEqual(jsi.call_function('f'), -11)
 
     def test_comments(self):
+        'Skipping: Not yet fully implemented'
+        return
         jsi = JSInterpreter('''
         function x() {
             var x = /* 1 + */ 2;
@@ -80,6 +82,15 @@ class TestJSInterpreter(unittest.TestCase):
         ''')
         self.assertEqual(jsi.call_function('x'), 52)
 
+        jsi = JSInterpreter('''
+        function f() {
+            var x = "/*";
+            var y = 1 /* comment */ + 2;
+            return y;
+        }
+        ''')
+        self.assertEqual(jsi.call_function('f'), 3)
+
     def test_precedence(self):
         jsi = JSInterpreter('''
         function x() {
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 09696e19a..060864434 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -64,6 +64,12 @@ _TESTS = [
         'js',
         '4646B5181C6C3020DF1D9C7FCFEA.AD80ABF70C39BD369CCCAE780AFBB98FA6B6CB42766249D9488C288',
         '82C8849D94266724DC6B6AF89BBFA087EACCD963.B93C07FBA084ACAEFCF7C9D1FD0203C6C1815B6B'
+    ),
+    (
+        'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js',
+        'js',
+        '312AA52209E3623129A412D56A40F11CB0AF14AE.3EE09501CB14E3BCDC3B2AE808BF3F1D14E7FBF12',
+        '112AA5220913623229A412D56A40F11CB0AF14AE.3EE0950FCB14EEBCDC3B2AE808BF331D14E7FBF3',
     )
 ]
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 453e2732c..0e0c7d90d 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -30,13 +30,10 @@ class JSInterpreter(object):
     def __init__(self, code, objects=None):
         if objects is None:
             objects = {}
-        self.code = self._remove_comments(code)
+        self.code = code
         self._functions = {}
         self._objects = objects
 
-    def _remove_comments(self, code):
-        return re.sub(r'(?s)/\*.*?\*/', '', code)
-
     def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if allow_recursion < 0:
             raise ExtractorError('Recursion limit reached')

From b8b928d5cb5d21390690cc21c29a3d679605015f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 10:54:45 +0100
Subject: [PATCH 1587/1937] [README] Add an FAQ entry for the player change in
 anticipation of many more bug reports

---
 README.md | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index acd0ef41b..a2c148311 100644
--- a/README.md
+++ b/README.md
@@ -515,11 +515,15 @@ If you want to play the video on a machine that is not running youtube-dl, you c
 
 ### ERROR: no fmt_url_map or conn information found in video info
 
-youtube has switched to a new video info format in July 2011 which is not supported by old versions of youtube-dl. You can update youtube-dl with `sudo youtube-dl --update`.
+YouTube has switched to a new video info format in July 2011 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
 
 ### ERROR: unable to download video ###
 
-youtube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. You can update youtube-dl with `sudo youtube-dl --update`.
+YouTube requires an additional signature since September 2012 which is not supported by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
+
+### ExtractorError: Could not find JS function u'OF'
+
+In February 2015, the new YouTube player contained a character sequence in a string that was misinterpreted by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
 
 ### SyntaxError: Non-ASCII character ###
 

From 81975f4693c15c1f97ad92e7fafb1463ed2aa4c1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 10:54:56 +0100
Subject: [PATCH 1588/1937] release 2015.02.18.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 635b6bdb8..1df05a6df 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.18'
+__version__ = '2015.02.18.1'

From 37dd5d4629ae955940265f245316c43cd0373a3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 16:54:36 +0100
Subject: [PATCH 1589/1937] [mit] Don't set the subtitles field

YouTube already provides them in more formats
---
 youtube_dl/extractor/mit.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py
index 3c61a850f..d7ab6a9ae 100644
--- a/youtube_dl/extractor/mit.py
+++ b/youtube_dl/extractor/mit.py
@@ -5,9 +5,6 @@ import json
 
 from .common import InfoExtractor
 from .youtube import YoutubeIE
-from ..compat import (
-    compat_urlparse,
-)
 from ..utils import (
     clean_html,
     ExtractorError,
@@ -108,7 +105,6 @@ class OCWMITIE(InfoExtractor):
                 'upload_date': '20121109',
                 'uploader_id': 'MIT',
                 'uploader': 'MIT OpenCourseWare',
-                # 'subtitles': 'http://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-041-probabilistic-systems-analysis-and-applied-probability-fall-2010/video-lectures/lecture-7-multiple-variables-expectations-independence/MIT6_041F11_lec07_300k.mp4.srt'
             }
         },
         {
@@ -121,7 +117,6 @@ class OCWMITIE(InfoExtractor):
                 'uploader_id': 'MIT',
                 'uploader': 'MIT OpenCourseWare',
                 'description': 'This section contains lecture video excerpts, lecture notes, an interactive mathlet with supporting documents, and problem solving videos.',
-                # 'subtitles': 'http://ocw.mit.edu//courses/mathematics/18-01sc-single-variable-calculus-fall-2010/ocw-18.01-f07-lec01_300k.SRT'
             }
         }
     ]
@@ -140,7 +135,6 @@ class OCWMITIE(InfoExtractor):
             metadata = re.sub(r'[\'"]', '', embed_chapter_media.group(1))
             metadata = re.split(r', ?', metadata)
             yt = metadata[1]
-            subs = compat_urlparse.urljoin(self._BASE_URL, metadata[7])
         else:
             # search for call to ocw_embed_chapter_media(container_id, media_url, provider, page_url, image_url, captions_file)
             embed_media = re.search(r'ocw_embed_media\((.+?)\)', webpage)
@@ -148,7 +142,6 @@ class OCWMITIE(InfoExtractor):
                 metadata = re.sub(r'[\'"]', '', embed_media.group(1))
                 metadata = re.split(r', ?', metadata)
                 yt = metadata[1]
-                subs = compat_urlparse.urljoin(self._BASE_URL, metadata[5])
             else:
                 raise ExtractorError('Unable to find embedded YouTube video.')
         video_id = YoutubeIE.extract_id(yt)
@@ -159,7 +152,5 @@ class OCWMITIE(InfoExtractor):
             'title': title,
             'description': description,
             'url': yt,
-            'url_transparent'
-            'subtitles': subs,
             'ie_key': 'Youtube',
         }

From 18c1c424057dd06f85f4420b14089e032fcb0000 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 17:20:22 +0100
Subject: [PATCH 1590/1937] [drtv] Convert to new subtitles system

---
 youtube_dl/extractor/drtv.py | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/drtv.py b/youtube_dl/extractor/drtv.py
index d5df18d7c..8257e35a4 100644
--- a/youtube_dl/extractor/drtv.py
+++ b/youtube_dl/extractor/drtv.py
@@ -1,11 +1,10 @@
 from __future__ import unicode_literals
 
-from .subtitles import SubtitlesInfoExtractor
-from .common import ExtractorError
+from .common import InfoExtractor, ExtractorError
 from ..utils import parse_iso8601
 
 
-class DRTVIE(SubtitlesInfoExtractor):
+class DRTVIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?dr\.dk/tv/se/(?:[^/]+/)*(?P<id>[\da-z-]+)(?:[/#?]|$)'
 
     _TEST = {
@@ -76,7 +75,7 @@ class DRTVIE(SubtitlesInfoExtractor):
                     }
                     for subs in subtitles_list:
                         lang = subs['Language']
-                        subtitles[LANGS.get(lang, lang)] = subs['Uri']
+                        subtitles[LANGS.get(lang, lang)] = [{'url': subs['Uri'], 'ext': 'vtt'}]
 
         if not formats and restricted_to_denmark:
             raise ExtractorError(
@@ -84,10 +83,6 @@ class DRTVIE(SubtitlesInfoExtractor):
 
         self._sort_formats(formats)
 
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
-
         return {
             'id': video_id,
             'title': title,
@@ -96,5 +91,5 @@ class DRTVIE(SubtitlesInfoExtractor):
             'timestamp': timestamp,
             'duration': duration,
             'formats': formats,
-            'subtitles': self.extract_subtitles(video_id, subtitles),
+            'subtitles': subtitles,
         }

From 311c39383827e42649a287633a67ef021476d23a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 17:46:33 +0100
Subject: [PATCH 1591/1937] [lynda] Convert to new subtitles system

---
 test/test_subtitles.py        | 13 ++++++++
 youtube_dl/extractor/lynda.py | 60 ++++++++++++++---------------------
 2 files changed, 37 insertions(+), 36 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 0ca510310..ee170879f 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -18,6 +18,7 @@ from youtube_dl.extractor import (
     VimeoIE,
     WallaIE,
     CeskaTelevizeIE,
+    LyndaIE,
 )
 
 
@@ -304,5 +305,17 @@ class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
         self.assertEqual(len(subtitles), 0)
 
 
+class TestLyndaSubtitles(BaseTestSubtitles):
+    url = 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html'
+    IE = LyndaIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(md5(subtitles['en']), '09bbe67222259bed60deaa26997d73a7')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 762cefa34..109055e72 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 import re
 import json
 
-from .subtitles import SubtitlesInfoExtractor
 from .common import InfoExtractor
 from ..compat import (
     compat_str,
@@ -16,7 +15,7 @@ from ..utils import (
 )
 
 
-class LyndaIE(SubtitlesInfoExtractor):
+class LyndaIE(InfoExtractor):
     IE_NAME = 'lynda'
     IE_DESC = 'lynda.com videos'
     _VALID_URL = r'https?://www\.lynda\.com/[^/]+/[^/]+/\d+/(\d+)-\d\.html'
@@ -88,11 +87,7 @@ class LyndaIE(SubtitlesInfoExtractor):
         self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, page)
-            return
-
-        subtitles = self._fix_subtitles(self.extract_subtitles(video_id, page))
+        subtitles = self.extract_subtitles(video_id, page)
 
         return {
             'id': video_id,
@@ -144,38 +139,31 @@ class LyndaIE(SubtitlesInfoExtractor):
         if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
             raise ExtractorError('Unable to log in')
 
-    def _fix_subtitles(self, subtitles):
-        if subtitles is None:
-            return subtitles  # subtitles not requested
-
-        fixed_subtitles = {}
-        for k, v in subtitles.items():
-            subs = json.loads(v)
-            if len(subs) == 0:
+    def _fix_subtitles(self, subs):
+        srt = ''
+        for pos in range(0, len(subs) - 1):
+            seq_current = subs[pos]
+            m_current = re.match(self._TIMECODE_REGEX, seq_current['Timecode'])
+            if m_current is None:
                 continue
-            srt = ''
-            for pos in range(0, len(subs) - 1):
-                seq_current = subs[pos]
-                m_current = re.match(self._TIMECODE_REGEX, seq_current['Timecode'])
-                if m_current is None:
-                    continue
-                seq_next = subs[pos + 1]
-                m_next = re.match(self._TIMECODE_REGEX, seq_next['Timecode'])
-                if m_next is None:
-                    continue
-                appear_time = m_current.group('timecode')
-                disappear_time = m_next.group('timecode')
-                text = seq_current['Caption']
-                srt += '%s\r\n%s --> %s\r\n%s' % (str(pos), appear_time, disappear_time, text)
-            if srt:
-                fixed_subtitles[k] = srt
-        return fixed_subtitles
+            seq_next = subs[pos + 1]
+            m_next = re.match(self._TIMECODE_REGEX, seq_next['Timecode'])
+            if m_next is None:
+                continue
+            appear_time = m_current.group('timecode')
+            disappear_time = m_next.group('timecode')
+            text = seq_current['Caption']
+            srt += '%s\r\n%s --> %s\r\n%s' % (str(pos), appear_time, disappear_time, text)
+        if srt:
+            return srt
 
-    def _get_available_subtitles(self, video_id, webpage):
+    def _get_subtitles(self, video_id, webpage):
         url = 'http://www.lynda.com/ajax/player?videoId=%s&type=transcript' % video_id
-        sub = self._download_webpage(url, None, False)
-        sub_json = json.loads(sub)
-        return {'en': url} if len(sub_json) > 0 else {}
+        subs = self._download_json(url, None, False)
+        if subs:
+            return {'en': [{'ext': 'srt', 'data': self._fix_subtitles(subs)}]}
+        else:
+            return {}
 
 
 class LyndaCourseIE(InfoExtractor):

From b9b42f2ea0c564f3e75a8f052bfe0dfe21cf320f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 17:57:10 +0100
Subject: [PATCH 1592/1937] [npo] Convert to new subtitles system

---
 test/test_subtitles.py      | 13 +++++++++++++
 youtube_dl/extractor/npo.py | 14 +++++---------
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index ee170879f..b2195cac4 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -19,6 +19,7 @@ from youtube_dl.extractor import (
     WallaIE,
     CeskaTelevizeIE,
     LyndaIE,
+    NPOIE,
 )
 
 
@@ -317,5 +318,17 @@ class TestLyndaSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['en']), '09bbe67222259bed60deaa26997d73a7')
 
 
+class TestNPOSubtitles(BaseTestSubtitles):
+    url = 'http://www.npo.nl/nos-journaal/28-08-2014/POW_00722860'
+    IE = NPOIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['nl']))
+        self.assertEqual(md5(subtitles['nl']), 'fc6435027572b63fb4ab143abd5ad3f4')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index c075618e8..9c01eb0af 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-from .subtitles import SubtitlesInfoExtractor
 from .common import InfoExtractor
 from ..utils import (
     fix_xml_ampersands,
@@ -12,7 +11,7 @@ from ..utils import (
 )
 
 
-class NPOBaseIE(SubtitlesInfoExtractor):
+class NPOBaseIE(InfoExtractor):
     def _get_token(self, video_id):
         token_page = self._download_webpage(
             'http://ida.omroep.nl/npoplayer/i.js',
@@ -164,13 +163,10 @@ class NPOIE(NPOBaseIE):
 
         subtitles = {}
         if metadata.get('tt888') == 'ja':
-            subtitles['nl'] = 'http://e.omroep.nl/tt888/%s' % video_id
-
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
-
-        subtitles = self.extract_subtitles(video_id, subtitles)
+            subtitles['nl'] = [{
+                'ext': 'vtt',
+                'url': 'http://e.omroep.nl/tt888/%s' % video_id,
+            }]
 
         return {
             'id': video_id,

From 0af25f784bc5bff7cbce2d4af725b4bf2d2262c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 18:27:45 +0100
Subject: [PATCH 1593/1937] [mtv] Convert to new subtitles system

---
 test/test_subtitles.py      | 16 ++++++++++++++++
 youtube_dl/extractor/mtv.py | 26 +++++++-------------------
 2 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index b2195cac4..c018d9b49 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -20,6 +20,7 @@ from youtube_dl.extractor import (
     CeskaTelevizeIE,
     LyndaIE,
     NPOIE,
+    ComedyCentralIE,
 )
 
 
@@ -330,5 +331,20 @@ class TestNPOSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['nl']), 'fc6435027572b63fb4ab143abd5ad3f4')
 
 
+class TestMTVSubtitles(BaseTestSubtitles):
+    url = 'http://www.cc.com/video-clips/kllhuv/stand-up-greg-fitzsimmons--uncensored---too-good-of-a-mother'
+    IE = ComedyCentralIE
+
+    def getInfoDict(self):
+        return super(TestMTVSubtitles, self).getInfoDict()['entries'][0]
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(md5(subtitles['en']), 'b9f6ca22a6acf597ec76f61749765e65')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index bc7f49ebb..c11de1cb6 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 import re
 
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
@@ -23,7 +23,7 @@ def _media_xml_tag(tag):
     return '{http://search.yahoo.com/mrss/}%s' % tag
 
 
-class MTVServicesInfoExtractor(SubtitlesInfoExtractor):
+class MTVServicesInfoExtractor(InfoExtractor):
     _MOBILE_TEMPLATE = None
 
     @staticmethod
@@ -95,25 +95,15 @@ class MTVServicesInfoExtractor(SubtitlesInfoExtractor):
 
     def _extract_subtitles(self, mdoc, mtvn_id):
         subtitles = {}
-        FORMATS = {
-            'scc': 'cea-608',
-            'eia-608': 'cea-608',
-            'xml': 'ttml',
-        }
-        subtitles_format = FORMATS.get(
-            self._downloader.params.get('subtitlesformat'), 'ttml')
         for transcript in mdoc.findall('.//transcript'):
             if transcript.get('kind') != 'captions':
                 continue
             lang = transcript.get('srclang')
-            for typographic in transcript.findall('./typographic'):
-                captions_format = typographic.get('format')
-                if captions_format == subtitles_format:
-                    subtitles[lang] = compat_str(typographic.get('src'))
-                    break
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(mtvn_id, subtitles)
-        return self.extract_subtitles(mtvn_id, subtitles)
+            subtitles[lang] = [{
+                'url': compat_str(typographic.get('src')),
+                'ext': typographic.get('format')
+            } for typographic in transcript.findall('./typographic')]
+        return subtitles
 
     def _get_video_info(self, itemdoc):
         uri = itemdoc.find('guid').text
@@ -196,8 +186,6 @@ class MTVServicesInfoExtractor(SubtitlesInfoExtractor):
                 webpage, 'mgid')
 
         videos_info = self._get_videos_info(mgid)
-        if self._downloader.params.get('listsubtitles', False):
-            return
         return videos_info
 
 

From 01561da142485a581e67ce98ef009ebe0ed7b4c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 18:57:01 +0100
Subject: [PATCH 1594/1937] [nrk] Convert to new subtitles system

---
 test/test_subtitles.py      | 13 +++++++++++++
 youtube_dl/extractor/nrk.py | 15 +++++++--------
 2 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index c018d9b49..1e2324232 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -21,6 +21,7 @@ from youtube_dl.extractor import (
     LyndaIE,
     NPOIE,
     ComedyCentralIE,
+    NRKTVIE,
 )
 
 
@@ -346,5 +347,17 @@ class TestMTVSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['en']), 'b9f6ca22a6acf597ec76f61749765e65')
 
 
+class TestNRKSubtitles(BaseTestSubtitles):
+    url = 'http://tv.nrk.no/serie/ikke-gjoer-dette-hjemme/DMPV73000411/sesong-2/episode-1'
+    IE = NRKTVIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['no']))
+        self.assertEqual(md5(subtitles['no']), '1d221e6458c95c5494dcd38e6a1f129a')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index f6de26022..46f493cfc 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -10,7 +10,6 @@ from ..utils import (
     parse_duration,
     unified_strdate,
 )
-from .subtitles import SubtitlesInfoExtractor
 
 
 class NRKIE(InfoExtractor):
@@ -73,7 +72,7 @@ class NRKIE(InfoExtractor):
         }
 
 
-class NRKTVIE(SubtitlesInfoExtractor):
+class NRKTVIE(InfoExtractor):
     _VALID_URL = r'(?P<baseurl>http://tv\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
 
     _TESTS = [
@@ -156,7 +155,7 @@ class NRKTVIE(SubtitlesInfoExtractor):
         if self._downloader.params.get('verbose', False):
             self.to_screen('[debug] %s' % txt)
 
-    def _extract_captions(self, subtitlesurl, video_id, baseurl):
+    def _get_subtitles(self, subtitlesurl, video_id, baseurl):
         url = "%s%s" % (baseurl, subtitlesurl)
         self._debug_print('%s: Subtitle url: %s' % (video_id, url))
         captions = self._download_xml(url, video_id, 'Downloading subtitles')
@@ -170,7 +169,10 @@ class NRKTVIE(SubtitlesInfoExtractor):
             endtime = self._seconds2str(begin + duration)
             text = '\n'.join(p.itertext())
             srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (str(pos), starttime, endtime, text)
-        return {lang: srt}
+        return {lang: [
+            {'ext': 'ttml', 'url': url},
+            {'ext': 'srt', 'data': srt},
+        ]}
 
     def _extract_f4m(self, manifest_url, video_id):
         return self._extract_f4m_formats(manifest_url + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124', video_id)
@@ -243,10 +245,7 @@ class NRKTVIE(SubtitlesInfoExtractor):
             webpage, 'subtitle URL', default=None)
         subtitles = None
         if subtitles_url:
-            subtitles = self._extract_captions(subtitles_url, video_id, baseurl)
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
+            subtitles = self.extract_subtitles(subtitles_url, video_id, baseurl)
 
         return {
             'id': video_id,

From d47c26e16803abc1d15a677d88bbee78f7680db6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 19:56:10 +0100
Subject: [PATCH 1595/1937] [brightcove] Correct keys in playlists

---
 youtube_dl/extractor/brightcove.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index ea0969d4d..0733bece7 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -95,6 +95,7 @@ class BrightcoveIE(InfoExtractor):
             'url': 'http://c.brightcove.com/services/viewer/htmlFederated?playerID=3550052898001&playerKey=AQ%7E%7E%2CAAABmA9XpXk%7E%2C-Kp7jNgisre1fG5OdqpAFUTcs0lP_ZoL',
             'info_dict': {
                 'title': 'Sealife',
+                'id': '3550319591001',
             },
             'playlist_mincount': 7,
         },
@@ -247,7 +248,7 @@ class BrightcoveIE(InfoExtractor):
         playlist_info = json_data['videoList']
         videos = [self._extract_video_info(video_info) for video_info in playlist_info['mediaCollectionDTO']['videoDTOs']]
 
-        return self.playlist_result(videos, playlist_id=playlist_info['id'],
+        return self.playlist_result(videos, playlist_id='%s' % playlist_info['id'],
                                     playlist_title=playlist_info['mediaCollectionDTO']['displayName'])
 
     def _extract_video_info(self, video_info):

From d8443cd3f7e50fafb918cff5e1676b8d6fef1a64 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 19:56:24 +0100
Subject: [PATCH 1596/1937] [wsj] Correct test case

---
 youtube_dl/extractor/wsj.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/wsj.py b/youtube_dl/extractor/wsj.py
index cbe3dc7be..2ddf29a69 100644
--- a/youtube_dl/extractor/wsj.py
+++ b/youtube_dl/extractor/wsj.py
@@ -18,8 +18,8 @@ class WSJIE(InfoExtractor):
             'id': '1BD01A4C-BFE8-40A5-A42F-8A8AF9898B1A',
             'ext': 'mp4',
             'upload_date': '20150202',
-            'uploader_id': 'bbright',
-            'creator': 'bbright',
+            'uploader_id': 'jdesai',
+            'creator': 'jdesai',
             'categories': list,  # a long list
             'duration': 90,
             'title': 'Bills Coach Rex Ryan Updates His Old Jets Tattoo',

From 9789d7535d6091f64dd688b0bb1f9869b6c6ca49 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 19:58:41 +0100
Subject: [PATCH 1597/1937] [xtube] Fix test case

---
 test/helper.py                | 10 ++++++++++
 youtube_dl/extractor/xtube.py |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index 570c91440..12afdf184 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -113,6 +113,16 @@ def expect_info_dict(self, got_dict, expected_dict):
             self.assertTrue(
                 got.startswith(start_str),
                 'field %s (value: %r) should start with %r' % (info_field, got, start_str))
+        elif isinstance(expected, compat_str) and expected.startswith('contains:'):
+            got = got_dict.get(info_field)
+            contains_str = expected[len('contains:'):]
+            self.assertTrue(
+                isinstance(got, compat_str),
+                'Expected a %s object, but got %s for field %s' % (
+                    compat_str.__name__, type(got).__name__, info_field))
+            self.assertTrue(
+                contains_str in got,
+                'field %s (value: %r) should contain %r' % (info_field, got, contains_str))
         elif isinstance(expected, type):
             got = got_dict.get(info_field)
             self.assertTrue(isinstance(got, expected),
diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index e8490b028..1644f53c8 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -22,7 +22,7 @@ class XTubeIE(InfoExtractor):
             'id': 'kVTUy_G222_',
             'ext': 'mp4',
             'title': 'strange erotica',
-            'description': 'http://www.xtube.com an ET kind of thing',
+            'description': 'contains:an ET kind of thing',
             'uploader': 'greenshowers',
             'duration': 450,
             'age_limit': 18,

From 85d586617750d38d742a24f141b099f6b898d269 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 20:03:00 +0100
Subject: [PATCH 1598/1937] [yahoo] Remove md5sum from test case

The md5 sum has changed repeatedly, and we check whether it looks like a video anyways nowadays.
---
 youtube_dl/extractor/yahoo.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index f8e7041a0..97dbac4cc 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -24,7 +24,6 @@ class YahooIE(InfoExtractor):
     _TESTS = [
         {
             'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html',
-            'md5': '4962b075c08be8690a922ee026d05e69',
             'info_dict': {
                 'id': '2d25e626-2378-391f-ada0-ddaf1417e588',
                 'ext': 'mp4',

From afbdd3acc36130d1a717b3cacab69c0dfc716622 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 20:14:42 +0100
Subject: [PATCH 1599/1937] [rai] Convert to new subtitles system

---
 test/test_subtitles.py      | 13 +++++++++++++
 youtube_dl/extractor/rai.py | 22 +++++++++-------------
 2 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 1e2324232..4dbb50515 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -22,6 +22,7 @@ from youtube_dl.extractor import (
     NPOIE,
     ComedyCentralIE,
     NRKTVIE,
+    RaiIE,
 )
 
 
@@ -359,5 +360,17 @@ class TestNRKSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['no']), '1d221e6458c95c5494dcd38e6a1f129a')
 
 
+class TestRaiSubtitles(BaseTestSubtitles):
+    url = 'http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-cb27157f-9dd0-4aee-b788-b1f67643a391.html'
+    IE = RaiIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['it']))
+        self.assertEqual(md5(subtitles['it']), 'b1d90a98755126b61e667567a1f6680a')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index aa26b7e0b..144e33982 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 import re
 
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
 )
@@ -12,7 +12,7 @@ from ..utils import (
 )
 
 
-class RaiIE(SubtitlesInfoExtractor):
+class RaiIE(InfoExtractor):
     _VALID_URL = r'(?P<url>http://(?:.+?\.)?(?:rai\.it|rai\.tv|rainews\.it)/dl/.+?-(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})(?:-.+?)?\.html)'
     _TESTS = [
         {
@@ -89,15 +89,7 @@ class RaiIE(SubtitlesInfoExtractor):
                 'ext': 'mp4',
             })
 
-        if self._downloader.params.get('listsubtitles', False):
-            page = self._download_webpage(url, video_id)
-            self._list_available_subtitles(video_id, page)
-            return
-
-        subtitles = {}
-        if self._have_to_download_any_subtitles:
-            page = self._download_webpage(url, video_id)
-            subtitles = self.extract_subtitles(video_id, page)
+        subtitles = self.extract_subtitles(video_id, url)
 
         return {
             'id': video_id,
@@ -111,7 +103,8 @@ class RaiIE(SubtitlesInfoExtractor):
             'subtitles': subtitles,
         }
 
-    def _get_available_subtitles(self, video_id, webpage):
+    def _get_subtitles(self, video_id, url):
+        webpage = self._download_webpage(url, video_id)
         subtitles = {}
         m = re.search(r'<meta name="closedcaption" content="(?P<captions>[^"]+)"', webpage)
         if m:
@@ -120,5 +113,8 @@ class RaiIE(SubtitlesInfoExtractor):
             SRT_EXT = '.srt'
             if captions.endswith(STL_EXT):
                 captions = captions[:-len(STL_EXT)] + SRT_EXT
-            subtitles['it'] = 'http://www.rai.tv%s' % compat_urllib_parse.quote(captions)
+            subtitles['it'] = [{
+                'ext': 'srt',
+                'url': 'http://www.rai.tv%s' % compat_urllib_parse.quote(captions),
+            }]
         return subtitles

From 4f7cea6c5327fdda79877e34e630c478f0e5d4dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 20:37:16 +0100
Subject: [PATCH 1600/1937] [viki] Convert to new subtitles system

---
 test/test_subtitles.py       | 13 +++++++++++++
 youtube_dl/extractor/viki.py | 17 +++++++++--------
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 4dbb50515..98d1afff4 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -23,6 +23,7 @@ from youtube_dl.extractor import (
     ComedyCentralIE,
     NRKTVIE,
     RaiIE,
+    VikiIE,
 )
 
 
@@ -372,5 +373,17 @@ class TestRaiSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['it']), 'b1d90a98755126b61e667567a1f6680a')
 
 
+class TestVikiSubtitles(BaseTestSubtitles):
+    url = 'http://www.viki.com/videos/1060846v-punch-episode-18'
+    IE = VikiIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(md5(subtitles['en']), 'b0b781eeb45efd3f6398a925b259150b')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 944901e14..6816dacb6 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -2,16 +2,17 @@ from __future__ import unicode_literals
 
 import re
 
+from ..compat import compat_urlparse
 from ..utils import (
     ExtractorError,
     unescapeHTML,
     unified_strdate,
     US_RATINGS,
 )
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 
 
-class VikiIE(SubtitlesInfoExtractor):
+class VikiIE(InfoExtractor):
     IE_NAME = 'viki'
 
     _VALID_URL = r'^https?://(?:www\.)?viki\.com/videos/(?P<id>[0-9]+v)'
@@ -69,9 +70,6 @@ class VikiIE(SubtitlesInfoExtractor):
 
         # subtitles
         video_subtitles = self.extract_subtitles(video_id, info_webpage)
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, info_webpage)
-            return
 
         return {
             'id': video_id,
@@ -85,12 +83,15 @@ class VikiIE(SubtitlesInfoExtractor):
             'upload_date': upload_date,
         }
 
-    def _get_available_subtitles(self, video_id, info_webpage):
+    def _get_subtitles(self, video_id, info_webpage):
         res = {}
-        for sturl_html in re.findall(r'<track src="([^"]+)"/>', info_webpage):
+        for sturl_html in re.findall(r'<track src="([^"]+)"', info_webpage):
             sturl = unescapeHTML(sturl_html)
             m = re.search(r'/(?P<lang>[a-z]+)\.vtt', sturl)
             if not m:
                 continue
-            res[m.group('lang')] = sturl
+            res[m.group('lang')] = [{
+                'url': compat_urlparse.urljoin('http://www.viki.com', sturl),
+                'ext': 'vtt',
+            }]
         return res

From 93a16ba238c25392ed66aa0f1d7cbd3980c45f2f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Feb 2015 22:00:12 +0100
Subject: [PATCH 1601/1937] [vimeo] Raise the ExtractorError with expected=True
 when no video password is given

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index e7284049d..78d287e0e 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -175,7 +175,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
     def _verify_video_password(self, url, video_id, webpage):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
-            raise ExtractorError('This video is protected by a password, use the --video-password option')
+            raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
         token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
         data = compat_urllib_parse.urlencode({
             'password': password,

From d305dd73a3d6927f0a2c63d08662a183fa173833 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Feb 2015 23:59:50 +0100
Subject: [PATCH 1602/1937] [utils] Fix js_to_json

Previously, the runtime could be atrocious for longer inputs.
---
 test/test_utils.py  | 4 ++++
 youtube_dl/utils.py | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 1c29d0889..c7373af1e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -370,6 +370,10 @@ class TestUtil(unittest.TestCase):
             "playlist":[{"controls":{"all":null}}]
         }''')
 
+        inp = '"SAND Number: SAND 2013-7800P\\nPresenter: Tom Russo\\nHabanero Software Training - Xyce Software\\nXyce, Sandia\\u0027s"'
+        json_code = js_to_json(inp)
+        self.assertEqual(json.loads(json_code), json.loads(inp))
+
     def test_js_to_json_edgecases(self):
         on = js_to_json("{abc_def:'1\\'\\\\2\\\\\\'3\"4'}")
         self.assertEqual(json.loads(on), {"abc_def": "1'\\2\\'3\"4"})
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 3eb6bc6d4..4358137a0 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1560,8 +1560,8 @@ def js_to_json(code):
         return '"%s"' % v
 
     res = re.sub(r'''(?x)
-        "(?:[^"\\]*(?:\\\\|\\")?)*"|
-        '(?:[^'\\]*(?:\\\\|\\')?)*'|
+        "(?:[^"\\]*(?:\\\\|\\['"nu]))*[^"\\]*"|
+        '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
         [a-zA-Z_][.a-zA-Z_0-9]*
         ''', fix_kv, code)
     res = re.sub(r',(\s*\])', lambda m: m.group(1), res)

From b81a359eb67aea83799f1306a441fd6163bf1840 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 00:28:58 +0100
Subject: [PATCH 1603/1937] [YoutubeDL] Use render_table for format listing

---
 youtube_dl/YoutubeDL.py | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b087d356f..88809783b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1534,29 +1534,18 @@ class YoutubeDL(object):
         return res
 
     def list_formats(self, info_dict):
-        def line(format, idlen=20):
-            return (('%-' + compat_str(idlen + 1) + 's%-10s%-12s%s') % (
-                format['format_id'],
-                format['ext'],
-                self.format_resolution(format),
-                self._format_note(format),
-            ))
-
         formats = info_dict.get('formats', [info_dict])
-        idlen = max(len('format code'),
-                    max(len(f['format_id']) for f in formats))
-        formats_s = [
-            line(f, idlen) for f in formats
+        table = [
+            [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
+            for f in formats
             if f.get('preference') is None or f['preference'] >= -1000]
         if len(formats) > 1:
-            formats_s[-1] += (' ' if self._format_note(formats[-1]) else '') + '(best)'
+            table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
 
-        header_line = line({
-            'format_id': 'format code', 'ext': 'extension',
-            'resolution': 'resolution', 'format_note': 'note'}, idlen=idlen)
+        header_line = ['format code', 'extension', 'resolution', 'note']
         self.to_screen(
-            '[info] Available formats for %s:\n%s\n%s' %
-            (info_dict['id'], header_line, '\n'.join(formats_s)))
+            '[info] Available formats for %s:\n%s' %
+            (info_dict['id'], render_table(header_line, table)))
 
     def list_thumbnails(self, info_dict):
         thumbnails = info_dict.get('thumbnails')

From c460bdd56b0420bb5175344c0f6523675fdae415 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 00:31:01 +0100
Subject: [PATCH 1604/1937] [sandia] Add new extractor (#4974)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/sandia.py   | 117 +++++++++++++++++++++++++++++++
 youtube_dl/utils.py              |  11 ++-
 3 files changed, 128 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/sandia.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7d0909b5d..c712b907f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -386,6 +386,7 @@ from .rutube import (
     RutubePersonIE,
 )
 from .rutv import RUTVIE
+from .sandia import SandiaIE
 from .sapo import SapoIE
 from .savefrom import SaveFromIE
 from .sbs import SBSIE
diff --git a/youtube_dl/extractor/sandia.py b/youtube_dl/extractor/sandia.py
new file mode 100644
index 000000000..9c88167f0
--- /dev/null
+++ b/youtube_dl/extractor/sandia.py
@@ -0,0 +1,117 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import itertools
+import json
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urllib_request,
+    compat_urlparse,
+)
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    mimetype2ext,
+    unified_strdate,
+)
+
+
+class SandiaIE(InfoExtractor):
+    IE_DESC = 'Sandia National Laboratories'
+    _VALID_URL = r'https?://digitalops\.sandia\.gov/Mediasite/Play/(?P<id>[0-9a-f]+)'
+    _TEST = {
+        'url': 'http://digitalops.sandia.gov/Mediasite/Play/24aace4429fc450fb5b38cdbf424a66e1d',
+        'md5': '9422edc9b9a60151727e4b6d8bef393d',
+        'info_dict': {
+            'id': '24aace4429fc450fb5b38cdbf424a66e1d',
+            'ext': 'mp4',
+            'title': 'Xyce Software Training - Section 1',
+            'description': 're:(?s)SAND Number: SAND 2013-7800.{200,}',
+            'upload_date': '20120904',
+            'duration': 7794,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        req = compat_urllib_request.Request(url)
+        req.add_header('Cookie', 'MediasitePlayerCaps=ClientPlugins=4')
+        webpage = self._download_webpage(req, video_id)
+
+        js_path = self._search_regex(
+            r'<script type="text/javascript" src="(/Mediasite/FileServer/Presentation/[^"]+)"',
+            webpage, 'JS code URL')
+        js_url = compat_urlparse.urljoin(url, js_path)
+
+        js_code = self._download_webpage(
+            js_url, video_id, note='Downloading player')
+
+        def extract_str(key, **args):
+            return self._search_regex(
+                r'Mediasite\.PlaybackManifest\.%s\s*=\s*(.+);\s*?\n' % re.escape(key),
+                js_code, key, **args)
+
+        def extract_data(key, **args):
+            data_json = extract_str(key, **args)
+            if data_json is None:
+                return data_json
+            return self._parse_json(
+                data_json, video_id, transform_source=js_to_json)
+
+        formats = []
+        for i in itertools.count():
+            fd = extract_data('VideoUrls[%d]' % i, default=None)
+            if fd is None:
+                break
+            formats.append({
+                'format_id': '%s' % i,
+                'format_note': fd['MimeType'].partition('/')[2],
+                'ext': mimetype2ext(fd['MimeType']),
+                'url': fd['Location'],
+                'protocol': 'f4m' if fd['MimeType'] == 'video/x-mp4-fragmented' else None,
+            })
+        self._sort_formats(formats)
+
+        slide_baseurl = compat_urlparse.urljoin(
+            url, extract_data('SlideBaseUrl'))
+        slide_template = slide_baseurl + re.sub(
+            r'\{0:D?([0-9+])\}', r'%0\1d', extract_data('SlideImageFileNameTemplate'))
+        slides = []
+        last_slide_time = 0
+        for i in itertools.count(1):
+            sd = extract_str('Slides[%d]' % i, default=None)
+            if sd is None:
+                break
+            timestamp = int_or_none(self._search_regex(
+                r'^Mediasite\.PlaybackManifest\.CreateSlide\("[^"]*"\s*,\s*([0-9]+),',
+                sd, 'slide %s timestamp' % i, fatal=False))
+            slides.append({
+                'url': slide_template % i,
+                'duration': timestamp - last_slide_time,
+            })
+            last_slide_time = timestamp
+        formats.append({
+            'format_id': 'slides',
+            'protocol': 'slideshow',
+            'url': json.dumps(slides),
+            'preference': -10000,  # Downloader not yet written
+        })
+        self._sort_formats(formats)
+
+        title = extract_data('Title')
+        description = extract_data('Description', fatal=False)
+        duration = int_or_none(extract_data(
+            'Duration', fatal=False), scale=1000)
+        upload_date = unified_strdate(extract_data('AirDate', fatal=False))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'formats': formats,
+            'upload_date': upload_date,
+            'duration': duration,
+        }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4358137a0..238b6556b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1616,6 +1616,15 @@ def args_to_str(args):
     return ' '.join(shlex_quote(a) for a in args)
 
 
+def mimetype2ext(mt):
+    _, _, res = mt.rpartition('/')
+
+    return {
+        'x-ms-wmv': 'wmv',
+        'x-mp4-fragmented': 'mp4',
+    }.get(res, res)
+
+
 def urlhandle_detect_ext(url_handle):
     try:
         url_handle.headers
@@ -1631,7 +1640,7 @@ def urlhandle_detect_ext(url_handle):
             if e:
                 return e
 
-    return getheader('Content-Type').split("/")[1]
+    return mimetype2ext(getheader('Content-Type'))
 
 
 def age_restricted(content_limit, age_limit):

From a025d3c5a529a3343d40886e9783fde0bf997b3f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 00:31:23 +0100
Subject: [PATCH 1605/1937] release 2015.02.19

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 4bb68fdc5..e28d7d47c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -352,6 +352,7 @@
  - **rutube:movie**: Rutube movies
  - **rutube:person**: Rutube person videos
  - **RUTV**: RUTV.RU
+ - **Sandia**: Sandia National Laboratories
  - **Sapo**: SAPO Vídeos
  - **savefrom.net**
  - **SBS**: sbs.com.au
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1df05a6df..8e6ff49d0 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.18.1'
+__version__ = '2015.02.19'

From 770700404365be7b16de89c75825f04e2cf15658 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 00:38:05 +0100
Subject: [PATCH 1606/1937] [patreon] Modernize

---
 youtube_dl/extractor/patreon.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py
index 5429592a7..e5c9822c5 100644
--- a/youtube_dl/extractor/patreon.py
+++ b/youtube_dl/extractor/patreon.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class PatreonIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?patreon\.com/creation\?hid=(.+)'
+    _VALID_URL = r'https?://(?:www\.)?patreon\.com/creation\?hid=(?P<id>[^&#]+)'
     _TESTS = [
         {
             'url': 'http://www.patreon.com/creation?hid=743933',
@@ -65,9 +65,7 @@ class PatreonIE(InfoExtractor):
     '''
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         title = self._og_search_title(webpage).strip()
 
@@ -80,11 +78,10 @@ class PatreonIE(InfoExtractor):
             uploader = self._html_search_regex(
                 r'<strong>(.*?)</strong> is creating', webpage, 'uploader')
         else:
-            playlist_js = self._search_regex(
+            playlist = self._parse_json(self._search_regex(
                 r'(?s)new\s+jPlayerPlaylist\(\s*\{\s*[^}]*},\s*(\[.*?,?\s*\])',
-                webpage, 'playlist JSON')
-            playlist_json = js_to_json(playlist_js)
-            playlist = json.loads(playlist_json)
+                webpage, 'playlist JSON'),
+                video_id, transform_source=js_to_json)
             data = playlist[0]
             video_url = self._proto_relative_url(data['mp3'])
             thumbnail = self._proto_relative_url(data.get('cover'))

From 6b961a85fdef8219913b7b97cb33ef8d59d15535 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 01:04:19 +0100
Subject: [PATCH 1607/1937] [patreon] Add support for embedlies (fixes #4969)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/embedly.py  | 16 ++++++++++++++++
 youtube_dl/extractor/patreon.py  | 26 +++++++++++++++++++++++---
 3 files changed, 40 insertions(+), 3 deletions(-)
 create mode 100644 youtube_dl/extractor/embedly.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c712b907f..84a7edffc 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -121,6 +121,7 @@ from .ellentv import (
     EllenTVClipsIE,
 )
 from .elpais import ElPaisIE
+from .embedly import EmbedlyIE
 from .empflix import EMPFlixIE
 from .engadget import EngadgetIE
 from .eporner import EpornerIE
diff --git a/youtube_dl/extractor/embedly.py b/youtube_dl/extractor/embedly.py
new file mode 100644
index 000000000..1cdb11e34
--- /dev/null
+++ b/youtube_dl/extractor/embedly.py
@@ -0,0 +1,16 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
+
+
+class EmbedlyIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www|cdn\.)?embedly\.com/widgets/media\.html\?(?:[^#]*?&)?url=(?P<id>[^#&]+)'
+    _TESTS = [{
+        'url': 'https://cdn.embedly.com/widgets/media.html?src=http%3A%2F%2Fwww.youtube.com%2Fembed%2Fvideoseries%3Flist%3DUUGLim4T2loE5rwCMdpCIPVg&url=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DSU4fj_aEMVw%26list%3DUUGLim4T2loE5rwCMdpCIPVg&image=http%3A%2F%2Fi.ytimg.com%2Fvi%2FSU4fj_aEMVw%2Fhqdefault.jpg&key=8ee8a2e6a8cc47aab1a5ee67f9a178e0&type=text%2Fhtml&schema=youtube&autoplay=1',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        return self.url_result(compat_urllib_parse_unquote(self._match_id(url)))
diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py
index e5c9822c5..f179ea200 100644
--- a/youtube_dl/extractor/patreon.py
+++ b/youtube_dl/extractor/patreon.py
@@ -1,9 +1,6 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
-import json
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     js_to_json,
@@ -35,6 +32,23 @@ class PatreonIE(InfoExtractor):
                 'thumbnail': 're:^https?://.*$',
             },
         },
+        {
+            'url': 'https://www.patreon.com/creation?hid=1682498',
+            'info_dict': {
+                'id': 'SU4fj_aEMVw',
+                'ext': 'mp4',
+                'title': 'I\'m on Patreon!',
+                'uploader': 'TraciJHines',
+                'thumbnail': 're:^https?://.*$',
+                'upload_date': '20150211',
+                'description': 'md5:c5a706b1f687817a3de09db1eb93acd4',
+                'uploader_id': 'TraciJHines',
+            },
+            'params': {
+                'noplaylist': True,
+                'skip_download': True,
+            }
+        }
     ]
 
     # Currently Patreon exposes download URL via hidden CSS, so login is not
@@ -72,11 +86,17 @@ class PatreonIE(InfoExtractor):
         attach_fn = self._html_search_regex(
             r'<div class="attach"><a target="_blank" href="([^"]+)">',
             webpage, 'attachment URL', default=None)
+        embed = self._html_search_regex(
+            r'<div id="watchCreation">\s*<iframe class="embedly-embed" src="([^"]+)"',
+            webpage, 'embedded URL', default=None)
+
         if attach_fn is not None:
             video_url = 'http://www.patreon.com' + attach_fn
             thumbnail = self._og_search_thumbnail(webpage)
             uploader = self._html_search_regex(
                 r'<strong>(.*?)</strong> is creating', webpage, 'uploader')
+        elif embed is not None:
+            return self.url_result(embed)
         else:
             playlist = self._parse_json(self._search_regex(
                 r'(?s)new\s+jPlayerPlaylist\(\s*\{\s*[^}]*},\s*(\[.*?,?\s*\])',

From 7bd15b1a0378cff6960cd55e40819ef3967f747c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 01:04:24 +0100
Subject: [PATCH 1608/1937] release 2015.02.19.1

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e28d7d47c..1fcec086a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -121,6 +121,7 @@
  - **EllenTV**
  - **EllenTV:clips**
  - **ElPais**: El País
+ - **Embedly**
  - **EMPFlix**
  - **Engadget**
  - **Eporner**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 8e6ff49d0..db0ed70da 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.19'
+__version__ = '2015.02.19.1'

From 1c2528c8a3c599740558d5d752dfb368c8714a89 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 01:22:50 +0100
Subject: [PATCH 1609/1937] [cbs] Modernize

---
 youtube_dl/extractor/cbs.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index e43756ec6..1ceb9d8d9 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 
 
@@ -39,8 +37,7 @@ class CBSIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
         real_id = self._search_regex(
             r"video\.settings\.pid\s*=\s*'([^']+)';",

From 3bf5705316adb4e486ec76a9308198b499787947 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 01:43:20 +0100
Subject: [PATCH 1610/1937] [imgur] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/imgur.py    | 84 ++++++++++++++++++++++++++++++++
 2 files changed, 85 insertions(+)
 create mode 100644 youtube_dl/extractor/imgur.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 84a7edffc..1d1f07ff5 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -205,6 +205,7 @@ from .imdb import (
     ImdbIE,
     ImdbListIE
 )
+from .imgur import ImgurIE
 from .ina import InaIE
 from .infoq import InfoQIE
 from .instagram import InstagramIE, InstagramUserIE
diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
new file mode 100644
index 000000000..16488e0c4
--- /dev/null
+++ b/youtube_dl/extractor/imgur.py
@@ -0,0 +1,84 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    js_to_json,
+    mimetype2ext,
+)
+
+
+class ImgurIE(InfoExtractor):
+    _VALID_URL = r'https?://i\.imgur\.com/(?P<id>[a-zA-Z0-9]+)\.(?:mp4|gifv)'
+
+    _TESTS = [{
+        'url': 'https://i.imgur.com/A61SaA1.gifv',
+        'info_dict': {
+            'id': 'A61SaA1',
+            'ext': 'mp4',
+            'title': 'MRW gifv is up and running without any bugs',
+            'description': 'The Internet\'s visual storytelling community. Explore, share, and discuss the best visual stories the Internet has to offer.',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        width = int_or_none(self._search_regex(
+            r'<param name="width" value="([0-9]+)"',
+            webpage, 'width', fatal=False))
+        height = int_or_none(self._search_regex(
+            r'<param name="height" value="([0-9]+)"',
+            webpage, 'height', fatal=False))
+
+        formats = []
+        video_elements = self._search_regex(
+            r'(?s)<div class="video-elements">(.*?)</div>',
+            webpage, 'video elements')
+        formats = []
+        for m in re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements):
+            formats.append({
+                'format_id': m.group('type').partition('/')[2],
+                'url': self._proto_relative_url(m.group('src')),
+                'ext': mimetype2ext(m.group('type')),
+                'acodec': 'none',
+                'width': width,
+                'height': height,
+                'http_headers': {
+                    'User-Agent': 'youtube-dl (like wget)',
+                },
+            })
+
+        gif_json = self._search_regex(
+            r'(?s)var\s+videoItem\s*=\s*(\{.*?\})',
+            webpage, 'GIF code', fatal=False)
+        if gif_json:
+            gifd = self._parse_json(
+                gif_json, video_id, transform_source=js_to_json)
+            formats.append({
+                'format_id': 'gif',
+                'preference': -10,
+                'width': width,
+                'height': height,
+                'ext': 'gif',
+                'acodec': 'none',
+                'vcodec': 'gif',
+                'container': 'gif',
+                'url': self._proto_relative_url(gifd['gifUrl']),
+                'filesize': gifd.get('size'),
+                'http_headers': {
+                    'User-Agent': 'youtube-dl (like wget)',
+                },
+            })
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'description': self._og_search_description(webpage),
+            'title': self._og_search_title(webpage),
+        }

From 1ac1af9b4702fa0b921ceb45f6e7b13b35e0a2de Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 01:43:28 +0100
Subject: [PATCH 1611/1937] release 2015.02.19.2

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 1fcec086a..607e958f6 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -191,6 +191,7 @@
  - **ign.com**
  - **imdb**: Internet Movie Database trailers
  - **imdb:list**: Internet Movie Database lists
+ - **Imgur**
  - **Ina**
  - **InfoQ**
  - **Instagram**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index db0ed70da..131dd571b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.19.1'
+__version__ = '2015.02.19.2'

From bd03ffc16e0e09c1ec6fb5c3f2c3083238af5349 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 01:58:52 +0100
Subject: [PATCH 1612/1937] [netzkino] Skip download in test case

Works fine from Germany, but fails from everywhere else
---
 youtube_dl/extractor/netzkino.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/netzkino.py b/youtube_dl/extractor/netzkino.py
index 93567d1e3..bc17e20aa 100644
--- a/youtube_dl/extractor/netzkino.py
+++ b/youtube_dl/extractor/netzkino.py
@@ -29,6 +29,9 @@ class NetzkinoIE(InfoExtractor):
             'timestamp': 1344858571,
             'age_limit': 12,
         },
+        'params': {
+            'skip_download': 'Download only works from Germany',
+        }
     }
 
     def _real_extract(self, url):

From f37e3f99f01df51af649a3544570b27facc5d92c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 02:00:50 +0100
Subject: [PATCH 1613/1937] [generic] Correct test case

Video has been reuploaded / edited
---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 36a1f65bf..8dce96a64 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -532,7 +532,7 @@ class GenericIE(InfoExtractor):
             'info_dict': {
                 'id': 'Mrj4DVp2zeA',
                 'ext': 'mp4',
-                'upload_date': '20150204',
+                'upload_date': '20150212',
                 'uploader': 'The National Archives UK',
                 'description': 'md5:a236581cd2449dd2df4f93412f3f01c6',
                 'uploader_id': 'NationalArchives08',

From 71d53ace2f5c4637f5d3390298f475bbeb2452f3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 02:04:28 +0100
Subject: [PATCH 1614/1937] [sockshare] Do not require thumbnail anymore

Thumbnail is not present on the website anymore.
---
 youtube_dl/extractor/sockshare.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/sockshare.py b/youtube_dl/extractor/sockshare.py
index 7d3c0e937..b5fa6f1da 100644
--- a/youtube_dl/extractor/sockshare.py
+++ b/youtube_dl/extractor/sockshare.py
@@ -25,7 +25,6 @@ class SockshareIE(InfoExtractor):
             'id': '437BE28B89D799D7',
             'title': 'big_buck_bunny_720p_surround.avi',
             'ext': 'avi',
-            'thumbnail': 're:^http://.*\.jpg$',
         }
     }
 
@@ -45,7 +44,7 @@ class SockshareIE(InfoExtractor):
             ''', webpage, 'hash')
 
         fields = {
-            "hash": confirm_hash,
+            "hash": confirm_hash.encode('utf-8'),
             "confirm": "Continue as Free User"
         }
 
@@ -68,7 +67,7 @@ class SockshareIE(InfoExtractor):
             webpage, 'title', default=None)
         thumbnail = self._html_search_regex(
             r'<img\s+src="([^"]*)".+?name="bg"',
-            webpage, 'thumbnail')
+            webpage, 'thumbnail', default=None)
 
         formats = [{
             'format_id': 'sd',

From 3d54788495aae464a4ce63b5f093d4409f600dd8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 02:12:08 +0100
Subject: [PATCH 1615/1937] [webofstories] Fix extraction

---
 youtube_dl/extractor/webofstories.py | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/webofstories.py b/youtube_dl/extractor/webofstories.py
index 396cf4e83..73077a312 100644
--- a/youtube_dl/extractor/webofstories.py
+++ b/youtube_dl/extractor/webofstories.py
@@ -45,19 +45,17 @@ class WebOfStoriesIE(InfoExtractor):
         description = self._html_search_meta('description', webpage)
         thumbnail = self._og_search_thumbnail(webpage)
 
-        story_filename = self._search_regex(
-            r'\.storyFileName\("([^"]+)"\)', webpage, 'story filename')
-        speaker_id = self._search_regex(
-            r'\.speakerId\("([^"]+)"\)', webpage, 'speaker ID')
-        story_id = self._search_regex(
-            r'\.storyId\((\d+)\)', webpage, 'story ID')
-        speaker_type = self._search_regex(
-            r'\.speakerType\("([^"]+)"\)', webpage, 'speaker type')
-        great_life = self._search_regex(
-            r'isGreatLifeStory\s*=\s*(true|false)', webpage, 'great life story')
+        embed_params = [s.strip(" \r\n\t'") for s in self._search_regex(
+            r'(?s)\$\("#embedCode"\).html\(getEmbedCode\((.*?)\)',
+            webpage, 'embed params').split(',')]
+
+        (
+            _, speaker_id, story_id, story_duration,
+            speaker_type, great_life, _thumbnail, _has_subtitles,
+            story_filename, _story_order) = embed_params
+
         is_great_life_series = great_life == 'true'
-        duration = int_or_none(self._search_regex(
-            r'\.duration\((\d+)\)', webpage, 'duration', fatal=False))
+        duration = int_or_none(story_duration)
 
         # URL building, see: http://www.webofstories.com/scripts/player.js
         ms_prefix = ''

From 1a13940c8dada638f8298b6c1406f38d4a3bf270 Mon Sep 17 00:00:00 2001
From: John Boehr <jbboehr@gmail.com>
Date: Wed, 18 Feb 2015 18:12:48 -0800
Subject: [PATCH 1616/1937] [imgur] support regular URL

---
 youtube_dl/extractor/imgur.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index 16488e0c4..8449c45f4 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -7,11 +7,11 @@ from ..utils import (
     int_or_none,
     js_to_json,
     mimetype2ext,
+    ExtractorError,
 )
 
-
 class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://i\.imgur\.com/(?P<id>[a-zA-Z0-9]+)\.(?:mp4|gifv)'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)(?:\.)?(?:mp4|gifv)?'
 
     _TESTS = [{
         'url': 'https://i.imgur.com/A61SaA1.gifv',
@@ -21,12 +21,25 @@ class ImgurIE(InfoExtractor):
             'title': 'MRW gifv is up and running without any bugs',
             'description': 'The Internet\'s visual storytelling community. Explore, share, and discuss the best visual stories the Internet has to offer.',
         },
+    }, {
+        'url': 'https://imgur.com/A61SaA1',
+        'info_dict': {
+            'id': 'A61SaA1',
+            'ext': 'mp4',
+            'title': 'MRW gifv is up and running without any bugs',
+            'description': 'The Internet\'s visual storytelling community. Explore, share, and discuss the best visual stories the Internet has to offer.',
+        },
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
+        sources = re.findall(r'<source src="([^"]+)" type="([^"]+)"', webpage)
+        if not sources:
+            raise ExtractorError(
+                'No sources found for video %s' % video_id, expected=True)
+
         width = int_or_none(self._search_regex(
             r'<param name="width" value="([0-9]+)"',
             webpage, 'width', fatal=False))

From 383456aa29c9c47066d422d3fdfdce29da2eee07 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 04:09:52 +0100
Subject: [PATCH 1617/1937] [Makefile] Also delete *.avi files in clean

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 0636fc4cb..573c82685 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites
 
 clean:
-	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe
+	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe
 
 PREFIX ?= /usr/local
 BINDIR ?= $(PREFIX)/bin

From e880c66bd800e4769ffdff41fb9dc88cf3206c12 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 04:12:40 +0100
Subject: [PATCH 1618/1937] [theonion] Modernize

---
 youtube_dl/extractor/theonion.py | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/theonion.py b/youtube_dl/extractor/theonion.py
index b65d8e03f..10239c906 100644
--- a/youtube_dl/extractor/theonion.py
+++ b/youtube_dl/extractor/theonion.py
@@ -4,11 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
 
 
 class TheOnionIE(InfoExtractor):
-    _VALID_URL = r'(?x)https?://(?:www\.)?theonion\.com/video/[^,]+,(?P<article_id>[0-9]+)/?'
+    _VALID_URL = r'https?://(?:www\.)?theonion\.com/video/[^,]+,(?P<id>[0-9]+)/?'
     _TEST = {
         'url': 'http://www.theonion.com/video/man-wearing-mm-jacket-gods-image,36918/',
         'md5': '19eaa9a39cf9b9804d982e654dc791ee',
@@ -22,10 +21,8 @@ class TheOnionIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        article_id = mobj.group('article_id')
-
-        webpage = self._download_webpage(url, article_id)
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
 
         video_id = self._search_regex(
             r'"videoId":\s(\d+),', webpage, 'video ID')
@@ -34,10 +31,6 @@ class TheOnionIE(InfoExtractor):
         thumbnail = self._og_search_thumbnail(webpage)
 
         sources = re.findall(r'<source src="([^"]+)" type="([^"]+)"', webpage)
-        if not sources:
-            raise ExtractorError(
-                'No sources found for video %s' % video_id, expected=True)
-
         formats = []
         for src, type_ in sources:
             if type_ == 'video/mp4':
@@ -54,15 +47,15 @@ class TheOnionIE(InfoExtractor):
                 })
             elif type_ == 'application/x-mpegURL':
                 formats.extend(
-                    self._extract_m3u8_formats(src, video_id, preference=-1))
+                    self._extract_m3u8_formats(src, display_id, preference=-1))
             else:
                 self.report_warning(
                     'Encountered unexpected format: %s' % type_)
-
         self._sort_formats(formats)
 
         return {
             'id': video_id,
+            'display_id': display_id,
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,

From d236b37ac94cd36657c881e18b8d9187483afa80 Mon Sep 17 00:00:00 2001
From: John Boehr <jbboehr@gmail.com>
Date: Wed, 18 Feb 2015 19:28:19 -0800
Subject: [PATCH 1619/1937] [imgur] improve regex #4998

---
 youtube_dl/extractor/imgur.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index 8449c45f4..38c961773 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -11,7 +11,7 @@ from ..utils import (
 )
 
 class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)(?:\.)?(?:mp4|gifv)?'
+    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)(?:\.mp4|\.gifv)?'
 
     _TESTS = [{
         'url': 'https://i.imgur.com/A61SaA1.gifv',

From 9e2d7dca87a15cf455fa6c4843a0241ba0b7ad77 Mon Sep 17 00:00:00 2001
From: John Boehr <jbboehr@gmail.com>
Date: Wed, 18 Feb 2015 19:47:54 -0800
Subject: [PATCH 1620/1937] [imgur] improve error check for non-video URLs

---
 youtube_dl/extractor/imgur.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index 38c961773..7937a5c81 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -5,6 +5,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
+    str_or_none,
     js_to_json,
     mimetype2ext,
     ExtractorError,
@@ -35,11 +36,6 @@ class ImgurIE(InfoExtractor):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        sources = re.findall(r'<source src="([^"]+)" type="([^"]+)"', webpage)
-        if not sources:
-            raise ExtractorError(
-                'No sources found for video %s' % video_id, expected=True)
-
         width = int_or_none(self._search_regex(
             r'<param name="width" value="([0-9]+)"',
             webpage, 'width', fatal=False))
@@ -47,10 +43,13 @@ class ImgurIE(InfoExtractor):
             r'<param name="height" value="([0-9]+)"',
             webpage, 'height', fatal=False))
 
-        formats = []
-        video_elements = self._search_regex(
+        video_elements = str_or_none(self._search_regex(
             r'(?s)<div class="video-elements">(.*?)</div>',
-            webpage, 'video elements')
+            webpage, 'video elements', fatal=False))
+        if not video_elements:
+            raise ExtractorError(
+                'No sources found for video %s' % video_id, expected=True)
+
         formats = []
         for m in re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements):
             formats.append({

From b88ba05356bdae07245dd5240b36415002e25d0b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 05:53:09 +0100
Subject: [PATCH 1621/1937] [imgur] Simplify

---
 youtube_dl/extractor/imgur.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index 7937a5c81..b16c7aed0 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -5,12 +5,12 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
-    str_or_none,
     js_to_json,
     mimetype2ext,
     ExtractorError,
 )
 
+
 class ImgurIE(InfoExtractor):
     _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?P<id>[a-zA-Z0-9]+)(?:\.mp4|\.gifv)?'
 
@@ -43,12 +43,13 @@ class ImgurIE(InfoExtractor):
             r'<param name="height" value="([0-9]+)"',
             webpage, 'height', fatal=False))
 
-        video_elements = str_or_none(self._search_regex(
+        video_elements = self._search_regex(
             r'(?s)<div class="video-elements">(.*?)</div>',
-            webpage, 'video elements', fatal=False))
+            webpage, 'video elements', default=None)
         if not video_elements:
             raise ExtractorError(
-                'No sources found for video %s' % video_id, expected=True)
+                'No sources found for video %s. Maybe an image?' % video_id,
+                expected=True)
 
         formats = []
         for m in re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements):

From a3b9157f499e0ba0ef9cfd44b70b0d43795f60f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 19 Feb 2015 13:06:53 +0100
Subject: [PATCH 1622/1937] [cbssports] Add extractor (closes #4996)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/cbssports.py | 30 ++++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+)
 create mode 100644 youtube_dl/extractor/cbssports.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1d1f07ff5..c2424ed48 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -58,6 +58,7 @@ from .canalplus import CanalplusIE
 from .canalc2 import Canalc2IE
 from .cbs import CBSIE
 from .cbsnews import CBSNewsIE
+from .cbssports import CBSSportsIE
 from .ccc import CCCIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
diff --git a/youtube_dl/extractor/cbssports.py b/youtube_dl/extractor/cbssports.py
new file mode 100644
index 000000000..ae47e74cc
--- /dev/null
+++ b/youtube_dl/extractor/cbssports.py
@@ -0,0 +1,30 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class CBSSportsIE(InfoExtractor):
+    _VALID_URL = r'http://www\.cbssports\.com/video/player/(?P<section>[^/]+)/(?P<id>[^/]+)'
+
+    _TEST = {
+        'url': 'http://www.cbssports.com/video/player/tennis/318462531970/0/us-open-flashbacks-1990s',
+        'info_dict': {
+            'id': '_d5_GbO8p1sT',
+            'ext': 'flv',
+            'title': 'US Open flashbacks: 1990s',
+            'description': 'Bill Macatee relives the best moments in US Open history from the 1990s.',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        section = mobj.group('section')
+        video_id = mobj.group('id')
+        all_videos = self._download_json(
+            'http://www.cbssports.com/data/video/player/getVideos/%s?as=json' % section,
+            video_id)
+        # The json file contains the info of all the videos in the section
+        video_info = next(v for v in all_videos if v['pcid'] == video_id)
+        return self.url_result('theplatform:%s' % video_info['pid'], 'ThePlatform')

From 8807f1277f8c69488046fc7215cc79165e976ff3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 19 Feb 2015 14:54:50 +0100
Subject: [PATCH 1623/1937] [theplatform] Convert to new subtitles system

---
 test/test_subtitles.py              | 15 +++++++++++++++
 youtube_dl/extractor/theplatform.py | 18 +++++++-----------
 2 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 98d1afff4..c04fe6f22 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -24,6 +24,7 @@ from youtube_dl.extractor import (
     NRKTVIE,
     RaiIE,
     VikiIE,
+    ThePlatformIE,
 )
 
 
@@ -385,5 +386,19 @@ class TestVikiSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['en']), 'b0b781eeb45efd3f6398a925b259150b')
 
 
+class TestThePlatformSubtitles(BaseTestSubtitles):
+    # from http://www.3playmedia.com/services-features/tools/integrations/theplatform/
+    # (see http://theplatform.com/about/partners/type/subtitles-closed-captioning/)
+    url = 'theplatform:JFUjUE1_ehvq'
+    IE = ThePlatformIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(md5(subtitles['en']), '97e7670cbae3c4d26ae8bcc7fdd78d4b')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 1579822f2..5f24189cc 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -8,7 +8,7 @@ import binascii
 import hashlib
 
 
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..compat import (
     compat_str,
 )
@@ -22,7 +22,7 @@ from ..utils import (
 _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language'})
 
 
-class ThePlatformIE(SubtitlesInfoExtractor):
+class ThePlatformIE(InfoExtractor):
     _VALID_URL = r'''(?x)
         (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
            (?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/)?
@@ -104,15 +104,11 @@ class ThePlatformIE(SubtitlesInfoExtractor):
         captions = info.get('captions')
         if isinstance(captions, list):
             for caption in captions:
-                lang, src = caption.get('lang'), caption.get('src')
-                if lang and src:
-                    subtitles[lang] = src
-
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id, subtitles)
-            return
-
-        subtitles = self.extract_subtitles(video_id, subtitles)
+                lang, src, mime = caption.get('lang', 'en'), caption.get('src'), caption.get('type')
+                subtitles[lang] = [{
+                    'ext': 'srt' if mime == 'text/srt' else 'ttml',
+                    'url': src,
+                }]
 
         head = meta.find(_x('smil:head'))
         body = meta.find(_x('smil:body'))

From 5c8a3f862a325987a9ddbdc7f08ea62aad80278e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 19 Feb 2015 15:00:39 +0100
Subject: [PATCH 1624/1937] [nbc] Use a test video that works outside the US

---
 youtube_dl/extractor/nbc.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 89a2845fe..3645d3033 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -18,13 +18,13 @@ class NBCIE(InfoExtractor):
 
     _TESTS = [
         {
-            'url': 'http://www.nbc.com/chicago-fire/video/i-am-a-firefighter/2734188',
+            'url': 'http://www.nbc.com/the-tonight-show/segments/112966',
             # md5 checksum is not stable
             'info_dict': {
-                'id': 'bTmnLCvIbaaH',
+                'id': 'c9xnCo0YPOPH',
                 'ext': 'flv',
-                'title': 'I Am a Firefighter',
-                'description': 'An emergency puts Dawson\'sf irefighter skills to the ultimate test in this four-part digital series.',
+                'title': 'Jimmy Fallon Surprises Fans at Ben & Jerry\'s',
+                'description': 'Jimmy gives out free scoops of his new "Tonight Dough" ice cream flavor by surprising customers at the Ben & Jerry\'s scoop shop.',
             },
         },
         {

From f13b1e7d7fd4a63c9ca4a0aa9930c540033cc408 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 19 Feb 2015 16:46:41 +0100
Subject: [PATCH 1625/1937] [bbccouk] Convert to new subtitles system

I haven't found any video available outside the UK, so I haven't added tests.

I have updated how the srt file is build, because (at least for www.bbc.co.uk/programmes/p02j9b69) the subtitles is inside 'span' elements.
---
 youtube_dl/extractor/bbccouk.py | 33 ++++++++++++++++++++++-----------
 1 file changed, 22 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/bbccouk.py b/youtube_dl/extractor/bbccouk.py
index f23e39545..abc34a576 100644
--- a/youtube_dl/extractor/bbccouk.py
+++ b/youtube_dl/extractor/bbccouk.py
@@ -2,12 +2,12 @@ from __future__ import unicode_literals
 
 import xml.etree.ElementTree
 
-from .subtitles import SubtitlesInfoExtractor
+from .common import InfoExtractor
 from ..utils import ExtractorError
 from ..compat import compat_HTTPError
 
 
-class BBCCoUkIE(SubtitlesInfoExtractor):
+class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
     _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/(?:(?:(?:programmes|iplayer(?:/[^/]+)?/(?:episode|playlist))/)|music/clips[/#])(?P<id>[\da-z]{8})'
@@ -215,17 +215,32 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
             formats.extend(conn_formats)
         return formats
 
-    def _extract_captions(self, media, programme_id):
+    def _get_subtitles(self, media, programme_id):
         subtitles = {}
         for connection in self._extract_connections(media):
             captions = self._download_xml(connection.get('href'), programme_id, 'Downloading captions')
             lang = captions.get('{http://www.w3.org/XML/1998/namespace}lang', 'en')
             ps = captions.findall('./{0}body/{0}div/{0}p'.format('{http://www.w3.org/2006/10/ttaf1}'))
             srt = ''
+
+            def _extract_text(p):
+                if p.text is not None:
+                    stripped_text = p.text.strip()
+                    if stripped_text:
+                        return stripped_text
+                return ' '.join(span.text.strip() for span in p.findall('{http://www.w3.org/2006/10/ttaf1}span'))
             for pos, p in enumerate(ps):
-                srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (str(pos), p.get('begin'), p.get('end'),
-                                                          p.text.strip() if p.text is not None else '')
-            subtitles[lang] = srt
+                srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (str(pos), p.get('begin'), p.get('end'), _extract_text(p))
+            subtitles[lang] = [
+                {
+                    'url': connection.get('href'),
+                    'ext': 'ttml',
+                },
+                {
+                    'data': srt,
+                    'ext': 'srt',
+                },
+            ]
         return subtitles
 
     def _download_media_selector(self, programme_id):
@@ -249,7 +264,7 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
             elif kind == 'video':
                 formats.extend(self._extract_video(media, programme_id))
             elif kind == 'captions':
-                subtitles = self._extract_captions(media, programme_id)
+                subtitles = self.extract_subtitles(media, programme_id)
 
         return formats, subtitles
 
@@ -324,10 +339,6 @@ class BBCCoUkIE(SubtitlesInfoExtractor):
         else:
             programme_id, title, description, duration, formats, subtitles = self._download_playlist(group_id)
 
-        if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(programme_id, subtitles)
-            return
-
         self._sort_formats(formats)
 
         return {

From d5c69f1da41f3e370c500482303196ae845472ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Feb 2015 21:47:11 +0600
Subject: [PATCH 1626/1937] [5min] Cover joystiq.com URLs (Closes #4962)

---
 youtube_dl/extractor/fivemin.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/fivemin.py b/youtube_dl/extractor/fivemin.py
index 5b24b921c..157094e8c 100644
--- a/youtube_dl/extractor/fivemin.py
+++ b/youtube_dl/extractor/fivemin.py
@@ -14,6 +14,7 @@ class FiveMinIE(InfoExtractor):
     IE_NAME = '5min'
     _VALID_URL = r'''(?x)
         (?:https?://[^/]*?5min\.com/Scripts/PlayerSeed\.js\?(?:.*?&)?playList=|
+            https?://(?:(?:massively|www)\.)?joystiq\.com/video/|
             5min:)
         (?P<id>\d+)
         '''

From e66e1a0046ca804461c4c9c3e890165ec9d865e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Feb 2015 22:15:19 +0600
Subject: [PATCH 1627/1937] [pornhub] Add support for playlists (Closes #4995)

---
 youtube_dl/extractor/__init__.py |  5 ++++-
 youtube_dl/extractor/pornhub.py  | 30 ++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c2424ed48..9b5855768 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -353,7 +353,10 @@ from .playfm import PlayFMIE
 from .playvid import PlayvidIE
 from .podomatic import PodomaticIE
 from .pornhd import PornHdIE
-from .pornhub import PornHubIE
+from .pornhub import (
+    PornHubIE,
+    PornHubPlaylistIE,
+)
 from .pornotube import PornotubeIE
 from .pornoxo import PornoXOIE
 from .promptfile import PromptFileIE
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index fb2032832..913995044 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -110,3 +110,33 @@ class PornHubIE(InfoExtractor):
             'formats': formats,
             'age_limit': 18,
         }
+
+
+class PornHubPlaylistIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?pornhub\.com/playlist/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://www.pornhub.com/playlist/6201671',
+        'info_dict': {
+            'id': '6201671',
+            'title': 'P0p4',
+        },
+        'playlist_mincount': 35,
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('http://www.pornhub.com/%s' % video_url, 'PornHub')
+            for video_url in set(re.findall('href="/?(view_video\.php\?viewkey=\d+[^"]*)"', webpage))
+        ]
+
+        playlist = self._parse_json(
+            self._search_regex(
+                r'playlistObject\s*=\s*({.+?});', webpage, 'playlist'),
+            playlist_id)
+
+        return self.playlist_result(
+            entries, playlist_id, playlist.get('title'), playlist.get('description'))

From 8fc642eb5b69f25d0488509c4c4f9d8bfe2b7225 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Feb 2015 22:15:49 +0600
Subject: [PATCH 1628/1937] [pornhub] Fix uploader regex

---
 youtube_dl/extractor/pornhub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 913995044..3a27e3789 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -56,7 +56,7 @@ class PornHubIE(InfoExtractor):
 
         video_title = self._html_search_regex(r'<h1 [^>]+>([^<]+)', webpage, 'title')
         video_uploader = self._html_search_regex(
-            r'(?s)From:&nbsp;.+?<(?:a href="/users/|a href="/channels/|<span class="username)[^>]+>(.+?)<',
+            r'(?s)From:&nbsp;.+?<(?:a href="/users/|a href="/channels/|span class="username)[^>]+>(.+?)<',
             webpage, 'uploader', fatal=False)
         thumbnail = self._html_search_regex(r'"image_url":"([^"]+)', webpage, 'thumbnail', fatal=False)
         if thumbnail:

From 6140baf4e1aa7015ac6b43e93236ac6a5813569e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 19 Feb 2015 18:17:31 +0100
Subject: [PATCH 1629/1937] [nationalgeographic] Add extractor (closes #4960)

---
 youtube_dl/extractor/__init__.py           |  1 +
 youtube_dl/extractor/nationalgeographic.py | 38 ++++++++++++++++++++++
 youtube_dl/extractor/theplatform.py        |  4 ++-
 3 files changed, 42 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/nationalgeographic.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 9b5855768..f225ac654 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -285,6 +285,7 @@ from .myspace import MySpaceIE, MySpaceAlbumIE
 from .myspass import MySpassIE
 from .myvideo import MyVideoIE
 from .myvidster import MyVidsterIE
+from .nationalgeographic import NationalGeographicIE
 from .naver import NaverIE
 from .nba import NBAIE
 from .nbc import (
diff --git a/youtube_dl/extractor/nationalgeographic.py b/youtube_dl/extractor/nationalgeographic.py
new file mode 100644
index 000000000..c18640c5a
--- /dev/null
+++ b/youtube_dl/extractor/nationalgeographic.py
@@ -0,0 +1,38 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    smuggle_url,
+    url_basename,
+)
+
+
+class NationalGeographicIE(InfoExtractor):
+    _VALID_URL = r'http://video\.nationalgeographic\.com/video/.*?'
+
+    _TEST = {
+        'url': 'http://video.nationalgeographic.com/video/news/150210-news-crab-mating-vin?source=featuredvideo',
+        'info_dict': {
+            'id': '4DmDACA6Qtk_',
+            'ext': 'flv',
+            'title': 'Mating Crabs Busted by Sharks',
+            'description': 'md5:16f25aeffdeba55aaa8ec37e093ad8b3',
+        },
+        'add_ie': ['ThePlatform'],
+    }
+
+    def _real_extract(self, url):
+        name = url_basename(url)
+
+        webpage = self._download_webpage(url, name)
+        feed_url = self._search_regex(r'data-feed-url="([^"]+)"', webpage, 'feed url')
+        guid = self._search_regex(r'data-video-guid="([^"]+)"', webpage, 'guid')
+
+        feed = self._download_xml('%s?byGuid=%s' % (feed_url, guid), name)
+        content = feed.find('.//{http://search.yahoo.com/mrss/}content')
+        theplatform_id = url_basename(content.attrib.get('url'))
+
+        return self.url_result(smuggle_url(
+            'http://link.theplatform.com/s/ngs/%s?format=SMIL&formats=MPEG4&manifest=f4m' % theplatform_id,
+            # For some reason, the normal links don't work and we must force the use of f4m
+            {'force_smil_url': True}))
diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py
index 1579822f2..f7b34bd26 100644
--- a/youtube_dl/extractor/theplatform.py
+++ b/youtube_dl/extractor/theplatform.py
@@ -71,7 +71,9 @@ class ThePlatformIE(SubtitlesInfoExtractor):
         if not provider_id:
             provider_id = 'dJ5BDC'
 
-        if mobj.group('config'):
+        if smuggled_data.get('force_smil_url', False):
+            smil_url = url
+        elif mobj.group('config'):
             config_url = url + '&form=json'
             config_url = config_url.replace('swf/', 'config/')
             config_url = config_url.replace('onsite/', 'onsite/config/')

From a21420389edf665bfeb9610888d2d39dae374945 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 19 Feb 2015 19:28:17 +0100
Subject: [PATCH 1630/1937] release 2015.02.19.3

---
 docs/supportedsites.md | 3 +++
 youtube_dl/version.py  | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 607e958f6..f6ba28e7a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -68,6 +68,7 @@
  - **Canalplus**: canalplus.fr, piwiplus.fr and d8.tv
  - **CBS**
  - **CBSNews**: CBS News
+ - **CBSSports**
  - **CeskaTelevize**
  - **channel9**: Channel 9
  - **Chilloutzone**
@@ -264,6 +265,7 @@
  - **myvideo**
  - **MyVidster**
  - **n-tv.de**
+ - **NationalGeographic**
  - **Naver**
  - **NBA**
  - **NBC**
@@ -321,6 +323,7 @@
  - **podomatic**
  - **PornHd**
  - **PornHub**
+ - **PornHubPlaylist**
  - **Pornotube**
  - **PornoXO**
  - **PromptFile**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 131dd571b..537e8cf60 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.19.2'
+__version__ = '2015.02.19.3'

From dd0a58f5f028980ba76450998b218ea7a4920420 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Feb 2015 01:19:38 +0600
Subject: [PATCH 1631/1937] [blinkx] Fix extraction

---
 youtube_dl/extractor/blinkx.py | 49 ++++++++++++++++------------------
 1 file changed, 23 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/blinkx.py b/youtube_dl/extractor/blinkx.py
index 3e461e715..3b8eabe8f 100644
--- a/youtube_dl/extractor/blinkx.py
+++ b/youtube_dl/extractor/blinkx.py
@@ -1,40 +1,35 @@
 from __future__ import unicode_literals
 
 import json
-import re
 
 from .common import InfoExtractor
-from ..utils import remove_start
+from ..utils import (
+    remove_start,
+    int_or_none,
+)
 
 
 class BlinkxIE(InfoExtractor):
-    _VALID_URL = r'^(?:https?://(?:www\.)blinkx\.com/#?ce/|blinkx:)(?P<id>[^?]+)'
+    _VALID_URL = r'(?:https?://(?:www\.)blinkx\.com/#?ce/|blinkx:)(?P<id>[^?]+)'
     IE_NAME = 'blinkx'
 
     _TEST = {
-        'url': 'http://www.blinkx.com/ce/8aQUy7GVFYgFzpKhT0oqsilwOGFRVXk3R1ZGWWdGenBLaFQwb3FzaWx3OGFRVXk3R1ZGWWdGenB',
-        'md5': '2e9a07364af40163a908edbf10bb2492',
+        'url': 'http://www.blinkx.com/ce/Da0Gw3xc5ucpNduzLuDDlv4WC9PuI4fDi1-t6Y3LyfdY2SZS5Urbvn-UPJvrvbo8LTKTc67Wu2rPKSQDJyZeeORCR8bYkhs8lI7eqddznH2ofh5WEEdjYXnoRtj7ByQwt7atMErmXIeYKPsSDuMAAqJDlQZ-3Ff4HJVeH_s3Gh8oQ',
+        'md5': '337cf7a344663ec79bf93a526a2e06c7',
         'info_dict': {
-            'id': '8aQUy7GV',
+            'id': 'Da0Gw3xc',
             'ext': 'mp4',
-            'title': 'Police Car Rolls Away',
-            'uploader': 'stupidvideos.com',
-            'upload_date': '20131215',
-            'timestamp': 1387068000,
-            'description': 'A police car gently rolls away from a fight. Maybe it felt weird being around a confrontation and just had to get out of there!',
-            'duration': 14.886,
-            'thumbnails': [{
-                'width': 100,
-                'height': 76,
-                'resolution': '100x76',
-                'url': 'http://cdn.blinkx.com/stream/b/41/StupidVideos/20131215/1873969261/1873969261_tn_0.jpg',
-            }],
+            'title': 'No Daily Show for John Oliver; HBO Show Renewed - IGN News',
+            'uploader': 'IGN News',
+            'upload_date': '20150217',
+            'timestamp': 1424215740,
+            'description': 'HBO has renewed Last Week Tonight With John Oliver for two more seasons.',
+            'duration': 47.743333,
         },
     }
 
-    def _real_extract(self, rl):
-        m = re.match(self._VALID_URL, rl)
-        video_id = m.group('id')
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
         display_id = video_id[:8]
 
         api_url = ('https://apib4.blinkx.com/api.php?action=play_video&' +
@@ -60,18 +55,20 @@ class BlinkxIE(InfoExtractor):
             elif m['type'] in ('flv', 'mp4'):
                 vcodec = remove_start(m['vcodec'], 'ff')
                 acodec = remove_start(m['acodec'], 'ff')
-                tbr = (int(m['vbr']) + int(m['abr'])) // 1000
+                vbr = int_or_none(m.get('vbr') or m.get('vbitrate'), 1000)
+                abr = int_or_none(m.get('abr') or m.get('abitrate'), 1000)
+                tbr = vbr + abr if vbr and abr else None
                 format_id = '%s-%sk-%s' % (vcodec, tbr, m['w'])
                 formats.append({
                     'format_id': format_id,
                     'url': m['link'],
                     'vcodec': vcodec,
                     'acodec': acodec,
-                    'abr': int(m['abr']) // 1000,
-                    'vbr': int(m['vbr']) // 1000,
+                    'abr': abr,
+                    'vbr': vbr,
                     'tbr': tbr,
-                    'width': int(m['w']),
-                    'height': int(m['h']),
+                    'width': int_or_none(m.get('w')),
+                    'height': int_or_none(m.get('h')),
                 })
 
         self._sort_formats(formats)

From fb7cb6823e5ace9437bc79f2e1928a30f317856b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 19 Feb 2015 23:24:24 +0100
Subject: [PATCH 1632/1937] Remove the SubtitlesInfoExtractor class

No longer needed
---
 youtube_dl/extractor/subtitles.py | 99 -------------------------------
 1 file changed, 99 deletions(-)
 delete mode 100644 youtube_dl/extractor/subtitles.py

diff --git a/youtube_dl/extractor/subtitles.py b/youtube_dl/extractor/subtitles.py
deleted file mode 100644
index 59a51268d..000000000
--- a/youtube_dl/extractor/subtitles.py
+++ /dev/null
@@ -1,99 +0,0 @@
-from __future__ import unicode_literals
-from .common import InfoExtractor
-
-from ..compat import compat_str
-from ..utils import (
-    ExtractorError,
-)
-
-
-class SubtitlesInfoExtractor(InfoExtractor):
-    @property
-    def _have_to_download_any_subtitles(self):
-        return any([self._downloader.params.get('writesubtitles', False),
-                    self._downloader.params.get('writeautomaticsub')])
-
-    def _list_available_subtitles(self, video_id, webpage):
-        """ outputs the available subtitles for the video """
-        sub_lang_list = self._get_available_subtitles(video_id, webpage)
-        auto_captions_list = self._get_available_automatic_caption(video_id, webpage)
-        sub_lang = ",".join(list(sub_lang_list.keys()))
-        self.to_screen('%s: Available subtitles for video: %s' %
-                       (video_id, sub_lang))
-        auto_lang = ",".join(auto_captions_list.keys())
-        self.to_screen('%s: Available automatic captions for video: %s' %
-                       (video_id, auto_lang))
-
-    def extract_subtitles(self, video_id, webpage):
-        """
-        returns {sub_lang: sub} ,{} if subtitles not found or None if the
-        subtitles aren't requested.
-        """
-        if not self._have_to_download_any_subtitles:
-            return None
-        available_subs_list = {}
-        if self._downloader.params.get('writeautomaticsub', False):
-            available_subs_list.update(self._get_available_automatic_caption(video_id, webpage))
-        if self._downloader.params.get('writesubtitles', False):
-            available_subs_list.update(self._get_available_subtitles(video_id, webpage))
-
-        if not available_subs_list:  # error, it didn't get the available subtitles
-            return {}
-        if self._downloader.params.get('allsubtitles', False):
-            sub_lang_list = available_subs_list
-        else:
-            if self._downloader.params.get('subtitleslangs', False):
-                requested_langs = self._downloader.params.get('subtitleslangs')
-            elif 'en' in available_subs_list:
-                requested_langs = ['en']
-            else:
-                requested_langs = [list(available_subs_list.keys())[0]]
-
-            sub_lang_list = {}
-            for sub_lang in requested_langs:
-                if sub_lang not in available_subs_list:
-                    self._downloader.report_warning('no closed captions found in the specified language "%s"' % sub_lang)
-                    continue
-                sub_lang_list[sub_lang] = available_subs_list[sub_lang]
-
-        subtitles = {}
-        for sub_lang, url in sub_lang_list.items():
-            subtitle = self._request_subtitle_url(sub_lang, url)
-            if subtitle:
-                subtitles[sub_lang] = subtitle
-        return subtitles
-
-    def _download_subtitle_url(self, sub_lang, url):
-        return self._download_webpage(url, None, note=False)
-
-    def _request_subtitle_url(self, sub_lang, url):
-        """ makes the http request for the subtitle """
-        try:
-            sub = self._download_subtitle_url(sub_lang, url)
-        except ExtractorError as err:
-            self._downloader.report_warning('unable to download video subtitles for %s: %s' % (sub_lang, compat_str(err)))
-            return
-        if not sub:
-            self._downloader.report_warning('Did not fetch video subtitles')
-            return
-        return sub
-
-    def _get_available_subtitles(self, video_id, webpage):
-        """
-        returns {sub_lang: url} or {} if not available
-        Must be redefined by the subclasses
-        """
-
-        # By default, allow implementations to simply pass in the result
-        assert isinstance(webpage, dict), \
-            '_get_available_subtitles not implemented'
-        return webpage
-
-    def _get_available_automatic_caption(self, video_id, webpage):
-        """
-        returns {sub_lang: url} or {} if not available
-        Must be redefined by the subclasses that support automatic captions,
-        otherwise it will return {}
-        """
-        self._downloader.report_warning('Automatic Captions not supported by this server')
-        return {}

From 5e9a033e6e8054605bb87c8448f95a5bb86a71bb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 20 Feb 2015 02:52:03 +0100
Subject: [PATCH 1633/1937] [imgur] Allow alternative values

Every now and then, imgur.com goes crazy and gives us a generic title and description (otherwise it looks all fine though).
Simply update the test case to allow for that craziness.
---
 youtube_dl/extractor/imgur.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index b16c7aed0..fe5d95e2c 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -19,16 +19,16 @@ class ImgurIE(InfoExtractor):
         'info_dict': {
             'id': 'A61SaA1',
             'ext': 'mp4',
-            'title': 'MRW gifv is up and running without any bugs',
-            'description': 'The Internet\'s visual storytelling community. Explore, share, and discuss the best visual stories the Internet has to offer.',
+            'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$',
+            'description': 're:The origin of the Internet\'s most viral images$|The Internet\'s visual storytelling community\. Explore, share, and discuss the best visual stories the Internet has to offer\.$',
         },
     }, {
         'url': 'https://imgur.com/A61SaA1',
         'info_dict': {
             'id': 'A61SaA1',
             'ext': 'mp4',
-            'title': 'MRW gifv is up and running without any bugs',
-            'description': 'The Internet\'s visual storytelling community. Explore, share, and discuss the best visual stories the Internet has to offer.',
+            'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$',
+            'description': 're:The origin of the Internet\'s most viral images$|The Internet\'s visual storytelling community\. Explore, share, and discuss the best visual stories the Internet has to offer\.$',
         },
     }]
 

From 5da6bd00837236cf8a5dc5aeeadae5cfed7f2021 Mon Sep 17 00:00:00 2001
From: "Leslie P. Polzer" <polzer@gnu.org>
Date: Fri, 20 Feb 2015 10:49:45 +0100
Subject: [PATCH 1634/1937] [chirbit] Add new extractor.

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/chirbit.py  | 34 ++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 youtube_dl/extractor/chirbit.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f225ac654..de08e69bc 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -63,6 +63,7 @@ from .ccc import CCCIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chilloutzone import ChilloutzoneIE
+from .chirbit import ChirbitIE
 from .cinchcast import CinchcastIE
 from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
diff --git a/youtube_dl/extractor/chirbit.py b/youtube_dl/extractor/chirbit.py
new file mode 100644
index 000000000..06a3e1a7a
--- /dev/null
+++ b/youtube_dl/extractor/chirbit.py
@@ -0,0 +1,34 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class ChirbitIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?chirb\.it/(?P<id>[^/]+)'
+    _TEST = {
+        'url': 'http://chirb.it/PrIPv5',
+        'md5': '9847b0dad6ac3e074568bf2cfb197de8',
+        'info_dict': {
+            'id': 'PrIPv5',
+            'display_id': 'kukushtv_1423231243',
+            'ext': 'mp3',
+            'title': 'Фасадстрой',
+            'url': 'http://audio.chirbit.com/kukushtv_1423231243.mp3'
+        }
+    }
+
+    def _real_extract(self, url):
+        audio_linkid = self._match_id(url)
+        webpage = self._download_webpage(url, audio_linkid)
+
+        audio_title = self._html_search_regex(r'<h2\s+itemprop="name">(.*?)</h2>', webpage, 'title')
+        audio_id = self._html_search_regex(r'\("setFile",\s+"http://audio.chirbit.com/(.*?).mp3"\)', webpage, 'audio ID')
+        audio_url = 'http://audio.chirbit.com/' + audio_id + '.mp3';
+
+        return {
+            'id': audio_linkid,
+            'display_id': audio_id,
+            'title': audio_title,
+            'url': audio_url
+        }

From 50efb383f01071fe5ae6ec6da2557031da54abde Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 20 Feb 2015 14:13:41 +0200
Subject: [PATCH 1635/1937] [tv4] Add new extractor (Closes #4839) (Closes
 #2103)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/tv4.py      | 100 +++++++++++++++++++++++++++++++
 2 files changed, 101 insertions(+)
 create mode 100644 youtube_dl/extractor/tv4.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f225ac654..7d0c6b5ac 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -490,6 +490,7 @@ from .tumblr import TumblrIE
 from .tunein import TuneInIE
 from .turbo import TurboIE
 from .tutv import TutvIE
+from .tv4 import TV4IE
 from .tvigle import TvigleIE
 from .tvp import TvpIE, TvpSeriesIE
 from .tvplay import TVPlayIE
diff --git a/youtube_dl/extractor/tv4.py b/youtube_dl/extractor/tv4.py
new file mode 100644
index 000000000..1c4b6d635
--- /dev/null
+++ b/youtube_dl/extractor/tv4.py
@@ -0,0 +1,100 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    parse_iso8601,
+)
+
+
+class TV4IE(InfoExtractor):
+    IE_DESC = 'tv4.se and tv4play.se'
+    _VALID_URL = r'''(?x)https?://(?:www\.)?
+        (?:
+            tv4\.se/(?:[^/]+)/klipp/(?:.*)-|
+            tv4play\.se/
+            (?:
+                (?:program|barn)/(?:[^\?]+)\?video_id=|
+                iframe/video/|
+                film/|
+                sport/|
+            )
+        )(?P<id>[0-9]+)'''
+    _TESTS = [
+        {
+            'url': 'http://www.tv4.se/kalla-fakta/klipp/kalla-fakta-5-english-subtitles-2491650',
+            'md5': '909d6454b87b10a25aa04c4bdd416a9b',
+            'info_dict': {
+                'id': '2491650',
+                'ext': 'mp4',
+                'title': 'Kalla Fakta 5 (english subtitles)',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'timestamp': int,
+                'upload_date': '20131125',
+            },
+        },
+        {
+            'url': 'http://www.tv4play.se/iframe/video/3054113',
+            'md5': '77f851c55139ffe0ebd41b6a5552489b',
+            'info_dict': {
+                'id': '3054113',
+                'ext': 'mp4',
+                'title': 'Så här jobbar ficktjuvarna - se avslöjande bilder',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'description': 'Unika bilder avslöjar hur turisternas fickor vittjas mitt på Stockholms central. Två experter på ficktjuvarna avslöjar knepen du ska se upp för.',
+                'timestamp': int,
+                'upload_date': '20150130',
+            },
+        },
+        {
+            'url': 'http://www.tv4play.se/sport/3060959',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.tv4play.se/film/2378136',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.tv4play.se/barn/looney-tunes?video_id=3062412',
+            'only_matching': True,
+        },
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        info = self._download_json(
+            'http://www.tv4play.se/player/assets/%s.json' % video_id, video_id, 'Downloading video info JSON')
+
+        # If is_geo_restricted is true, it doesn't neceserally mean we can't download it
+        if info['is_geo_restricted']:
+            self.report_warning('This content might not be available in your country due to licensing restrictions.')
+        if info['requires_subscription']:
+            raise ExtractorError('This content requires subscription.', expected=True)
+
+        sources_data = self._download_json(
+            'https://prima.tv4play.se/api/web/asset/%s/play.json?protocol=http&videoFormat=MP4' % video_id, video_id, 'Downloading sources JSON')
+        sources = sources_data['playback']
+
+        formats = []
+        for item in sources.get('items', {}).get('item', []):
+            ext, bitrate = item['mediaFormat'], item['bitrate']
+            formats.append({
+                'format_id': '%s_%s' % (ext, bitrate),
+                'tbr': bitrate,
+                'ext': ext,
+                'url': item['url'],
+            })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': info['title'],
+            'formats': formats,
+            'description': info.get('description'),
+            'timestamp': parse_iso8601(info.get('broadcast_date_time')),
+            'duration': info.get('duration'),
+            'thumbnail': info.get('image'),
+            'is_live': sources.get('live'),
+        }

From 365577f5676d63089cb834855dd4cdce7d0dc8aa Mon Sep 17 00:00:00 2001
From: "Leslie P. Polzer" <polzer@gnu.org>
Date: Fri, 20 Feb 2015 14:48:12 +0100
Subject: [PATCH 1636/1937] [chirbit] add profile extractor.

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/chirbit.py  | 63 ++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index de08e69bc..94e150826 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -63,7 +63,7 @@ from .ccc import CCCIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chilloutzone import ChilloutzoneIE
-from .chirbit import ChirbitIE
+from .chirbit import ChirbitIE, ChirbitProfileIE
 from .cinchcast import CinchcastIE
 from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
diff --git a/youtube_dl/extractor/chirbit.py b/youtube_dl/extractor/chirbit.py
index 06a3e1a7a..47ce94aa0 100644
--- a/youtube_dl/extractor/chirbit.py
+++ b/youtube_dl/extractor/chirbit.py
@@ -1,7 +1,10 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..utils import clean_html
 
 
 class ChirbitIE(InfoExtractor):
@@ -32,3 +35,63 @@ class ChirbitIE(InfoExtractor):
             'title': audio_title,
             'url': audio_url
         }
+
+class ChirbitProfileIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?chirbit.com/(?P<id>[^/]+)'
+    _TEST = {
+        'url': 'http://chirbit.com/ScarletBeauty',
+        'playlist_count': 3,
+        'info_dict': {
+            '_type': 'playlist',
+            'title': 'ScarletBeauty',
+            'id': 'ScarletBeauty'
+        }
+    }
+
+    def _real_extract(self, url):
+        profile_id = self._match_id(url)
+
+        # Chirbit has a pretty weird "Last Page" navigation behavior.
+        # We grab the profile's oldest entry to determine when to
+        # stop fetching entries.
+        oldestpage = self._download_webpage(url + '/24599', profile_id)
+        oldest_page_entries = re.findall(
+            r'''soundFile:\s*"http://audio.chirbit.com/(.*?).mp3"''',
+            oldestpage);
+        oldestentry = clean_html(oldest_page_entries[-1]);
+
+        ids = []
+        titles = []
+        n = 0
+        while True:
+            page = self._download_webpage(url + '/' + str(n), profile_id)
+            page_ids = re.findall(
+                r'''soundFile:\s*"http://audio.chirbit.com/(.*?).mp3"''',
+                page);
+            page_titles = re.findall(
+                r'''<div\s+class="chirbit_title"\s*>(.*?)</div>''',
+                page);
+            ids += page_ids
+            titles += page_titles
+            if oldestentry in page_ids:
+                break
+            n += 1
+
+        entries = []
+        i = 0
+        for id in ids:
+            entries.append({
+                'id': id,
+                'title': titles[i],
+                'url': 'http://audio.chirbit.com/' + id + '.mp3'
+            });
+            i += 1
+
+        info_dict = {
+            '_type': 'playlist',
+            'id': profile_id,
+            'title': profile_id,
+            'entries': entries
+        }
+
+        return info_dict;

From 1bd838608fda2da789c8991ec7fc194ed5ec0a1f Mon Sep 17 00:00:00 2001
From: minusf <minusf@gmail.com>
Date: Fri, 20 Feb 2015 18:24:20 +0100
Subject: [PATCH 1637/1937] prefer 'code' to 'uri' if present

---
 youtube_dl/extractor/ted.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 10b3b706a..5e5322974 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -134,9 +134,13 @@ class TEDIE(SubtitlesInfoExtractor):
 
         if talk_info.get('external') is not None:
             self.to_screen('Found video from %s' % talk_info['external']['service'])
+            if 'code' in talk_info['external']:
+                ext_url = talk_info['external']['code']
+            else:
+                ext_url = talk_info['external']['uri']
             return {
                 '_type': 'url',
-                'url': talk_info['external']['uri'],
+                'url': ext_url,
             }
 
         formats = [{

From a461a11989e147c30e3611005861cc63dcd16af3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Feb 2015 00:14:38 +0600
Subject: [PATCH 1638/1937] [ted] Improve external video handling and add test

---
 youtube_dl/extractor/ted.py | 31 ++++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 5e5322974..59678399d 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -83,6 +83,22 @@ class TEDIE(SubtitlesInfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # YouTube video
+        'url': 'http://www.ted.com/talks/jeffrey_kluger_the_sibling_bond',
+        'add_ie': ['Youtube'],
+        'info_dict': {
+            'id': 'aFBIPO-P7LM',
+            'ext': 'mp4',
+            'title': 'The hidden power of siblings: Jeff Kluger at TEDxAsheville',
+            'description': 'md5:3d7a4f50d95ca5dd67104e2a20f43fe1',
+            'uploader': 'TEDx Talks',
+            'uploader_id': 'TEDxTalks',
+            'upload_date': '20111216',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     _NATIVE_FORMATS = {
@@ -132,15 +148,16 @@ class TEDIE(SubtitlesInfoExtractor):
 
         talk_info = self._extract_info(webpage)['talks'][0]
 
-        if talk_info.get('external') is not None:
-            self.to_screen('Found video from %s' % talk_info['external']['service'])
-            if 'code' in talk_info['external']:
-                ext_url = talk_info['external']['code']
-            else:
-                ext_url = talk_info['external']['uri']
+        external = talk_info.get('external')
+        if external:
+            service = external['service']
+            self.to_screen('Found video from %s' % service)
+            ext_url = None
+            if service.lower() == 'youtube':
+                ext_url = external.get('code')
             return {
                 '_type': 'url',
-                'url': ext_url,
+                'url': ext_url or external['uri'],
             }
 
         formats = [{

From 3da0db62e6c4122b151349e7a05f59803da4fbbc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 20 Feb 2015 23:22:47 +0100
Subject: [PATCH 1639/1937] [escapist] Fix extraction (fixes #5017)

---
 youtube_dl/extractor/escapist.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 4303feccd..6b693b3b6 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -22,6 +22,7 @@ class EscapistIE(InfoExtractor):
             'uploader_id': 'the-escapist-presents',
             'uploader': 'The Escapist Presents',
             'title': "Breaking Down Baldur's Gate",
+            'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
 
@@ -40,9 +41,8 @@ class EscapistIE(InfoExtractor):
         raw_title = self._html_search_meta('title', webpage, fatal=True)
         title = raw_title.partition(' : ')[2]
 
-        player_url = self._og_search_video_url(webpage, name='player URL')
-        config_url = compat_urllib_parse.unquote(self._search_regex(
-            r'config=(.*)$', player_url, 'config URL'))
+        config_url = compat_urllib_parse.unquote(self._html_search_regex(
+            r'<param name="flashvars" value="config=([^"&]+)', webpage, 'config URL'))
 
         formats = []
 
@@ -81,5 +81,4 @@ class EscapistIE(InfoExtractor):
             'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
             'description': description,
-            'player_url': player_url,
         }

From 1e9a9e167d5e518f44402394bb4cc23712ff32fc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 20 Feb 2015 23:23:12 +0100
Subject: [PATCH 1640/1937] release 2015.02.20

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index f6ba28e7a..dbff5e270 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -451,6 +451,7 @@
  - **Turbo**
  - **Tutv**
  - **tv.dfb.de**
+ - **TV4**: tv4.se and tv4play.se
  - **tvigle**: Интернет-телевидение Tvigle.ru
  - **tvp.pl**
  - **tvp.pl:Series**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 537e8cf60..9fd0ee963 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.19.3'
+__version__ = '2015.02.20'

From a00a8bcc8a131e0bea605ab3cfa0f09bedde9e0c Mon Sep 17 00:00:00 2001
From: CyberJacob <cyberjacob@gmail.com>
Date: Fri, 20 Feb 2015 22:43:51 +0000
Subject: [PATCH 1641/1937] Change example URLs in readme (fixes #5018y)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a2c148311..8ea31d605 100644
--- a/README.md
+++ b/README.md
@@ -571,7 +571,7 @@ Support requests for services that **do** purchase the rights to distribute thei
 
 ### How can I detect whether a given URL is supported by youtube-dl?
 
-For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/v/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
+For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/video/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
 
 It is *not* possible to detect whether a URL is supported or not. That's because youtube-dl contains a generic extractor which matches **all** URLs. You may be tempted to disable, exclude, or remove the generic extractor, but the generic extractor not only allows users to extract videos from lots of websites that embed a video from another service, but may also be used to extract video from a service that it's hosting itself. Therefore, we neither recommend nor support disabling, excluding, or removing the generic extractor.
 

From fad6768bd1a67dccbb153ac371d3e82575321ea9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Feb 2015 18:00:25 +0600
Subject: [PATCH 1642/1937] [vimeo] Fix password protected videos (Closes
 #5001)

---
 youtube_dl/extractor/vimeo.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 78d287e0e..5f8649e35 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import json
 import re
 import itertools
+import hashlib
 
 from .common import InfoExtractor
 from .subtitles import SubtitlesInfoExtractor
@@ -225,6 +226,10 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
         if mobj.group('pro') or mobj.group('player'):
             url = 'http://player.vimeo.com/video/' + video_id
 
+        password = self._downloader.params.get('videopassword', None)
+        if password:
+            headers['Cookie'] = '%s_password=%s' % (video_id, hashlib.md5(password).hexdigest())
+
         # Retrieve video webpage to extract further information
         request = compat_urllib_request.Request(url, None, headers)
         try:

From 62b013df0dcb5f902d745b77b3de62b64b828863 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Feb 2015 18:31:10 +0600
Subject: [PATCH 1643/1937] [vimeo] Encode password before hash calculation

---
 youtube_dl/extractor/vimeo.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 5f8649e35..4cd2f73d9 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -228,7 +228,8 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor):
 
         password = self._downloader.params.get('videopassword', None)
         if password:
-            headers['Cookie'] = '%s_password=%s' % (video_id, hashlib.md5(password).hexdigest())
+            headers['Cookie'] = '%s_password=%s' % (
+                video_id, hashlib.md5(password.encode('utf-8')).hexdigest())
 
         # Retrieve video webpage to extract further information
         request = compat_urllib_request.Request(url, None, headers)

From 77b2986b5b0246234b72ae9dd78fb40f9d37374f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 21 Feb 2015 14:51:07 +0100
Subject: [PATCH 1644/1937] [extractor/common] Recognize Indian censorship
 (#5021)

---
 youtube_dl/extractor/common.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 08b8ad37c..ee64ad329 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -391,6 +391,16 @@ class InfoExtractor(object):
             if blocked_iframe:
                 msg += ' Visit %s for more details' % blocked_iframe
             raise ExtractorError(msg, expected=True)
+        if '<title>The URL you requested has been blocked</title>' in content[:512]:
+            msg = (
+                'Access to this webpage has been blocked by Indian censorship. '
+                'Use a VPN or proxy server (with --proxy) to route around it.')
+            block_msg = self._html_search_regex(
+                r'</h1><p>(.*?)</p>',
+                content, 'block message', default=None)
+            if block_msg:
+                msg += ' (Message: "%s")' % block_msg.replace('\n', ' ')
+            raise ExtractorError(msg, expected=True)
 
         return content
 

From 8fb3ac3649ca7df6f328971f58afa84dd9d05cc6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 21 Feb 2015 14:55:13 +0100
Subject: [PATCH 1645/1937] PEP8: W503

---
 devscripts/check-porn.py        |  8 ++++----
 test/test_swfinterp.py          |  4 ++--
 youtube_dl/YoutubeDL.py         |  8 ++++----
 youtube_dl/__init__.py          | 16 ++++++++--------
 youtube_dl/downloader/common.py | 10 +++++-----
 youtube_dl/downloader/f4m.py    |  4 ++--
 youtube_dl/extractor/adobetv.py |  7 ++++---
 youtube_dl/extractor/common.py  |  4 ++--
 youtube_dl/extractor/defense.py |  5 +++--
 youtube_dl/utils.py             |  4 ++--
 10 files changed, 36 insertions(+), 34 deletions(-)

diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py
index 216282712..6a5bd9eda 100644
--- a/devscripts/check-porn.py
+++ b/devscripts/check-porn.py
@@ -45,12 +45,12 @@ for test in get_testcases():
 
         RESULT = ('.' + domain + '\n' in LIST or '\n' + domain + '\n' in LIST)
 
-    if RESULT and ('info_dict' not in test or 'age_limit' not in test['info_dict']
-                   or test['info_dict']['age_limit'] != 18):
+    if RESULT and ('info_dict' not in test or 'age_limit' not in test['info_dict'] or
+                   test['info_dict']['age_limit'] != 18):
         print('\nPotential missing age_limit check: {0}'.format(test['name']))
 
-    elif not RESULT and ('info_dict' in test and 'age_limit' in test['info_dict']
-                         and test['info_dict']['age_limit'] == 18):
+    elif not RESULT and ('info_dict' in test and 'age_limit' in test['info_dict'] and
+                         test['info_dict']['age_limit'] == 18):
         print('\nPotential false negative: {0}'.format(test['name']))
 
     else:
diff --git a/test/test_swfinterp.py b/test/test_swfinterp.py
index 9f18055e6..f1e899819 100644
--- a/test/test_swfinterp.py
+++ b/test/test_swfinterp.py
@@ -34,8 +34,8 @@ def _make_testfunc(testfile):
     def test_func(self):
         as_file = os.path.join(TEST_DIR, testfile)
         swf_file = os.path.join(TEST_DIR, test_id + '.swf')
-        if ((not os.path.exists(swf_file))
-                or os.path.getmtime(swf_file) < os.path.getmtime(as_file)):
+        if ((not os.path.exists(swf_file)) or
+                os.path.getmtime(swf_file) < os.path.getmtime(as_file)):
             # Recompile
             try:
                 subprocess.check_call([
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 88809783b..ca7c3f5c6 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -308,8 +308,8 @@ class YoutubeDL(object):
                     raise
 
         if (sys.version_info >= (3,) and sys.platform != 'win32' and
-                sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
-                and not params.get('restrictfilenames', False)):
+                sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
+                not params.get('restrictfilenames', False)):
             # On Python 3, the Unicode filesystem API will throw errors (#1474)
             self.report_warning(
                 'Assuming --restrict-filenames since file system encoding '
@@ -1366,8 +1366,8 @@ class YoutubeDL(object):
         """Download a given list of URLs."""
         outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
         if (len(url_list) > 1 and
-                '%' not in outtmpl
-                and self.params.get('max_downloads') != 1):
+                '%' not in outtmpl and
+                self.params.get('max_downloads') != 1):
             raise SameFileError(outtmpl)
 
         for url in url_list:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index eac2a26ec..25ab3fdfe 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -189,14 +189,14 @@ def _real_main(argv=None):
         # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
         if opts.outtmpl is not None:
             opts.outtmpl = opts.outtmpl.decode(preferredencoding())
-    outtmpl = ((opts.outtmpl is not None and opts.outtmpl)
-               or (opts.format == '-1' and opts.usetitle and '%(title)s-%(id)s-%(format)s.%(ext)s')
-               or (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s')
-               or (opts.usetitle and opts.autonumber and '%(autonumber)s-%(title)s-%(id)s.%(ext)s')
-               or (opts.usetitle and '%(title)s-%(id)s.%(ext)s')
-               or (opts.useid and '%(id)s.%(ext)s')
-               or (opts.autonumber and '%(autonumber)s-%(id)s.%(ext)s')
-               or DEFAULT_OUTTMPL)
+    outtmpl = ((opts.outtmpl is not None and opts.outtmpl) or
+               (opts.format == '-1' and opts.usetitle and '%(title)s-%(id)s-%(format)s.%(ext)s') or
+               (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s') or
+               (opts.usetitle and opts.autonumber and '%(autonumber)s-%(title)s-%(id)s.%(ext)s') or
+               (opts.usetitle and '%(title)s-%(id)s.%(ext)s') or
+               (opts.useid and '%(id)s.%(ext)s') or
+               (opts.autonumber and '%(autonumber)s-%(id)s.%(ext)s') or
+               DEFAULT_OUTTMPL)
     if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
         parser.error('Cannot download a video and extract audio into the same'
                      ' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 45e55b99c..3ae90021a 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -311,14 +311,14 @@ class FileDownloader(object):
         """
 
         nooverwrites_and_exists = (
-            self.params.get('nooverwrites', False)
-            and os.path.exists(encodeFilename(filename))
+            self.params.get('nooverwrites', False) and
+            os.path.exists(encodeFilename(filename))
         )
 
         continuedl_and_exists = (
-            self.params.get('continuedl', False)
-            and os.path.isfile(encodeFilename(filename))
-            and not self.params.get('nopart', False)
+            self.params.get('continuedl', False) and
+            os.path.isfile(encodeFilename(filename)) and
+            not self.params.get('nopart', False)
         )
 
         # Check file already present
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index b40ebfa50..7b8fe8cf5 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -325,8 +325,8 @@ class F4mFD(FileDownloader):
                 state['frag_index'] += 1
 
             estimated_size = (
-                (state['downloaded_bytes'] + frag_total_bytes)
-                / (state['frag_index'] + 1) * total_frags)
+                (state['downloaded_bytes'] + frag_total_bytes) /
+                (state['frag_index'] + 1) * total_frags)
             time_now = time.time()
             state['total_bytes_estimate'] = estimated_size
             state['elapsed'] = time_now - start
diff --git a/youtube_dl/extractor/adobetv.py b/youtube_dl/extractor/adobetv.py
index 28e07f8b0..97d128560 100644
--- a/youtube_dl/extractor/adobetv.py
+++ b/youtube_dl/extractor/adobetv.py
@@ -28,7 +28,6 @@ class AdobeTVIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-
         webpage = self._download_webpage(url, video_id)
 
         player = self._parse_json(
@@ -44,8 +43,10 @@ class AdobeTVIE(InfoExtractor):
             self._html_search_meta('datepublished', webpage, 'upload date'))
 
         duration = parse_duration(
-            self._html_search_meta('duration', webpage, 'duration')
-            or self._search_regex(r'Runtime:\s*(\d{2}:\d{2}:\d{2})', webpage, 'duration'))
+            self._html_search_meta('duration', webpage, 'duration') or
+            self._search_regex(
+                r'Runtime:\s*(\d{2}:\d{2}:\d{2})',
+                webpage, 'duration', fatal=False))
 
         view_count = str_to_int(self._search_regex(
             r'<div class="views">\s*Views?:\s*([\d,.]+)\s*</div>',
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index ee64ad329..d3f86cf4a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -808,8 +808,8 @@ class InfoExtractor(object):
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
             if manifest_version == '2.0':
-                manifest_url = ('/'.join(manifest_url.split('/')[:-1]) + '/'
-                                + (media_el.attrib.get('href') or media_el.attrib.get('url')))
+                manifest_url = ('/'.join(manifest_url.split('/')[:-1]) + '/' +
+                                (media_el.attrib.get('href') or media_el.attrib.get('url')))
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
                 'format_id': '-'.join(filter(None, [f4m_id, 'f4m-%d' % (i if tbr is None else tbr)])),
diff --git a/youtube_dl/extractor/defense.py b/youtube_dl/extractor/defense.py
index 2b90bf4fc..98e3aedfd 100644
--- a/youtube_dl/extractor/defense.py
+++ b/youtube_dl/extractor/defense.py
@@ -25,8 +25,9 @@ class DefenseGouvFrIE(InfoExtractor):
             r"flashvars.pvg_id=\"(\d+)\";",
             webpage, 'ID')
 
-        json_url = ('http://static.videos.gouv.fr/brightcovehub/export/json/'
-                    + video_id)
+        json_url = (
+            'http://static.videos.gouv.fr/brightcovehub/export/json/%s' %
+            video_id)
         info = self._download_json(json_url, title, 'Downloading JSON config')
         video_url = info['renditions'][0]['url']
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 238b6556b..475fad3c9 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -900,8 +900,8 @@ def _windows_write_string(s, out):
     def not_a_console(handle):
         if handle == INVALID_HANDLE_VALUE or handle is None:
             return True
-        return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR
-                or GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
+        return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
+                GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
 
     if not_a_console(h):
         return False

From 93540ee10e4143f8de7885af2d68c213aab7d8cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Feb 2015 19:31:39 +0100
Subject: [PATCH 1646/1937] [rtve] Fix the video url

Changing mvod to mvod1 fixes the url, we don't need to add the query.
---
 youtube_dl/extractor/rtve.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 3469d9578..e60f85b5b 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -6,6 +6,7 @@ import re
 import time
 
 from .common import InfoExtractor
+from ..compat import compat_urlparse
 from ..utils import (
     struct_unpack,
     remove_end,
@@ -96,12 +97,10 @@ class RTVEALaCartaIE(InfoExtractor):
             ).replace('.net.rtve', '.multimedia.cdn.rtve')
             video_path = self._download_webpage(
                 auth_url, video_id, 'Getting video url')
-            # Use mvod.akcdn instead of flash.akamaihd.multimedia.cdn to get
+            # Use mvod1.akcdn instead of flash.akamaihd.multimedia.cdn to get
             # the right Content-Length header and the mp4 format
-            video_url = (
-                'http://mvod.akcdn.rtve.es/{0}&v=2.6.8'
-                '&fp=MAC%2016,0,0,296&r=MRUGG&g=OEOJWFXNFGCP'.format(video_path)
-            )
+            video_url = compat_urlparse.urljoin(
+                'http://mvod1.akcdn.rtve.es/', video_path)
 
         return {
             'id': video_id,

From 4aeccadf4ef8528c252c917d071e98a091f7766c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Feb 2015 01:38:57 +0600
Subject: [PATCH 1647/1937] [zapiks] Add extractor (#5014)

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/zapiks.py   | 102 +++++++++++++++++++++++++++++++
 2 files changed, 103 insertions(+)
 create mode 100644 youtube_dl/extractor/zapiks.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7d0c6b5ac..ef0adfd87 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -612,6 +612,7 @@ from .youtube import (
     YoutubeUserIE,
     YoutubeWatchLaterIE,
 )
+from .zapiks import ZapiksIE
 from .zdf import ZDFIE, ZDFChannelIE
 from .zingmp3 import (
     ZingMp3SongIE,
diff --git a/youtube_dl/extractor/zapiks.py b/youtube_dl/extractor/zapiks.py
new file mode 100644
index 000000000..12810637e
--- /dev/null
+++ b/youtube_dl/extractor/zapiks.py
@@ -0,0 +1,102 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    parse_duration,
+    parse_iso8601,
+    xpath_with_ns,
+    xpath_text,
+    int_or_none,
+)
+
+
+class ZapiksIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?zapiks\.fr/(?:(?P<display_id>.+?)\.html|index\.php\?.*\bmedia_id=(?P<id>\d+))'
+    _TESTS = [
+        {
+            'url': 'http://www.zapiks.fr/ep2s3-bon-appetit-eh-be-viva.html',
+            'md5': 'aeb3c473b2d564b2d46d664d28d5f050',
+            'info_dict': {
+                'id': '80798',
+                'ext': 'mp4',
+                'title': 'EP2S3 - Bon Appétit - Eh bé viva les pyrénées con!',
+                'description': 'md5:7054d6f6f620c6519be1fe710d4da847',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 528,
+                'timestamp': 1359044972,
+                'upload_date': '20130124',
+                'view_count': int,
+                'comment_count': int,
+            },
+        },
+        {
+            'url': 'http://www.zapiks.fr/index.php?action=playerIframe&amp;media_id=118046&amp;width=640&amp;height=360&amp;autoStart=false&amp;language=fr',
+            'only_matching': True,
+        },
+    ]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id') or video_id
+
+        webpage = self._download_webpage(url, display_id)
+
+        if not video_id:
+            video_id = self._search_regex(
+                r'data-media-id="(\d+)"', webpage, 'video id')
+
+        playlist = self._download_xml(
+            'http://www.zapiks.fr/view/index.php?action=playlist&media_id=%s&lang=en' % video_id,
+            display_id)
+
+        NS_MAP = {
+            'jwplayer': 'http://rss.jwpcdn.com/'
+        }
+
+        def ns(path):
+            return xpath_with_ns(path, NS_MAP)
+
+        item = playlist.find('./channel/item')
+
+        title = xpath_text(item, 'title', 'title') or self._og_search_title(webpage)
+        description = self._og_search_description(webpage, default=None)
+        thumbnail = xpath_text(
+            item, ns('./jwplayer:image'), 'thumbnail') or self._og_search_thumbnail(webpage, default=None)
+        duration = parse_duration(self._html_search_meta(
+            'duration', webpage, 'duration', default=None))
+        timestamp = parse_iso8601(self._html_search_meta(
+            'uploadDate', webpage, 'upload date', default=None), ' ')
+
+        view_count = int_or_none(self._search_regex(
+            r'UserPlays:(\d+)', webpage, 'view count', default=None))
+        comment_count = int_or_none(self._search_regex(
+            r'UserComments:(\d+)', webpage, 'comment count', default=None))
+
+        formats = []
+        for source in item.findall(ns('./jwplayer:source')):
+            format_id = source.attrib['label']
+            f = {
+                'url': source.attrib['file'],
+                'format_id': format_id,
+            }
+            m = re.search(r'^(?P<height>\d+)[pP]', format_id)
+            if m:
+                f['height'] = int(m.group('height'))
+            formats.append(f)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'timestamp': timestamp,
+            'view_count': view_count,
+            'comment_count': comment_count,
+            'formats': formats,
+        }

From 255fca5eea70a171530a5a0f2af143362f0211cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Feb 2015 01:39:26 +0600
Subject: [PATCH 1648/1937] [generic] Add support for Zapiks embeds (#5014)

---
 youtube_dl/extractor/generic.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8dce96a64..875e1bf05 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -547,7 +547,16 @@ class GenericIE(InfoExtractor):
                 'id': 'aanslagen-kopenhagen',
                 'title': 'Aanslagen Kopenhagen | RTL Nieuws',
             }
-        }
+        },
+        # Zapiks embed
+        {
+            'url': 'http://www.skipass.com/news/116090-bon-appetit-s5ep3-baqueira-mi-cor.html',
+            'info_dict': {
+                'id': '118046',
+                'ext': 'mp4',
+                'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !',
+            }
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -1098,6 +1107,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'Livestream')
 
+        # Look for Zapiks embed
+        mobj = re.search(
+            r'<iframe[^>]+src="(?P<url>https?://(?:www\.)?zapiks\.fr/index\.php\?.+?)"', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'), 'Zapiks')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From ea5152cae110d55b82c755c23926f077b90c071c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Feb 2015 01:42:47 +0600
Subject: [PATCH 1649/1937] [zapiks] Extend _VALID_URL (#5014)

---
 youtube_dl/extractor/zapiks.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/zapiks.py b/youtube_dl/extractor/zapiks.py
index 12810637e..22a9a57e8 100644
--- a/youtube_dl/extractor/zapiks.py
+++ b/youtube_dl/extractor/zapiks.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class ZapiksIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?zapiks\.fr/(?:(?P<display_id>.+?)\.html|index\.php\?.*\bmedia_id=(?P<id>\d+))'
+    _VALID_URL = r'https?://(?:www\.)?zapiks\.(?:fr|com)/(?:(?:[a-z]{2}/)?(?P<display_id>.+?)\.html|index\.php\?.*\bmedia_id=(?P<id>\d+))'
     _TESTS = [
         {
             'url': 'http://www.zapiks.fr/ep2s3-bon-appetit-eh-be-viva.html',
@@ -32,6 +32,14 @@ class ZapiksIE(InfoExtractor):
                 'comment_count': int,
             },
         },
+        {
+            'url': 'http://www.zapiks.com/ep3s5-bon-appetit-baqueira-m-1.html',
+            'only_matching': True,
+        },
+        {
+            'url': 'http://www.zapiks.com/nl/ep3s5-bon-appetit-baqueira-m-1.html',
+            'only_matching': True,
+        },
         {
             'url': 'http://www.zapiks.fr/index.php?action=playerIframe&amp;media_id=118046&amp;width=640&amp;height=360&amp;autoStart=false&amp;language=fr',
             'only_matching': True,

From c5181ab4101323de94bdb20850c64711c625c3ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Feb 2015 02:10:26 +0600
Subject: [PATCH 1650/1937] [gdcvault] Fix rtmp streams (Closes #5024)

---
 youtube_dl/extractor/gdcvault.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index fed968f51..05f58f1af 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -7,6 +7,7 @@ from ..compat import (
     compat_urllib_parse,
     compat_urllib_request,
 )
+from ..utils import remove_end
 
 
 class GDCVaultIE(InfoExtractor):
@@ -68,7 +69,9 @@ class GDCVaultIE(InfoExtractor):
         akami_url = xml_description.find('./metadata/akamaiHost').text
         slide_video_path = xml_description.find('./metadata/slideVideo').text
         video_formats.append({
-            'url': 'rtmp://' + akami_url + '/' + slide_video_path,
+            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % 'fms.digitallyspeaking.com/cfx/st',
+            'play_path': remove_end(slide_video_path, '.flv'),
+            'ext': 'flv',
             'format_note': 'slide deck video',
             'quality': -2,
             'preference': -2,
@@ -76,7 +79,9 @@ class GDCVaultIE(InfoExtractor):
         })
         speaker_video_path = xml_description.find('./metadata/speakerVideo').text
         video_formats.append({
-            'url': 'rtmp://' + akami_url + '/' + speaker_video_path,
+            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % 'fms.digitallyspeaking.com/cfx/st',
+            'play_path': remove_end(speaker_video_path, '.flv'),
+            'ext': 'flv',
             'format_note': 'speaker video',
             'quality': -1,
             'preference': -1,

From 314368c822428437e60bbc24af65d5415717632c Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sat, 21 Feb 2015 22:19:39 +0200
Subject: [PATCH 1651/1937] [teamcoco] Fix extraction

Also, use a single style of quotes
---
 youtube_dl/extractor/teamcoco.py | 49 ++++++++++++++++----------------
 1 file changed, 24 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index a73da1c9c..5793dbc10 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -1,8 +1,10 @@
 from __future__ import unicode_literals
 
+import base64
 import re
 
 from .common import InfoExtractor
+from ..utils import qualities
 
 
 class TeamcocoIE(InfoExtractor):
@@ -24,8 +26,8 @@ class TeamcocoIE(InfoExtractor):
             'info_dict': {
                 'id': '19705',
                 'ext': 'mp4',
-                "description": "Louis C.K. got starstruck by George W. Bush, so what? Part one.",
-                "title": "Louis C.K. Interview Pt. 1 11/3/11",
+                'description': 'Louis C.K. got starstruck by George W. Bush, so what? Part one.',
+                'title': 'Louis C.K. Interview Pt. 1 11/3/11',
                 'age_limit': 0,
             }
         }
@@ -42,42 +44,39 @@ class TeamcocoIE(InfoExtractor):
         display_id = mobj.group('display_id')
         webpage = self._download_webpage(url, display_id)
 
-        video_id = mobj.group("video_id")
+        video_id = mobj.group('video_id')
         if not video_id:
             video_id = self._html_search_regex(
                 self._VIDEO_ID_REGEXES, webpage, 'video id')
 
-        data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id
-        data = self._download_xml(
-            data_url, display_id, 'Downloading data webpage')
+        embed_url = 'http://teamcoco.com/embed/v/%s' % video_id
+        embed = self._download_webpage(
+            embed_url, video_id, 'Downloading embed page')
+
+        encoded_data = self._search_regex(
+            r'"preload"\s*:\s*"([^"]+)"', embed, 'encoded data')
+        data = self._parse_json(
+            base64.b64decode(encoded_data.encode('ascii')).decode('utf-8'), video_id)
 
-        qualities = ['500k', '480p', '1000k', '720p', '1080p']
         formats = []
-        for filed in data.findall('files/file'):
-            if filed.attrib.get('playmode') == 'all':
-                # it just duplicates one of the entries
-                break
-            file_url = filed.text
-            m_format = re.search(r'(\d+(k|p))\.mp4', file_url)
+        get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])
+        for filed in data['files']:
+            m_format = re.search(r'(\d+(k|p))\.mp4', filed['url'])
             if m_format is not None:
                 format_id = m_format.group(1)
             else:
-                format_id = filed.attrib['bitrate']
+                format_id = filed['bitrate']
             tbr = (
-                int(filed.attrib['bitrate'])
-                if filed.attrib['bitrate'].isdigit()
+                int(filed['bitrate'])
+                if filed['bitrate'].isdigit()
                 else None)
 
-            try:
-                quality = qualities.index(format_id)
-            except ValueError:
-                quality = -1
             formats.append({
-                'url': file_url,
+                'url': filed['url'],
                 'ext': 'mp4',
                 'tbr': tbr,
                 'format_id': format_id,
-                'quality': quality,
+                'quality': get_quality(format_id),
             })
 
         self._sort_formats(formats)
@@ -86,8 +85,8 @@ class TeamcocoIE(InfoExtractor):
             'id': video_id,
             'display_id': display_id,
             'formats': formats,
-            'title': self._og_search_title(webpage),
-            'thumbnail': self._og_search_thumbnail(webpage),
-            'description': self._og_search_description(webpage),
+            'title': data['title'],
+            'thumbnail': data.get('thumb', {}).get('href'),
+            'description': data.get('teaser'),
             'age_limit': self._family_friendly_search(webpage),
         }

From e086e0eb6cef80db2d4ab44572a1a5d6b6f1dee0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 21 Feb 2015 21:25:29 +0100
Subject: [PATCH 1652/1937] release 2015.02.21

---
 docs/supportedsites.md | 1 +
 youtube_dl/version.py  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index dbff5e270..5fe3e47cd 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -559,6 +559,7 @@
  - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)
  - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword)
  - **youtube:watch_later**: Youtube watch later list, ":ytwatchlater" for short (requires authentication)
+ - **Zapiks**
  - **ZDF**
  - **ZDFChannel**
  - **zingmp3:album**: mp3.zing.vn albums
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9fd0ee963..7c8b29c3b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.20'
+__version__ = '2015.02.21'

From 4d1718481755dde078678b6e55d457fc6351fcdd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Feb 2015 22:31:53 +0100
Subject: [PATCH 1653/1937] [YoutubeDL] don't set the 'requested_subtitles'
 without writesubtitles or writeautomaticsub

---
 test/test_subtitles.py  | 10 +++++-----
 youtube_dl/YoutubeDL.py |  6 ++++--
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index c04fe6f22..457f268fa 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -113,7 +113,7 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(len(subtitles), 0)
+        self.assertFalse(subtitles)
 
     def test_youtube_multiple_langs(self):
         self.url = 'QRS8MkLhQmM'
@@ -152,7 +152,7 @@ class TestDailymotionSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(len(subtitles), 0)
+        self.assertFalse(subtitles)
 
     def test_multiple_langs(self):
         self.DL.params['writesubtitles'] = True
@@ -246,7 +246,7 @@ class TestVimeoSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(len(subtitles), 0)
+        self.assertFalse(subtitles)
 
     def test_multiple_langs(self):
         self.DL.params['writesubtitles'] = True
@@ -281,7 +281,7 @@ class TestWallaSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(len(subtitles), 0)
+        self.assertFalse(subtitles)
 
 
 class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
@@ -308,7 +308,7 @@ class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(len(subtitles), 0)
+        self.assertFalse(subtitles)
 
 
 class TestLyndaSubtitles(BaseTestSubtitles):
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index f8b8fb0c1..088b111eb 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1164,8 +1164,10 @@ class YoutubeDL(object):
                 if lang not in available_subs:
                     available_subs[lang] = cap_info
 
-        if not available_subs:
-            return available_subs
+        if (not self.params.get('writesubtitles') and not
+                self.params.get('writeautomaticsub') or not
+                available_subs):
+            return None
 
         if self.params.get('allsubtitles', False):
             requested_langs = available_subs.keys()

From 03091e372f7033fa52c7961b1a99cd3790c0f60f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Feb 2015 22:33:11 +0100
Subject: [PATCH 1654/1937] [ted] Always extract the subtitles

The required info is already in the webpage
---
 youtube_dl/extractor/ted.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 1809eaae4..0c38c8f89 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -163,8 +163,6 @@ class TEDIE(InfoExtractor):
         self._sort_formats(formats)
 
         video_id = compat_str(talk_info['id'])
-        # subtitles
-        video_subtitles = self.extract_subtitles(video_id, talk_info)
 
         thumbnail = talk_info['thumb']
         if not thumbnail.startswith('http'):
@@ -175,7 +173,7 @@ class TEDIE(InfoExtractor):
             'uploader': talk_info['speaker'],
             'thumbnail': thumbnail,
             'description': self._og_search_description(webpage),
-            'subtitles': video_subtitles,
+            'subtitles': self._get_subtitles(video_id, talk_info),
             'formats': formats,
             'duration': talk_info.get('duration'),
         }
@@ -194,7 +192,6 @@ class TEDIE(InfoExtractor):
                 ]
             return sub_lang_list
         else:
-            self._downloader.report_warning('video doesn\'t have subtitles')
             return {}
 
     def _watch_info(self, url, name):

From ab84349b16b3c94775543a04855fc77005f8237e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 22 Feb 2015 11:26:27 +0100
Subject: [PATCH 1655/1937] [test/YoutubeDL] Add test for subtitles

Updated the offlinetest make target to not skip it
---
 Makefile               |  2 +-
 test/test_YoutubeDL.py | 52 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 0636fc4cb..07c90c225 100644
--- a/Makefile
+++ b/Makefile
@@ -43,7 +43,7 @@ test:
 ot: offlinetest
 
 offlinetest: codetest
-	nosetests --verbose test --exclude test_download --exclude test_age_restriction --exclude test_subtitles --exclude test_write_annotations --exclude test_youtube_lists
+	nosetests --verbose test --exclude test_download.py --exclude test_age_restriction.py --exclude test_subtitles.py --exclude test_write_annotations.py --exclude test_youtube_lists.py
 
 tar: youtube-dl.tar.gz
 
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index b1cd6a69f..e11292211 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -337,6 +337,58 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], 'G')
 
+    def test_subtitles(self):
+        def s_formats(lang, autocaption=False):
+            return [{
+                'ext': ext,
+                'url': 'http://localhost/video.%s.%s' % (lang, ext),
+                '_auto': autocaption,
+            } for ext in ['vtt', 'srt', 'ass']]
+        subtitles = dict((l, s_formats(l)) for l in ['en', 'fr', 'es'])
+        auto_captions = dict((l, s_formats(l, True)) for l in ['it', 'pt', 'es'])
+        info_dict = {
+            'id': 'test',
+            'title': 'Test',
+            'url': 'http://localhost/video.mp4',
+            'subtitles': subtitles,
+            'automatic_captions': auto_captions,
+            'extractor': 'TEST',
+        }
+
+        def get_info(params={}):
+            params.setdefault('simulate', True)
+            ydl = YDL(params)
+            ydl.report_warning = lambda *args, **kargs: None
+            return ydl.process_video_result(info_dict, download=False)
+
+        result = get_info()
+        self.assertFalse(result.get('requested_subtitles'))
+        self.assertEqual(result['subtitles'], subtitles)
+        self.assertEqual(result['automatic_captions'], auto_captions)
+
+        result = get_info({'writesubtitles': True})
+        subs = result['requested_subtitles']
+        self.assertTrue(subs)
+        self.assertEqual(set(subs.keys()), set(['en']))
+        self.assertTrue(subs['en'].get('data') is None)
+        self.assertEqual(subs['en']['ext'], 'ass')
+
+        result = get_info({'writesubtitles': True, 'subtitlesformat': 'foo/srt'})
+        subs = result['requested_subtitles']
+        self.assertEqual(subs['en']['ext'], 'srt')
+
+        result = get_info({'writesubtitles': True, 'subtitleslangs': ['es', 'fr', 'it']})
+        subs = result['requested_subtitles']
+        self.assertTrue(subs)
+        self.assertEqual(set(subs.keys()), set(['es', 'fr']))
+
+        result = get_info({'writesubtitles': True, 'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
+        subs = result['requested_subtitles']
+        self.assertTrue(subs)
+        self.assertEqual(set(subs.keys()), set(['es', 'pt']))
+        self.assertFalse(subs['es']['_auto'])
+        self.assertTrue(subs['pt']['_auto'])
+
     def test_add_extra_info(self):
         test_dict = {
             'extractor': 'Foo',

From 98c70d6fc7006c8cbbd76fb1b8661d758fc4f5d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 22 Feb 2015 11:37:27 +0100
Subject: [PATCH 1656/1937] [YoutubeDL] only add normal subtitles to the
 'requested_subtitles' field if 'writesubtitles' is True

---
 test/test_YoutubeDL.py  |  7 +++++++
 youtube_dl/YoutubeDL.py | 10 ++++++----
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index e11292211..055e42555 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -389,6 +389,13 @@ class TestFormatSelection(unittest.TestCase):
         self.assertFalse(subs['es']['_auto'])
         self.assertTrue(subs['pt']['_auto'])
 
+        result = get_info({'writeautomaticsub': True, 'subtitleslangs': ['es', 'pt']})
+        subs = result['requested_subtitles']
+        self.assertTrue(subs)
+        self.assertEqual(set(subs.keys()), set(['es', 'pt']))
+        self.assertTrue(subs['es']['_auto'])
+        self.assertTrue(subs['pt']['_auto'])
+
     def test_add_extra_info(self):
         test_dict = {
             'extractor': 'Foo',
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 088b111eb..7319323e5 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1156,11 +1156,13 @@ class YoutubeDL(object):
         info_dict.update(formats_to_download[-1])
         return info_dict
 
-    def process_subtitles(self, video_id, available_subs, available_autocaps):
+    def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
         """Select the requested subtitles and their format"""
-        if available_autocaps and self.params.get('writeautomaticsub'):
-            available_subs = available_subs.copy()
-            for lang, cap_info in available_autocaps.items():
+        available_subs = {}
+        if normal_subtitles and self.params.get('writesubtitles'):
+            available_subs.update(normal_subtitles)
+        if automatic_captions and self.params.get('writeautomaticsub'):
+            for lang, cap_info in automatic_captions.items():
                 if lang not in available_subs:
                     available_subs[lang] = cap_info
 

From b7bb76df05f53d4fc0570d07be5abcee238745e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 22 Feb 2015 11:49:27 +0100
Subject: [PATCH 1657/1937] [test/subtitles] Remove some tests

Test only with 'allsubtitles', the language selection is already tested in test_YoutubeDL.py
---
 test/test_subtitles.py | 121 +++++------------------------------------
 1 file changed, 13 insertions(+), 108 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 457f268fa..aa4e2bec4 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -57,22 +57,15 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
     url = 'QRS8MkLhQmM'
     IE = YoutubeIE
 
-    def test_youtube_subtitles(self):
-        self.DL.params['writesubtitles'] = True
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '4cd9278a35ba2305f47354ee13472260')
-
-    def test_youtube_subtitles_lang(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['subtitleslangs'] = ['it']
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['it']), '164a51f16f260476a05b50fe4c2f161d')
-
     def test_youtube_allsubtitles(self):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(len(subtitles.keys()), 13)
+        self.assertEqual(md5(subtitles['en']), '4cd9278a35ba2305f47354ee13472260')
+        self.assertEqual(md5(subtitles['it']), '164a51f16f260476a05b50fe4c2f161d')
+        for lang in ['it', 'fr', 'de']:
+            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
     def test_youtube_subtitles_sbv_format(self):
         self.DL.params['writesubtitles'] = True
@@ -86,12 +79,6 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
         subtitles = self.getSubtitles()
         self.assertEqual(md5(subtitles['en']), '3cb210999d3e021bd6c7f0ea751eab06')
 
-    def test_youtube_list_subtitles(self):
-        self.DL.expect_warning('Video doesn\'t have automatic captions')
-        self.DL.params['listsubtitles'] = True
-        info_dict = self.getInfoDict()
-        self.assertEqual(info_dict, None)
-
     def test_youtube_automatic_captions(self):
         self.url = '8YoUxe5ncPo'
         self.DL.params['writeautomaticsub'] = True
@@ -115,36 +102,20 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
         subtitles = self.getSubtitles()
         self.assertFalse(subtitles)
 
-    def test_youtube_multiple_langs(self):
-        self.url = 'QRS8MkLhQmM'
-        self.DL.params['writesubtitles'] = True
-        langs = ['it', 'fr', 'de']
-        self.DL.params['subtitleslangs'] = langs
-        subtitles = self.getSubtitles()
-        for lang in langs:
-            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
-
 
 class TestDailymotionSubtitles(BaseTestSubtitles):
     url = 'http://www.dailymotion.com/video/xczg00'
     IE = DailymotionIE
 
-    def test_subtitles(self):
-        self.DL.params['writesubtitles'] = True
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '976553874490cba125086bbfea3ff76f')
-
-    def test_subtitles_lang(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['subtitleslangs'] = ['fr']
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['fr']), '594564ec7d588942e384e920e5341792')
-
     def test_allsubtitles(self):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertTrue(len(subtitles.keys()) >= 6)
+        self.assertEqual(md5(subtitles['en']), '976553874490cba125086bbfea3ff76f')
+        self.assertEqual(md5(subtitles['fr']), '594564ec7d588942e384e920e5341792')
+        for lang in ['es', 'fr', 'de']:
+            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
     def test_nosubtitles(self):
         self.DL.expect_warning('video doesn\'t have subtitles')
@@ -154,51 +125,19 @@ class TestDailymotionSubtitles(BaseTestSubtitles):
         subtitles = self.getSubtitles()
         self.assertFalse(subtitles)
 
-    def test_multiple_langs(self):
-        self.DL.params['writesubtitles'] = True
-        langs = ['es', 'fr', 'de']
-        self.DL.params['subtitleslangs'] = langs
-        subtitles = self.getSubtitles()
-        for lang in langs:
-            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
-
 
 class TestTedSubtitles(BaseTestSubtitles):
     url = 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html'
     IE = TEDIE
 
-    def test_no_writesubtitles(self):
-        subtitles = self.getSubtitles()
-        self.assertFalse(subtitles)
-
-    def test_subtitles(self):
-        self.DL.params['writesubtitles'] = True
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '4262c1665ff928a2dada178f62cb8d14')
-
-    def test_subtitles_lang(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['subtitleslangs'] = ['fr']
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['fr']), '66a63f7f42c97a50f8c0e90bc7797bb5')
-
     def test_allsubtitles(self):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertTrue(len(subtitles.keys()) >= 28)
-
-    def test_list_subtitles(self):
-        self.DL.params['listsubtitles'] = True
-        info_dict = self.getInfoDict()
-        self.assertEqual(info_dict, None)
-
-    def test_multiple_langs(self):
-        self.DL.params['writesubtitles'] = True
-        langs = ['es', 'fr', 'de']
-        self.DL.params['subtitleslangs'] = langs
-        subtitles = self.getSubtitles()
-        for lang in langs:
+        self.assertEqual(md5(subtitles['en']), '4262c1665ff928a2dada178f62cb8d14')
+        self.assertEqual(md5(subtitles['fr']), '66a63f7f42c97a50f8c0e90bc7797bb5')
+        for lang in ['es', 'fr', 'de']:
             self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
 
@@ -206,11 +145,6 @@ class TestBlipTVSubtitles(BaseTestSubtitles):
     url = 'http://blip.tv/a/a-6603250'
     IE = BlipTVIE
 
-    def test_list_subtitles(self):
-        self.DL.params['listsubtitles'] = True
-        info_dict = self.getInfoDict()
-        self.assertEqual(info_dict, None)
-
     def test_allsubtitles(self):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
@@ -223,22 +157,13 @@ class TestVimeoSubtitles(BaseTestSubtitles):
     url = 'http://vimeo.com/76979871'
     IE = VimeoIE
 
-    def test_subtitles(self):
-        self.DL.params['writesubtitles'] = True
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '8062383cf4dec168fc40a088aa6d5888')
-
-    def test_subtitles_lang(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['subtitleslangs'] = ['fr']
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['fr']), 'b6191146a6c5d3a452244d853fde6dc8')
-
     def test_allsubtitles(self):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['de', 'en', 'es', 'fr']))
+        self.assertEqual(md5(subtitles['en']), '8062383cf4dec168fc40a088aa6d5888')
+        self.assertEqual(md5(subtitles['fr']), 'b6191146a6c5d3a452244d853fde6dc8')
 
     def test_nosubtitles(self):
         self.DL.expect_warning('video doesn\'t have subtitles')
@@ -248,25 +173,11 @@ class TestVimeoSubtitles(BaseTestSubtitles):
         subtitles = self.getSubtitles()
         self.assertFalse(subtitles)
 
-    def test_multiple_langs(self):
-        self.DL.params['writesubtitles'] = True
-        langs = ['es', 'fr', 'de']
-        self.DL.params['subtitleslangs'] = langs
-        subtitles = self.getSubtitles()
-        for lang in langs:
-            self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
-
 
 class TestWallaSubtitles(BaseTestSubtitles):
     url = 'http://vod.walla.co.il/movie/2705958/the-yes-men'
     IE = WallaIE
 
-    def test_list_subtitles(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
-        self.DL.params['listsubtitles'] = True
-        info_dict = self.getInfoDict()
-        self.assertEqual(info_dict, None)
-
     def test_allsubtitles(self):
         self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['writesubtitles'] = True
@@ -288,12 +199,6 @@ class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
     url = 'http://www.ceskatelevize.cz/ivysilani/10600540290-u6-uzasny-svet-techniky'
     IE = CeskaTelevizeIE
 
-    def test_list_subtitles(self):
-        self.DL.expect_warning('Automatic Captions not supported by this server')
-        self.DL.params['listsubtitles'] = True
-        info_dict = self.getInfoDict()
-        self.assertEqual(info_dict, None)
-
     def test_allsubtitles(self):
         self.DL.expect_warning('Automatic Captions not supported by this server')
         self.DL.params['writesubtitles'] = True

From 80970e531bd377e1952ac358e7e345cfbf23593d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 22 Feb 2015 11:52:22 +0100
Subject: [PATCH 1658/1937] [test/subtitles] Update checksum for Viki

---
 test/test_subtitles.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index aa4e2bec4..7f93f0a75 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -288,7 +288,7 @@ class TestVikiSubtitles(BaseTestSubtitles):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['en']))
-        self.assertEqual(md5(subtitles['en']), 'b0b781eeb45efd3f6398a925b259150b')
+        self.assertEqual(md5(subtitles['en']), '53cb083a5914b2d84ef1ab67b880d18a')
 
 
 class TestThePlatformSubtitles(BaseTestSubtitles):

From f311cfa23153fee51f94f14d1ab1f7f8b6a74702 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Feb 2015 19:53:32 +0600
Subject: [PATCH 1659/1937] [appletrailers] Extend _VALID_URL (Closes #5027)

---
 youtube_dl/extractor/appletrailers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 43e82847f..9c718ea66 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 
 class AppleTrailersIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/trailers/(?P<company>[^/]+)/(?P<movie>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
     _TEST = {
         "url": "http://trailers.apple.com/trailers/wb/manofsteel/",
         'info_dict': {

From 35b798230334e984977090ae03a307eaf7eedbc1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Feb 2015 19:58:39 +0600
Subject: [PATCH 1660/1937] [appletrailers] Add test (#5027)

---
 youtube_dl/extractor/appletrailers.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py
index 9c718ea66..576f03b5b 100644
--- a/youtube_dl/extractor/appletrailers.py
+++ b/youtube_dl/extractor/appletrailers.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 class AppleTrailersIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/(?:trailers|ca)/(?P<company>[^/]+)/(?P<movie>[^/]+)'
-    _TEST = {
+    _TESTS = [{
         "url": "http://trailers.apple.com/trailers/wb/manofsteel/",
         'info_dict': {
             'id': 'manofsteel',
@@ -63,7 +63,10 @@ class AppleTrailersIE(InfoExtractor):
                 },
             },
         ]
-    }
+    }, {
+        'url': 'http://trailers.apple.com/ca/metropole/autrui/',
+        'only_matching': True,
+    }]
 
     _JSON_RE = r'iTunes.playURL\((.*?)\);'
 

From c010af6f195c2e84aec7d0ddec060fcbe9c45089 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Feb 2015 23:11:33 +0600
Subject: [PATCH 1661/1937] [escapist] Make regexes more robust (Closes #5028)

---
 youtube_dl/extractor/escapist.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 6b693b3b6..b49b9869f 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -31,10 +31,10 @@ class EscapistIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         uploader_id = self._html_search_regex(
-            r"<h1 class='headline'><a href='/videos/view/(.*?)'",
+            r"<h1\s+class='headline'>\s*<a\s+href='/videos/view/(.*?)'",
             webpage, 'uploader ID', fatal=False)
         uploader = self._html_search_regex(
-            r"<h1 class='headline'>(.*?)</a>",
+            r"<h1\s+class='headline'>(.*?)</a>",
             webpage, 'uploader', fatal=False)
         description = self._html_search_meta('description', webpage)
 
@@ -42,7 +42,7 @@ class EscapistIE(InfoExtractor):
         title = raw_title.partition(' : ')[2]
 
         config_url = compat_urllib_parse.unquote(self._html_search_regex(
-            r'<param name="flashvars" value="config=([^"&]+)', webpage, 'config URL'))
+            r'<param\s+name="flashvars"\s+value="config=([^"&]+)', webpage, 'config URL'))
 
         formats = []
 

From 9fe6ef7ab249af1bd8f302d37dc6c13a2e4817f1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 03:30:10 +0600
Subject: [PATCH 1662/1937] [extractor/common] Fix preference for m3u8 quality
 selection URL

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d3f86cf4a..79f6d199b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -833,7 +833,7 @@ class InfoExtractor(object):
             'url': m3u8_url,
             'ext': ext,
             'protocol': 'm3u8',
-            'preference': -1,
+            'preference': preference - 1 if preference else -1,
             'resolution': 'multiple',
             'format_note': 'Quality selection URL',
         }]

From fcc3e6138b372e13578949dc724f456ae76dd065 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 03:32:53 +0600
Subject: [PATCH 1663/1937] [r7] Add extractor (Closes #4405, closes #5004)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/r7.py       | 88 ++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+)
 create mode 100644 youtube_dl/extractor/r7.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ef0adfd87..4d3e79de9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -364,6 +364,7 @@ from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
 from .pyvideo import PyvideoIE
 from .quickvid import QuickVidIE
+from .r7 import R7IE
 from .radiode import RadioDeIE
 from .radiobremen import RadioBremenIE
 from .radiofrance import RadioFranceIE
diff --git a/youtube_dl/extractor/r7.py b/youtube_dl/extractor/r7.py
new file mode 100644
index 000000000..976c8feec
--- /dev/null
+++ b/youtube_dl/extractor/r7.py
@@ -0,0 +1,88 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    js_to_json,
+    unescapeHTML,
+    int_or_none,
+)
+
+
+class R7IE(InfoExtractor):
+    _VALID_URL = r'''(?x)https?://
+                        (?:
+                            (?:[a-zA-Z]+)\.r7\.com(?:/[^/]+)+/idmedia/|
+                            noticias\.r7\.com(?:/[^/]+)+/[^/]+-|
+                            player\.r7\.com/video/i/
+                        )
+                        (?P<id>[\da-f]{24})
+                        '''
+    _TESTS = [{
+        'url': 'http://videos.r7.com/policiais-humilham-suspeito-a-beira-da-morte-morre-com-dignidade-/idmedia/54e7050b0cf2ff57e0279389.html',
+        'md5': '403c4e393617e8e8ddc748978ee8efde',
+        'info_dict': {
+            'id': '54e7050b0cf2ff57e0279389',
+            'ext': 'mp4',
+            'title': 'Policiais humilham suspeito à beira da morte: "Morre com dignidade"',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 98,
+            'like_count': int,
+            'view_count': int,
+        },
+    }, {
+        'url': 'http://esportes.r7.com/videos/cigano-manda-recado-aos-fas/idmedia/4e176727b51a048ee6646a1b.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://noticias.r7.com/record-news/video/representante-do-instituto-sou-da-paz-fala-sobre-fim-do-estatuto-do-desarmamento-5480fc580cf2285b117f438d/',
+        'only_matching': True,
+    }, {
+        'url': 'http://player.r7.com/video/i/54e7050b0cf2ff57e0279389?play=true&video=http://vsh.r7.com/54e7050b0cf2ff57e0279389/ER7_RE_BG_MORTE_JOVENS_570kbps_2015-02-2009f17818-cc82-4c8f-86dc-89a66934e633-ATOS_copy.mp4&linkCallback=http://videos.r7.com/policiais-humilham-suspeito-a-beira-da-morte-morre-com-dignidade-/idmedia/54e7050b0cf2ff57e0279389.html&thumbnail=http://vtb.r7.com/ER7_RE_BG_MORTE_JOVENS_570kbps_2015-02-2009f17818-cc82-4c8f-86dc-89a66934e633-thumb.jpg&idCategory=192&share=true&layout=full&full=true',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'http://player.r7.com/video/i/%s' % video_id, video_id)
+
+        item = self._parse_json(js_to_json(self._search_regex(
+            r'(?s)var\s+item\s*=\s*({.+?});', webpage, 'player')), video_id)
+
+        title = unescapeHTML(item['title'])
+        thumbnail = item.get('init', {}).get('thumbUri')
+        duration = None
+
+        statistics = item.get('statistics', {})
+        like_count = int_or_none(statistics.get('likes'))
+        view_count = int_or_none(statistics.get('views'))
+
+        formats = []
+        for format_key, format_dict in item['playlist'][0].items():
+            src = format_dict.get('src')
+            if not src:
+                continue
+            format_id = format_dict.get('format') or format_key
+            if duration is None:
+                duration = format_dict.get('duration')
+            if '.f4m' in src:
+                formats.extend(self._extract_f4m_formats(src, video_id, preference=-1))
+            elif src.endswith('.m3u8'):
+                formats.extend(self._extract_m3u8_formats(src, video_id, 'mp4', preference=-2))
+            else:
+                formats.append({
+                    'url': src,
+                    'format_id': format_id,
+                })
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'like_count': like_count,
+            'view_count': view_count,
+            'formats': formats,
+        }

From 1b40dc92eb27b2a3f299157f83bfc8e95ca42268 Mon Sep 17 00:00:00 2001
From: Duncan Keall <duncan@duncankeall.com>
Date: Mon, 23 Feb 2015 16:10:08 +1300
Subject: [PATCH 1664/1937] [airmozilla] Add new extractor

---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/airmozilla.py | 73 ++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+)
 create mode 100644 youtube_dl/extractor/airmozilla.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ef0adfd87..7e1864315 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -8,6 +8,7 @@ from .adobetv import AdobeTVIE
 from .adultswim import AdultSwimIE
 from .aftenposten import AftenpostenIE
 from .aftonbladet import AftonbladetIE
+from .airmozilla import AirMozillaIE
 from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE
 from .anitube import AnitubeIE
diff --git a/youtube_dl/extractor/airmozilla.py b/youtube_dl/extractor/airmozilla.py
new file mode 100644
index 000000000..44c20f886
--- /dev/null
+++ b/youtube_dl/extractor/airmozilla.py
@@ -0,0 +1,73 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import parse_iso8601
+
+
+class AirMozillaIE(InfoExtractor):
+    _VALID_URL = r'https?://air\.mozilla\.org/(?P<id>[0-9a-z-]+)/?'
+    _TEST = {
+        'url': 'https://air.mozilla.org/privacy-lab-a-meetup-for-privacy-minded-people-in-san-francisco/',
+        'md5': '2e3e7486ba5d180e829d453875b9b8bf',
+        'info_dict': {
+            'id': '6x4q2w',
+            'ext': 'mp4',
+            'title': 'Privacy Lab - a meetup for privacy minded people in San Francisco',
+            'thumbnail': 're:https://\w+\.cloudfront\.net/6x4q2w/poster\.jpg\?t=\d+',
+            'description': 'Brings together privacy professionals and others interested in privacy at for-profits, non-profits, and NGOs in an effort to contribute to the state of the ecosystem...',
+            'timestamp': 1422487800,
+            'upload_date': '20150128',
+            'location': 'SFO Commons',
+            'duration': 3780,
+            'view_count': int,
+            'categories': ['Main'],
+        }
+    }
+
+    _QUALITY_MAP = {
+        '360p': 0,
+        '576p': 1,
+        '640p': 2,
+        '720p': 3,
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._html_search_regex(r'//vid.ly/(.*?)/embed', webpage, 'id')
+
+        embed_script = self._download_webpage('https://vid.ly/{0}/embed'.format(video_id), video_id)
+        jwconfig = self._search_regex(r'\svar jwconfig = (\{.*?\});\s', embed_script, 'metadata')
+        metadata = self._parse_json(jwconfig, video_id)
+
+        formats = []
+        for source in metadata['playlist'][0]['sources']:
+            fmt = {
+                'url': source['file'],
+                'ext': source['type'],
+                'format_id': self._search_regex(r'&format=(.*)$', source['file'], 'video format'),
+                'resolution': source['label'],
+                'quality': self._QUALITY_MAP.get(source['label'], -1),
+            }
+            formats.append(fmt)
+        self._sort_formats(formats)
+
+        duration_match = re.search(r'Duration:(?: (?P<H>\d+) hours?)?(?: (?P<M>\d+) minutes?)?', webpage)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'formats': formats,
+            'url': self._og_search_url(webpage),
+            'display_id': display_id,
+            'thumbnail': metadata['playlist'][0]['image'],
+            'description': self._og_search_description(webpage),
+            'timestamp': parse_iso8601(self._html_search_regex(r'<time datetime="(.*?)"', webpage, 'timestamp')),
+            'location': self._html_search_regex(r'Location: (.*)', webpage, 'location', default=None),
+            'duration': int(duration_match.groupdict()['H'] or 0) * 3600 + int(duration_match.groupdict()['M'] or 0) * 60,
+            'view_count': int(self._html_search_regex(r'Views since archived: ([0-9]+)', webpage, 'view count')),
+            'categories': re.findall(r'<a href=".*?" class="channel">(.*?)</a>', webpage),
+        }

From ddc369f073fda4ddd429c2d9a104e561cefd417f Mon Sep 17 00:00:00 2001
From: "Leslie P. Polzer" <polzer@gnu.org>
Date: Mon, 23 Feb 2015 12:00:43 +0100
Subject: [PATCH 1665/1937] [chirbit] fix profile downloader regex.

---
 youtube_dl/extractor/chirbit.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/chirbit.py b/youtube_dl/extractor/chirbit.py
index 47ce94aa0..443192f43 100644
--- a/youtube_dl/extractor/chirbit.py
+++ b/youtube_dl/extractor/chirbit.py
@@ -37,7 +37,7 @@ class ChirbitIE(InfoExtractor):
         }
 
 class ChirbitProfileIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?chirbit.com/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?chirbit.com/(?P<id>[^/]+)/?$'
     _TEST = {
         'url': 'http://chirbit.com/ScarletBeauty',
         'playlist_count': 3,

From 93b5071f73738d788c878b38a57f2b6efe0da883 Mon Sep 17 00:00:00 2001
From: "Leslie P. Polzer" <polzer@gnu.org>
Date: Mon, 23 Feb 2015 12:11:19 +0100
Subject: [PATCH 1666/1937] [soundgasm] add profile IE.

---
 youtube_dl/extractor/__init__.py  |  5 ++++-
 youtube_dl/extractor/soundgasm.py | 36 +++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 94e150826..cf58f0800 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -425,7 +425,10 @@ from .soundcloud import (
     SoundcloudUserIE,
     SoundcloudPlaylistIE
 )
-from .soundgasm import SoundgasmIE
+from .soundgasm import (
+    SoundgasmIE,
+    SoundgasmProfileIE
+)
 from .southpark import (
     SouthParkIE,
     SouthparkDeIE,
diff --git a/youtube_dl/extractor/soundgasm.py b/youtube_dl/extractor/soundgasm.py
index a4f8ce6c3..e568ff18c 100644
--- a/youtube_dl/extractor/soundgasm.py
+++ b/youtube_dl/extractor/soundgasm.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import clean_html
 
 
 class SoundgasmIE(InfoExtractor):
@@ -38,3 +39,38 @@ class SoundgasmIE(InfoExtractor):
             'title': audio_title,
             'description': description
         }
+
+class SoundgasmProfileIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<id>[0-9a-zA-Z_\-]+)/?$'
+    _TEST = {
+        'url': 'http://soundgasm.net/u/ytdl',
+        'playlist_count': 1,
+        'info_dict': {
+            '_type': 'playlist',
+            'id': 'ytdl',
+            'title': 'ytdl'
+        }
+    }
+
+    def _real_extract(self, url):
+        profile_id = self._match_id(url)
+        webpage = self._download_webpage(url, profile_id)
+
+        ids = re.findall(r'''<a\s+href=".+?/u/%s/([^/]+)">''' % re.escape(profile_id), webpage)
+        ids = [clean_html(id) for id in ids]
+
+        entries = []
+        for id in ids:
+            entries.append({
+                '_type': 'url',
+                'url': ('http://soundgasm.net/u/%s/%s' % (profile_id, id))
+            })
+
+        info_dict = {
+            '_type': 'playlist',
+            'id': profile_id,
+            'title': profile_id,
+            'entries': entries
+        }
+
+        return info_dict;

From b531cfc019576b682f930bd269f68eb87cfd5abf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 23 Feb 2015 16:12:35 +0100
Subject: [PATCH 1667/1937] [YoutubeDL] remove compatiblity with the old
 subtitles system

---
 youtube_dl/YoutubeDL.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 7319323e5..70b364c9b 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1189,14 +1189,6 @@ class YoutubeDL(object):
             if formats is None:
                 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
                 continue
-            if isinstance(formats, compat_str):
-                # TODO: convert all IE with subtitles support to the new format
-                # and remove this
-                subs[lang] = {
-                    'ext': formats_preference[0],
-                    'data': formats,
-                }
-                continue
             for ext in formats_preference:
                 if ext == 'best':
                     f = formats[-1]

From a65d4e7f1458a681f250d6e2e0190644b50d6793 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 21:15:16 +0600
Subject: [PATCH 1668/1937] [chirbit] Simplify and extract profile from RSS
 (#5032)

---
 youtube_dl/extractor/__init__.py |   5 +-
 youtube_dl/extractor/chirbit.py  | 113 ++++++++++++++-----------------
 2 files changed, 53 insertions(+), 65 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index c3088fba2..40fc92cf7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -63,7 +63,10 @@ from .ccc import CCCIE
 from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .chilloutzone import ChilloutzoneIE
-from .chirbit import ChirbitIE, ChirbitProfileIE
+from .chirbit import (
+    ChirbitIE,
+    ChirbitProfileIE,
+)
 from .cinchcast import CinchcastIE
 from .clipfish import ClipfishIE
 from .cliphunter import CliphunterIE
diff --git a/youtube_dl/extractor/chirbit.py b/youtube_dl/extractor/chirbit.py
index 443192f43..124307b7c 100644
--- a/youtube_dl/extractor/chirbit.py
+++ b/youtube_dl/extractor/chirbit.py
@@ -1,97 +1,82 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import clean_html
+from ..utils import (
+    parse_duration,
+    int_or_none,
+)
 
 
 class ChirbitIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?chirb\.it/(?P<id>[^/]+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?chirb\.it/(?:(?:wp|pl)/|fb_chirbit_player\.swf\?key=)?(?P<id>[\da-zA-Z]+)'
+    _TESTS = [{
         'url': 'http://chirb.it/PrIPv5',
         'md5': '9847b0dad6ac3e074568bf2cfb197de8',
         'info_dict': {
             'id': 'PrIPv5',
-            'display_id': 'kukushtv_1423231243',
             'ext': 'mp3',
             'title': 'Фасадстрой',
-            'url': 'http://audio.chirbit.com/kukushtv_1423231243.mp3'
+            'duration': 52,
+            'view_count': int,
+            'comment_count': int,
         }
-    }
+    }, {
+        'url': 'https://chirb.it/fb_chirbit_player.swf?key=PrIPv5',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        audio_linkid = self._match_id(url)
-        webpage = self._download_webpage(url, audio_linkid)
+        audio_id = self._match_id(url)
 
-        audio_title = self._html_search_regex(r'<h2\s+itemprop="name">(.*?)</h2>', webpage, 'title')
-        audio_id = self._html_search_regex(r'\("setFile",\s+"http://audio.chirbit.com/(.*?).mp3"\)', webpage, 'audio ID')
-        audio_url = 'http://audio.chirbit.com/' + audio_id + '.mp3';
+        webpage = self._download_webpage(
+            'http://chirb.it/%s' % audio_id, audio_id)
+
+        audio_url = self._search_regex(
+            r'"setFile"\s*,\s*"([^"]+)"', webpage, 'audio url')
+
+        title = self._search_regex(
+            r'itemprop="name">([^<]+)', webpage, 'title')
+        duration = parse_duration(self._html_search_meta(
+            'duration', webpage, 'duration', fatal=False))
+        view_count = int_or_none(self._search_regex(
+            r'itemprop="playCount"\s*>(\d+)', webpage,
+            'listen count', fatal=False))
+        comment_count = int_or_none(self._search_regex(
+            r'>(\d+) Comments?:', webpage,
+            'comment count', fatal=False))
 
         return {
-            'id': audio_linkid,
-            'display_id': audio_id,
-            'title': audio_title,
-            'url': audio_url
+            'id': audio_id,
+            'url': audio_url,
+            'title': title,
+            'duration': duration,
+            'view_count': view_count,
+            'comment_count': comment_count,
         }
 
+
 class ChirbitProfileIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?chirbit.com/(?P<id>[^/]+)/?$'
+    _VALID_URL = r'https?://(?:www\.)?chirbit.com/(?:rss/)?(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://chirbit.com/ScarletBeauty',
-        'playlist_count': 3,
         'info_dict': {
-            '_type': 'playlist',
-            'title': 'ScarletBeauty',
-            'id': 'ScarletBeauty'
-        }
+            'id': 'ScarletBeauty',
+            'title': 'Chirbits by ScarletBeauty',
+        },
+        'playlist_mincount': 3,
     }
 
     def _real_extract(self, url):
         profile_id = self._match_id(url)
 
-        # Chirbit has a pretty weird "Last Page" navigation behavior.
-        # We grab the profile's oldest entry to determine when to
-        # stop fetching entries.
-        oldestpage = self._download_webpage(url + '/24599', profile_id)
-        oldest_page_entries = re.findall(
-            r'''soundFile:\s*"http://audio.chirbit.com/(.*?).mp3"''',
-            oldestpage);
-        oldestentry = clean_html(oldest_page_entries[-1]);
+        rss = self._download_xml(
+            'http://chirbit.com/rss/%s' % profile_id, profile_id)
 
-        ids = []
-        titles = []
-        n = 0
-        while True:
-            page = self._download_webpage(url + '/' + str(n), profile_id)
-            page_ids = re.findall(
-                r'''soundFile:\s*"http://audio.chirbit.com/(.*?).mp3"''',
-                page);
-            page_titles = re.findall(
-                r'''<div\s+class="chirbit_title"\s*>(.*?)</div>''',
-                page);
-            ids += page_ids
-            titles += page_titles
-            if oldestentry in page_ids:
-                break
-            n += 1
+        entries = [
+            self.url_result(audio_url.text, 'Chirbit')
+            for audio_url in rss.findall('./channel/item/link')]
 
-        entries = []
-        i = 0
-        for id in ids:
-            entries.append({
-                'id': id,
-                'title': titles[i],
-                'url': 'http://audio.chirbit.com/' + id + '.mp3'
-            });
-            i += 1
+        title = rss.find('./channel/title').text
 
-        info_dict = {
-            '_type': 'playlist',
-            'id': profile_id,
-            'title': profile_id,
-            'entries': entries
-        }
-
-        return info_dict;
+        return self.playlist_result(entries, profile_id, title)

From 3cc57f96455ce14cc5c72264a25b8d434174f7dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 21:27:24 +0600
Subject: [PATCH 1669/1937] [soundgasm:profile] Simplify

---
 youtube_dl/extractor/soundgasm.py | 30 ++++++++----------------------
 1 file changed, 8 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/soundgasm.py b/youtube_dl/extractor/soundgasm.py
index e568ff18c..e11d999f3 100644
--- a/youtube_dl/extractor/soundgasm.py
+++ b/youtube_dl/extractor/soundgasm.py
@@ -41,36 +41,22 @@ class SoundgasmIE(InfoExtractor):
         }
 
 class SoundgasmProfileIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<id>[0-9a-zA-Z_\-]+)/?$'
+    _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://soundgasm.net/u/ytdl',
-        'playlist_count': 1,
         'info_dict': {
-            '_type': 'playlist',
             'id': 'ytdl',
-            'title': 'ytdl'
-        }
+        },
+        'playlist_count': 1,
     }
 
     def _real_extract(self, url):
         profile_id = self._match_id(url)
+
         webpage = self._download_webpage(url, profile_id)
 
-        ids = re.findall(r'''<a\s+href=".+?/u/%s/([^/]+)">''' % re.escape(profile_id), webpage)
-        ids = [clean_html(id) for id in ids]
+        entries = [
+            self.url_result(audio_url, 'Soundgasm')
+            for audio_url in re.findall(r'href="([^"]+/u/%s/[^"]+)' % profile_id, webpage)]
 
-        entries = []
-        for id in ids:
-            entries.append({
-                '_type': 'url',
-                'url': ('http://soundgasm.net/u/%s/%s' % (profile_id, id))
-            })
-
-        info_dict = {
-            '_type': 'playlist',
-            'id': profile_id,
-            'title': profile_id,
-            'entries': entries
-        }
-
-        return info_dict;
+        return self.playlist_result(entries, profile_id)

From 80af2b73ab0b51e4416500301948caa71ec39cb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 21:27:56 +0600
Subject: [PATCH 1670/1937] [soundgasm] Clarify extractors' IE_NAMEs

---
 youtube_dl/extractor/soundgasm.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/soundgasm.py b/youtube_dl/extractor/soundgasm.py
index e11d999f3..26e96a120 100644
--- a/youtube_dl/extractor/soundgasm.py
+++ b/youtube_dl/extractor/soundgasm.py
@@ -8,6 +8,7 @@ from ..utils import clean_html
 
 
 class SoundgasmIE(InfoExtractor):
+    IE_NAME = 'soundgasm'
     _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<user>[0-9a-zA-Z_\-]+)/(?P<title>[0-9a-zA-Z_\-]+)'
     _TEST = {
         'url': 'http://soundgasm.net/u/ytdl/Piano-sample',
@@ -41,6 +42,7 @@ class SoundgasmIE(InfoExtractor):
         }
 
 class SoundgasmProfileIE(InfoExtractor):
+    IE_NAME = 'soundgasm:profile'
     _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://soundgasm.net/u/ytdl',

From 04e8c1108023d9fe5c466d16f988a469e04f326e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 21:28:14 +0600
Subject: [PATCH 1671/1937] [chirbit] Clarify extractors' IE_NAMEs

---
 youtube_dl/extractor/chirbit.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/chirbit.py b/youtube_dl/extractor/chirbit.py
index 124307b7c..b1eeaf101 100644
--- a/youtube_dl/extractor/chirbit.py
+++ b/youtube_dl/extractor/chirbit.py
@@ -9,6 +9,7 @@ from ..utils import (
 
 
 class ChirbitIE(InfoExtractor):
+    IE_NAME = 'chirbit'
     _VALID_URL = r'https?://(?:www\.)?chirb\.it/(?:(?:wp|pl)/|fb_chirbit_player\.swf\?key=)?(?P<id>[\da-zA-Z]+)'
     _TESTS = [{
         'url': 'http://chirb.it/PrIPv5',
@@ -57,6 +58,7 @@ class ChirbitIE(InfoExtractor):
 
 
 class ChirbitProfileIE(InfoExtractor):
+    IE_NAME = 'chirbit:profile'
     _VALID_URL = r'https?://(?:www\.)?chirbit.com/(?:rss/)?(?P<id>[^/]+)'
     _TEST = {
         'url': 'http://chirbit.com/ScarletBeauty',

From 409693984f0acb8fbbf006c0d7965bc138211ac6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 21:30:30 +0600
Subject: [PATCH 1672/1937] [soundgasm:profile] Fix _VALID_URL

---
 youtube_dl/extractor/soundgasm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundgasm.py b/youtube_dl/extractor/soundgasm.py
index 26e96a120..9e992c9b7 100644
--- a/youtube_dl/extractor/soundgasm.py
+++ b/youtube_dl/extractor/soundgasm.py
@@ -43,7 +43,7 @@ class SoundgasmIE(InfoExtractor):
 
 class SoundgasmProfileIE(InfoExtractor):
     IE_NAME = 'soundgasm:profile'
-    _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<id>[^/]+)/?(?:\#.*)?$'
     _TEST = {
         'url': 'http://soundgasm.net/u/ytdl',
         'info_dict': {

From 09c200acf258de115caeda210741a59f2b971b65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 21:31:57 +0600
Subject: [PATCH 1673/1937] Credit @skypher for chirbit and soundgasm:profile
 (#5032)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 47f12a9ee..bdd2a15dc 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -111,3 +111,4 @@ Paul Hartmann
 Frans de Jonge
 Robin de Rooij
 Ryan Schmidt
+Leslie P. Polzer

From 3438e7acd27d89d83d41e722d21d7660dbad7eea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 21:40:50 +0600
Subject: [PATCH 1674/1937] [soundgasm] Remove unused import

---
 youtube_dl/extractor/soundgasm.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/soundgasm.py b/youtube_dl/extractor/soundgasm.py
index 9e992c9b7..1c48478a6 100644
--- a/youtube_dl/extractor/soundgasm.py
+++ b/youtube_dl/extractor/soundgasm.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import clean_html
 
 
 class SoundgasmIE(InfoExtractor):

From bd61a9e770506283f82e2ddf9e53b587169c2f04 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 23 Feb 2015 16:47:19 +0100
Subject: [PATCH 1675/1937] release 2015.02.23

---
 docs/supportedsites.md | 6 +++++-
 youtube_dl/version.py  | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 5fe3e47cd..9f70db80a 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -72,6 +72,8 @@
  - **CeskaTelevize**
  - **channel9**: Channel 9
  - **Chilloutzone**
+ - **chirbit**
+ - **chirbit:profile**
  - **Cinchcast**
  - **Cinemassacre**
  - **clipfish**
@@ -330,6 +332,7 @@
  - **prosiebensat1**: ProSiebenSat.1 Digital
  - **Pyvideo**
  - **QuickVid**
+ - **R7**
  - **radio.de**
  - **radiobremen**
  - **radiofrance**
@@ -385,7 +388,8 @@
  - **soundcloud:playlist**
  - **soundcloud:set**
  - **soundcloud:user**
- - **Soundgasm**
+ - **soundgasm**
+ - **soundgasm:profile**
  - **southpark.cc.com**
  - **southpark.de**
  - **Space**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7c8b29c3b..17317b29c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.21'
+__version__ = '2015.02.23'

From 5bca2424bc2dfb15b5394a51fa5befd7148edc41 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 23 Feb 2015 16:51:09 +0100
Subject: [PATCH 1676/1937] [gdcvault] Remove dead code

---
 youtube_dl/extractor/gdcvault.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 05f58f1af..e5011a5dc 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -66,7 +66,6 @@ class GDCVaultIE(InfoExtractor):
 
     def _parse_flv(self, xml_description):
         video_formats = []
-        akami_url = xml_description.find('./metadata/akamaiHost').text
         slide_video_path = xml_description.find('./metadata/slideVideo').text
         video_formats.append({
             'url': 'rtmp://%s/ondemand?ovpfv=1.1' % 'fms.digitallyspeaking.com/cfx/st',

From 591ab1dff913d7ff88f30487c54c1e9c5d44d0cb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 23 Feb 2015 16:51:21 +0100
Subject: [PATCH 1677/1937] [soundgasm] PEP8

---
 youtube_dl/extractor/soundgasm.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/soundgasm.py b/youtube_dl/extractor/soundgasm.py
index 1c48478a6..3a4ddf57e 100644
--- a/youtube_dl/extractor/soundgasm.py
+++ b/youtube_dl/extractor/soundgasm.py
@@ -40,6 +40,7 @@ class SoundgasmIE(InfoExtractor):
             'description': description
         }
 
+
 class SoundgasmProfileIE(InfoExtractor):
     IE_NAME = 'soundgasm:profile'
     _VALID_URL = r'https?://(?:www\.)?soundgasm\.net/u/(?P<id>[^/]+)/?(?:\#.*)?$'

From 4432db35d9ddd0e6777df6c596d8637514ba0b56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Feb 2015 21:59:11 +0600
Subject: [PATCH 1678/1937] [gdcvault] Restore akamai host for rtmp videos

---
 youtube_dl/extractor/gdcvault.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index e5011a5dc..f7b467b0a 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -66,9 +66,10 @@ class GDCVaultIE(InfoExtractor):
 
     def _parse_flv(self, xml_description):
         video_formats = []
+        akamai_url = xml_description.find('./metadata/akamaiHost').text
         slide_video_path = xml_description.find('./metadata/slideVideo').text
         video_formats.append({
-            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % 'fms.digitallyspeaking.com/cfx/st',
+            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
             'play_path': remove_end(slide_video_path, '.flv'),
             'ext': 'flv',
             'format_note': 'slide deck video',
@@ -78,7 +79,7 @@ class GDCVaultIE(InfoExtractor):
         })
         speaker_video_path = xml_description.find('./metadata/speakerVideo').text
         video_formats.append({
-            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % 'fms.digitallyspeaking.com/cfx/st',
+            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
             'play_path': remove_end(speaker_video_path, '.flv'),
             'ext': 'flv',
             'format_note': 'speaker video',

From 459e5fbd5fa61064076534c4d6d8a1d010acb1b3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 23 Feb 2015 18:17:39 +0100
Subject: [PATCH 1679/1937] release 2015.02.23.1

---
 README.md             | 4 ++--
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 8ea31d605..699401b49 100644
--- a/README.md
+++ b/README.md
@@ -351,8 +351,8 @@ which means you can modify it, redistribute it or use it however you like.
     --all-subs                       downloads all the available subtitles of
                                      the video
     --list-subs                      lists all available subtitles for the video
-    --sub-format FORMAT              subtitle format (default=srt) ([sbv/vtt]
-                                     youtube only)
+    --sub-format FORMAT              subtitle format, accepts formats
+                                     preference, for example: "ass/srt/best"
     --sub-lang LANGS                 languages of the subtitles to download
                                      (optional) separated by commas, use IETF
                                      language tags like 'en,pt'
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 17317b29c..1852d834b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.23'
+__version__ = '2015.02.23.1'

From ffdf972b9115d6d8c86439bf0828e945823bdcf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 23 Feb 2015 18:54:15 +0100
Subject: [PATCH 1680/1937] [facebook] Extract all the formats (closes #5037)

---
 youtube_dl/extractor/facebook.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 1ad4e77a8..f0e575320 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -126,11 +126,17 @@ class FacebookIE(InfoExtractor):
         params_raw = compat_urllib_parse.unquote(data['params'])
         params = json.loads(params_raw)
         video_data = params['video_data'][0]
-        video_url = video_data.get('hd_src')
-        if not video_url:
-            video_url = video_data['sd_src']
-        if not video_url:
-            raise ExtractorError('Cannot find video URL')
+
+        formats = []
+        for quality in ['sd', 'hd']:
+            src = video_data.get('%s_src' % quality)
+            if src is not None:
+                formats.append({
+                    'format_id': quality,
+                    'url': src,
+                })
+        if not formats:
+            raise ExtractorError('Cannot find video formats')
 
         video_title = self._html_search_regex(
             r'<h2 class="uiHeaderTitle">([^<]*)</h2>', webpage, 'title',
@@ -146,7 +152,7 @@ class FacebookIE(InfoExtractor):
         return {
             'id': video_id,
             'title': video_title,
-            'url': video_url,
+            'formats': formats,
             'duration': int_or_none(video_data.get('video_duration')),
             'thumbnail': video_data.get('thumbnail_src'),
         }

From 3037b91e05e68a4ab3420cbbdb23cfb0739011d3 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 23 Feb 2015 20:45:36 +0200
Subject: [PATCH 1681/1937] [laola1tv] Improve extraction and update test case
 (#3742)

---
 youtube_dl/extractor/laola1tv.py | 47 +++++++++++++++++++++-----------
 1 file changed, 31 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index 2fd3b4699..135421406 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -1,23 +1,26 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import random
 import re
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    ExtractorError,
+    xpath_text,
+)
 
 
 class Laola1TvIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?laola1\.tv/(?P<lang>[a-z]+)-(?P<portal>[a-z]+)/.*?/(?P<id>[0-9]+)\.html'
     _TEST = {
-        'url': 'http://www.laola1.tv/de-de/live/bwf-bitburger-open-grand-prix-gold-court-1/250019.html',
+        'url': 'http://www.laola1.tv/de-de/video/straubing-tigers-koelner-haie/227883.html',
         'info_dict': {
-            'id': '250019',
+            'id': '227883',
             'ext': 'mp4',
-            'title': 'Bitburger Open Grand Prix Gold - Court 1',
-            'categories': ['Badminton'],
-            'uploader': 'BWF - Badminton World Federation',
-            'is_live': True,
+            'title': 'Straubing Tigers - Kölner Haie',
+            'categories': ['Eishockey'],
+            'is_live': False,
         },
         'params': {
             'skip_download': True,
@@ -43,15 +46,26 @@ class Laola1TvIE(InfoExtractor):
             r'flashvars\.([_a-zA-Z0-9]+)\s*=\s*"([^"]*)";', iframe)
         flashvars = dict((m[0], m[1]) for m in flashvars_m)
 
+        partner_id = self._search_regex(
+            'partnerid\s*:\s*"([^"]+)"', iframe, 'partner id')
+
         xml_url = ('http://www.laola1.tv/server/hd_video.php?' +
-                   'play=%s&partner=1&portal=%s&v5ident=&lang=%s' % (
-                       video_id, portal, lang))
+                   'play=%s&partner=%s&portal=%s&v5ident=&lang=%s' % (
+                       video_id, partner_id, portal, lang))
         hd_doc = self._download_xml(xml_url, video_id)
 
-        title = hd_doc.find('.//video/title').text
-        flash_url = hd_doc.find('.//video/url').text
-        categories = hd_doc.find('.//video/meta_sports').text.split(',')
-        uploader = hd_doc.find('.//video/meta_organistation').text
+        title = xpath_text(hd_doc, './/video/title', fatal=True)
+        flash_url = xpath_text(hd_doc, './/video/url', fatal=True)
+        uploader = xpath_text(hd_doc, './/video/meta_organistation')
+
+        is_live = xpath_text(hd_doc, './/video/islive') == 'true'
+        if is_live:
+            raise ExtractorError(
+                'Live streams are not supported by the f4m downloader.')
+
+        categories = xpath_text(hd_doc, './/video/meta_sports')
+        if categories:
+            categories = categories.split(',')
 
         ident = random.randint(10000000, 99999999)
         token_url = '%s&ident=%s&klub=0&unikey=0&timestamp=%s&auth=%s' % (
@@ -60,15 +74,16 @@ class Laola1TvIE(InfoExtractor):
         token_doc = self._download_xml(
             token_url, video_id, note='Downloading token')
         token_attrib = token_doc.find('.//token').attrib
-        if token_attrib.get('auth') == 'blocked':
-            raise ExtractorError('Token error: ' % token_attrib.get('comment'))
+        if token_attrib.get('auth') in ('blocked', 'restricted'):
+            raise ExtractorError(
+                'Token error: %s' % token_attrib.get('comment'), expected=True)
 
         video_url = '%s?hdnea=%s&hdcore=3.2.0' % (
             token_attrib['url'], token_attrib['auth'])
 
         return {
             'id': video_id,
-            'is_live': True,
+            'is_live': is_live,
             'title': title,
             'url': video_url,
             'uploader': uploader,

From 1fbaa0a5210976a2a8fc0c20207708c35621416a Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Mon, 23 Feb 2015 20:51:30 +0200
Subject: [PATCH 1682/1937] [laola1tv] Use raw strings for regular expressions

Oops
---
 youtube_dl/extractor/laola1tv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index 135421406..e8ca49fd1 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -47,7 +47,7 @@ class Laola1TvIE(InfoExtractor):
         flashvars = dict((m[0], m[1]) for m in flashvars_m)
 
         partner_id = self._search_regex(
-            'partnerid\s*:\s*"([^"]+)"', iframe, 'partner id')
+            r'partnerid\s*:\s*"([^"]+)"', iframe, 'partner id')
 
         xml_url = ('http://www.laola1.tv/server/hd_video.php?' +
                    'play=%s&partner=%s&portal=%s&v5ident=&lang=%s' % (

From 99209c2916753799e9c68e8d466c5253113f25bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 24 Feb 2015 01:35:15 +0600
Subject: [PATCH 1683/1937] [youtube] Extract UL playlists as mixes (Closes
 #5040)

---
 youtube_dl/extractor/youtube.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 22db896b1..3690f8021 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1153,13 +1153,13 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
                         |  p/
                         )
                         (
-                            (?:PL|LL|EC|UU|FL|RD)?[0-9A-Za-z-_]{10,}
+                            (?:PL|LL|EC|UU|FL|RD|UL)?[0-9A-Za-z-_]{10,}
                             # Top tracks, they can also include dots
                             |(?:MC)[\w\.]*
                         )
                         .*
                      |
-                        ((?:PL|LL|EC|UU|FL|RD)[0-9A-Za-z-_]{10,})
+                        ((?:PL|LL|EC|UU|FL|RD|UL)[0-9A-Za-z-_]{10,})
                      )"""
     _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
     _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)'
@@ -1244,7 +1244,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
             for vid_id in ids]
 
     def _extract_mix(self, playlist_id):
-        # The mixes are generated from a a single video
+        # The mixes are generated from a single video
         # the id of the playlist is just 'RD' + video_id
         url = 'https://youtube.com/watch?v=%s&list=%s' % (playlist_id[-11:], playlist_id)
         webpage = self._download_webpage(
@@ -1280,7 +1280,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
             else:
                 self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
 
-        if playlist_id.startswith('RD'):
+        if playlist_id.startswith('RD') or playlist_id.startswith('UL'):
             # Mixes require a custom extraction process
             return self._extract_mix(playlist_id)
 

From 25ac63ed71bdc2a82842a593db9a150a0b8b7a6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 23 Feb 2015 21:52:07 +0100
Subject: [PATCH 1684/1937] [rtve] Extract subtitles

---
 test/test_subtitles.py       | 15 +++++++++++++++
 youtube_dl/extractor/rtve.py | 12 ++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 7f93f0a75..3f2d8a2ba 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -25,6 +25,7 @@ from youtube_dl.extractor import (
     RaiIE,
     VikiIE,
     ThePlatformIE,
+    RTVEALaCartaIE,
 )
 
 
@@ -305,5 +306,19 @@ class TestThePlatformSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['en']), '97e7670cbae3c4d26ae8bcc7fdd78d4b')
 
 
+class TestRtveSubtitles(BaseTestSubtitles):
+    url = 'http://www.rtve.es/alacarta/videos/los-misterios-de-laura/misterios-laura-capitulo-32-misterio-del-numero-17-2-parte/2428621/'
+    IE = RTVEALaCartaIE
+
+    def test_allsubtitles(self):
+        print('Skipping, only available from Spain')
+        return
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['es']))
+        self.assertEqual(md5(subtitles['es']), '69e70cae2d40574fb7316f31d6eb7fca')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index e60f85b5b..27cd34b7d 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -102,14 +102,26 @@ class RTVEALaCartaIE(InfoExtractor):
             video_url = compat_urlparse.urljoin(
                 'http://mvod1.akcdn.rtve.es/', video_path)
 
+        subtitles = None
+        if info.get('sbtFile') is not None:
+            subtitles = self.extract_subtitles(video_id, info['sbtFile'])
+
         return {
             'id': video_id,
             'title': info['title'],
             'url': video_url,
             'thumbnail': info.get('image'),
             'page_url': url,
+            'subtitles': subtitles,
         }
 
+    def _get_subtitles(self, video_id, sub_file):
+        subs = self._download_json(
+            sub_file + '.json', video_id,
+            'Downloading subtitles info')['page']['items']
+        return dict((s['lang'], [{'ext': 'vtt', 'url': s['src']}])
+            for s in subs)
+
 
 class RTVELiveIE(InfoExtractor):
     IE_NAME = 'rtve.es:live'

From ec5913b5cd92dfb8607ec535e02b04bdc09ff804 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Feb 2015 11:08:00 +0100
Subject: [PATCH 1685/1937] [bloomberg] Modernize

---
 youtube_dl/extractor/bloomberg.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py
index c51a97ce4..4a88ccd13 100644
--- a/youtube_dl/extractor/bloomberg.py
+++ b/youtube_dl/extractor/bloomberg.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 
 class BloombergIE(InfoExtractor):
-    _VALID_URL = r'https?://www\.bloomberg\.com/video/(?P<name>.+?)\.html'
+    _VALID_URL = r'https?://www\.bloomberg\.com/video/(?P<id>.+?)\.html'
 
     _TEST = {
         'url': 'http://www.bloomberg.com/video/shah-s-presentation-on-foreign-exchange-strategies-qurhIVlJSB6hzkVi229d8g.html',
@@ -20,9 +20,9 @@ class BloombergIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        name = mobj.group('name')
+        name = self._match_id(url)
         webpage = self._download_webpage(url, name)
+
         f4m_url = self._search_regex(
             r'<source src="(https?://[^"]+\.f4m.*?)"', webpage,
             'f4m url')

From b665ba6aa6551243aa1a5b707ee7034be356f1bb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Feb 2015 11:24:26 +0100
Subject: [PATCH 1686/1937] release 2015.02.24

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1852d834b..589f38834 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.23.1'
+__version__ = '2015.02.24'

From 9c665ab72e5fc99989800109cdada5acc3af56c5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Feb 2015 11:37:27 +0100
Subject: [PATCH 1687/1937] [rtve] PEP8

---
 youtube_dl/extractor/rtve.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index 27cd34b7d..c0fd23ff1 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -119,7 +119,8 @@ class RTVEALaCartaIE(InfoExtractor):
         subs = self._download_json(
             sub_file + '.json', video_id,
             'Downloading subtitles info')['page']['items']
-        return dict((s['lang'], [{'ext': 'vtt', 'url': s['src']}])
+        return dict(
+            (s['lang'], [{'ext': 'vtt', 'url': s['src']}])
             for s in subs)
 
 

From 5a42414b9c4718f83f28fbc0e5a4a01ab67f23f6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Feb 2015 11:38:01 +0100
Subject: [PATCH 1688/1937] [utils] Prevent hyphen at beginning of filename
 (Fixes #5035)

---
 test/test_utils.py  | 2 ++
 youtube_dl/utils.py | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index c7373af1e..2f8996d7b 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -85,6 +85,8 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(
             sanitize_filename('New World record at 0:12:34'),
             'New World record at 0_12_34')
+        self.assertEqual(sanitize_filename('--gasdgf'), '_-gasdgf')
+        self.assertEqual(sanitize_filename('--gasdgf', is_id=True), '--gasdgf')
 
         forbidden = '"\0\\/'
         for fc in forbidden:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 475fad3c9..e2631dccd 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -304,6 +304,8 @@ def sanitize_filename(s, restricted=False, is_id=False):
         # Common case of "Foreign band name - English song title"
         if restricted and result.startswith('-_'):
             result = result[2:]
+        if result.startswith('-'):
+            result = '_' + result[len('-'):]
         if not result:
             result = '_'
     return result

From db8e13ef714544574691f3dd4255dd0f12c1cf77 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Feb 2015 11:38:21 +0100
Subject: [PATCH 1689/1937] release 2015.02.24.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 589f38834..a420860ed 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.24'
+__version__ = '2015.02.24.1'

From 54233c9080c1956f53802988fd8d5328cb38b7d7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Feb 2015 16:33:07 +0100
Subject: [PATCH 1690/1937] [escapist] Support JavaScript player (Fixes #5034)

---
 youtube_dl/extractor/escapist.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index b49b9869f..51ffec7ee 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -42,7 +42,14 @@ class EscapistIE(InfoExtractor):
         title = raw_title.partition(' : ')[2]
 
         config_url = compat_urllib_parse.unquote(self._html_search_regex(
-            r'<param\s+name="flashvars"\s+value="config=([^"&]+)', webpage, 'config URL'))
+            r'''(?x)
+            (?:
+                <param\s+name="flashvars"\s+value="config=|
+                flashvars=&quot;config=
+            )
+            ([^"&]+)
+            ''',
+            webpage, 'config URL'))
 
         formats = []
 

From 4f3b21e1c738a7dacd514eb59242da43e81b5ae1 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Feb 2015 16:34:42 +0100
Subject: [PATCH 1691/1937] release 2015.02.24.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a420860ed..d23c6ae3d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.24.1'
+__version__ = '2015.02.24.2'

From 7f09a662a0db2ad0a338645e4f2dbd43056a6fa6 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 24 Feb 2015 23:55:57 +0800
Subject: [PATCH 1692/1937] [Letv] Add new extractor. Single video only

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/letv.py     | 120 +++++++++++++++++++++++++++++++
 2 files changed, 121 insertions(+)
 create mode 100644 youtube_dl/extractor/letv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 40fc92cf7..829ab18a9 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -237,6 +237,7 @@ from .krasview import KrasViewIE
 from .ku6 import Ku6IE
 from .la7 import LA7IE
 from .laola1tv import Laola1TvIE
+from .letv import LetvIE
 from .lifenews import LifeNewsIE
 from .liveleak import LiveLeakIE
 from .livestream import (
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
new file mode 100644
index 000000000..3eb20678e
--- /dev/null
+++ b/youtube_dl/extractor/letv.py
@@ -0,0 +1,120 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import os.path
+import time
+import datetime
+
+from .common import InfoExtractor
+from ..compat import (compat_urlparse, compat_urllib_parse)
+from ..utils import (ExtractorError, parse_iso8601)
+
+
+class LetvIE(InfoExtractor):
+    _VALID_URL = r'http://www.letv.com/ptv/vplay/(?P<id>\d+).html'
+
+    _TESTS = [{
+        'url': 'http://www.letv.com/ptv/vplay/22005890.html',
+        'md5': 'cab23bd68d5a8db9be31c9a222c1e8df',
+        'info_dict': {
+            'id': '22005890',
+            'ext': 'mp4',
+            'title': '第87届奥斯卡颁奖礼完美落幕 《鸟人》成最大赢家',
+            'timestamp': 1424747397,
+            'upload_date': '20150224',
+        }
+    }, {
+        'url': 'http://www.letv.com/ptv/vplay/1118082.html',
+        'info_dict': {
+            'id': '1118082',
+            'ext': 'mp4',
+        }
+    }]
+
+    @staticmethod
+    def urshift(val, n):
+        return val >> n if val >= 0 else (val + 0x100000000) >> n
+
+    # ror() and calcTimeKey() are reversed from a embedded swf file in KLetvPlayer.swf
+    def ror(self, param1, param2):
+        _loc3_ = 0
+        while _loc3_ < param2:
+            param1 = self.urshift(param1, 1) + ((param1 & 1) << 31)
+            _loc3_ += 1
+        return param1
+
+    def calcTimeKey(self, param1):
+        _loc2_ = 773625421
+        _loc3_ = self.ror(param1, _loc2_ % 13)
+        _loc3_ = _loc3_ ^ _loc2_
+        _loc3_ = self.ror(_loc3_, _loc2_ % 17)
+        return _loc3_
+
+    def _real_extract(self, url):
+        media_id = self._match_id(url)
+        page = self._download_webpage(url, media_id)
+        params = {
+            'id': media_id,
+            'platid': 1,
+            'splatid': 101,
+            'format': 1,
+            'tkey': self.calcTimeKey(int(time.time())),
+            'domain': 'www.letv.com'
+        }
+        play_json = self._download_json(
+            'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params),
+            media_id, 'playJson data')
+
+        # Check for errors
+        playstatus = play_json['playstatus']
+        if playstatus['status'] == 0:
+            flag = playstatus['flag']
+            if flag == 1:
+                msg = 'Country %s auth error' % playstatus['country']
+            else:
+                msg = 'Generic error. flag = %d' % flag
+            raise ExtractorError(msg, expected=True)
+
+        playurl = play_json['playurl']
+
+        formats = ['350', '1000', '1300', '720p', '1080p']
+        dispatch = playurl['dispatch']
+
+        urls = []
+        for format_id in formats:
+            if format_id in dispatch:
+                media_url = playurl['domain'][0] + dispatch[format_id][0]
+
+                # Mimic what flvxz.com do
+                url_parts = list(compat_urlparse.urlparse(media_url))
+                qs = dict(compat_urlparse.parse_qs(url_parts[4]))
+                qs.update({
+                    'platid': '14',
+                    'splatid': '1401',
+                    'tss': 'no',
+                    'retry': 1
+                })
+                url_parts[4] = compat_urllib_parse.urlencode(qs)
+                media_url = compat_urlparse.urlunparse(url_parts)
+
+                url_info_dict = {
+                    'url': media_url,
+                    'ext': os.path.splitext(dispatch[format_id][1])[1][1:]
+                }
+
+                if format_id[-1:] == 'p':
+                    url_info_dict['height'] = format_id[:-1]
+
+                urls.append(url_info_dict)
+
+        publish_time = parse_iso8601(self._html_search_regex(
+            r'发布时间&nbsp;([^<>]+) ', page, 'publish time', fatal=False),
+            delimiter=' ', timezone=datetime.timedelta(hours=8))
+
+        return {
+            'id': media_id,
+            'formats': urls,
+            'title': playurl['title'],
+            'thumbnail': playurl['pic'],
+            'timestamp': publish_time,
+        }

From df4bd0d53ff4baff6ce25ad04a1e87f37777c3ff Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Feb 2015 17:25:02 +0100
Subject: [PATCH 1693/1937] [options] Add --yes-playlist as inverse of
 --no-playlist (Fixes #5051)

---
 youtube_dl/options.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 5c2d153b1..886ce9613 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -272,6 +272,10 @@ def parseOpts(overrideArguments=None):
         '--no-playlist',
         action='store_true', dest='noplaylist', default=False,
         help='If the URL refers to a video and a playlist, download only the video.')
+    selection.add_option(
+        '--yes-playlist',
+        action='store_false', dest='noplaylist', default=False,
+        help='If the URL refers to a video and a playlist, download the playlist.')
     selection.add_option(
         '--age-limit',
         metavar='YEARS', dest='age_limit', default=None, type=int,

From 41b264e77cd357444b632a132ea11ff7ddc3de1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 24 Feb 2015 23:06:44 +0600
Subject: [PATCH 1694/1937] [nrktv] Workaround subtitles conversion issues on
 python 2.6 (Closes #5036)

---
 youtube_dl/extractor/nrk.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 46f493cfc..1e4cfa2e7 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     float_or_none,
@@ -158,7 +159,9 @@ class NRKTVIE(InfoExtractor):
     def _get_subtitles(self, subtitlesurl, video_id, baseurl):
         url = "%s%s" % (baseurl, subtitlesurl)
         self._debug_print('%s: Subtitle url: %s' % (video_id, url))
-        captions = self._download_xml(url, video_id, 'Downloading subtitles')
+        captions = self._download_xml(
+            url, video_id, 'Downloading subtitles',
+            transform_source=lambda s: s.replace(r'<br />', '\r\n'))
         lang = captions.get('lang', 'no')
         ps = captions.findall('./{0}body/{0}div/{0}p'.format('{http://www.w3.org/ns/ttml}'))
         srt = ''
@@ -167,8 +170,7 @@ class NRKTVIE(InfoExtractor):
             duration = parse_duration(p.get('dur'))
             starttime = self._seconds2str(begin)
             endtime = self._seconds2str(begin + duration)
-            text = '\n'.join(p.itertext())
-            srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (str(pos), starttime, endtime, text)
+            srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (compat_str(pos), starttime, endtime, p.text)
         return {lang: [
             {'ext': 'ttml', 'url': url},
             {'ext': 'srt', 'data': srt},

From 570311610e33eb67d5a65a86705d54daaee17361 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 25 Feb 2015 00:45:11 +0800
Subject: [PATCH 1695/1937] [Letv] Add playlist support

---
 youtube_dl/extractor/__init__.py |  6 +++-
 youtube_dl/extractor/letv.py     | 54 +++++++++++++++++++++++++++++++-
 2 files changed, 58 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 829ab18a9..7b7d41adf 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -237,7 +237,11 @@ from .krasview import KrasViewIE
 from .ku6 import Ku6IE
 from .la7 import LA7IE
 from .laola1tv import Laola1TvIE
-from .letv import LetvIE
+from .letv import (
+    LetvIE,
+    LetvTvIE,
+    LetvPlaylistIE
+)
 from .lifenews import LifeNewsIE
 from .liveleak import LiveLeakIE
 from .livestream import (
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 3eb20678e..d7d315634 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import os.path
+import re
 import time
 import datetime
 
@@ -11,7 +12,7 @@ from ..utils import (ExtractorError, parse_iso8601)
 
 
 class LetvIE(InfoExtractor):
-    _VALID_URL = r'http://www.letv.com/ptv/vplay/(?P<id>\d+).html'
+    _VALID_URL = r'http://www\.letv\.com/ptv/vplay/(?P<id>\d+).html'
 
     _TESTS = [{
         'url': 'http://www.letv.com/ptv/vplay/22005890.html',
@@ -118,3 +119,54 @@ class LetvIE(InfoExtractor):
             'thumbnail': playurl['pic'],
             'timestamp': publish_time,
         }
+
+
+class LetvTvIE(InfoExtractor):
+    _VALID_URL = r'http://www.letv.com/tv/(?P<id>\d+).html'
+    _TESTS = [{
+        'url': 'http://www.letv.com/tv/46177.html',
+        'info_dict': {
+            'id': '46177',
+            'title': '美人天下',
+            'description': 'md5:395666ff41b44080396e59570dbac01c'
+        },
+        'playlist_count': 35
+    }]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        page = self._download_webpage(url, playlist_id)
+
+        media_urls = list(set(re.findall(
+            r'http://www.letv.com/ptv/vplay/\d+.html', page)))
+        entries = [self.url_result(media_url, ie='Letv')
+                   for media_url in media_urls]
+
+        title = self._html_search_meta('keywords', page, fatal=False).split('，')[0]
+        description = self._html_search_meta('description', page, fatal=False)
+
+        return self.playlist_result(entries, playlist_id, playlist_title=title,
+                                    playlist_description=description)
+
+
+class LetvPlaylistIE(LetvTvIE):
+    _VALID_URL = r'http://tv.letv.com/[a-z]+/(?P<id>[a-z]+)/index.s?html'
+    _TESTS = [{
+        'url': 'http://tv.letv.com/izt/wuzetian/index.html',
+        'info_dict': {
+            'id': 'wuzetian',
+            'title': '武媚娘传奇',
+            'description': 'md5:e12499475ab3d50219e5bba00b3cb248'
+        },
+        'playlist_count': 96
+    }, {
+        'url': 'http://tv.letv.com/pzt/lswjzzjc/index.shtml',
+        'info_dict': {
+            'id': 'lswjzzjc',
+            # should be "劲舞青春", but I can't find a simple way to determine
+            # the playlist title
+            'title': '乐视午间自制剧场',
+            'description': 'md5:b1eef244f45589a7b5b1af9ff25a4489'
+        },
+        'playlist_mincount': 7
+    }]

From 59c7cbd482ba82248cd1bdca3569da6035720f21 Mon Sep 17 00:00:00 2001
From: logon84 <rubenlogon@yahoo.es>
Date: Tue, 24 Feb 2015 18:58:32 +0100
Subject: [PATCH 1696/1937] Update eporner.py

Updated to work. Old version shows an error about being unable to extract "redirect_code"
---
 youtube_dl/extractor/eporner.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/eporner.py b/youtube_dl/extractor/eporner.py
index 4de8d4bc5..9ae28855b 100644
--- a/youtube_dl/extractor/eporner.py
+++ b/youtube_dl/extractor/eporner.py
@@ -35,10 +35,7 @@ class EpornerIE(InfoExtractor):
         title = self._html_search_regex(
             r'<title>(.*?) - EPORNER', webpage, 'title')
 
-        redirect_code = self._html_search_regex(
-            r'<script type="text/javascript" src="/config5/%s/([a-f\d]+)/">' % video_id,
-            webpage, 'redirect_code')
-        redirect_url = 'http://www.eporner.com/config5/%s/%s' % (video_id, redirect_code)
+        redirect_url = 'http://www.eporner.com/config5/%s/%s' % (video_id, display_id)
         player_code = self._download_webpage(
             redirect_url, display_id, note='Downloading player config')
 

From 677063594e28d11ada9d2bfd3a30397d4de24360 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 25 Feb 2015 02:10:55 +0800
Subject: [PATCH 1697/1937] [Letv] Update testcases

---
 youtube_dl/extractor/letv.py | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index d7d315634..d5839263c 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -23,14 +23,22 @@ class LetvIE(InfoExtractor):
             'title': '第87届奥斯卡颁奖礼完美落幕 《鸟人》成最大赢家',
             'timestamp': 1424747397,
             'upload_date': '20150224',
+            'description': 'md5:a9cb175fd753e2962176b7beca21a47c',
         }
     }, {
-        'url': 'http://www.letv.com/ptv/vplay/1118082.html',
+        'url': 'http://www.letv.com/ptv/vplay/1415246.html',
         'info_dict': {
-            'id': '1118082',
+            'id': '1415246',
             'ext': 'mp4',
-        }
+            'title': '美人天下01',
+            'description': 'md5:f88573d9d7225ada1359eaf0dbf8bcda',
+        },
+        'expected_warnings': [
+            'publish time'
+        ]
     }]
+    # http://www.letv.com/ptv/vplay/1118082.html
+    # This video is available only in Mainland China
 
     @staticmethod
     def urshift(val, n):
@@ -111,12 +119,14 @@ class LetvIE(InfoExtractor):
         publish_time = parse_iso8601(self._html_search_regex(
             r'发布时间&nbsp;([^<>]+) ', page, 'publish time', fatal=False),
             delimiter=' ', timezone=datetime.timedelta(hours=8))
+        description = self._html_search_meta('description', page, fatal=False)
 
         return {
             'id': media_id,
             'formats': urls,
             'title': playurl['title'],
             'thumbnail': playurl['pic'],
+            'description': description,
             'timestamp': publish_time,
         }
 
@@ -142,7 +152,8 @@ class LetvTvIE(InfoExtractor):
         entries = [self.url_result(media_url, ie='Letv')
                    for media_url in media_urls]
 
-        title = self._html_search_meta('keywords', page, fatal=False).split('，')[0]
+        title = self._html_search_meta('keywords', page,
+                                       fatal=False).split('，')[0]
         description = self._html_search_meta('description', page, fatal=False)
 
         return self.playlist_result(entries, playlist_id, playlist_title=title,
@@ -158,13 +169,14 @@ class LetvPlaylistIE(LetvTvIE):
             'title': '武媚娘传奇',
             'description': 'md5:e12499475ab3d50219e5bba00b3cb248'
         },
-        'playlist_count': 96
+        # This playlist contains some extra videos other than the drama itself
+        'playlist_mincount': 96
     }, {
         'url': 'http://tv.letv.com/pzt/lswjzzjc/index.shtml',
         'info_dict': {
             'id': 'lswjzzjc',
-            # should be "劲舞青春", but I can't find a simple way to determine
-            # the playlist title
+            # The title should be "劲舞青春", but I can't find a simple way to
+            # determine the playlist title
             'title': '乐视午间自制剧场',
             'description': 'md5:b1eef244f45589a7b5b1af9ff25a4489'
         },

From e765ed3a9c4ba52d4709a4a696881eae3401efa0 Mon Sep 17 00:00:00 2001
From: logon84 <rubenlogon@yahoo.es>
Date: Tue, 24 Feb 2015 19:41:46 +0100
Subject: [PATCH 1698/1937] [eporner] Fix redirect_code error

---
 youtube_dl/extractor/eporner.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/eporner.py b/youtube_dl/extractor/eporner.py
index 9ae28855b..f5943caa5 100644
--- a/youtube_dl/extractor/eporner.py
+++ b/youtube_dl/extractor/eporner.py
@@ -35,6 +35,7 @@ class EpornerIE(InfoExtractor):
         title = self._html_search_regex(
             r'<title>(.*?) - EPORNER', webpage, 'title')
 
+
         redirect_url = 'http://www.eporner.com/config5/%s/%s' % (video_id, display_id)
         player_code = self._download_webpage(
             redirect_url, display_id, note='Downloading player config')

From 37f885650c323e040a200bda9376bc7dbdf2ca25 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 25 Feb 2015 01:08:54 +0600
Subject: [PATCH 1699/1937] [eporner] Simplify and hardcode age limit

---
 youtube_dl/extractor/eporner.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/eporner.py b/youtube_dl/extractor/eporner.py
index f5943caa5..e006921ec 100644
--- a/youtube_dl/extractor/eporner.py
+++ b/youtube_dl/extractor/eporner.py
@@ -35,8 +35,7 @@ class EpornerIE(InfoExtractor):
         title = self._html_search_regex(
             r'<title>(.*?) - EPORNER', webpage, 'title')
 
-
-        redirect_url = 'http://www.eporner.com/config5/%s/%s' % (video_id, display_id)
+        redirect_url = 'http://www.eporner.com/config5/%s' % video_id
         player_code = self._download_webpage(
             redirect_url, display_id, note='Downloading player config')
 
@@ -67,5 +66,5 @@ class EpornerIE(InfoExtractor):
             'duration': duration,
             'view_count': view_count,
             'formats': formats,
-            'age_limit': self._rta_search(webpage),
+            'age_limit': 18,
         }

From 637570326bfa12575fe210e52e2a39d6585891d8 Mon Sep 17 00:00:00 2001
From: Antti Ajanki <antti.ajanki@iki.fi>
Date: Sun, 22 Feb 2015 10:16:51 +0200
Subject: [PATCH 1700/1937] [extractor/common] Extract the first of a seq of
 videos in a .smil file

---
 youtube_dl/extractor/common.py | 68 +++++++++++++++++++---------------
 1 file changed, 39 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 87fce9cd8..4fe99d25d 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -921,39 +921,49 @@ class InfoExtractor(object):
 
         formats = []
         rtmp_count = 0
-        for video in smil.findall('./body/switch/video'):
-            src = video.get('src')
-            if not src:
-                continue
-            bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
-            width = int_or_none(video.get('width'))
-            height = int_or_none(video.get('height'))
-            proto = video.get('proto')
-            if not proto:
-                if base:
-                    if base.startswith('rtmp'):
-                        proto = 'rtmp'
-                    elif base.startswith('http'):
-                        proto = 'http'
-            ext = video.get('ext')
-            if proto == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(src, video_id, ext))
-            elif proto == 'rtmp':
-                rtmp_count += 1
-                streamer = video.get('streamer') or base
-                formats.append({
-                    'url': streamer,
-                    'play_path': src,
-                    'ext': 'flv',
-                    'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
-                    'tbr': bitrate,
-                    'width': width,
-                    'height': height,
-                })
+        if smil.findall('./body/seq/video'):
+            video = smil.findall('./body/seq/video')[0]
+            fmts, rtmp_count = self._parse_smil_video(video, base, rtmp_count)
+            formats.extend(fmts)
+        else:
+            for video in smil.findall('./body/switch/video'):
+                fmts, rtmp_count = self._parse_smil_video(video, base, rtmp_count)
+                formats.extend(fmts)
+
         self._sort_formats(formats)
 
         return formats
 
+    def _parse_smil_video(self, video, base, rtmp_count):
+        src = video.get('src')
+        if not src:
+            return ([], rtmp_count)
+        bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
+        width = int_or_none(video.get('width'))
+        height = int_or_none(video.get('height'))
+        proto = video.get('proto')
+        if not proto:
+            if base:
+                if base.startswith('rtmp'):
+                    proto = 'rtmp'
+                elif base.startswith('http'):
+                    proto = 'http'
+        ext = video.get('ext')
+        if proto == 'm3u8':
+            return (self._extract_m3u8_formats(src, video_id, ext), rtmp_count)
+        elif proto == 'rtmp':
+            rtmp_count += 1
+            streamer = video.get('streamer') or base
+            return ([{
+                'url': streamer,
+                'play_path': src,
+                'ext': 'flv',
+                'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
+                'tbr': bitrate,
+                'width': width,
+                'height': height,
+            }], rtmp_count)
+
     def _live_title(self, name):
         """ Generate the title for a live video """
         now = datetime.datetime.now()

From 6f4ba54079893a09c6aa78fe3420523fb96df858 Mon Sep 17 00:00:00 2001
From: Antti Ajanki <antti.ajanki@iki.fi>
Date: Sun, 22 Feb 2015 10:18:36 +0200
Subject: [PATCH 1701/1937] [extractor/common] Extract HTTP (possibly f4m) URLs
 from a .smil file

---
 youtube_dl/extractor/common.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 4fe99d25d..313688208 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -963,6 +963,14 @@ class InfoExtractor(object):
                 'width': width,
                 'height': height,
             }], rtmp_count)
+        elif proto.startswith('http'):
+            return ([{
+                'url': base + src,
+                'ext': ext or 'flv',
+                'tbr': bitrate,
+                'width': width,
+                'height': height,
+            }], rtmp_count)
 
     def _live_title(self, name):
         """ Generate the title for a live video """

From c4f8c453ae2f735fc2320856e15e66510d74fd72 Mon Sep 17 00:00:00 2001
From: Antti Ajanki <antti.ajanki@iki.fi>
Date: Sun, 22 Feb 2015 21:03:49 +0200
Subject: [PATCH 1702/1937] [f4m] Refresh fragment list periodically on live
 streams

---
 youtube_dl/downloader/f4m.py | 59 ++++++++++++++++++++++++++++++------
 1 file changed, 49 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 7b8fe8cf5..1df9ebe5b 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -121,7 +121,8 @@ class FlvReader(io.BytesIO):
 
         self.read_unsigned_int()  # BootstrapinfoVersion
         # Profile,Live,Update,Reserved
-        self.read(1)
+        flags = self.read_unsigned_char()
+        live = flags & 0x20 != 0
         # time scale
         self.read_unsigned_int()
         # CurrentMediaTime
@@ -160,6 +161,7 @@ class FlvReader(io.BytesIO):
         return {
             'segments': segments,
             'fragments': fragments,
+            'live': live,
         }
 
     def read_bootstrap_info(self):
@@ -182,6 +184,10 @@ def build_fragments_list(boot_info):
     for segment, fragments_count in segment_run_table['segment_run']:
         for _ in range(fragments_count):
             res.append((segment, next(fragments_counter)))
+
+    if boot_info['live']:
+        res = res[-2:]
+
     return res
 
 
@@ -246,6 +252,38 @@ class F4mFD(FileDownloader):
             self.report_error('Unsupported DRM')
         return media
 
+    def _get_bootstrap_from_url(self, bootstrap_url):
+        bootstrap = self.ydl.urlopen(bootstrap_url).read()
+        return read_bootstrap_info(bootstrap)
+
+    def _update_live_fragments(self, bootstrap_url, latest_fragment):
+        fragments_list = []
+        retries = 30
+        while (not fragments_list) and (retries > 0):
+            boot_info = self._get_bootstrap_from_url(bootstrap_url)
+            fragments_list = build_fragments_list(boot_info)
+            fragments_list = [f for f in fragments_list if f[1] > latest_fragment]
+            if not fragments_list:
+                # Retry after a while
+                time.sleep(5.0)
+                retries -= 1
+
+        if not fragments_list:
+            self.report_error('Failed to update fragments')
+
+        return fragments_list
+
+    def _parse_bootstrap_node(self, node, base_url):
+        if node.text is None:
+            bootstrap_url = compat_urlparse.urljoin(
+                base_url, node.attrib['url'])
+            boot_info = self._get_bootstrap_from_url(bootstrap_url)
+        else:
+            bootstrap_url = None
+            bootstrap = base64.b64decode(node.text)
+            boot_info = read_bootstrap_info(bootstrap)
+        return (boot_info, bootstrap_url)
+
     def real_download(self, filename, info_dict):
         man_url = info_dict['url']
         requested_bitrate = info_dict.get('tbr')
@@ -265,18 +303,13 @@ class F4mFD(FileDownloader):
 
         base_url = compat_urlparse.urljoin(man_url, media.attrib['url'])
         bootstrap_node = doc.find(_add_ns('bootstrapInfo'))
-        if bootstrap_node.text is None:
-            bootstrap_url = compat_urlparse.urljoin(
-                base_url, bootstrap_node.attrib['url'])
-            bootstrap = self.ydl.urlopen(bootstrap_url).read()
-        else:
-            bootstrap = base64.b64decode(bootstrap_node.text)
+        boot_info, bootstrap_url = self._parse_bootstrap_node(bootstrap_node, base_url)
+        live = boot_info['live']
         metadata_node = media.find(_add_ns('metadata'))
         if metadata_node is not None:
             metadata = base64.b64decode(metadata_node.text)
         else:
             metadata = None
-        boot_info = read_bootstrap_info(bootstrap)
 
         fragments_list = build_fragments_list(boot_info)
         if self.params.get('test', False):
@@ -301,7 +334,8 @@ class F4mFD(FileDownloader):
         (dest_stream, tmpfilename) = sanitize_open(tmpfilename, 'wb')
 
         write_flv_header(dest_stream)
-        write_metadata_tag(dest_stream, metadata)
+        if not live:
+            write_metadata_tag(dest_stream, metadata)
 
         # This dict stores the download progress, it's updated by the progress
         # hook
@@ -348,7 +382,8 @@ class F4mFD(FileDownloader):
         http_dl.add_progress_hook(frag_progress_hook)
 
         frags_filenames = []
-        for (seg_i, frag_i) in fragments_list:
+        while fragments_list:
+            seg_i, frag_i = fragments_list.pop(0)
             name = 'Seg%d-Frag%d' % (seg_i, frag_i)
             url = base_url + name
             if akamai_pv:
@@ -367,6 +402,10 @@ class F4mFD(FileDownloader):
                         break
             frags_filenames.append(frag_filename)
 
+            if not fragments_list and live and bootstrap_url:
+                fragments_list = self._update_live_fragments(bootstrap_url, frag_i)
+                self.to_screen('Updated available fragments: %d' % len(fragments_list))
+
         dest_stream.close()
 
         elapsed = time.time() - start

From 5eaaeb7c317a543af4bde5eb7d465f3695fc97d9 Mon Sep 17 00:00:00 2001
From: Antti Ajanki <antti.ajanki@iki.fi>
Date: Mon, 23 Feb 2015 21:56:35 +0200
Subject: [PATCH 1703/1937] [f4m] Tolerate missed fragments on live streams

---
 youtube_dl/downloader/f4m.py | 43 +++++++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 1df9ebe5b..3dc796faa 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -11,6 +11,7 @@ from .common import FileDownloader
 from .http import HttpFD
 from ..compat import (
     compat_urlparse,
+    compat_urllib_error,
 )
 from ..utils import (
     struct_pack,
@@ -389,22 +390,38 @@ class F4mFD(FileDownloader):
             if akamai_pv:
                 url += '?' + akamai_pv.strip(';')
             frag_filename = '%s-%s' % (tmpfilename, name)
-            success = http_dl.download(frag_filename, {'url': url})
-            if not success:
-                return False
-            with open(frag_filename, 'rb') as down:
-                down_data = down.read()
-                reader = FlvReader(down_data)
-                while True:
-                    _, box_type, box_data = reader.read_box_info()
-                    if box_type == b'mdat':
-                        dest_stream.write(box_data)
-                        break
-            frags_filenames.append(frag_filename)
+            try:
+                success = http_dl.download(frag_filename, {'url': url})
+                if not success:
+                    return False
+                with open(frag_filename, 'rb') as down:
+                    down_data = down.read()
+                    reader = FlvReader(down_data)
+                    while True:
+                        _, box_type, box_data = reader.read_box_info()
+                        if box_type == b'mdat':
+                            dest_stream.write(box_data)
+                            break
+                if live:
+                    os.remove(frag_filename)
+                else:
+                    frags_filenames.append(frag_filename)
+            except (compat_urllib_error.HTTPError, ) as err:
+                if live and (err.code == 404 or err.code == 410):
+                    # We didn't keep up with the live window. Continue
+                    # with the next available fragment.
+                    msg = 'Fragment %d unavailable' % frag_i
+                    self.report_warning(msg)
+                    fragments_list = []
+                else:
+                    raise
 
             if not fragments_list and live and bootstrap_url:
                 fragments_list = self._update_live_fragments(bootstrap_url, frag_i)
-                self.to_screen('Updated available fragments: %d' % len(fragments_list))
+                total_frags += len(fragments_list)
+                if fragments_list and (fragments_list[0][1] > frag_i + 1):
+                    msg = 'Missed %d fragments' % (fragments_list[0][1] - (frag_i + 1))
+                    self.report_warning(msg)
 
         dest_stream.close()
 

From b8988b63a6baa206b7f5e35d99a9f4eff6ec7b5e Mon Sep 17 00:00:00 2001
From: Antti Ajanki <antti.ajanki@iki.fi>
Date: Tue, 24 Feb 2015 21:23:59 +0200
Subject: [PATCH 1704/1937] [wdr] Download a live stream

---
 youtube_dl/extractor/wdr.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index c90488500..b46802306 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -28,6 +28,7 @@ class WDRIE(InfoExtractor):
                 'title': 'Servicezeit',
                 'description': 'md5:c8f43e5e815eeb54d0b96df2fba906cb',
                 'upload_date': '20140310',
+                'is_live': False
             },
             'params': {
                 'skip_download': True,
@@ -41,6 +42,7 @@ class WDRIE(InfoExtractor):
                 'title': 'Marga Spiegel ist tot',
                 'description': 'md5:2309992a6716c347891c045be50992e4',
                 'upload_date': '20140311',
+                'is_live': False
             },
             'params': {
                 'skip_download': True,
@@ -55,6 +57,7 @@ class WDRIE(InfoExtractor):
                 'title': 'Erlebte Geschichten: Marga Spiegel (29.11.2009)',
                 'description': 'md5:2309992a6716c347891c045be50992e4',
                 'upload_date': '20091129',
+                'is_live': False
             },
         },
         {
@@ -66,6 +69,7 @@ class WDRIE(InfoExtractor):
                 'title': 'Flavia Coelho: Amar é Amar',
                 'description': 'md5:7b29e97e10dfb6e265238b32fa35b23a',
                 'upload_date': '20140717',
+                'is_live': False
             },
         },
         {
@@ -74,6 +78,20 @@ class WDRIE(InfoExtractor):
             'info_dict': {
                 'id': 'mediathek/video/sendungen/quarks_und_co/filterseite-quarks-und-co100',
             }
+        },
+        {
+            'url': 'http://www1.wdr.de/mediathek/video/livestream/index.html',
+            'info_dict': {
+                'id': 'mdb-103364',
+                'title': 're:^WDR Fernsehen [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+                'description': 'md5:ae2ff888510623bf8d4b115f95a9b7c9',
+                'ext': 'flv',
+                'upload_date': '20150212',
+                'is_live': True
+            },
+            'params': {
+                'skip_download': True,
+            },
         }
     ]
 
@@ -119,6 +137,10 @@ class WDRIE(InfoExtractor):
         video_url = flashvars['dslSrc'][0]
         title = flashvars['trackerClipTitle'][0]
         thumbnail = flashvars['startPicture'][0] if 'startPicture' in flashvars else None
+        is_live = flashvars.get('isLive', ['0'])[0] == '1'
+
+        if is_live:
+            title = self._live_title(title)
 
         if 'trackerClipAirTime' in flashvars:
             upload_date = flashvars['trackerClipAirTime'][0]
@@ -131,6 +153,13 @@ class WDRIE(InfoExtractor):
         if video_url.endswith('.f4m'):
             video_url += '?hdcore=3.2.0&plugin=aasp-3.2.0.77.18'
             ext = 'flv'
+        elif video_url.endswith('.smil'):
+            fmt = self._extract_smil_formats(video_url, page_id)[0]
+            video_url = fmt['url']
+            sep = '&' if '?' in video_url else '?'
+            video_url += sep
+            video_url += 'hdcore=3.3.0&plugin=aasp-3.3.0.99.43'
+            ext = fmt['ext']
         else:
             ext = determine_ext(video_url)
 
@@ -144,6 +173,7 @@ class WDRIE(InfoExtractor):
             'description': description,
             'thumbnail': thumbnail,
             'upload_date': upload_date,
+            'is_live': is_live
         }
 
 

From 13d8fbef30f1d7608b02e78818fa2ec211d69e21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 25 Feb 2015 17:56:51 +0100
Subject: [PATCH 1705/1937] [generic] Don't set the 'title' if it's not defined
 in the entry (closes #5061)

Some of them may be an 'url' result, which in general don't have the 'title' field.
---
 youtube_dl/extractor/generic.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 875e1bf05..3aff57e30 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1208,7 +1208,9 @@ class GenericIE(InfoExtractor):
             return entries[0]
         else:
             for num, e in enumerate(entries, start=1):
-                e['title'] = '%s (%d)' % (e['title'], num)
+                # 'url' results don't have a title
+                if e.get('title') is not None:
+                    e['title'] = '%s (%d)' % (e['title'], num)
             return {
                 '_type': 'playlist',
                 'entries': entries,

From 9504fc21b583d6ff968eec9d5843fc40b48725e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Markus=20M=C3=BCller?= <mmu@grummel.net>
Date: Wed, 25 Feb 2015 23:27:19 +0100
Subject: [PATCH 1706/1937] Fix the RTL extractor for new episodes by using a
 different hostname

---
 youtube_dl/extractor/rtlnow.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py
index fd93cc66f..785a8045e 100644
--- a/youtube_dl/extractor/rtlnow.py
+++ b/youtube_dl/extractor/rtlnow.py
@@ -146,7 +146,7 @@ class RTLnowIE(InfoExtractor):
                 mobj = re.search(r'.*/(?P<hoster>[^/]+)/videos/(?P<play_path>.+)\.f4m', filename.text)
                 if mobj:
                     fmt = {
-                        'url': 'rtmpe://fmspay-fra2.rtl.de/' + mobj.group('hoster'),
+                        'url': 'rtmpe://fms.rtl.de/' + mobj.group('hoster'),
                         'play_path': 'mp4:' + mobj.group('play_path'),
                         'page_url': url,
                         'player_url': video_page_url + 'includes/vodplayer.swf',

From c10ea454dc7a06bf4d911774fd12b49ce7550845 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 25 Feb 2015 23:52:54 +0100
Subject: [PATCH 1707/1937] [telecinco] Recognize more urls (closes #5065)

---
 youtube_dl/extractor/mitele.py    | 4 ++--
 youtube_dl/extractor/telecinco.py | 9 ++++++---
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index 256758323..d8897eb90 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -18,7 +18,7 @@ class MiTeleIE(InfoExtractor):
     IE_NAME = 'mitele.es'
     _VALID_URL = r'http://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<id>[^/]+)/'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/',
         'md5': '6a75fe9d0d3275bead0cb683c616fddb',
         'info_dict': {
@@ -29,7 +29,7 @@ class MiTeleIE(InfoExtractor):
             'display_id': 'programa-144',
             'duration': 2913,
         },
-    }
+    }]
 
     def _real_extract(self, url):
         episode = self._match_id(url)
diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index be3f72df7..251a68680 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -6,9 +6,9 @@ from .mitele import MiTeleIE
 
 class TelecincoIE(MiTeleIE):
     IE_NAME = 'telecinco.es'
-    _VALID_URL = r'https?://www\.telecinco\.es/[^/]+/[^/]+/[^/]+/(?P<id>.*?)\.html'
+    _VALID_URL = r'https?://www\.telecinco\.es/[^/]+/[^/]+/(?:[^/]+/)?(?P<id>.*?)\.html'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.telecinco.es/robinfood/temporada-01/t01xp14/Bacalao-cocochas-pil-pil_0_1876350223.html',
         'info_dict': {
             'id': 'MDSVID20141015_0058',
@@ -16,4 +16,7 @@ class TelecincoIE(MiTeleIE):
             'title': 'Con Martín Berasategui, hacer un bacalao al ...',
             'duration': 662,
         },
-    }
+    }, {
+        'url': 'http://www.telecinco.es/informativos/nacional/Pablo_Iglesias-Informativos_Telecinco-entrevista-Pedro_Piqueras_2_1945155182.html',
+        'only_matching': True,
+    }]

From 1c69bca25871ebb8b54ebc5cb9bb4b2124bb12ab Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 00:24:54 +0100
Subject: [PATCH 1708/1937] [escapist] Fix config URL matching

---
 youtube_dl/extractor/escapist.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 51ffec7ee..3244306dc 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -44,10 +44,10 @@ class EscapistIE(InfoExtractor):
         config_url = compat_urllib_parse.unquote(self._html_search_regex(
             r'''(?x)
             (?:
-                <param\s+name="flashvars"\s+value="config=|
+                <param\s+name="flashvars".*?\s+value="config=|
                 flashvars=&quot;config=
             )
-            ([^"&]+)
+            (https?://[^"&]+)
             ''',
             webpage, 'config URL'))
 

From 183139340b4841f22357fcf2b6a5ce2dedd71146 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 00:40:12 +0100
Subject: [PATCH 1709/1937] [utils] Bump our user agent

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e2631dccd..506c896de 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -54,7 +54,7 @@ from .compat import (
 compiled_regex_type = type(re.compile(''))
 
 std_headers = {
-    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0 (Chrome)',
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',
     'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
     'Accept-Encoding': 'gzip, deflate',

From 13cd97f3dfd87373cfc501224692352cb20e8194 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 00:42:02 +0100
Subject: [PATCH 1710/1937] release 2015.02.26

---
 README.md             | 2 ++
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 699401b49..2c53e2211 100644
--- a/README.md
+++ b/README.md
@@ -139,6 +139,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      dislike_count <? 50 & description" .
     --no-playlist                    If the URL refers to a video and a
                                      playlist, download only the video.
+    --yes-playlist                   If the URL refers to a video and a
+                                     playlist, download the playlist.
     --age-limit YEARS                download only videos suitable for the given
                                      age
     --download-archive FILE          Download only videos not listed in the
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d23c6ae3d..0cbf66ed1 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.24.2'
+__version__ = '2015.02.26'

From 9a48926a5734e0c1530c4788680961610090a837 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 00:59:53 +0100
Subject: [PATCH 1711/1937] [escapist] Add support for advertisements

---
 youtube_dl/extractor/escapist.py | 41 ++++++++++++++++++++++++--------
 1 file changed, 31 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 3244306dc..b45c1dbd0 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -52,6 +52,7 @@ class EscapistIE(InfoExtractor):
             webpage, 'config URL'))
 
         formats = []
+        ad_formats = []
 
         def _add_format(name, cfgurl, quality):
             config = self._download_json(
@@ -61,14 +62,19 @@ class EscapistIE(InfoExtractor):
                 transform_source=js_to_json)
 
             playlist = config['playlist']
-            video_url = next(
-                p['url'] for p in playlist
-                if p.get('eventCategory') == 'Video')
-            formats.append({
-                'url': video_url,
-                'format_id': name,
-                'quality': quality,
-            })
+            for p in playlist:
+                if p.get('eventCategory') == 'Video':
+                    ar = formats
+                elif p.get('eventCategory') == 'Video Postroll':
+                    ar = ad_formats
+                else:
+                    continue
+
+                ar.append({
+                    'url': p['url'],
+                    'format_id': name,
+                    'quality': quality,
+                })
 
         _add_format('normal', config_url, quality=0)
         hq_url = (config_url +
@@ -77,10 +83,9 @@ class EscapistIE(InfoExtractor):
             _add_format('hq', hq_url, quality=1)
         except ExtractorError:
             pass  # That's fine, we'll just use normal quality
-
         self._sort_formats(formats)
 
-        return {
+        res = {
             'id': video_id,
             'formats': formats,
             'uploader': uploader,
@@ -89,3 +94,19 @@ class EscapistIE(InfoExtractor):
             'thumbnail': self._og_search_thumbnail(webpage),
             'description': description,
         }
+
+        if self._downloader.params.get('include_ads') and ad_formats:
+            self._sort_formats(ad_formats)
+            ad_res = {
+                'id': '%s-ad' % video_id,
+                'title': '%s (Postroll)' % title,
+                'formats': ad_formats,
+            }
+            return {
+                '_type': 'playlist',
+                'entries': [res, ad_res],
+                'title': title,
+                'id': video_id,
+            }
+
+        return res

From 3e675fabe027ecf20e662454ff7a174596801256 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 01:25:00 +0100
Subject: [PATCH 1712/1937] [airmozilla] Be more tolerant when nonessential
 items are missing (#5030)

---
 test/test_utils.py                 |  1 +
 youtube_dl/extractor/airmozilla.py | 47 +++++++++++++++---------------
 youtube_dl/utils.py                |  5 ++++
 3 files changed, 30 insertions(+), 23 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 2f8996d7b..3fba8ae11 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -246,6 +246,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(parse_duration('2.5 hours'), 9000)
         self.assertEqual(parse_duration('02:03:04'), 7384)
         self.assertEqual(parse_duration('01:02:03:04'), 93784)
+        self.assertEqual(parse_duration('1 hour 3 minutes'), 3780)
 
     def test_fix_xml_ampersands(self):
         self.assertEqual(
diff --git a/youtube_dl/extractor/airmozilla.py b/youtube_dl/extractor/airmozilla.py
index 44c20f886..611ad1e9d 100644
--- a/youtube_dl/extractor/airmozilla.py
+++ b/youtube_dl/extractor/airmozilla.py
@@ -4,7 +4,11 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import parse_iso8601
+from ..utils import (
+    int_or_none,
+    parse_duration,
+    parse_iso8601,
+)
 
 
 class AirMozillaIE(InfoExtractor):
@@ -27,13 +31,6 @@ class AirMozillaIE(InfoExtractor):
         }
     }
 
-    _QUALITY_MAP = {
-        '360p': 0,
-        '576p': 1,
-        '640p': 2,
-        '720p': 3,
-    }
-
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
@@ -43,19 +40,23 @@ class AirMozillaIE(InfoExtractor):
         jwconfig = self._search_regex(r'\svar jwconfig = (\{.*?\});\s', embed_script, 'metadata')
         metadata = self._parse_json(jwconfig, video_id)
 
-        formats = []
-        for source in metadata['playlist'][0]['sources']:
-            fmt = {
-                'url': source['file'],
-                'ext': source['type'],
-                'format_id': self._search_regex(r'&format=(.*)$', source['file'], 'video format'),
-                'resolution': source['label'],
-                'quality': self._QUALITY_MAP.get(source['label'], -1),
-            }
-            formats.append(fmt)
+        formats = [{
+            'url': source['file'],
+            'ext': source['type'],
+            'format_id': self._search_regex(r'&format=(.*)$', source['file'], 'video format'),
+            'format': source['label'],
+            'height': int(source['label'].rstrip('p')),
+        } for source in metadata['playlist'][0]['sources']]
         self._sort_formats(formats)
 
-        duration_match = re.search(r'Duration:(?: (?P<H>\d+) hours?)?(?: (?P<M>\d+) minutes?)?', webpage)
+        view_count = int_or_none(self._html_search_regex(
+            r'Views since archived: ([0-9]+)',
+            webpage, 'view count', fatal=False))
+        timestamp = parse_iso8601(self._html_search_regex(
+            r'<time datetime="(.*?)"', webpage, 'timestamp', fatal=False))
+        duration = parse_duration(self._search_regex(
+            r'Duration:\s*(\d+\s*hours?\s*\d+\s*minutes?)',
+            webpage, 'duration', fatal=False))
 
         return {
             'id': video_id,
@@ -63,11 +64,11 @@ class AirMozillaIE(InfoExtractor):
             'formats': formats,
             'url': self._og_search_url(webpage),
             'display_id': display_id,
-            'thumbnail': metadata['playlist'][0]['image'],
+            'thumbnail': metadata['playlist'][0].get('image'),
             'description': self._og_search_description(webpage),
-            'timestamp': parse_iso8601(self._html_search_regex(r'<time datetime="(.*?)"', webpage, 'timestamp')),
+            'timestamp': timestamp,
             'location': self._html_search_regex(r'Location: (.*)', webpage, 'location', default=None),
-            'duration': int(duration_match.groupdict()['H'] or 0) * 3600 + int(duration_match.groupdict()['M'] or 0) * 60,
-            'view_count': int(self._html_search_regex(r'Views since archived: ([0-9]+)', webpage, 'view count')),
+            'duration': duration,
+            'view_count': view_count,
             'categories': re.findall(r'<a href=".*?" class="channel">(.*?)</a>', webpage),
         }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 506c896de..1f3bfef7d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1290,6 +1290,7 @@ def parse_duration(s):
             (?P<only_mins>[0-9.]+)\s*(?:mins?|minutes?)\s*|
             (?P<only_hours>[0-9.]+)\s*(?:hours?)|
 
+            \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*|
             (?:
                 (?:
                     (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
@@ -1308,10 +1309,14 @@ def parse_duration(s):
         return float_or_none(m.group('only_hours'), invscale=60 * 60)
     if m.group('secs'):
         res += int(m.group('secs'))
+    if m.group('mins_reversed'):
+        res += int(m.group('mins_reversed')) * 60
     if m.group('mins'):
         res += int(m.group('mins')) * 60
     if m.group('hours'):
         res += int(m.group('hours')) * 60 * 60
+    if m.group('hours_reversed'):
+        res += int(m.group('hours_reversed')) * 60 * 60
     if m.group('days'):
         res += int(m.group('days')) * 24 * 60 * 60
     if m.group('ms'):

From 84be7c230cd7b17c54926928e4108a1118f9892e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 01:25:54 +0100
Subject: [PATCH 1713/1937] Cred @duncankl for airmozilla

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index bdd2a15dc..4674a5af3 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -112,3 +112,4 @@ Frans de Jonge
 Robin de Rooij
 Ryan Schmidt
 Leslie P. Polzer
+Duncan Keall

From 265bfa2c79abc8f233132126be313ed2d4b18dc8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 01:30:18 +0100
Subject: [PATCH 1714/1937] [letv] Simplify

---
 youtube_dl/extractor/letv.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index d5839263c..583ce35b9 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -1,14 +1,20 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import os.path
+import datetime
 import re
 import time
-import datetime
 
 from .common import InfoExtractor
-from ..compat import (compat_urlparse, compat_urllib_parse)
-from ..utils import (ExtractorError, parse_iso8601)
+from ..compat import (
+    compat_urlparse,
+    compat_urllib_parse,
+)
+from ..utils import (
+    determine_ext,
+    ExtractorError,
+    parse_iso8601,
+)
 
 
 class LetvIE(InfoExtractor):
@@ -44,7 +50,7 @@ class LetvIE(InfoExtractor):
     def urshift(val, n):
         return val >> n if val >= 0 else (val + 0x100000000) >> n
 
-    # ror() and calcTimeKey() are reversed from a embedded swf file in KLetvPlayer.swf
+    # ror() and calc_time_key() are reversed from a embedded swf file in KLetvPlayer.swf
     def ror(self, param1, param2):
         _loc3_ = 0
         while _loc3_ < param2:
@@ -52,7 +58,7 @@ class LetvIE(InfoExtractor):
             _loc3_ += 1
         return param1
 
-    def calcTimeKey(self, param1):
+    def calc_time_key(self, param1):
         _loc2_ = 773625421
         _loc3_ = self.ror(param1, _loc2_ % 13)
         _loc3_ = _loc3_ ^ _loc2_
@@ -67,7 +73,7 @@ class LetvIE(InfoExtractor):
             'platid': 1,
             'splatid': 101,
             'format': 1,
-            'tkey': self.calcTimeKey(int(time.time())),
+            'tkey': self.calc_time_key(int(time.time())),
             'domain': 'www.letv.com'
         }
         play_json = self._download_json(
@@ -108,7 +114,7 @@ class LetvIE(InfoExtractor):
 
                 url_info_dict = {
                     'url': media_url,
-                    'ext': os.path.splitext(dispatch[format_id][1])[1][1:]
+                    'ext': determine_ext(dispatch[format_id][1])
                 }
 
                 if format_id[-1:] == 'p':

From 72a406e7aa6ef751f5f2ad4b110b749ce40d70a4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 01:35:43 +0100
Subject: [PATCH 1715/1937] [extractor/common] Pass in video_id (#5057)

---
 youtube_dl/extractor/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 313688208..7977fa8d0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -923,18 +923,18 @@ class InfoExtractor(object):
         rtmp_count = 0
         if smil.findall('./body/seq/video'):
             video = smil.findall('./body/seq/video')[0]
-            fmts, rtmp_count = self._parse_smil_video(video, base, rtmp_count)
+            fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
             formats.extend(fmts)
         else:
             for video in smil.findall('./body/switch/video'):
-                fmts, rtmp_count = self._parse_smil_video(video, base, rtmp_count)
+                fmts, rtmp_count = self._parse_smil_video(video, video_id, base, rtmp_count)
                 formats.extend(fmts)
 
         self._sort_formats(formats)
 
         return formats
 
-    def _parse_smil_video(self, video, base, rtmp_count):
+    def _parse_smil_video(self, video, video_id, base, rtmp_count):
         src = video.get('src')
         if not src:
             return ([], rtmp_count)

From 2a15a98a6a077381cba52207b791d03506b0416c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 01:44:20 +0100
Subject: [PATCH 1716/1937] [rmtp] Encode filename before invoking subprocess

This fixes #5066.
Reproducible with
LC_ALL=C youtube-dl "http://www.prosieben.de/tv/germanys-next-topmodel/video/playlist/ganze-folge-episode-2-das-casting-in-muenchen"
---
 youtube_dl/downloader/rtmp.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index 0a52c34c7..89e98ae61 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -119,7 +119,9 @@ class RtmpFD(FileDownloader):
         # Download using rtmpdump. rtmpdump returns exit code 2 when
         # the connection was interrumpted and resuming appears to be
         # possible. This is part of rtmpdump's normal usage, AFAIK.
-        basic_args = ['rtmpdump', '--verbose', '-r', url, '-o', tmpfilename]
+        basic_args = [
+            'rtmpdump', '--verbose', '-r', url,
+            '-o', encodeFilename(tmpfilename, True)]
         if player_url is not None:
             basic_args += ['--swfVfy', player_url]
         if page_url is not None:

From 1c6510f57a458aa9634923c0f48596b57d084649 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 01:47:12 +0100
Subject: [PATCH 1717/1937] [Makefile] clean pyc files in clean target

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index 708732956..c6c76274f 100644
--- a/Makefile
+++ b/Makefile
@@ -2,6 +2,7 @@ all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bas
 
 clean:
 	rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe
+	find -name "*.pyc" -delete
 
 PREFIX ?= /usr/local
 BINDIR ?= $(PREFIX)/bin

From 5c340b038740061c61f55be1935e364a883a3138 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 01:47:16 +0100
Subject: [PATCH 1718/1937] release 2015.02.26.1

---
 docs/supportedsites.md | 4 ++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 9f70db80a..49b4ac8c1 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -17,6 +17,7 @@
  - **AdultSwim**
  - **Aftenposten**
  - **Aftonbladet**
+ - **AirMozilla**
  - **AlJazeera**
  - **Allocine**
  - **AlphaPorno**
@@ -220,6 +221,9 @@
  - **Ku6**
  - **la7.tv**
  - **Laola1Tv**
+ - **Letv**
+ - **LetvPlaylist**
+ - **LetvTv**
  - **lifenews**: LIFE | NEWS
  - **LiveLeak**
  - **livestream**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0cbf66ed1..9043ca695 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.26'
+__version__ = '2015.02.26.1'

From 63a562f95ee62e05e689bf7fbf8e923749248b05 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 09:19:26 +0100
Subject: [PATCH 1719/1937] [escapist] Detect IP blocking and use another UA
 (Fixes #5069)

---
 youtube_dl/extractor/escapist.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index b45c1dbd0..80e9084f4 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import (
     compat_urllib_parse,
+    compat_urllib_request,
 )
 from ..utils import (
     ExtractorError,
@@ -12,6 +13,7 @@ from ..utils import (
 
 class EscapistIE(InfoExtractor):
     _VALID_URL = r'https?://?(www\.)?escapistmagazine\.com/videos/view/[^/?#]+/(?P<id>[0-9]+)-[^/?#]*(?:$|[?#])'
+    _USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko'
     _TEST = {
         'url': 'http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
         'md5': 'ab3a706c681efca53f0a35f1415cf0d1',
@@ -28,7 +30,9 @@ class EscapistIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        webpage_req = compat_urllib_request.Request(url)
+        webpage_req.add_header('User-Agent', self._USER_AGENT)
+        webpage = self._download_webpage(webpage_req, video_id)
 
         uploader_id = self._html_search_regex(
             r"<h1\s+class='headline'>\s*<a\s+href='/videos/view/(.*?)'",
@@ -54,9 +58,11 @@ class EscapistIE(InfoExtractor):
         formats = []
         ad_formats = []
 
-        def _add_format(name, cfgurl, quality):
+        def _add_format(name, cfg_url, quality):
+            cfg_req = compat_urllib_request.Request(cfg_url)
+            cfg_req.add_header('User-Agent', self._USER_AGENT)
             config = self._download_json(
-                cfgurl, video_id,
+                cfg_req, video_id,
                 'Downloading ' + name + ' configuration',
                 'Unable to download ' + name + ' configuration',
                 transform_source=js_to_json)
@@ -74,6 +80,9 @@ class EscapistIE(InfoExtractor):
                     'url': p['url'],
                     'format_id': name,
                     'quality': quality,
+                    'http_headers': {
+                        'User-Agent': self._USER_AGENT,
+                    },
                 })
 
         _add_format('normal', config_url, quality=0)
@@ -85,6 +94,9 @@ class EscapistIE(InfoExtractor):
             pass  # That's fine, we'll just use normal quality
         self._sort_formats(formats)
 
+        if '/escapist/sales-marketing/' in formats[-1]['url']:
+            raise ExtractorError('This IP address has been blocked by The Escapist', expected=True)
+
         res = {
             'id': video_id,
             'formats': formats,

From 9724e5d33661b6d6c84b6da64b78c0b96221ab24 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 26 Feb 2015 09:45:11 +0100
Subject: [PATCH 1720/1937] release 2015.02.26.2

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 9043ca695..cf3e28bbe 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.26.1'
+__version__ = '2015.02.26.2'

From 2e241242a3ee6338cafd515c9cd7481eb5c6f928 Mon Sep 17 00:00:00 2001
From: "PishPosh.McGee" <pishposh.mcgee@gmail.com>
Date: Thu, 26 Feb 2015 03:59:35 -0600
Subject: [PATCH 1721/1937] Adding subtitles

---
 youtube_dl/extractor/comedycentral.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index b24538981..e5edcc84b 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -250,6 +250,8 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
                 })
                 self._sort_formats(formats)
 
+            subtitles = self._extract_subtitles(cdoc, guid)
+
             virtual_id = show_name + ' ' + epTitle + ' part ' + compat_str(part_num + 1)
             entries.append({
                 'id': guid,
@@ -260,6 +262,7 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
                 'duration': duration,
                 'thumbnail': thumbnail,
                 'description': description,
+                'subtitles': subtitles,
             })
 
         return {

From e129c5bc0de4913564c1e1a62baae4bd0073824c Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 26 Feb 2015 14:35:48 +0200
Subject: [PATCH 1722/1937] [laola1tv] Allow live stream downloads

---
 youtube_dl/extractor/laola1tv.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py
index e8ca49fd1..b459559b0 100644
--- a/youtube_dl/extractor/laola1tv.py
+++ b/youtube_dl/extractor/laola1tv.py
@@ -27,8 +27,6 @@ class Laola1TvIE(InfoExtractor):
         }
     }
 
-    _BROKEN = True  # Not really - extractor works fine, but f4m downloader does not support live streams yet.
-
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -57,11 +55,7 @@ class Laola1TvIE(InfoExtractor):
         title = xpath_text(hd_doc, './/video/title', fatal=True)
         flash_url = xpath_text(hd_doc, './/video/url', fatal=True)
         uploader = xpath_text(hd_doc, './/video/meta_organistation')
-
         is_live = xpath_text(hd_doc, './/video/islive') == 'true'
-        if is_live:
-            raise ExtractorError(
-                'Live streams are not supported by the f4m downloader.')
 
         categories = xpath_text(hd_doc, './/video/meta_sports')
         if categories:

From 7ab7c9e93290f575747c1790c4aecf2055227a8d Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 26 Feb 2015 16:22:05 +0200
Subject: [PATCH 1723/1937] [gamestar] Fix title extraction

---
 youtube_dl/extractor/gamestar.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamestar.py b/youtube_dl/extractor/gamestar.py
index 7591a151e..590ccf526 100644
--- a/youtube_dl/extractor/gamestar.py
+++ b/youtube_dl/extractor/gamestar.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -31,7 +33,7 @@ class GameStarIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         og_title = self._og_search_title(webpage)
-        title = og_title.replace(' - Video bei GameStar.de', '').strip()
+        title = re.sub(r'\s*- Video (bei|-) GameStar\.de$', '', og_title)
 
         url = 'http://gamestar.de/_misc/videos/portal/getVideoUrl.cfm?premium=0&videoId=' + video_id
 

From 6317a3e9da42f5d60e514fddcb89eec7f5768294 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 26 Feb 2015 21:10:49 +0600
Subject: [PATCH 1724/1937] [mpora] Fix extraction

---
 youtube_dl/extractor/mpora.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mpora.py b/youtube_dl/extractor/mpora.py
index 6db3c67a5..a07d28928 100644
--- a/youtube_dl/extractor/mpora.py
+++ b/youtube_dl/extractor/mpora.py
@@ -25,7 +25,9 @@ class MporaIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         data_json = self._search_regex(
-            r"new FM\.Player\('[^']+',\s*(\{.*?)\).player;", webpage, 'json')
+            [r"new FM\.Player\('[^']+',\s*(\{.*?)\).player;",
+             r"new\s+FM\.Kaltura\.Player\('[^']+'\s*,\s*({.+?})\);"],
+            webpage, 'json')
         data = self._parse_json(data_json, video_id)
 
         uploader = data['info_overlay'].get('username')

From 250a9bdfe27ecf3d0acf179b4c21ff2126c64990 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 26 Feb 2015 21:16:35 +0600
Subject: [PATCH 1725/1937] [mpora] Improve _VALID_URL

---
 youtube_dl/extractor/mpora.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mpora.py b/youtube_dl/extractor/mpora.py
index a07d28928..5a1bee5c8 100644
--- a/youtube_dl/extractor/mpora.py
+++ b/youtube_dl/extractor/mpora.py
@@ -5,7 +5,7 @@ from ..utils import int_or_none
 
 
 class MporaIE(InfoExtractor):
-    _VALID_URL = r'https?://(www\.)?mpora\.(?:com|de)/videos/(?P<id>[^?#/]+)'
+    _VALID_URL = r'https?://(?:www\.)?mpora\.(?:com|de)/videos/(?P<id>[^?#/]+)'
     IE_NAME = 'MPORA'
 
     _TEST = {

From 1a2313a6f2d83c9e0a5c0c25bf5a2e3167994dbe Mon Sep 17 00:00:00 2001
From: anovicecodemonkey <anovicecodemonkey13435@mailinator.com>
Date: Fri, 27 Feb 2015 02:36:45 +1030
Subject: [PATCH 1726/1937] [TheChiveIE] added support for TheChive.com (Closes
 #5016)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/thechive.py | 60 ++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 youtube_dl/extractor/thechive.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ddb9d6670..b064a3a7c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -481,6 +481,7 @@ from .tenplay import TenPlayIE
 from .testurl import TestURLIE
 from .testtube import TestTubeIE
 from .tf1 import TF1IE
+from .thechive import TheChiveIE
 from .theonion import TheOnionIE
 from .theplatform import ThePlatformIE
 from .thesixtyone import TheSixtyOneIE
diff --git a/youtube_dl/extractor/thechive.py b/youtube_dl/extractor/thechive.py
new file mode 100644
index 000000000..df1a7998d
--- /dev/null
+++ b/youtube_dl/extractor/thechive.py
@@ -0,0 +1,60 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import unified_strdate
+
+
+
+class TheChiveIE(InfoExtractor):
+    _VALID_URL = r'http://(www\.)?thechive\.com/[^/]+/[^/]+/[^/]+/(?P<video_id>[A-Za-z\-]+)'
+    _TEST = {
+        'url': "http://thechive.com/2015/02/20/so-thats-what-a-set-of-redneck-bagpipes-sound-like-video/",
+        'md5': "366710dda77cfa727bdef3523ba8466f",
+        'info_dict': {
+            'id': "so-thats-what-a-set-of-redneck-bagpipes-sound-like-video",
+            'title': "So that's what a set of redneck bagpipes sound like... (Video)",
+            'description': "Okay that was pretty good. Now play Freebird!...",
+            'thumbnail': "https://thechive.files.wordpress.com/2015/02/0_07dghz0w-thumbnail2.jpg",
+            'author': "Ben",
+            'upload_date': "20150220",
+            'ext': "mp4"
+        }
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('video_id')
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._og_search_title(webpage)
+        description = self._html_search_regex(r'(?s)<meta name="description" content="(.*?)" />', webpage, 'description')
+        thumbnail = self._og_search_thumbnail(webpage)
+        author = self._html_search_regex(
+            r'(?s)itemprop="author">(.+?)</span>', webpage, 'author', fatal=False).capitalize() 
+        upload_date = unified_strdate(self._html_search_regex(
+            r'(?s)itemprop="datePublished" datetime="(.+?)">', webpage, 'upload_date', fatal=False))
+
+        # Adapted from extractor/musicvault.py
+        VIDEO_URL_TEMPLATE = 'http://cdnapi.kaltura.com/p/%(uid)s/sp/%(wid)s/playManifest/entryId/%(entry_id)s/format/url/protocol/http'
+
+        kaltura_id = self._search_regex(
+            r'entry_id=([^"]+)',
+            webpage, 'kaltura ID')
+        video_url = VIDEO_URL_TEMPLATE % {
+            'entry_id': kaltura_id,
+            'wid': self._search_regex(r'partner_id/([0-9]+)\?', webpage, 'wid'),
+            'uid': self._search_regex(r'uiconf_id/([0-9]+)/', webpage, 'uid'),
+        }
+
+        return {
+            'url': video_url,
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'author': author,
+            'upload_date': upload_date,
+            'ext': 'mp4'
+        }
\ No newline at end of file

From 0d97ef43bec006157870fd4a5cedfac1eaebf3a9 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 26 Feb 2015 23:45:54 +0200
Subject: [PATCH 1727/1937] [kaltura] Add new extractor

---
 youtube_dl/extractor/__init__.py |   1 +
 youtube_dl/extractor/kaltura.py  | 134 +++++++++++++++++++++++++++++++
 2 files changed, 135 insertions(+)
 create mode 100644 youtube_dl/extractor/kaltura.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ddb9d6670..e3b2cb54f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -227,6 +227,7 @@ from .jeuxvideo import JeuxVideoIE
 from .jove import JoveIE
 from .jukebox import JukeboxIE
 from .jpopsukitv import JpopsukiIE
+from .kaltura import KalturaIE
 from .kankan import KankanIE
 from .karaoketv import KaraoketvIE
 from .keezmovies import KeezMoviesIE
diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
new file mode 100644
index 000000000..2aff410c5
--- /dev/null
+++ b/youtube_dl/extractor/kaltura.py
@@ -0,0 +1,134 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_parse
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+)
+
+
+class KalturaIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+    (?:kaltura:|
+       https?://(:?www\.)?kaltura\.com/index\.php/kwidget/(?:[^/]+/)*?wid/_
+    )(?P<partner_id>\d+)
+    (?::|
+       /(?:[^/]+/)*?entry_id/
+    )(?P<id>[0-9a-z_]+)'''
+    _API_BASE = 'http://cdnapi.kaltura.com/api_v3/index.php?'
+    _TESTS = [
+        {
+            'url': 'kaltura:269692:1_1jc2y3e4',
+            'md5': '3adcbdb3dcc02d647539e53f284ba171',
+            'info_dict': {
+                'id': '1_1jc2y3e4',
+                'ext': 'mp4',
+                'title': 'Track 4',
+                'upload_date': '20131219',
+                'uploader_id': 'mlundberg@wolfgangsvault.com',
+                'description': 'The Allman Brothers Band, 12/16/1981',
+                'thumbnail': 're:^https?://.*/thumbnail/.*',
+                'timestamp': int,
+            },
+        },
+        {
+            'url': 'http://www.kaltura.com/index.php/kwidget/cache_st/1300318621/wid/_269692/uiconf_id/3873291/entry_id/1_1jc2y3e4',
+            'only_matching': True,
+        },
+    ]
+
+    def _kaltura_api_call(self, video_id, actions, *args, **kwargs):
+        params = actions[0]
+        if len(actions) > 1:
+            for i, a in enumerate(actions[1:], start=1):
+                for k, v in a.items():
+                    params['%d:%s' % (i, k)] = v
+
+        query = compat_urllib_parse.urlencode(params)
+        url = self._API_BASE + query
+        data = self._download_json(url, video_id, *args, **kwargs)
+
+        status = data if len(actions) == 1 else data[0]
+        if status.get('objectType') == 'KalturaAPIException':
+            raise ExtractorError(
+                '%s said: %s' % (self.IE_NAME, status['message']))
+
+        return data
+
+    def _get_kaltura_signature(self, video_id, partner_id):
+        actions = [{
+            'apiVersion': '3.1',
+            'expiry': 86400,
+            'format': 1,
+            'service': 'session',
+            'action': 'startWidgetSession',
+            'widgetId': '_%s' % partner_id,
+        }]
+        return self._kaltura_api_call(
+            video_id, actions, note='Downloading Kaltura signature')['ks']
+
+    def _get_video_info(self, video_id, partner_id):
+        signature = self._get_kaltura_signature(video_id, partner_id)
+        actions = [
+            {
+                'action': 'null',
+                'apiVersion': '3.1.5',
+                'clientTag': 'kdp:v3.8.5',
+                'format': 1,  # JSON, 2 = XML, 3 = PHP
+                'service': 'multirequest',
+                'ks': signature,
+            },
+            {
+                'action': 'get',
+                'entryId': video_id,
+                'service': 'baseentry',
+                'version': '-1',
+            },
+            {
+                'action': 'getContextData',
+                'contextDataParams:objectType': 'KalturaEntryContextDataParams',
+                'contextDataParams:referrer': 'http://www.kaltura.com/',
+                'contextDataParams:streamerType': 'http',
+                'entryId': video_id,
+                'service': 'baseentry',
+            },
+        ]
+        return self._kaltura_api_call(
+            video_id, actions, note='Downloading video info JSON')
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        partner_id, entry_id = mobj.group('partner_id'), mobj.group('id')
+
+        info, source_data = self._get_video_info(entry_id, partner_id)
+
+        formats = [{
+            'format_id': '%(fileExt)s-%(bitrate)s' % f,
+            'ext': f['fileExt'],
+            'tbr': f['bitrate'],
+            'fps': f.get('frameRate'),
+            'filesize_approx': int_or_none(f.get('size'), invscale=1024),
+            'container': f.get('containerFormat'),
+            'vcodec': f.get('videoCodecId'),
+            'height': f.get('height'),
+            'width': f.get('width'),
+            'url': '%s/flavorId/%s' % (info['dataUrl'], f['id']),
+        } for f in source_data['flavorAssets']]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': info['name'],
+            'formats': formats,
+            'description': info.get('description'),
+            'thumbnail': info.get('thumbnailUrl'),
+            'duration': info.get('duration'),
+            'timestamp': info.get('createdAt'),
+            'uploader_id': info.get('userId'),
+            'view_count': info.get('plays'),
+        }

From da419e23320aef560c05f543c5ee375e8956c95a Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 26 Feb 2015 23:47:45 +0200
Subject: [PATCH 1728/1937] [musicvault] Use the Kaltura extractor

---
 youtube_dl/extractor/musicvault.py | 27 +++++++--------------------
 1 file changed, 7 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/musicvault.py b/youtube_dl/extractor/musicvault.py
index ebb1eb8e9..0e46ac7c1 100644
--- a/youtube_dl/extractor/musicvault.py
+++ b/youtube_dl/extractor/musicvault.py
@@ -3,17 +3,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    parse_duration,
-    unified_strdate,
-)
 
 
 class MusicVaultIE(InfoExtractor):
     _VALID_URL = r'https?://www\.musicvault\.com/(?P<uploader_id>[^/?#]*)/video/(?P<display_id>[^/?#]*)_(?P<id>[0-9]+)\.html'
     _TEST = {
         'url': 'http://www.musicvault.com/the-allman-brothers-band/video/straight-from-the-heart_1010863.html',
-        'md5': '2cdbb3ae75f7fb3519821507d2fb3c15',
+        'md5': '3adcbdb3dcc02d647539e53f284ba171',
         'info_dict': {
             'id': '1010863',
             'ext': 'mp4',
@@ -22,9 +18,10 @@ class MusicVaultIE(InfoExtractor):
             'duration': 244,
             'uploader': 'The Allman Brothers Band',
             'thumbnail': 're:^https?://.*/thumbnail/.*',
-            'upload_date': '19811216',
+            'upload_date': '20131219',
             'location': 'Capitol Theatre (Passaic, NJ)',
             'description': 'Listen to The Allman Brothers Band perform Straight from the Heart at Capitol Theatre (Passaic, NJ) on Dec 16, 1981',
+            'timestamp': int,
         }
     }
 
@@ -43,34 +40,24 @@ class MusicVaultIE(InfoExtractor):
             r'<h1.*?>(.*?)</h1>', data_div, 'uploader', fatal=False)
         title = self._html_search_regex(
             r'<h2.*?>(.*?)</h2>', data_div, 'title')
-        upload_date = unified_strdate(self._html_search_regex(
-            r'<h3.*?>(.*?)</h3>', data_div, 'uploader', fatal=False))
         location = self._html_search_regex(
             r'<h4.*?>(.*?)</h4>', data_div, 'location', fatal=False)
 
-        duration = parse_duration(self._html_search_meta('duration', webpage))
-
-        VIDEO_URL_TEMPLATE = 'http://cdnapi.kaltura.com/p/%(uid)s/sp/%(wid)s/playManifest/entryId/%(entry_id)s/format/url/protocol/http'
         kaltura_id = self._search_regex(
             r'<div id="video-detail-player" data-kaltura-id="([^"]+)"',
             webpage, 'kaltura ID')
-        video_url = VIDEO_URL_TEMPLATE % {
-            'entry_id': kaltura_id,
-            'wid': self._search_regex(r'/wid/_([0-9]+)/', webpage, 'wid'),
-            'uid': self._search_regex(r'uiconf_id/([0-9]+)/', webpage, 'uid'),
-        }
+        wid = self._search_regex(r'/wid/_([0-9]+)/', webpage, 'wid')
 
         return {
             'id': mobj.group('id'),
-            'url': video_url,
-            'ext': 'mp4',
+            '_type': 'url_transparent',
+            'url': 'kaltura:%s:%s' % (wid, kaltura_id),
+            'ie_key': 'Kaltura',
             'display_id': display_id,
             'uploader_id': mobj.group('uploader_id'),
             'thumbnail': thumbnail,
             'description': self._html_search_meta('description', webpage),
-            'upload_date': upload_date,
             'location': location,
             'title': title,
             'uploader': uploader,
-            'duration': duration,
         }

From e3216b82bf6ef54db63984f7fece4e95fbc3b981 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 27 Feb 2015 00:34:19 +0200
Subject: [PATCH 1729/1937] [generic] Support dynamic Kaltura embeds (#5016)
 (#5073)

---
 youtube_dl/extractor/generic.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 3aff57e30..27e2bc300 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -557,6 +557,18 @@ class GenericIE(InfoExtractor):
                 'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !',
             }
         },
+        # Kaltura embed
+        {
+            'url': 'http://www.monumentalnetwork.com/videos/john-carlson-postgame-2-25-15',
+            'info_dict': {
+                'id': '1_eergr3h1',
+                'ext': 'mp4',
+                'upload_date': '20150226',
+                'uploader_id': 'MonumentalSports-Kaltura@perfectsensedigital.com',
+                'timestamp': int,
+                'title': 'John Carlson Postgame 2/25/15',
+            },
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -1113,6 +1125,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'Zapiks')
 
+        # Look for Kaltura embeds
+        mobj = re.search(
+            r"(?s)kWidget\.(?:thumb)?[Ee]mbed\(\{.*?'wid'\s*:\s*'_?(?P<partner_id>[^']+)',.*?'entry_id'\s*:\s*'(?P<id>[^']+)',", webpage)
+        if mobj is not None:
+            return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From 0eba1e178230a88d1b316f54edbb671d216c1d02 Mon Sep 17 00:00:00 2001
From: Sergey <Ftornik@users.noreply.github.com>
Date: Fri, 27 Feb 2015 00:51:22 +0200
Subject: [PATCH 1730/1937] [lynda] Fixed subtitles broken file

---
 youtube_dl/extractor/lynda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 109055e72..8bb21ee1d 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -152,7 +152,7 @@ class LyndaIE(InfoExtractor):
                 continue
             appear_time = m_current.group('timecode')
             disappear_time = m_next.group('timecode')
-            text = seq_current['Caption']
+            text = seq_current['Caption'].strip()
             srt += '%s\r\n%s --> %s\r\n%s' % (str(pos), appear_time, disappear_time, text)
         if srt:
             return srt

From f3bff94cf96587462baa8da6d7e5f17801381753 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 27 Feb 2015 12:24:51 +0100
Subject: [PATCH 1731/1937] [rtve] Extract duration

---
 youtube_dl/extractor/rtve.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index c0fd23ff1..b42442d12 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -8,8 +8,9 @@ import time
 from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
-    struct_unpack,
+    float_or_none,
     remove_end,
+    struct_unpack,
 )
 
 
@@ -67,6 +68,7 @@ class RTVEALaCartaIE(InfoExtractor):
             'id': '2491869',
             'ext': 'mp4',
             'title': 'Balonmano - Swiss Cup masculina. Final: España-Suecia',
+            'duration': 5024.566,
         },
     }, {
         'note': 'Live stream',
@@ -113,6 +115,7 @@ class RTVEALaCartaIE(InfoExtractor):
             'thumbnail': info.get('image'),
             'page_url': url,
             'subtitles': subtitles,
+            'duration': float_or_none(info.get('duration'), scale=1000),
         }
 
     def _get_subtitles(self, video_id, sub_file):

From 7862ad88b725daae957ad27ae60993e360c01e13 Mon Sep 17 00:00:00 2001
From: HanYOLO <HanYOLO@users.noreply.github.com>
Date: Fri, 27 Feb 2015 15:41:58 +0100
Subject: [PATCH 1732/1937] puls4 Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/puls4.py    | 61 ++++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)
 create mode 100644 youtube_dl/extractor/puls4.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e3b2cb54f..d137e1104 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -373,6 +373,7 @@ from .pornotube import PornotubeIE
 from .pornoxo import PornoXOIE
 from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
+from .puls4 import Puls4IE
 from .pyvideo import PyvideoIE
 from .quickvid import QuickVidIE
 from .r7 import R7IE
diff --git a/youtube_dl/extractor/puls4.py b/youtube_dl/extractor/puls4.py
new file mode 100644
index 000000000..70dedbff3
--- /dev/null
+++ b/youtube_dl/extractor/puls4.py
@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+import re
+
+
+class Puls4IE(InfoExtractor):
+
+    _VALID_URL = r'https?://www.puls4.com/video/.+?/play/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'http://www.puls4.com/video/pro-und-contra/play/2716816',
+        'md5': '49f6a6629747eeec43cef6a46b5df81d',
+        'info_dict': {
+            'id': '2716816',
+            'ext': 'mp4',
+            'title': 'Pro und Contra vom 23.02.2015'}},
+        {
+        'url': 'http://www.puls4.com/video/kult-spielfilme/play/1298106',
+        'md5': '6a48316c8903ece8dab9b9a7bf7a59ec',
+        'info_dict': {
+            'id': '1298106',
+            'ext': 'mp4',
+            'title': 'Lucky Fritz'}}
+    ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        # if fsk-button
+        real_url = self._html_search_regex(r'\"fsk-button\".+?href=\"([^"]+)',
+                                           webpage, 'fsk_button', default=None)
+        if real_url:
+            webpage = self._download_webpage(real_url, video_id)
+
+        title = self._html_search_regex(
+            r'<div id="bg_brandableContent">.+?<h1>(.+?)</h1>',
+            webpage, 'title', flags=re.DOTALL)
+
+        sd_url = self._html_search_regex(
+            r'{\"url\":\"([^"]+?)\",\"hd\":false',
+            webpage, 'sd_url').replace('\\', '')
+
+        formats = [{'format_id': 'sd', 'url': sd_url, 'quality': -2}]
+
+        hd_url = self._html_search_regex(
+            r'{\"url\":\"([^"]+?)\",\"hd\":true',
+            webpage, 'hd_url', default=None)
+        if hd_url:
+            hd_url = hd_url.replace('\\', '')
+            formats.append({'format_id': 'hd', 'url': hd_url, 'quality': -1})
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'ext': 'mp4'
+        }

From a0d646135aaf417e0aa000419974c676335d164a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 27 Feb 2015 20:56:06 +0600
Subject: [PATCH 1733/1937] [lynda] Extend _VALID_URL

---
 youtube_dl/extractor/lynda.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 109055e72..e7e9d80de 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -18,7 +18,7 @@ from ..utils import (
 class LyndaIE(InfoExtractor):
     IE_NAME = 'lynda'
     IE_DESC = 'lynda.com videos'
-    _VALID_URL = r'https?://www\.lynda\.com/[^/]+/[^/]+/\d+/(\d+)-\d\.html'
+    _VALID_URL = r'https?://www\.lynda\.com/(?:[^/]+/[^/]+/\d+|player/embed)/(\d+)'
     _LOGIN_URL = 'https://www.lynda.com/login/login.aspx'
     _NETRC_MACHINE = 'lynda'
 
@@ -27,7 +27,7 @@ class LyndaIE(InfoExtractor):
 
     ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html',
         'md5': 'ecfc6862da89489161fb9cd5f5a6fac1',
         'info_dict': {
@@ -36,7 +36,10 @@ class LyndaIE(InfoExtractor):
             'title': 'Using the exercise files',
             'duration': 68
         }
-    }
+    }, {
+        'url': 'https://www.lynda.com/player/embed/133770?tr=foo=1;bar=g;fizz=rt&fs=0',
+        'only_matching': True,
+    }]
 
     def _real_initialize(self):
         self._login()

From 781a7ef60ab03cf1e646a80ecf4cbd70f1532db6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 27 Feb 2015 16:18:18 +0100
Subject: [PATCH 1734/1937] [lynda] Use 'lstrip' for the subtitles

The newlines at the end are important, they separate each piece of text.
---
 youtube_dl/extractor/lynda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index bfd9b73d2..5dc22da22 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -155,7 +155,7 @@ class LyndaIE(InfoExtractor):
                 continue
             appear_time = m_current.group('timecode')
             disappear_time = m_next.group('timecode')
-            text = seq_current['Caption'].strip()
+            text = seq_current['Caption'].lstrip()
             srt += '%s\r\n%s --> %s\r\n%s' % (str(pos), appear_time, disappear_time, text)
         if srt:
             return srt

From 4ffbf77886b1c6c3f8d767223ca088395d08ab14 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Feb 2015 00:15:03 +0600
Subject: [PATCH 1735/1937] [odnoklassniki] Add extractor (Closes #5075)

---
 youtube_dl/extractor/__init__.py      |  1 +
 youtube_dl/extractor/odnoklassniki.py | 85 +++++++++++++++++++++++++++
 2 files changed, 86 insertions(+)
 create mode 100644 youtube_dl/extractor/odnoklassniki.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e3b2cb54f..aecb67bf4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -346,6 +346,7 @@ from .ntvde import NTVDeIE
 from .ntvru import NTVRuIE
 from .nytimes import NYTimesIE
 from .nuvid import NuvidIE
+from .odnoklassniki import OdnoklassnikiIE
 from .oktoberfesttv import OktoberfestTVIE
 from .ooyala import OoyalaIE
 from .openfilm import OpenFilmIE
diff --git a/youtube_dl/extractor/odnoklassniki.py b/youtube_dl/extractor/odnoklassniki.py
new file mode 100644
index 000000000..155d0ee6a
--- /dev/null
+++ b/youtube_dl/extractor/odnoklassniki.py
@@ -0,0 +1,85 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate,
+    int_or_none,
+    qualities,
+)
+
+
+class OdnoklassnikiIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:odnoklassniki|ok)\.ru/(?:video|web-api/video/moviePlayer)/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://ok.ru/video/20079905452',
+        'md5': '8e24ad2da6f387948e7a7d44eb8668fe',
+        'info_dict': {
+            'id': '20079905452',
+            'ext': 'mp4',
+            'title': 'Культура меняет нас (прекрасный ролик!))',
+            'duration': 100,
+            'upload_date': '20141207',
+            'uploader_id': '330537914540',
+            'uploader': 'Виталий Добровольский',
+            'like_count': int,
+            'age_limit': 0,
+        },
+    }, {
+        'url': 'http://ok.ru/web-api/video/moviePlayer/20079905452',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        player = self._parse_json(
+            self._search_regex(
+                r"OKVideo\.start\(({.+?})\s*,\s*'VideoAutoplay_player'", webpage, 'player'),
+            video_id)
+
+        metadata = self._parse_json(player['flashvars']['metadata'], video_id)
+
+        movie = metadata['movie']
+        title = movie['title']
+        thumbnail = movie.get('poster')
+        duration = int_or_none(movie.get('duration'))
+
+        author = metadata.get('author', {})
+        uploader_id = author.get('id')
+        uploader = author.get('name')
+
+        upload_date = unified_strdate(self._html_search_meta(
+            'ya:ovs:upload_date', webpage, 'upload date'))
+
+        age_limit = None
+        adult = self._html_search_meta(
+            'ya:ovs:adult', webpage, 'age limit')
+        if adult:
+            age_limit = 18 if adult == 'true' else 0
+
+        like_count = int_or_none(metadata.get('likeCount'))
+
+        quality = qualities(('mobile', 'lowest', 'low', 'sd', 'hd'))
+
+        formats = [{
+            'url': f['url'],
+            'ext': 'mp4',
+            'format_id': f['name'],
+            'quality': quality(f['name']),
+        } for f in metadata['videos']]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'upload_date': upload_date,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'like_count': like_count,
+            'age_limit': age_limit,
+            'formats': formats,
+        }

From bd3749ed693ae96becd3832f20e765e1efe01476 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Feb 2015 00:19:31 +0600
Subject: [PATCH 1736/1937] [kaltura] Extend _VALID_URL (Closes #5081)

---
 youtube_dl/extractor/kaltura.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 2aff410c5..d28730492 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -14,7 +14,7 @@ from ..utils import (
 class KalturaIE(InfoExtractor):
     _VALID_URL = r'''(?x)
     (?:kaltura:|
-       https?://(:?www\.)?kaltura\.com/index\.php/kwidget/(?:[^/]+/)*?wid/_
+       https?://(:?(?:www|cdnapisec)\.)?kaltura\.com/index\.php/kwidget/(?:[^/]+/)*?wid/_
     )(?P<partner_id>\d+)
     (?::|
        /(?:[^/]+/)*?entry_id/
@@ -39,6 +39,10 @@ class KalturaIE(InfoExtractor):
             'url': 'http://www.kaltura.com/index.php/kwidget/cache_st/1300318621/wid/_269692/uiconf_id/3873291/entry_id/1_1jc2y3e4',
             'only_matching': True,
         },
+        {
+            'url': 'https://cdnapisec.kaltura.com/index.php/kwidget/wid/_557781/uiconf_id/22845202/entry_id/1_plr1syf3',
+            'only_matching': True,
+        },
     ]
 
     def _kaltura_api_call(self, video_id, actions, *args, **kwargs):

From 40b077bc7ee2722463f23974ce488acb7f11815a Mon Sep 17 00:00:00 2001
From: thc202 <thc202@gmail.com>
Date: Fri, 27 Feb 2015 22:27:30 +0000
Subject: [PATCH 1737/1937] [oppetarkiv] Add new extractor

Some, if not all, of the videos appear to be geo-blocked (Sweden).
Test might fail (403 Forbidden) if not run through a Swedish connection.
---
 youtube_dl/extractor/__init__.py   |  1 +
 youtube_dl/extractor/oppetarkiv.py | 56 ++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 youtube_dl/extractor/oppetarkiv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index aecb67bf4..1544f1059 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -350,6 +350,7 @@ from .odnoklassniki import OdnoklassnikiIE
 from .oktoberfesttv import OktoberfestTVIE
 from .ooyala import OoyalaIE
 from .openfilm import OpenFilmIE
+from .oppetarkiv import OppetArkivIE
 from .orf import (
     ORFTVthekIE,
     ORFOE1IE,
diff --git a/youtube_dl/extractor/oppetarkiv.py b/youtube_dl/extractor/oppetarkiv.py
new file mode 100644
index 000000000..6dd1fad3f
--- /dev/null
+++ b/youtube_dl/extractor/oppetarkiv.py
@@ -0,0 +1,56 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+)
+
+
+class OppetArkivIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?oppetarkiv.se/video/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318',
+        'md5': '7b95ca9bedeead63012b2d7c3992c28f',
+        'info_dict': {
+            'id': '1058509',
+            'ext': 'mp4',
+            'title': 'Farlig kryssning',
+            'duration': 2566,
+            'thumbnail': 're:^https?://.*[\.-]jpg$',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        info = self._download_json(
+            'http://www.oppetarkiv.se/video/%s?output=json' % video_id, video_id)
+
+        title = info['context']['title']
+        thumbnail = info['context'].get('thumbnailImage')
+
+        video_info = info['video']
+        formats = []
+        for vr in video_info['videoReferences']:
+            vurl = vr['url']
+            if determine_ext(vurl) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    vurl, video_id,
+                    ext='mp4', entry_protocol='m3u8_native',
+                    m3u8_id=vr.get('playerType')))
+            else:
+                formats.append({
+                    'format_id': vr.get('playerType'),
+                    'url': vurl,
+                })
+        self._sort_formats(formats)
+
+        duration = video_info.get('materialLength')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': thumbnail,
+            'duration': duration,
+        }

From 0f2c0d335b14ba1596e6608db7a6f29a0d9e1c86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 28 Feb 2015 14:03:27 +0100
Subject: [PATCH 1738/1937] [YoutubeDL] Use the InfoExtractor._download_webpage
 method for getting the subtitles

It handles encodings better, for example for 'http://www.npo.nl/nos-journaal/14-02-2015/POW_00942207'
---
 youtube_dl/YoutubeDL.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 76fc394bc..74e426168 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1300,17 +1300,18 @@ class YoutubeDL(object):
             # subtitles download errors are already managed as troubles in relevant IE
             # that way it will silently go on when used with unsupporting IE
             subtitles = info_dict['requested_subtitles']
+            ie = self.get_info_extractor(info_dict['extractor_key'])
             for sub_lang, sub_info in subtitles.items():
                 sub_format = sub_info['ext']
                 if sub_info.get('data') is not None:
                     sub_data = sub_info['data']
                 else:
                     try:
-                        uf = self.urlopen(sub_info['url'])
-                        sub_data = uf.read().decode('utf-8')
-                    except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+                        sub_data = ie._download_webpage(
+                            sub_info['url'], info_dict['id'], note=False)
+                    except ExtractorError as err:
                         self.report_warning('Unable to download subtitle for "%s": %s' %
-                                            (sub_lang, compat_str(err)))
+                                            (sub_lang, compat_str(err.cause)))
                         continue
                 try:
                     sub_filename = subtitles_filename(filename, sub_lang, sub_format)

From e9fade72f347cbcce779ff176d516467a425b43a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 28 Feb 2015 14:43:24 +0100
Subject: [PATCH 1739/1937] Add postprocessor for converting subtitles (closes
 #4954)

---
 youtube_dl/__init__.py               |  8 ++++++
 youtube_dl/options.py                |  4 +++
 youtube_dl/postprocessor/__init__.py |  2 ++
 youtube_dl/postprocessor/ffmpeg.py   | 38 ++++++++++++++++++++++++++++
 4 files changed, 52 insertions(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 5ce201800..49f382695 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -170,6 +170,9 @@ def _real_main(argv=None):
     if opts.recodevideo is not None:
         if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
             parser.error('invalid video recode format specified')
+    if opts.convertsubtitles is not None:
+        if opts.convertsubtitles not in ['srt', 'vtt', 'ass']:
+            parser.error('invalid subtitle format specified')
 
     if opts.date is not None:
         date = DateRange.day(opts.date)
@@ -223,6 +226,11 @@ def _real_main(argv=None):
             'key': 'FFmpegVideoConvertor',
             'preferedformat': opts.recodevideo,
         })
+    if opts.convertsubtitles:
+        postprocessors.append({
+            'key': 'FFmpegSubtitlesConvertor',
+            'format': opts.convertsubtitles,
+        })
     if opts.embedsubtitles:
         postprocessors.append({
             'key': 'FFmpegEmbedSubtitle',
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 886ce9613..58f811162 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -751,6 +751,10 @@ def parseOpts(overrideArguments=None):
         '--exec',
         metavar='CMD', dest='exec_cmd',
         help='Execute a command on the file after downloading, similar to find\'s -exec syntax. Example: --exec \'adb push {} /sdcard/Music/ && rm {}\'')
+    postproc.add_option(
+        '--convert-subtitles', '--convert-subs',
+        metavar='FORMAT', dest='convertsubtitles', default=None,
+        help='Convert the subtitles to other format (currently supported: srt|ass|vtt)')
 
     parser.add_option_group(general)
     parser.add_option_group(network)
diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index 0ffbca258..708df3dd4 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -11,6 +11,7 @@ from .ffmpeg import (
     FFmpegMergerPP,
     FFmpegMetadataPP,
     FFmpegVideoConvertorPP,
+    FFmpegSubtitlesConvertorPP,
 )
 from .xattrpp import XAttrMetadataPP
 from .execafterdownload import ExecAfterDownloadPP
@@ -31,6 +32,7 @@ __all__ = [
     'FFmpegMergerPP',
     'FFmpegMetadataPP',
     'FFmpegPostProcessor',
+    'FFmpegSubtitlesConvertorPP',
     'FFmpegVideoConvertorPP',
     'XAttrMetadataPP',
 ]
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 398fe050e..30094c2f3 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import io
 import os
 import subprocess
 import sys
@@ -635,3 +636,40 @@ class FFmpegFixupM4aPP(FFmpegPostProcessor):
         os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
         return True, info
+
+
+class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
+    def __init__(self, downloader=None, format=None):
+        super(FFmpegSubtitlesConvertorPP, self).__init__(downloader)
+        self.format = format
+
+    def run(self, info):
+        subs = info.get('requested_subtitles')
+        filename = info['filepath']
+        new_ext = self.format
+        new_format = new_ext
+        if new_format == 'vtt':
+            new_format = 'webvtt'
+        if subs is None:
+            self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to convert')
+            return True, info
+        self._downloader.to_screen('[ffmpeg] Converting subtitles')
+        for lang, sub in subs.items():
+            ext = sub['ext']
+            if ext == new_ext:
+                self._downloader.to_screen(
+                    '[ffmpeg] Subtitle file for %s is already in the requested'
+                    'format' % new_ext)
+                continue
+            new_file = subtitles_filename(filename, lang, new_ext)
+            self.run_ffmpeg(
+                subtitles_filename(filename, lang, ext),
+                new_file, ['-f', new_format])
+
+            with io.open(new_file, 'rt', encoding='utf-8') as f:
+                subs[lang] = {
+                    'ext': ext,
+                    'data': f.read(),
+                }
+
+        return True, info

From e143f5dae9c767529b8b522a9df63ac0ee8fc356 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Feb 2015 21:12:06 +0600
Subject: [PATCH 1740/1937] [oppetarkiv] Extract f4m formats and age limit

---
 youtube_dl/extractor/oppetarkiv.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/oppetarkiv.py b/youtube_dl/extractor/oppetarkiv.py
index 6dd1fad3f..ae6a28308 100644
--- a/youtube_dl/extractor/oppetarkiv.py
+++ b/youtube_dl/extractor/oppetarkiv.py
@@ -8,17 +8,19 @@ from ..utils import (
 
 
 class OppetArkivIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?oppetarkiv.se/video/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?oppetarkiv\.se/video/(?P<id>[0-9]+)'
     _TEST = {
         'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318',
-        'md5': '7b95ca9bedeead63012b2d7c3992c28f',
+        'md5': '5c1eb616e59f733d4af77edc5177d2fe',
         'info_dict': {
             'id': '1058509',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'Farlig kryssning',
             'duration': 2566,
             'thumbnail': 're:^https?://.*[\.-]jpg$',
+            'age_limit': 0,
         },
+        'skip': 'Only works from Sweden',
     }
 
     def _real_extract(self, url):
@@ -33,11 +35,16 @@ class OppetArkivIE(InfoExtractor):
         formats = []
         for vr in video_info['videoReferences']:
             vurl = vr['url']
-            if determine_ext(vurl) == 'm3u8':
+            ext = determine_ext(vurl)
+            if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
                     vurl, video_id,
                     ext='mp4', entry_protocol='m3u8_native',
                     m3u8_id=vr.get('playerType')))
+            elif ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    vurl + '?hdcore=3.3.0', video_id,
+                    f4m_id=vr.get('playerType')))
             else:
                 formats.append({
                     'format_id': vr.get('playerType'),
@@ -47,10 +54,13 @@ class OppetArkivIE(InfoExtractor):
 
         duration = video_info.get('materialLength')
 
+        age_limit = 18 if video_info.get('inappropriateForChildren') else 0
+
         return {
             'id': video_id,
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
             'duration': duration,
+            'age_limit': age_limit,
         }

From df5ae3eb16effec4dc2609ed3520ddc068305efe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Feb 2015 21:25:04 +0600
Subject: [PATCH 1741/1937] [oppetarkiv] Merge with svtplay

---
 youtube_dl/extractor/__init__.py   |  1 -
 youtube_dl/extractor/oppetarkiv.py | 66 ------------------------------
 youtube_dl/extractor/svtplay.py    | 42 +++++++++++++++----
 3 files changed, 34 insertions(+), 75 deletions(-)
 delete mode 100644 youtube_dl/extractor/oppetarkiv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1544f1059..aecb67bf4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -350,7 +350,6 @@ from .odnoklassniki import OdnoklassnikiIE
 from .oktoberfesttv import OktoberfestTVIE
 from .ooyala import OoyalaIE
 from .openfilm import OpenFilmIE
-from .oppetarkiv import OppetArkivIE
 from .orf import (
     ORFTVthekIE,
     ORFOE1IE,
diff --git a/youtube_dl/extractor/oppetarkiv.py b/youtube_dl/extractor/oppetarkiv.py
deleted file mode 100644
index ae6a28308..000000000
--- a/youtube_dl/extractor/oppetarkiv.py
+++ /dev/null
@@ -1,66 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    determine_ext,
-)
-
-
-class OppetArkivIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?oppetarkiv\.se/video/(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318',
-        'md5': '5c1eb616e59f733d4af77edc5177d2fe',
-        'info_dict': {
-            'id': '1058509',
-            'ext': 'flv',
-            'title': 'Farlig kryssning',
-            'duration': 2566,
-            'thumbnail': 're:^https?://.*[\.-]jpg$',
-            'age_limit': 0,
-        },
-        'skip': 'Only works from Sweden',
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        info = self._download_json(
-            'http://www.oppetarkiv.se/video/%s?output=json' % video_id, video_id)
-
-        title = info['context']['title']
-        thumbnail = info['context'].get('thumbnailImage')
-
-        video_info = info['video']
-        formats = []
-        for vr in video_info['videoReferences']:
-            vurl = vr['url']
-            ext = determine_ext(vurl)
-            if ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    vurl, video_id,
-                    ext='mp4', entry_protocol='m3u8_native',
-                    m3u8_id=vr.get('playerType')))
-            elif ext == 'f4m':
-                formats.extend(self._extract_f4m_formats(
-                    vurl + '?hdcore=3.3.0', video_id,
-                    f4m_id=vr.get('playerType')))
-            else:
-                formats.append({
-                    'format_id': vr.get('playerType'),
-                    'url': vurl,
-                })
-        self._sort_formats(formats)
-
-        duration = video_info.get('materialLength')
-
-        age_limit = 18 if video_info.get('inappropriateForChildren') else 0
-
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'age_limit': age_limit,
-        }
diff --git a/youtube_dl/extractor/svtplay.py b/youtube_dl/extractor/svtplay.py
index eadb9ccb4..433dfd1cb 100644
--- a/youtube_dl/extractor/svtplay.py
+++ b/youtube_dl/extractor/svtplay.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
@@ -8,23 +10,40 @@ from ..utils import (
 
 
 class SVTPlayIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?svtplay\.se/video/(?P<id>[0-9]+)'
-    _TEST = {
+    IE_DESC = 'SVT Play and Öppet arkiv'
+    _VALID_URL = r'https?://(?:www\.)?(?P<host>svtplay|oppetarkiv)\.se/video/(?P<id>[0-9]+)'
+    _TESTS = [{
         'url': 'http://www.svtplay.se/video/2609989/sm-veckan/sm-veckan-rally-final-sasong-1-sm-veckan-rally-final',
-        'md5': 'f4a184968bc9c802a9b41316657aaa80',
+        'md5': 'ade3def0643fa1c40587a422f98edfd9',
         'info_dict': {
             'id': '2609989',
-            'ext': 'mp4',
+            'ext': 'flv',
             'title': 'SM veckan vinter, Örebro - Rally, final',
             'duration': 4500,
             'thumbnail': 're:^https?://.*[\.-]jpg$',
+            'age_limit': 0,
         },
-    }
+    }, {
+        'url': 'http://www.oppetarkiv.se/video/1058509/rederiet-sasong-1-avsnitt-1-av-318',
+        'md5': 'c3101a17ce9634f4c1f9800f0746c187',
+        'info_dict': {
+            'id': '1058509',
+            'ext': 'flv',
+            'title': 'Farlig kryssning',
+            'duration': 2566,
+            'thumbnail': 're:^https?://.*[\.-]jpg$',
+            'age_limit': 0,
+        },
+        'skip': 'Only works from Sweden',
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        host = mobj.group('host')
+
         info = self._download_json(
-            'http://www.svtplay.se/video/%s?output=json' % video_id, video_id)
+            'http://www.%s.se/video/%s?output=json' % (host, video_id), video_id)
 
         title = info['context']['title']
         thumbnail = info['context'].get('thumbnailImage')
@@ -33,11 +52,16 @@ class SVTPlayIE(InfoExtractor):
         formats = []
         for vr in video_info['videoReferences']:
             vurl = vr['url']
-            if determine_ext(vurl) == 'm3u8':
+            ext = determine_ext(vurl)
+            if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
                     vurl, video_id,
                     ext='mp4', entry_protocol='m3u8_native',
                     m3u8_id=vr.get('playerType')))
+            elif ext == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    vurl + '?hdcore=3.3.0', video_id,
+                    f4m_id=vr.get('playerType')))
             else:
                 formats.append({
                     'format_id': vr.get('playerType'),
@@ -46,6 +70,7 @@ class SVTPlayIE(InfoExtractor):
         self._sort_formats(formats)
 
         duration = video_info.get('materialLength')
+        age_limit = 18 if video_info.get('inappropriateForChildren') else 0
 
         return {
             'id': video_id,
@@ -53,4 +78,5 @@ class SVTPlayIE(InfoExtractor):
             'formats': formats,
             'thumbnail': thumbnail,
             'duration': duration,
+            'age_limit': age_limit,
         }

From 6c87c2eea8b7d14c4178aaae3d74559347a772e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Feb 2015 22:25:57 +0600
Subject: [PATCH 1742/1937] [puls4] Improve and extract more metadata

---
 youtube_dl/extractor/puls4.py | 89 +++++++++++++++++++++++------------
 1 file changed, 58 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/puls4.py b/youtube_dl/extractor/puls4.py
index 70dedbff3..cce84b9e4 100644
--- a/youtube_dl/extractor/puls4.py
+++ b/youtube_dl/extractor/puls4.py
@@ -1,61 +1,88 @@
 # -*- coding: utf-8 -*-
-
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-
-import re
+from ..utils import (
+    ExtractorError,
+    unified_strdate,
+    int_or_none,
+)
 
 
 class Puls4IE(InfoExtractor):
-
-    _VALID_URL = r'https?://www.puls4.com/video/.+?/play/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?puls4\.com/video/[^/]+/play/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://www.puls4.com/video/pro-und-contra/play/2716816',
         'md5': '49f6a6629747eeec43cef6a46b5df81d',
         'info_dict': {
             'id': '2716816',
             'ext': 'mp4',
-            'title': 'Pro und Contra vom 23.02.2015'}},
-        {
+            'title': 'Pro und Contra vom 23.02.2015',
+            'description': 'md5:293e44634d9477a67122489994675db6',
+            'duration': 2989,
+            'upload_date': '20150224',
+            'uploader': 'PULS_4',
+        },
+        'skip': 'Only works from Germany',
+    }, {
         'url': 'http://www.puls4.com/video/kult-spielfilme/play/1298106',
         'md5': '6a48316c8903ece8dab9b9a7bf7a59ec',
         'info_dict': {
             'id': '1298106',
             'ext': 'mp4',
-            'title': 'Lucky Fritz'}}
-    ]
+            'title': 'Lucky Fritz',
+        },
+        'skip': 'Only works from Germany',
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        # if fsk-button
-        real_url = self._html_search_regex(r'\"fsk-button\".+?href=\"([^"]+)',
-                                           webpage, 'fsk_button', default=None)
+        error_message = self._html_search_regex(
+            r'<div class="message-error">(.+?)</div>',
+            webpage, 'error message', default=None)
+        if error_message:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error_message), expected=True)
+
+        real_url = self._html_search_regex(
+            r'\"fsk-button\".+?href=\"([^"]+)',
+            webpage, 'fsk_button', default=None)
         if real_url:
             webpage = self._download_webpage(real_url, video_id)
 
-        title = self._html_search_regex(
-            r'<div id="bg_brandableContent">.+?<h1>(.+?)</h1>',
-            webpage, 'title', flags=re.DOTALL)
+        player = self._search_regex(
+            r'p4_video_player(?:_iframe)?\("video_\d+_container"\s*,(.+?)\);\s*\}',
+            webpage, 'player')
 
-        sd_url = self._html_search_regex(
-            r'{\"url\":\"([^"]+?)\",\"hd\":false',
-            webpage, 'sd_url').replace('\\', '')
+        player_json = self._parse_json(
+            '[%s]' % player, video_id,
+            transform_source=lambda s: s.replace('undefined,', ''))
 
-        formats = [{'format_id': 'sd', 'url': sd_url, 'quality': -2}]
+        formats = None
+        result = None
 
-        hd_url = self._html_search_regex(
-            r'{\"url\":\"([^"]+?)\",\"hd\":true',
-            webpage, 'hd_url', default=None)
-        if hd_url:
-            hd_url = hd_url.replace('\\', '')
-            formats.append({'format_id': 'hd', 'url': hd_url, 'quality': -1})
+        for v in player_json:
+            if isinstance(v, list) and not formats:
+                formats = [{
+                    'url': f['url'],
+                    'format': 'hd' if f.get('hd') else 'sd',
+                    'width': int_or_none(f.get('size_x')),
+                    'height': int_or_none(f.get('size_y')),
+                    'tbr': int_or_none(f.get('bitrate')),
+                } for f in v]
+                self._sort_formats(formats)
+            elif isinstance(v, dict) and not result:
+                result = {
+                    'id': video_id,
+                    'title': v['videopartname'].strip(),
+                    'description': v.get('videotitle'),
+                    'duration': int_or_none(v.get('videoduration') or v.get('episodeduration')),
+                    'upload_date': unified_strdate(v.get('clipreleasetime')),
+                    'uploader': v.get('channel'),
+                }
 
-        return {
-            'id': video_id,
-            'title': title,
-            'formats': formats,
-            'ext': 'mp4'
-        }
+        result['formats'] = formats
+
+        return result

From 0d103de3b0b03c5027f0015327c2c44b9073513a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 28 Feb 2015 22:59:55 +0600
Subject: [PATCH 1743/1937] [twitch] Pass api_token along with every request
 (Closes #3986)

---
 youtube_dl/extractor/twitch.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 4b0d8988d..4b0ce54df 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -34,7 +34,15 @@ class TwitchBaseIE(InfoExtractor):
                 expected=True)
 
     def _download_json(self, url, video_id, note='Downloading JSON metadata'):
-        response = super(TwitchBaseIE, self)._download_json(url, video_id, note)
+        headers = {
+            'Referer': 'http://api.twitch.tv/crossdomain/receiver.html?v=2',
+            'X-Requested-With': 'XMLHttpRequest',
+        }
+        for cookie in self._downloader.cookiejar:
+            if cookie.name == 'api_token':
+                headers['Twitch-Api-Token'] = cookie.value
+        request = compat_urllib_request.Request(url, headers=headers)
+        response = super(TwitchBaseIE, self)._download_json(request, video_id, note)
         self._handle_error(response)
         return response
 

From 8237bec4f05d6930146a7f0b087ae1e259917799 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 28 Feb 2015 20:52:52 +0100
Subject: [PATCH 1744/1937] [escapist] Extract duration

---
 youtube_dl/extractor/escapist.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 80e9084f4..e47f3e27a 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -8,6 +8,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     js_to_json,
+    parse_duration,
 )
 
 
@@ -25,6 +26,7 @@ class EscapistIE(InfoExtractor):
             'uploader': 'The Escapist Presents',
             'title': "Breaking Down Baldur's Gate",
             'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 264,
         }
     }
 
@@ -41,6 +43,7 @@ class EscapistIE(InfoExtractor):
             r"<h1\s+class='headline'>(.*?)</a>",
             webpage, 'uploader', fatal=False)
         description = self._html_search_meta('description', webpage)
+        duration = parse_duration(self._html_search_meta('duration', webpage))
 
         raw_title = self._html_search_meta('title', webpage, fatal=True)
         title = raw_title.partition(' : ')[2]
@@ -105,6 +108,7 @@ class EscapistIE(InfoExtractor):
             'title': title,
             'thumbnail': self._og_search_thumbnail(webpage),
             'description': description,
+            'duration': duration,
         }
 
         if self._downloader.params.get('include_ads') and ad_formats:

From eee6293d572b85c700ef645ac96b1816cb45901d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 28 Feb 2015 20:55:44 +0100
Subject: [PATCH 1745/1937] [thechive] remove in favor of Kaltura (#5072)

---
 youtube_dl/extractor/__init__.py |  1 -
 youtube_dl/extractor/thechive.py | 60 --------------------------------
 2 files changed, 61 deletions(-)
 delete mode 100644 youtube_dl/extractor/thechive.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f3967ff7b..ffcc7d9ab 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -484,7 +484,6 @@ from .tenplay import TenPlayIE
 from .testurl import TestURLIE
 from .testtube import TestTubeIE
 from .tf1 import TF1IE
-from .thechive import TheChiveIE
 from .theonion import TheOnionIE
 from .theplatform import ThePlatformIE
 from .thesixtyone import TheSixtyOneIE
diff --git a/youtube_dl/extractor/thechive.py b/youtube_dl/extractor/thechive.py
deleted file mode 100644
index df1a7998d..000000000
--- a/youtube_dl/extractor/thechive.py
+++ /dev/null
@@ -1,60 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import unified_strdate
-
-
-
-class TheChiveIE(InfoExtractor):
-    _VALID_URL = r'http://(www\.)?thechive\.com/[^/]+/[^/]+/[^/]+/(?P<video_id>[A-Za-z\-]+)'
-    _TEST = {
-        'url': "http://thechive.com/2015/02/20/so-thats-what-a-set-of-redneck-bagpipes-sound-like-video/",
-        'md5': "366710dda77cfa727bdef3523ba8466f",
-        'info_dict': {
-            'id': "so-thats-what-a-set-of-redneck-bagpipes-sound-like-video",
-            'title': "So that's what a set of redneck bagpipes sound like... (Video)",
-            'description': "Okay that was pretty good. Now play Freebird!...",
-            'thumbnail': "https://thechive.files.wordpress.com/2015/02/0_07dghz0w-thumbnail2.jpg",
-            'author': "Ben",
-            'upload_date': "20150220",
-            'ext': "mp4"
-        }
-    }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
-        webpage = self._download_webpage(url, video_id)
-
-        title = self._og_search_title(webpage)
-        description = self._html_search_regex(r'(?s)<meta name="description" content="(.*?)" />', webpage, 'description')
-        thumbnail = self._og_search_thumbnail(webpage)
-        author = self._html_search_regex(
-            r'(?s)itemprop="author">(.+?)</span>', webpage, 'author', fatal=False).capitalize() 
-        upload_date = unified_strdate(self._html_search_regex(
-            r'(?s)itemprop="datePublished" datetime="(.+?)">', webpage, 'upload_date', fatal=False))
-
-        # Adapted from extractor/musicvault.py
-        VIDEO_URL_TEMPLATE = 'http://cdnapi.kaltura.com/p/%(uid)s/sp/%(wid)s/playManifest/entryId/%(entry_id)s/format/url/protocol/http'
-
-        kaltura_id = self._search_regex(
-            r'entry_id=([^"]+)',
-            webpage, 'kaltura ID')
-        video_url = VIDEO_URL_TEMPLATE % {
-            'entry_id': kaltura_id,
-            'wid': self._search_regex(r'partner_id/([0-9]+)\?', webpage, 'wid'),
-            'uid': self._search_regex(r'uiconf_id/([0-9]+)/', webpage, 'uid'),
-        }
-
-        return {
-            'url': video_url,
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'author': author,
-            'upload_date': upload_date,
-            'ext': 'mp4'
-        }
\ No newline at end of file

From 01349011088b10861e283c245f5da56aa3d5fba0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 28 Feb 2015 21:24:25 +0100
Subject: [PATCH 1746/1937] release 2015.02.28

---
 README.md              | 2 ++
 docs/supportedsites.md | 5 ++++-
 youtube_dl/version.py  | 2 +-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 2c53e2211..04f664cd3 100644
--- a/README.md
+++ b/README.md
@@ -408,6 +408,8 @@ which means you can modify it, redistribute it or use it however you like.
                                      downloading, similar to find's -exec
                                      syntax. Example: --exec 'adb push {}
                                      /sdcard/Music/ && rm {}'
+    --convert-subtitles FORMAT       Convert the subtitles to other format
+                                     (currently supported: srt|ass|vtt)
 
 # CONFIGURATION
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 49b4ac8c1..062cb3d62 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -210,6 +210,7 @@
  - **Jove**
  - **jpopsuki.tv**
  - **Jukebox**
+ - **Kaltura**
  - **Kankan**
  - **Karaoketv**
  - **keek**
@@ -308,6 +309,7 @@
  - **Nuvid**
  - **NYTimes**
  - **ocw.mit.edu**
+ - **Odnoklassniki**
  - **OktoberfestTV**
  - **on.aol.com**
  - **Ooyala**
@@ -334,6 +336,7 @@
  - **PornoXO**
  - **PromptFile**
  - **prosiebensat1**: ProSiebenSat.1 Digital
+ - **Puls4**
  - **Pyvideo**
  - **QuickVid**
  - **R7**
@@ -412,7 +415,7 @@
  - **StreamCZ**
  - **StreetVoice**
  - **SunPorno**
- - **SVTPlay**
+ - **SVTPlay**: SVT Play and Öppet arkiv
  - **SWRMediathek**
  - **Syfy**
  - **SztvHu**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index cf3e28bbe..5582348ba 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.26.2'
+__version__ = '2015.02.28'

From 003c69a84b68cadb46aeb8e03115848a722fd675 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 28 Feb 2015 21:42:16 +0100
Subject: [PATCH 1747/1937] Use shutil.get_terminal_size for getting the
 terminal width if it's available (python >= 3.3)

---
 youtube_dl/YoutubeDL.py |  4 ++--
 youtube_dl/compat.py    | 30 ++++++++++++++++++++++++++++++
 youtube_dl/options.py   |  4 ++--
 youtube_dl/utils.py     | 17 -----------------
 4 files changed, 34 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 74e426168..d7c6db0ff 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -28,6 +28,7 @@ from .compat import (
     compat_basestring,
     compat_cookiejar,
     compat_expanduser,
+    compat_get_terminal_size,
     compat_http_client,
     compat_kwargs,
     compat_str,
@@ -46,7 +47,6 @@ from .utils import (
     ExtractorError,
     format_bytes,
     formatSeconds,
-    get_term_width,
     locked_file,
     make_HTTPS_handler,
     MaxDownloadsReached,
@@ -284,7 +284,7 @@ class YoutubeDL(object):
             try:
                 import pty
                 master, slave = pty.openpty()
-                width = get_term_width()
+                width = compat_get_terminal_size().columns
                 if width is None:
                     width_args = []
                 else:
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index e989cdbbd..b2bf149ef 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -1,9 +1,11 @@
 from __future__ import unicode_literals
 
+import collections
 import getpass
 import optparse
 import os
 import re
+import shutil
 import socket
 import subprocess
 import sys
@@ -364,6 +366,33 @@ def workaround_optparse_bug9161():
             return real_add_option(self, *bargs, **bkwargs)
         optparse.OptionGroup.add_option = _compat_add_option
 
+if hasattr(shutil, 'get_terminal_size'):  # Python >= 3.3
+    compat_get_terminal_size = shutil.get_terminal_size
+else:
+    _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
+
+    def compat_get_terminal_size():
+        columns = compat_getenv('COLUMNS', None)
+        if columns:
+            columns = int(columns)
+        else:
+            columns = None
+        lines = compat_getenv('LINES', None)
+        if lines:
+            lines = int(lines)
+        else:
+            lines = None
+
+        try:
+            sp = subprocess.Popen(
+                ['stty', 'size'],
+                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            out, err = sp.communicate()
+            lines, columns = map(int, out.split())
+        except:
+            pass
+        return _terminal_size(columns, lines)
+
 
 __all__ = [
     'compat_HTTPError',
@@ -371,6 +400,7 @@ __all__ = [
     'compat_chr',
     'compat_cookiejar',
     'compat_expanduser',
+    'compat_get_terminal_size',
     'compat_getenv',
     'compat_getpass',
     'compat_html_entities',
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 58f811162..a2ffe96bc 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -8,11 +8,11 @@ import sys
 from .downloader.external import list_external_downloaders
 from .compat import (
     compat_expanduser,
+    compat_get_terminal_size,
     compat_getenv,
     compat_kwargs,
 )
 from .utils import (
-    get_term_width,
     write_string,
 )
 from .version import __version__
@@ -100,7 +100,7 @@ def parseOpts(overrideArguments=None):
         return opts
 
     # No need to wrap help messages if we're on a wide console
-    columns = get_term_width()
+    columns = compat_get_terminal_size().columns
     max_width = columns if columns else 80
     max_help_position = 80
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1f3bfef7d..d4938ec36 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -35,7 +35,6 @@ import zlib
 from .compat import (
     compat_basestring,
     compat_chr,
-    compat_getenv,
     compat_html_entities,
     compat_http_client,
     compat_parse_qs,
@@ -1173,22 +1172,6 @@ def parse_filesize(s):
     return int(float(num_str) * mult)
 
 
-def get_term_width():
-    columns = compat_getenv('COLUMNS', None)
-    if columns:
-        return int(columns)
-
-    try:
-        sp = subprocess.Popen(
-            ['stty', 'size'],
-            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out, err = sp.communicate()
-        return int(out.split()[1])
-    except:
-        pass
-    return None
-
-
 def month_by_name(name):
     """ Return the number of a month by (locale-independently) English name """
 

From 31bd39256b153391f521169ae449fd8c7c6e51d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 1 Mar 2015 11:46:57 +0100
Subject: [PATCH 1748/1937] --load-info: Use the fileinput module

It automatically handles the '-' filename as stdin
---
 youtube_dl/YoutubeDL.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d7c6db0ff..15367c4e3 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -4,8 +4,10 @@
 from __future__ import absolute_import, unicode_literals
 
 import collections
+import contextlib
 import datetime
 import errno
+import fileinput
 import io
 import itertools
 import json
@@ -1452,8 +1454,11 @@ class YoutubeDL(object):
         return self._download_retcode
 
     def download_with_info_file(self, info_filename):
-        with io.open(info_filename, 'r', encoding='utf-8') as f:
-            info = json.load(f)
+        with contextlib.closing(fileinput.FileInput(
+                [info_filename], mode='r',
+                openhook=fileinput.hook_encoded('utf-8'))) as f:
+            # FileInput doesn't have a read method, we can't call json.load
+            info = json.loads('\n'.join(f))
         try:
             self.process_ie_result(info, download=True)
         except DownloadError:

From 66dc9a3701cd97af44cf478e813d4f7eb0947edc Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 1 Mar 2015 12:04:39 +0100
Subject: [PATCH 1749/1937] [README] Document HTTP 429 (Closes #5092)

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index 04f664cd3..f2909e8d6 100644
--- a/README.md
+++ b/README.md
@@ -529,6 +529,10 @@ YouTube requires an additional signature since September 2012 which is not suppo
 
 In February 2015, the new YouTube player contained a character sequence in a string that was misinterpreted by old versions of youtube-dl. See [above](#how-do-i-update-youtube-dl) for how to update youtube-dl.
 
+### HTTP Error 429: Too Many Requests or 402: Payment Required
+
+These two error codes indicate that the service is blocking your IP address because of overuse. Contact the service and ask them to unblock your IP address, or - if you have acquired a whitelisted IP address already - use the [`--proxy` or `--network-address` options](#network-options) to select another IP address.
+
 ### SyntaxError: Non-ASCII character ###
 
 The error

From 1aa5172f56eca82ccb473859c6850ada677e6551 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Mar 2015 21:55:43 +0600
Subject: [PATCH 1750/1937] [vk] Catch temporarily unavailable video error
 message

---
 youtube_dl/extractor/vk.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 7dea8c59d..08c3830a2 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -152,7 +152,10 @@ class VKIE(InfoExtractor):
             'use --username and --password options to provide account credentials.',
 
             r'<!>Unknown error':
-            'Video %s does not exist.'
+            'Video %s does not exist.',
+
+            r'<!>Видео временно недоступно':
+            'Video %s is temporarily unavailable.',
         }
 
         for error_re, error_msg in ERRORS.items():

From 4e01501bbf36519ab94dd16ba3486947b2785ee8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 1 Mar 2015 21:56:30 +0600
Subject: [PATCH 1751/1937] [vk] Fix extraction (Closes #4967, closes #4686)

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 08c3830a2..842263f34 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -140,7 +140,7 @@ class VKIE(InfoExtractor):
         if not video_id:
             video_id = '%s_%s' % (mobj.group('oid'), mobj.group('id'))
 
-        info_url = 'http://vk.com/al_video.php?act=show&al=1&video=%s' % video_id
+        info_url = 'http://vk.com/al_video.php?act=show&al=1&module=video&video=%s' % video_id
         info_page = self._download_webpage(info_url, video_id)
 
         ERRORS = {

From 36300346092937a8320fa4ae7303bc1b746a69ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Mar 2015 03:30:18 +0600
Subject: [PATCH 1752/1937] [vk] Fix test (Closes #5100)

---
 youtube_dl/extractor/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 842263f34..cc384adbf 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -31,7 +31,7 @@ class VKIE(InfoExtractor):
                 'id': '162222515',
                 'ext': 'flv',
                 'title': 'ProtivoGunz - Хуёвая песня',
-                'uploader': 're:Noize MC.*',
+                'uploader': 're:(?:Noize MC|Alexander Ilyashenko).*',
                 'duration': 195,
                 'upload_date': '20120212',
             },

From 7594be85ffebee0c1ba457855ee7f4917e96499b Mon Sep 17 00:00:00 2001
From: Sergey <Ftornik@users.noreply.github.com>
Date: Mon, 2 Mar 2015 11:49:39 +0200
Subject: [PATCH 1753/1937] [lynda] Check for the empty subtitle

---
 youtube_dl/extractor/lynda.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 5dc22da22..a84019bdf 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -144,6 +144,7 @@ class LyndaIE(InfoExtractor):
 
     def _fix_subtitles(self, subs):
         srt = ''
+        seq_counter = 0
         for pos in range(0, len(subs) - 1):
             seq_current = subs[pos]
             m_current = re.match(self._TIMECODE_REGEX, seq_current['Timecode'])
@@ -155,8 +156,10 @@ class LyndaIE(InfoExtractor):
                 continue
             appear_time = m_current.group('timecode')
             disappear_time = m_next.group('timecode')
-            text = seq_current['Caption'].lstrip()
-            srt += '%s\r\n%s --> %s\r\n%s' % (str(pos), appear_time, disappear_time, text)
+            text = seq_current['Caption'].strip()
+            if text:
+                seq_counter += 1
+                srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (seq_counter, appear_time, disappear_time, text)
         if srt:
             return srt
 

From 30cbd4e0d68361a20b036fc90bd53a8fa9a463dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Mar 2015 22:12:10 +0600
Subject: [PATCH 1754/1937] [lynda] Completely skip videos we don't have access
 to, extract base class and modernize (Closes #5093)

---
 youtube_dl/extractor/lynda.py | 149 ++++++++++++++++++----------------
 1 file changed, 81 insertions(+), 68 deletions(-)

diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index a84019bdf..1f02bef44 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -15,18 +15,72 @@ from ..utils import (
 )
 
 
-class LyndaIE(InfoExtractor):
+class LyndaBaseIE(InfoExtractor):
+    _LOGIN_URL = 'https://www.lynda.com/login/login.aspx'
+    _SUCCESSFUL_LOGIN_REGEX = r'isLoggedIn: true'
+    _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
+
+    def _real_initialize(self):
+        self._login()
+
+    def _login(self):
+        (username, password) = self._get_login_info()
+        if username is None:
+            return
+
+        login_form = {
+            'username': username,
+            'password': password,
+            'remember': 'false',
+            'stayPut': 'false'
+        }
+        request = compat_urllib_request.Request(
+            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
+        login_page = self._download_webpage(
+            request, None, 'Logging in as %s' % username)
+
+        # Not (yet) logged in
+        m = re.search(r'loginResultJson = \'(?P<json>[^\']+)\';', login_page)
+        if m is not None:
+            response = m.group('json')
+            response_json = json.loads(response)
+            state = response_json['state']
+
+            if state == 'notlogged':
+                raise ExtractorError(
+                    'Unable to login, incorrect username and/or password',
+                    expected=True)
+
+            # This is when we get popup:
+            # > You're already logged in to lynda.com on two devices.
+            # > If you log in here, we'll log you out of another device.
+            # So, we need to confirm this.
+            if state == 'conflicted':
+                confirm_form = {
+                    'username': '',
+                    'password': '',
+                    'resolve': 'true',
+                    'remember': 'false',
+                    'stayPut': 'false',
+                }
+                request = compat_urllib_request.Request(
+                    self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form))
+                login_page = self._download_webpage(
+                    request, None,
+                    'Confirming log in and log out from another device')
+
+        if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
+            raise ExtractorError('Unable to log in')
+
+
+class LyndaIE(LyndaBaseIE):
     IE_NAME = 'lynda'
     IE_DESC = 'lynda.com videos'
-    _VALID_URL = r'https?://www\.lynda\.com/(?:[^/]+/[^/]+/\d+|player/embed)/(\d+)'
-    _LOGIN_URL = 'https://www.lynda.com/login/login.aspx'
+    _VALID_URL = r'https?://www\.lynda\.com/(?:[^/]+/[^/]+/\d+|player/embed)/(?P<id>\d+)'
     _NETRC_MACHINE = 'lynda'
 
-    _SUCCESSFUL_LOGIN_REGEX = r'isLoggedIn: true'
     _TIMECODE_REGEX = r'\[(?P<timecode>\d+:\d+:\d+[\.,]\d+)\]'
 
-    ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
-
     _TESTS = [{
         'url': 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html',
         'md5': 'ecfc6862da89489161fb9cd5f5a6fac1',
@@ -41,23 +95,22 @@ class LyndaIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    def _real_initialize(self):
-        self._login()
-
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
+        video_id = self._match_id(url)
 
-        page = self._download_webpage('http://www.lynda.com/ajax/player?videoId=%s&type=video' % video_id, video_id,
-                                      'Downloading video JSON')
+        page = self._download_webpage(
+            'http://www.lynda.com/ajax/player?videoId=%s&type=video' % video_id,
+            video_id, 'Downloading video JSON')
         video_json = json.loads(page)
 
         if 'Status' in video_json:
-            raise ExtractorError('lynda returned error: %s' % video_json['Message'], expected=True)
+            raise ExtractorError(
+                'lynda returned error: %s' % video_json['Message'], expected=True)
 
         if video_json['HasAccess'] is False:
             raise ExtractorError(
-                'Video %s is only available for members. ' % video_id + self.ACCOUNT_CREDENTIALS_HINT, expected=True)
+                'Video %s is only available for members. '
+                % video_id + self._ACCOUNT_CREDENTIALS_HINT, expected=True)
 
         video_id = compat_str(video_json['ID'])
         duration = video_json['DurationInSeconds']
@@ -100,48 +153,6 @@ class LyndaIE(InfoExtractor):
             'formats': formats
         }
 
-    def _login(self):
-        (username, password) = self._get_login_info()
-        if username is None:
-            return
-
-        login_form = {
-            'username': username,
-            'password': password,
-            'remember': 'false',
-            'stayPut': 'false'
-        }
-        request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
-        login_page = self._download_webpage(request, None, 'Logging in as %s' % username)
-
-        # Not (yet) logged in
-        m = re.search(r'loginResultJson = \'(?P<json>[^\']+)\';', login_page)
-        if m is not None:
-            response = m.group('json')
-            response_json = json.loads(response)
-            state = response_json['state']
-
-            if state == 'notlogged':
-                raise ExtractorError('Unable to login, incorrect username and/or password', expected=True)
-
-            # This is when we get popup:
-            # > You're already logged in to lynda.com on two devices.
-            # > If you log in here, we'll log you out of another device.
-            # So, we need to confirm this.
-            if state == 'conflicted':
-                confirm_form = {
-                    'username': '',
-                    'password': '',
-                    'resolve': 'true',
-                    'remember': 'false',
-                    'stayPut': 'false',
-                }
-                request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(confirm_form))
-                login_page = self._download_webpage(request, None, 'Confirming log in and log out from another device')
-
-        if re.search(self._SUCCESSFUL_LOGIN_REGEX, login_page) is None:
-            raise ExtractorError('Unable to log in')
-
     def _fix_subtitles(self, subs):
         srt = ''
         seq_counter = 0
@@ -172,7 +183,7 @@ class LyndaIE(InfoExtractor):
             return {}
 
 
-class LyndaCourseIE(InfoExtractor):
+class LyndaCourseIE(LyndaBaseIE):
     IE_NAME = 'lynda:course'
     IE_DESC = 'lynda.com online courses'
 
@@ -185,35 +196,37 @@ class LyndaCourseIE(InfoExtractor):
         course_path = mobj.group('coursepath')
         course_id = mobj.group('courseid')
 
-        page = self._download_webpage('http://www.lynda.com/ajax/player?courseId=%s&type=course' % course_id,
-                                      course_id, 'Downloading course JSON')
+        page = self._download_webpage(
+            'http://www.lynda.com/ajax/player?courseId=%s&type=course' % course_id,
+            course_id, 'Downloading course JSON')
         course_json = json.loads(page)
 
         if 'Status' in course_json and course_json['Status'] == 'NotFound':
-            raise ExtractorError('Course %s does not exist' % course_id, expected=True)
+            raise ExtractorError(
+                'Course %s does not exist' % course_id, expected=True)
 
         unaccessible_videos = 0
         videos = []
-        (username, _) = self._get_login_info()
 
         # Might want to extract videos right here from video['Formats'] as it seems 'Formats' is not provided
         # by single video API anymore
 
         for chapter in course_json['Chapters']:
             for video in chapter['Videos']:
-                if username is None and video['HasAccess'] is False:
+                if video['HasAccess'] is False:
                     unaccessible_videos += 1
                     continue
                 videos.append(video['ID'])
 
         if unaccessible_videos > 0:
-            self._downloader.report_warning('%s videos are only available for members and will not be downloaded. '
-                                            % unaccessible_videos + LyndaIE.ACCOUNT_CREDENTIALS_HINT)
+            self._downloader.report_warning(
+                '%s videos are only available for members (or paid members) and will not be downloaded. '
+                % unaccessible_videos + self._ACCOUNT_CREDENTIALS_HINT)
 
         entries = [
-            self.url_result('http://www.lynda.com/%s/%s-4.html' %
-                            (course_path, video_id),
-                            'Lynda')
+            self.url_result(
+                'http://www.lynda.com/%s/%s-4.html' % (course_path, video_id),
+                'Lynda')
             for video_id in videos]
 
         course_title = course_json['Title']

From 2f0f6578c3e4b82a5decc6f906a5f58b7b34c1a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Mar 2015 22:38:44 +0600
Subject: [PATCH 1755/1937] [extractor/common] Assume non HTTP(S) URLs valid

---
 youtube_dl/extractor/common.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7977fa8d0..cf39c0c21 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -767,6 +767,10 @@ class InfoExtractor(object):
                 formats)
 
     def _is_valid_url(self, url, video_id, item='video'):
+        url = self._proto_relative_url(url, scheme='http:')
+        # For now assume non HTTP(S) URLs always valid
+        if not (url.startswith('http://') or url.startswith('https://')):
+            return True
         try:
             self._request_webpage(url, video_id, 'Checking %s URL' % item)
             return True

From 562ceab13dc3a19243d5ab9d4d5927031a608334 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Mar 2015 22:39:32 +0600
Subject: [PATCH 1756/1937] [soundcloud] Check direct links validity (Closes
 #5101)

---
 youtube_dl/extractor/soundcloud.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index c5284fa67..95788098e 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -200,8 +200,9 @@ class SoundcloudIE(InfoExtractor):
                 if f['format_id'].startswith('rtmp'):
                     f['protocol'] = 'rtmp'
 
-            self._sort_formats(formats)
-            result['formats'] = formats
+        self._check_formats(formats, track_id)
+        self._sort_formats(formats)
+        result['formats'] = formats
 
         return result
 

From 295df4edb97e3c0dc0ecd95746bb2c455607a4a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 2 Mar 2015 22:47:07 +0600
Subject: [PATCH 1757/1937] [soundcloud] Fix glitches (#5101)

---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 95788098e..9d4505972 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -180,7 +180,7 @@ class SoundcloudIE(InfoExtractor):
                     'format_id': key,
                     'url': url,
                     'play_path': 'mp3:' + path,
-                    'ext': ext,
+                    'ext': 'flv',
                     'vcodec': 'none',
                 })
 

From c75f0b361a1b00f6ac1298615d6fee101994b2b9 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Mar 2015 15:06:09 +0100
Subject: [PATCH 1758/1937] [downloader/external] Add support for custom
 options (Fixes #4885, closes #5098)

---
 youtube_dl/YoutubeDL.py           |  4 ++--
 youtube_dl/__init__.py            |  5 +++++
 youtube_dl/downloader/common.py   |  2 ++
 youtube_dl/downloader/external.py | 15 ++++++++++++---
 youtube_dl/options.py             |  6 +++++-
 5 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 15367c4e3..e0baa9872 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -249,10 +249,10 @@ class YoutubeDL(object):
     hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
 
     The following parameters are not used by YoutubeDL itself, they are used by
-    the FileDownloader:
+    the downloader (see youtube_dl/downloader/common.py):
     nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
     noresizebuffer, retries, continuedl, noprogress, consoletitle,
-    xattr_set_filesize.
+    xattr_set_filesize, external_downloader_args.
 
     The following options are used by the post processors:
     prefer_ffmpeg:     If True, use ffmpeg instead of avconv if both are available,
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 49f382695..6056da1be 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -9,6 +9,7 @@ import codecs
 import io
 import os
 import random
+import shlex
 import sys
 
 
@@ -255,6 +256,9 @@ def _real_main(argv=None):
             xattr  # Confuse flake8
         except ImportError:
             parser.error('setting filesize xattr requested but python-xattr is not available')
+    external_downloader_args = None
+    if opts.external_downloader_args:
+        external_downloader_args = shlex.split(opts.external_downloader_args)
     match_filter = (
         None if opts.match_filter is None
         else match_filter_func(opts.match_filter))
@@ -359,6 +363,7 @@ def _real_main(argv=None):
         'no_color': opts.no_color,
         'ffmpeg_location': opts.ffmpeg_location,
         'hls_prefer_native': opts.hls_prefer_native,
+        'external_downloader_args': external_downloader_args,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 3ae90021a..8ed5c19a6 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -42,6 +42,8 @@ class FileDownloader(object):
     max_filesize:       Skip files larger than this size
     xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
                         (experimenatal)
+    external_downloader_args:  A list of additional command-line arguments for the
+                        external downloader.
 
     Subclasses of this one must re-define the real_download method.
     """
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 51c41c704..1673b2382 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -51,6 +51,13 @@ class ExternalFD(FileDownloader):
             return []
         return [command_option, source_address]
 
+    def _configuration_args(self, default=[]):
+        ex_args = self.params.get('external_downloader_args')
+        if ex_args is None:
+            return default
+        assert isinstance(ex_args, list)
+        return ex_args
+
     def _call_downloader(self, tmpfilename, info_dict):
         """ Either overwrite this or implement _make_cmd """
         cmd = self._make_cmd(tmpfilename, info_dict)
@@ -79,6 +86,7 @@ class CurlFD(ExternalFD):
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._source_address('--interface')
+        cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd
 
@@ -89,15 +97,16 @@ class WgetFD(ExternalFD):
         for key, val in info_dict['http_headers'].items():
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._source_address('--bind-address')
+        cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd
 
 
 class Aria2cFD(ExternalFD):
     def _make_cmd(self, tmpfilename, info_dict):
-        cmd = [
-            self.exe, '-c',
-            '--min-split-size', '1M', '--max-connection-per-server', '4']
+        cmd = [self.exe, '-c']
+        cmd += self._configuration_args([
+            '--min-split-size', '1M', '--max-connection-per-server', '4'])
         dn = os.path.dirname(tmpfilename)
         if dn:
             cmd += ['--dir', dn]
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index a2ffe96bc..df2be7b74 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -435,8 +435,12 @@ def parseOpts(overrideArguments=None):
     downloader.add_option(
         '--external-downloader',
         dest='external_downloader', metavar='COMMAND',
-        help='(experimental) Use the specified external downloader. '
+        help='Use the specified external downloader. '
              'Currently supports %s' % ','.join(list_external_downloaders()))
+    downloader.add_option(
+        '--external-downloader-args',
+        dest='external_downloader_args', metavar='ARGS',
+        help='Give these arguments to the external downloader.')
 
     workarounds = optparse.OptionGroup(parser, 'Workarounds')
     workarounds.add_option(

From 76c73715fb1e0eee61ace5ff7855d8237abdcd54 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Mar 2015 15:21:11 +0100
Subject: [PATCH 1759/1937] [generic] Parse RSS enclosure URLs (Fixes #5091)

---
 youtube_dl/extractor/generic.py | 34 ++++++++++++++++++++++++++++-----
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 27e2bc300..5dc53685c 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -26,6 +26,7 @@ from ..utils import (
     unsmuggle_url,
     UnsupportedError,
     url_basename,
+    xpath_text,
 )
 from .brightcove import BrightcoveIE
 from .ooyala import OoyalaIE
@@ -569,6 +570,16 @@ class GenericIE(InfoExtractor):
                 'title': 'John Carlson Postgame 2/25/15',
             },
         },
+        # RSS feed with enclosure
+        {
+            'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
+            'info_dict': {
+                'id': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
+                'ext': 'm4v',
+                'upload_date': '20150228',
+                'title': 'pdv_maddow_netcast_m4v-02-27-2015-201624',
+            }
+        }
     ]
 
     def report_following_redirect(self, new_url):
@@ -580,11 +591,24 @@ class GenericIE(InfoExtractor):
         playlist_desc_el = doc.find('./channel/description')
         playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text
 
-        entries = [{
-            '_type': 'url',
-            'url': e.find('link').text,
-            'title': e.find('title').text,
-        } for e in doc.findall('./channel/item')]
+        entries = []
+        for it in doc.findall('./channel/item'):
+            next_url = xpath_text(it, 'link', fatal=False)
+            if not next_url:
+                enclosure_nodes = it.findall('./enclosure')
+                for e in enclosure_nodes:
+                    next_url = e.attrib.get('url')
+                    if next_url:
+                        break
+
+            if not next_url:
+                continue
+
+            entries.append({
+                '_type': 'url',
+                'url': next_url,
+                'title': it.find('title').text,
+            })
 
         return {
             '_type': 'playlist',

From a7440261c52d6518a429210aaae598095112c61b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 2 Mar 2015 19:07:17 +0100
Subject: [PATCH 1760/1937] [utils] Streap leading dots

Fixes #2865, closes #5087
---
 test/test_utils.py  | 3 +++
 youtube_dl/utils.py | 1 +
 2 files changed, 4 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 3fba8ae11..64fad58ad 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -85,8 +85,11 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(
             sanitize_filename('New World record at 0:12:34'),
             'New World record at 0_12_34')
+
         self.assertEqual(sanitize_filename('--gasdgf'), '_-gasdgf')
         self.assertEqual(sanitize_filename('--gasdgf', is_id=True), '--gasdgf')
+        self.assertEqual(sanitize_filename('.gasdgf'), 'gasdgf')
+        self.assertEqual(sanitize_filename('.gasdgf', is_id=True), '.gasdgf')
 
         forbidden = '"\0\\/'
         for fc in forbidden:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d4938ec36..1d3401bc2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -305,6 +305,7 @@ def sanitize_filename(s, restricted=False, is_id=False):
             result = result[2:]
         if result.startswith('-'):
             result = '_' + result[len('-'):]
+        result = result.lstrip('.')
         if not result:
             result = '_'
     return result

From 91410c9bfa9fd8f01fb817474bcc7b0db5cabf95 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Mar 2015 00:03:06 +0100
Subject: [PATCH 1761/1937] [letv] Add --cn-verification-proxy (Closes #5077)

---
 youtube_dl/YoutubeDL.py      |  5 ++++-
 youtube_dl/__init__.py       |  1 +
 youtube_dl/extractor/letv.py | 34 ++++++++++++++++++++++++++++------
 youtube_dl/options.py        |  6 ++++++
 youtube_dl/utils.py          | 10 ++++++++++
 5 files changed, 49 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e0baa9872..915963d96 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -54,6 +54,7 @@ from .utils import (
     MaxDownloadsReached,
     PagedList,
     parse_filesize,
+    PerRequestProxyHandler,
     PostProcessingError,
     platform_name,
     preferredencoding,
@@ -183,6 +184,8 @@ class YoutubeDL(object):
     prefer_insecure:   Use HTTP instead of HTTPS to retrieve information.
                        At the moment, this is only supported by YouTube.
     proxy:             URL of the proxy server to use
+    cn_verification_proxy:  URL of the proxy to use for IP address verification
+                       on Chinese sites. (Experimental)
     socket_timeout:    Time to wait for unresponsive hosts, in seconds
     bidi_workaround:   Work around buggy terminals without bidirectional text
                        support, using fridibi
@@ -1762,7 +1765,7 @@ class YoutubeDL(object):
             # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
             if 'http' in proxies and 'https' not in proxies:
                 proxies['https'] = proxies['http']
-        proxy_handler = compat_urllib_request.ProxyHandler(proxies)
+        proxy_handler = PerRequestProxyHandler(proxies)
 
         debuglevel = 1 if self.params.get('debug_printtraffic') else 0
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 6056da1be..a08ddd670 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -364,6 +364,7 @@ def _real_main(argv=None):
         'ffmpeg_location': opts.ffmpeg_location,
         'hls_prefer_native': opts.hls_prefer_native,
         'external_downloader_args': external_downloader_args,
+        'cn_verification_proxy': opts.cn_verification_proxy,
     }
 
     with YoutubeDL(ydl_opts) as ydl:
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 583ce35b9..fd5fd260e 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -7,8 +7,9 @@ import time
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urlparse,
     compat_urllib_parse,
+    compat_urllib_request,
+    compat_urlparse,
 )
 from ..utils import (
     determine_ext,
@@ -42,9 +43,23 @@ class LetvIE(InfoExtractor):
         'expected_warnings': [
             'publish time'
         ]
+    }, {
+        'note': 'This video is available only in Mainland China, thus a proxy is needed',
+        'url': 'http://www.letv.com/ptv/vplay/1118082.html',
+        'md5': 'f80936fbe20fb2f58648e81386ff7927',
+        'info_dict': {
+            'id': '1118082',
+            'ext': 'mp4',
+            'title': '与龙共舞 完整版',
+            'description': 'md5:7506a5eeb1722bb9d4068f85024e3986',
+        },
+        'expected_warnings': [
+            'publish time'
+        ],
+        'params': {
+            'cn_verification_proxy': 'proxy.uku.im:8888'
+        },
     }]
-    # http://www.letv.com/ptv/vplay/1118082.html
-    # This video is available only in Mainland China
 
     @staticmethod
     def urshift(val, n):
@@ -76,8 +91,14 @@ class LetvIE(InfoExtractor):
             'tkey': self.calc_time_key(int(time.time())),
             'domain': 'www.letv.com'
         }
+        play_json_req = compat_urllib_request.Request(
+            'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
+        )
+        play_json_req.add_header(
+            'Ytdl-Request-Proxy',
+            self._downloader.params.get('cn_verification_proxy'))
         play_json = self._download_json(
-            'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params),
+            play_json_req,
             media_id, 'playJson data')
 
         # Check for errors
@@ -114,7 +135,8 @@ class LetvIE(InfoExtractor):
 
                 url_info_dict = {
                     'url': media_url,
-                    'ext': determine_ext(dispatch[format_id][1])
+                    'ext': determine_ext(dispatch[format_id][1]),
+                    'format_id': format_id,
                 }
 
                 if format_id[-1:] == 'p':
@@ -123,7 +145,7 @@ class LetvIE(InfoExtractor):
                 urls.append(url_info_dict)
 
         publish_time = parse_iso8601(self._html_search_regex(
-            r'发布时间&nbsp;([^<>]+) ', page, 'publish time', fatal=False),
+            r'发布时间&nbsp;([^<>]+) ', page, 'publish time', default=None),
             delimiter=' ', timezone=datetime.timedelta(hours=8))
         description = self._html_search_meta('description', page, fatal=False)
 
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index df2be7b74..a4ca8adc4 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -195,6 +195,12 @@ def parseOpts(overrideArguments=None):
         action='store_const', const='::', dest='source_address',
         help='Make all connections via IPv6 (experimental)',
     )
+    network.add_option(
+        '--cn-verification-proxy',
+        dest='cn_verification_proxy', default=None, metavar='URL',
+        help='Use this proxy to verify the IP address for some Chinese sites. '
+        'The default proxy specified by --proxy (or none, if the options is not present) is used for the actual downloading. (experimental)'
+    )
 
     selection = optparse.OptionGroup(parser, 'Video Selection')
     selection.add_option(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1d3401bc2..b568288fa 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1768,3 +1768,13 @@ def match_filter_func(filter_str):
             video_title = info_dict.get('title', info_dict.get('id', 'video'))
             return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
     return _match_func
+
+
+class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
+    def proxy_open(self, req, proxy, type):
+        req_proxy = req.headers.get('Ytdl-Request-Proxy')
+        if req_proxy is not None:
+            proxy = req_proxy
+            del req.headers['Ytdl-Request-Proxy']
+        return compat_urllib_request.ProxyHandler.proxy_open(
+            self, req, proxy, type)

From 07490f8017a83b7607686499074c41212fa0a44f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Mar 2015 00:05:05 +0100
Subject: [PATCH 1762/1937] release 2015.03.03

---
 README.md             | 411 ++++++++++++------------------------------
 youtube_dl/version.py |   2 +-
 2 files changed, 119 insertions(+), 294 deletions(-)

diff --git a/README.md b/README.md
index f2909e8d6..d05549125 100644
--- a/README.md
+++ b/README.md
@@ -47,211 +47,107 @@ which means you can modify it, redistribute it or use it however you like.
 # OPTIONS
     -h, --help                       print this help text and exit
     --version                        print program version and exit
-    -U, --update                     update this program to latest version. Make
-                                     sure that you have sufficient permissions
-                                     (run with sudo if needed)
-    -i, --ignore-errors              continue on download errors, for example to
-                                     skip unavailable videos in a playlist
-    --abort-on-error                 Abort downloading of further videos (in the
-                                     playlist or the command line) if an error
-                                     occurs
+    -U, --update                     update this program to latest version. Make sure that you have sufficient permissions (run with sudo if needed)
+    -i, --ignore-errors              continue on download errors, for example to skip unavailable videos in a playlist
+    --abort-on-error                 Abort downloading of further videos (in the playlist or the command line) if an error occurs
     --dump-user-agent                display the current browser identification
-    --list-extractors                List all supported extractors and the URLs
-                                     they would handle
-    --extractor-descriptions         Output descriptions of all supported
-                                     extractors
-    --default-search PREFIX          Use this prefix for unqualified URLs. For
-                                     example "gvsearch2:" downloads two videos
-                                     from google videos for  youtube-dl "large
-                                     apple". Use the value "auto" to let
-                                     youtube-dl guess ("auto_warning" to emit a
-                                     warning when guessing). "error" just throws
-                                     an error. The default value "fixup_error"
-                                     repairs broken URLs, but emits an error if
-                                     this is not possible instead of searching.
-    --ignore-config                  Do not read configuration files. When given
-                                     in the global configuration file /etc
-                                     /youtube-dl.conf: Do not read the user
-                                     configuration in ~/.config/youtube-
-                                     dl/config (%APPDATA%/youtube-dl/config.txt
-                                     on Windows)
-    --flat-playlist                  Do not extract the videos of a playlist,
-                                     only list them.
+    --list-extractors                List all supported extractors and the URLs they would handle
+    --extractor-descriptions         Output descriptions of all supported extractors
+    --default-search PREFIX          Use this prefix for unqualified URLs. For example "gvsearch2:" downloads two videos from google videos for  youtube-dl "large apple".
+                                     Use the value "auto" to let youtube-dl guess ("auto_warning" to emit a warning when guessing). "error" just throws an error. The
+                                     default value "fixup_error" repairs broken URLs, but emits an error if this is not possible instead of searching.
+    --ignore-config                  Do not read configuration files. When given in the global configuration file /etc/youtube-dl.conf: Do not read the user configuration
+                                     in ~/.config/youtube-dl/config (%APPDATA%/youtube-dl/config.txt on Windows)
+    --flat-playlist                  Do not extract the videos of a playlist, only list them.
     --no-color                       Do not emit color codes in output.
 
 ## Network Options:
-    --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in
-                                     an empty string (--proxy "") for direct
-                                     connection
+    --proxy URL                      Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection
     --socket-timeout SECONDS         Time to wait before giving up, in seconds
-    --source-address IP              Client-side IP address to bind to
-                                     (experimental)
-    -4, --force-ipv4                 Make all connections via IPv4
-                                     (experimental)
-    -6, --force-ipv6                 Make all connections via IPv6
-                                     (experimental)
+    --source-address IP              Client-side IP address to bind to (experimental)
+    -4, --force-ipv4                 Make all connections via IPv4 (experimental)
+    -6, --force-ipv6                 Make all connections via IPv6 (experimental)
 
 ## Video Selection:
     --playlist-start NUMBER          playlist video to start at (default is 1)
     --playlist-end NUMBER            playlist video to end at (default is last)
-    --playlist-items ITEM_SPEC       playlist video items to download. Specify
-                                     indices of the videos in the playlist
-                                     seperated by commas like: "--playlist-items
-                                     1,2,5,8" if you want to download videos
-                                     indexed 1, 2, 5, 8 in the playlist. You can
-                                     specify range: "--playlist-items
-                                     1-3,7,10-13", it will download the videos
-                                     at index 1, 2, 3, 7, 10, 11, 12 and 13.
-    --match-title REGEX              download only matching titles (regex or
-                                     caseless sub-string)
-    --reject-title REGEX             skip download for matching titles (regex or
-                                     caseless sub-string)
+    --playlist-items ITEM_SPEC       playlist video items to download. Specify indices of the videos in the playlist seperated by commas like: "--playlist-items 1,2,5,8"
+                                     if you want to download videos indexed 1, 2, 5, 8 in the playlist. You can specify range: "--playlist-items 1-3,7,10-13", it will
+                                     download the videos at index 1, 2, 3, 7, 10, 11, 12 and 13.
+    --match-title REGEX              download only matching titles (regex or caseless sub-string)
+    --reject-title REGEX             skip download for matching titles (regex or caseless sub-string)
     --max-downloads NUMBER           Abort after downloading NUMBER files
-    --min-filesize SIZE              Do not download any videos smaller than
-                                     SIZE (e.g. 50k or 44.6m)
-    --max-filesize SIZE              Do not download any videos larger than SIZE
-                                     (e.g. 50k or 44.6m)
+    --min-filesize SIZE              Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)
+    --max-filesize SIZE              Do not download any videos larger than SIZE (e.g. 50k or 44.6m)
     --date DATE                      download only videos uploaded in this date
-    --datebefore DATE                download only videos uploaded on or before
-                                     this date (i.e. inclusive)
-    --dateafter DATE                 download only videos uploaded on or after
-                                     this date (i.e. inclusive)
-    --min-views COUNT                Do not download any videos with less than
-                                     COUNT views
-    --max-views COUNT                Do not download any videos with more than
-                                     COUNT views
-    --match-filter FILTER            (Experimental) Generic video filter.
-                                     Specify any key (see help for -o for a list
-                                     of available keys) to match if the key is
-                                     present, !key to check if the key is not
-                                     present,key > NUMBER (like "comment_count >
-                                     12", also works with >=, <, <=, !=, =) to
-                                     compare against a number, and & to require
-                                     multiple matches. Values which are not
-                                     known are excluded unless you put a
-                                     question mark (?) after the operator.For
-                                     example, to only match videos that have
-                                     been liked more than 100 times and disliked
-                                     less than 50 times (or the dislike
-                                     functionality is not available at the given
-                                     service), but who also have a description,
-                                     use  --match-filter "like_count > 100 &
+    --datebefore DATE                download only videos uploaded on or before this date (i.e. inclusive)
+    --dateafter DATE                 download only videos uploaded on or after this date (i.e. inclusive)
+    --min-views COUNT                Do not download any videos with less than COUNT views
+    --max-views COUNT                Do not download any videos with more than COUNT views
+    --match-filter FILTER            (Experimental) Generic video filter. Specify any key (see help for -o for a list of available keys) to match if the key is present,
+                                     !key to check if the key is not present,key > NUMBER (like "comment_count > 12", also works with >=, <, <=, !=, =) to compare against
+                                     a number, and & to require multiple matches. Values which are not known are excluded unless you put a question mark (?) after the
+                                     operator.For example, to only match videos that have been liked more than 100 times and disliked less than 50 times (or the dislike
+                                     functionality is not available at the given service), but who also have a description, use  --match-filter "like_count > 100 &
                                      dislike_count <? 50 & description" .
-    --no-playlist                    If the URL refers to a video and a
-                                     playlist, download only the video.
-    --yes-playlist                   If the URL refers to a video and a
-                                     playlist, download the playlist.
-    --age-limit YEARS                download only videos suitable for the given
-                                     age
-    --download-archive FILE          Download only videos not listed in the
-                                     archive file. Record the IDs of all
-                                     downloaded videos in it.
-    --include-ads                    Download advertisements as well
-                                     (experimental)
+    --no-playlist                    If the URL refers to a video and a playlist, download only the video.
+    --yes-playlist                   If the URL refers to a video and a playlist, download the playlist.
+    --age-limit YEARS                download only videos suitable for the given age
+    --download-archive FILE          Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.
+    --include-ads                    Download advertisements as well (experimental)
 
 ## Download Options:
-    -r, --rate-limit LIMIT           maximum download rate in bytes per second
-                                     (e.g. 50K or 4.2M)
-    -R, --retries RETRIES            number of retries (default is 10), or
-                                     "infinite".
-    --buffer-size SIZE               size of download buffer (e.g. 1024 or 16K)
-                                     (default is 1024)
-    --no-resize-buffer               do not automatically adjust the buffer
-                                     size. By default, the buffer size is
-                                     automatically resized from an initial value
-                                     of SIZE.
+    -r, --rate-limit LIMIT           maximum download rate in bytes per second (e.g. 50K or 4.2M)
+    -R, --retries RETRIES            number of retries (default is 10), or "infinite".
+    --buffer-size SIZE               size of download buffer (e.g. 1024 or 16K) (default is 1024)
+    --no-resize-buffer               do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.
     --playlist-reverse               Download playlist videos in reverse order
-    --xattr-set-filesize             (experimental) set file xattribute
-                                     ytdl.filesize with expected filesize
-    --hls-prefer-native              (experimental) Use the native HLS
-                                     downloader instead of ffmpeg.
-    --external-downloader COMMAND    (experimental) Use the specified external
-                                     downloader. Currently supports
-                                     aria2c,curl,wget
+    --xattr-set-filesize             (experimental) set file xattribute ytdl.filesize with expected filesize
+    --hls-prefer-native              (experimental) Use the native HLS downloader instead of ffmpeg.
+    --external-downloader COMMAND    Use the specified external downloader. Currently supports aria2c,curl,wget
+    --external-downloader-args ARGS  Give these arguments to the external downloader.
 
 ## Filesystem Options:
-    -a, --batch-file FILE            file containing URLs to download ('-' for
-                                     stdin)
+    -a, --batch-file FILE            file containing URLs to download ('-' for stdin)
     --id                             use only video ID in file name
-    -o, --output TEMPLATE            output filename template. Use %(title)s to
-                                     get the title, %(uploader)s for the
-                                     uploader name, %(uploader_id)s for the
-                                     uploader nickname if different,
-                                     %(autonumber)s to get an automatically
-                                     incremented number, %(ext)s for the
-                                     filename extension, %(format)s for the
-                                     format description (like "22 - 1280x720" or
-                                     "HD"), %(format_id)s for the unique id of
-                                     the format (like Youtube's itags: "137"),
-                                     %(upload_date)s for the upload date
-                                     (YYYYMMDD), %(extractor)s for the provider
-                                     (youtube, metacafe, etc), %(id)s for the
-                                     video id, %(playlist_title)s,
-                                     %(playlist_id)s, or %(playlist)s (=title if
-                                     present, ID otherwise) for the playlist the
-                                     video is in, %(playlist_index)s for the
-                                     position in the playlist. %(height)s and
-                                     %(width)s for the width and height of the
-                                     video format. %(resolution)s for a textual
-                                     description of the resolution of the video
-                                     format. %% for a literal percent. Use - to
-                                     output to stdout. Can also be used to
-                                     download to a different directory, for
-                                     example with -o '/my/downloads/%(uploader)s
-                                     /%(title)s-%(id)s.%(ext)s' .
-    --autonumber-size NUMBER         Specifies the number of digits in
-                                     %(autonumber)s when it is present in output
-                                     filename template or --auto-number option
-                                     is given
-    --restrict-filenames             Restrict filenames to only ASCII
-                                     characters, and avoid "&" and spaces in
-                                     filenames
-    -A, --auto-number                [deprecated; use  -o
-                                     "%(autonumber)s-%(title)s.%(ext)s" ] number
-                                     downloaded files starting from 00000
-    -t, --title                      [deprecated] use title in file name
-                                     (default)
+    -o, --output TEMPLATE            output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(uploader_id)s for the uploader
+                                     nickname if different, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(format)s for
+                                     the format description (like "22 - 1280x720" or "HD"), %(format_id)s for the unique id of the format (like Youtube's itags: "137"),
+                                     %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id,
+                                     %(playlist_title)s, %(playlist_id)s, or %(playlist)s (=title if present, ID otherwise) for the playlist the video is in,
+                                     %(playlist_index)s for the position in the playlist. %(height)s and %(width)s for the width and height of the video format.
+                                     %(resolution)s for a textual description of the resolution of the video format. %% for a literal percent. Use - to output to stdout.
+                                     Can also be used to download to a different directory, for example with -o '/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s' .
+    --autonumber-size NUMBER         Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given
+    --restrict-filenames             Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames
+    -A, --auto-number                [deprecated; use  -o "%(autonumber)s-%(title)s.%(ext)s" ] number downloaded files starting from 00000
+    -t, --title                      [deprecated] use title in file name (default)
     -l, --literal                    [deprecated] alias of --title
     -w, --no-overwrites              do not overwrite files
-    -c, --continue                   force resume of partially downloaded files.
-                                     By default, youtube-dl will resume
-                                     downloads if possible.
-    --no-continue                    do not resume partially downloaded files
-                                     (restart from beginning)
-    --no-part                        do not use .part files - write directly
-                                     into output file
-    --no-mtime                       do not use the Last-modified header to set
-                                     the file modification time
-    --write-description              write video description to a .description
-                                     file
+    -c, --continue                   force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.
+    --no-continue                    do not resume partially downloaded files (restart from beginning)
+    --no-part                        do not use .part files - write directly into output file
+    --no-mtime                       do not use the Last-modified header to set the file modification time
+    --write-description              write video description to a .description file
     --write-info-json                write video metadata to a .info.json file
-    --write-annotations              write video annotations to a .annotation
-                                     file
-    --load-info FILE                 json file containing the video information
-                                     (created with the "--write-json" option)
-    --cookies FILE                   file to read cookies from and dump cookie
-                                     jar in
-    --cache-dir DIR                  Location in the filesystem where youtube-dl
-                                     can store some downloaded information
-                                     permanently. By default $XDG_CACHE_HOME
-                                     /youtube-dl or ~/.cache/youtube-dl . At the
-                                     moment, only YouTube player files (for
-                                     videos with obfuscated signatures) are
-                                     cached, but that may change.
+    --write-annotations              write video annotations to a .annotation file
+    --load-info FILE                 json file containing the video information (created with the "--write-json" option)
+    --cookies FILE                   file to read cookies from and dump cookie jar in
+    --cache-dir DIR                  Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl
+                                     or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may
+                                     change.
     --no-cache-dir                   Disable filesystem caching
     --rm-cache-dir                   Delete all filesystem cache files
 
 ## Thumbnail images:
     --write-thumbnail                write thumbnail image to disk
     --write-all-thumbnails           write all thumbnail image formats to disk
-    --list-thumbnails                Simulate and list all available thumbnail
-                                     formats
+    --list-thumbnails                Simulate and list all available thumbnail formats
 
 ## Verbosity / Simulation Options:
     -q, --quiet                      activates quiet mode
     --no-warnings                    Ignore warnings
-    -s, --simulate                   do not download the video and do not write
-                                     anything to disk
+    -s, --simulate                   do not download the video and do not write anything to disk
     --skip-download                  do not download the video
     -g, --get-url                    simulate, quiet but print URL
     -e, --get-title                  simulate, quiet but print title
@@ -261,155 +157,84 @@ which means you can modify it, redistribute it or use it however you like.
     --get-duration                   simulate, quiet but print video length
     --get-filename                   simulate, quiet but print output filename
     --get-format                     simulate, quiet but print output format
-    -j, --dump-json                  simulate, quiet but print JSON information.
-                                     See --output for a description of available
-                                     keys.
-    -J, --dump-single-json           simulate, quiet but print JSON information
-                                     for each command-line argument. If the URL
-                                     refers to a playlist, dump the whole
-                                     playlist information in a single line.
-    --print-json                     Be quiet and print the video information as
-                                     JSON (video is still being downloaded).
+    -j, --dump-json                  simulate, quiet but print JSON information. See --output for a description of available keys.
+    -J, --dump-single-json           simulate, quiet but print JSON information for each command-line argument. If the URL refers to a playlist, dump the whole playlist
+                                     information in a single line.
+    --print-json                     Be quiet and print the video information as JSON (video is still being downloaded).
     --newline                        output progress bar as new lines
     --no-progress                    do not print progress bar
     --console-title                  display progress in console titlebar
     -v, --verbose                    print various debugging information
-    --dump-intermediate-pages        print downloaded pages to debug problems
-                                     (very verbose)
-    --write-pages                    Write downloaded intermediary pages to
-                                     files in the current directory to debug
-                                     problems
+    --dump-intermediate-pages        print downloaded pages to debug problems (very verbose)
+    --write-pages                    Write downloaded intermediary pages to files in the current directory to debug problems
     --print-traffic                  Display sent and read HTTP traffic
-    -C, --call-home                  Contact the youtube-dl server for
-                                     debugging.
-    --no-call-home                   Do NOT contact the youtube-dl server for
-                                     debugging.
+    -C, --call-home                  Contact the youtube-dl server for debugging.
+    --no-call-home                   Do NOT contact the youtube-dl server for debugging.
 
 ## Workarounds:
     --encoding ENCODING              Force the specified encoding (experimental)
     --no-check-certificate           Suppress HTTPS certificate validation.
-    --prefer-insecure                Use an unencrypted connection to retrieve
-                                     information about the video. (Currently
-                                     supported only for YouTube)
+    --prefer-insecure                Use an unencrypted connection to retrieve information about the video. (Currently supported only for YouTube)
     --user-agent UA                  specify a custom user agent
-    --referer URL                    specify a custom referer, use if the video
-                                     access is restricted to one domain
-    --add-header FIELD:VALUE         specify a custom HTTP header and its value,
-                                     separated by a colon ':'. You can use this
-                                     option multiple times
-    --bidi-workaround                Work around terminals that lack
-                                     bidirectional text support. Requires bidiv
-                                     or fribidi executable in PATH
-    --sleep-interval SECONDS         Number of seconds to sleep before each
-                                     download.
+    --referer URL                    specify a custom referer, use if the video access is restricted to one domain
+    --add-header FIELD:VALUE         specify a custom HTTP header and its value, separated by a colon ':'. You can use this option multiple times
+    --bidi-workaround                Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH
+    --sleep-interval SECONDS         Number of seconds to sleep before each download.
 
 ## Video Format Options:
-    -f, --format FORMAT              video format code, specify the order of
-                                     preference using slashes, as in -f 22/17/18
-                                     .  Instead of format codes, you can select
-                                     by extension for the extensions aac, m4a,
-                                     mp3, mp4, ogg, wav, webm. You can also use
-                                     the special names "best", "bestvideo",
-                                     "bestaudio", "worst".  You can filter the
-                                     video results by putting a condition in
-                                     brackets, as in -f "best[height=720]" (or
-                                     -f "[filesize>10M]").  This works for
-                                     filesize, height, width, tbr, abr, vbr,
-                                     asr, and fps and the comparisons <, <=, >,
-                                     >=, =, != and for ext, acodec, vcodec,
-                                     container, and protocol and the comparisons
-                                     =, != . Formats for which the value is not
-                                     known are excluded unless you put a
-                                     question mark (?) after the operator. You
-                                     can combine format filters, so  -f "[height
-                                     <=? 720][tbr>500]" selects up to 720p
-                                     videos (or videos where the height is not
-                                     known) with a bitrate of at least 500
-                                     KBit/s. By default, youtube-dl will pick
-                                     the best quality. Use commas to download
-                                     multiple audio formats, such as -f
-                                     136/137/mp4/bestvideo,140/m4a/bestaudio.
-                                     You can merge the video and audio of two
-                                     formats into a single file using -f <video-
-                                     format>+<audio-format> (requires ffmpeg or
-                                     avconv), for example -f
+    -f, --format FORMAT              video format code, specify the order of preference using slashes, as in -f 22/17/18 .  Instead of format codes, you can select by
+                                     extension for the extensions aac, m4a, mp3, mp4, ogg, wav, webm. You can also use the special names "best", "bestvideo", "bestaudio",
+                                     "worst".  You can filter the video results by putting a condition in brackets, as in -f "best[height=720]" (or -f "[filesize>10M]").
+                                     This works for filesize, height, width, tbr, abr, vbr, asr, and fps and the comparisons <, <=, >, >=, =, != and for ext, acodec,
+                                     vcodec, container, and protocol and the comparisons =, != . Formats for which the value is not known are excluded unless you put a
+                                     question mark (?) after the operator. You can combine format filters, so  -f "[height <=? 720][tbr>500]" selects up to 720p videos
+                                     (or videos where the height is not known) with a bitrate of at least 500 KBit/s. By default, youtube-dl will pick the best quality.
+                                     Use commas to download multiple audio formats, such as -f  136/137/mp4/bestvideo,140/m4a/bestaudio. You can merge the video and audio
+                                     of two formats into a single file using -f <video-format>+<audio-format> (requires ffmpeg or avconv), for example -f
                                      bestvideo+bestaudio.
     --all-formats                    download all available video formats
-    --prefer-free-formats            prefer free video formats unless a specific
-                                     one is requested
+    --prefer-free-formats            prefer free video formats unless a specific one is requested
     --max-quality FORMAT             highest quality format to download
     -F, --list-formats               list all available formats
-    --youtube-skip-dash-manifest     Do not download the DASH manifest on
-                                     YouTube videos
-    --merge-output-format FORMAT     If a merge is required (e.g.
-                                     bestvideo+bestaudio), output to given
-                                     container format. One of mkv, mp4, ogg,
-                                     webm, flv.Ignored if no merge is required
+    --youtube-skip-dash-manifest     Do not download the DASH manifest on YouTube videos
+    --merge-output-format FORMAT     If a merge is required (e.g. bestvideo+bestaudio), output to given container format. One of mkv, mp4, ogg, webm, flv.Ignored if no
+                                     merge is required
 
 ## Subtitle Options:
     --write-sub                      write subtitle file
-    --write-auto-sub                 write automatic subtitle file (youtube
-                                     only)
-    --all-subs                       downloads all the available subtitles of
-                                     the video
+    --write-auto-sub                 write automatic subtitle file (youtube only)
+    --all-subs                       downloads all the available subtitles of the video
     --list-subs                      lists all available subtitles for the video
-    --sub-format FORMAT              subtitle format, accepts formats
-                                     preference, for example: "ass/srt/best"
-    --sub-lang LANGS                 languages of the subtitles to download
-                                     (optional) separated by commas, use IETF
-                                     language tags like 'en,pt'
+    --sub-format FORMAT              subtitle format, accepts formats preference, for example: "ass/srt/best"
+    --sub-lang LANGS                 languages of the subtitles to download (optional) separated by commas, use IETF language tags like 'en,pt'
 
 ## Authentication Options:
     -u, --username USERNAME          login with this account ID
-    -p, --password PASSWORD          account password. If this option is left
-                                     out, youtube-dl will ask interactively.
+    -p, --password PASSWORD          account password. If this option is left out, youtube-dl will ask interactively.
     -2, --twofactor TWOFACTOR        two-factor auth code
     -n, --netrc                      use .netrc authentication data
     --video-password PASSWORD        video password (vimeo, smotri)
 
 ## Post-processing Options:
-    -x, --extract-audio              convert video files to audio-only files
-                                     (requires ffmpeg or avconv and ffprobe or
-                                     avprobe)
-    --audio-format FORMAT            "best", "aac", "vorbis", "mp3", "m4a",
-                                     "opus", or "wav"; "best" by default
-    --audio-quality QUALITY          ffmpeg/avconv audio quality specification,
-                                     insert a value between 0 (better) and 9
-                                     (worse) for VBR or a specific bitrate like
-                                     128K (default 5)
-    --recode-video FORMAT            Encode the video to another format if
-                                     necessary (currently supported:
-                                     mp4|flv|ogg|webm|mkv)
-    -k, --keep-video                 keeps the video file on disk after the
-                                     post-processing; the video is erased by
-                                     default
-    --no-post-overwrites             do not overwrite post-processed files; the
-                                     post-processed files are overwritten by
-                                     default
-    --embed-subs                     embed subtitles in the video (only for mp4
-                                     videos)
+    -x, --extract-audio              convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)
+    --audio-format FORMAT            "best", "aac", "vorbis", "mp3", "m4a", "opus", or "wav"; "best" by default
+    --audio-quality QUALITY          ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K
+                                     (default 5)
+    --recode-video FORMAT            Encode the video to another format if necessary (currently supported: mp4|flv|ogg|webm|mkv)
+    -k, --keep-video                 keeps the video file on disk after the post-processing; the video is erased by default
+    --no-post-overwrites             do not overwrite post-processed files; the post-processed files are overwritten by default
+    --embed-subs                     embed subtitles in the video (only for mp4 videos)
     --embed-thumbnail                embed thumbnail in the audio as cover art
     --add-metadata                   write metadata to the video file
-    --xattrs                         write metadata to the video file's xattrs
-                                     (using dublin core and xdg standards)
-    --fixup POLICY                   Automatically correct known faults of the
-                                     file. One of never (do nothing), warn (only
-                                     emit a warning), detect_or_warn(the
-                                     default; fix file if we can, warn
-                                     otherwise)
-    --prefer-avconv                  Prefer avconv over ffmpeg for running the
-                                     postprocessors (default)
-    --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the
-                                     postprocessors
-    --ffmpeg-location PATH           Location of the ffmpeg/avconv binary;
-                                     either the path to the binary or its
-                                     containing directory.
-    --exec CMD                       Execute a command on the file after
-                                     downloading, similar to find's -exec
-                                     syntax. Example: --exec 'adb push {}
-                                     /sdcard/Music/ && rm {}'
-    --convert-subtitles FORMAT       Convert the subtitles to other format
-                                     (currently supported: srt|ass|vtt)
+    --xattrs                         write metadata to the video file's xattrs (using dublin core and xdg standards)
+    --fixup POLICY                   Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn(the default;
+                                     fix file if we can, warn otherwise)
+    --prefer-avconv                  Prefer avconv over ffmpeg for running the postprocessors (default)
+    --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the postprocessors
+    --ffmpeg-location PATH           Location of the ffmpeg/avconv binary; either the path to the binary or its containing directory.
+    --exec CMD                       Execute a command on the file after downloading, similar to find's -exec syntax. Example: --exec 'adb push {} /sdcard/Music/ && rm
+                                     {}'
+    --convert-subtitles FORMAT       Convert the subtitles to other format (currently supported: srt|ass|vtt)
 
 # CONFIGURATION
 
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5582348ba..3a49e52c9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.02.28'
+__version__ = '2015.03.03'

From 499bfcbfd09e85f053d7e8943a8d47fed9349b0e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Mar 2015 12:59:17 +0100
Subject: [PATCH 1763/1937] Make sure netrc works for all extractors with login
 support

Fixes #5112
---
 test/test_netrc.py                  | 26 ++++++++++++++++++++++++++
 youtube_dl/extractor/atresplayer.py |  1 +
 youtube_dl/extractor/crunchyroll.py |  1 +
 youtube_dl/extractor/gdcvault.py    |  1 +
 youtube_dl/extractor/lynda.py       |  1 +
 youtube_dl/extractor/twitch.py      |  1 +
 6 files changed, 31 insertions(+)
 create mode 100644 test/test_netrc.py

diff --git a/test/test_netrc.py b/test/test_netrc.py
new file mode 100644
index 000000000..7cf3a6a2e
--- /dev/null
+++ b/test/test_netrc.py
@@ -0,0 +1,26 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+from youtube_dl.extractor import (
+    gen_extractors,
+)
+
+
+class TestNetRc(unittest.TestCase):
+    def test_netrc_present(self):
+        for ie in gen_extractors():
+            if not hasattr(ie, '_login'):
+                continue
+            self.assertTrue(
+                hasattr(ie, '_NETRC_MACHINE'),
+                'Extractor %s supports login, but is missing a _NETRC_MACHINE property' % ie.IE_NAME)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/youtube_dl/extractor/atresplayer.py b/youtube_dl/extractor/atresplayer.py
index 7669e0e3d..29f8795d3 100644
--- a/youtube_dl/extractor/atresplayer.py
+++ b/youtube_dl/extractor/atresplayer.py
@@ -19,6 +19,7 @@ from ..utils import (
 
 class AtresPlayerIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?atresplayer\.com/television/[^/]+/[^/]+/[^/]+/(?P<id>.+?)_\d+\.html'
+    _NETRC_MACHINE = 'atresplayer'
     _TESTS = [
         {
             'url': 'http://www.atresplayer.com/television/programas/el-club-de-la-comedia/temporada-4/capitulo-10-especial-solidario-nochebuena_2014122100174.html',
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index f1da7d09b..e64b88fbc 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -29,6 +29,7 @@ from ..aes import (
 
 class CrunchyrollIE(InfoExtractor):
     _VALID_URL = r'https?://(?:(?P<prefix>www|m)\.)?(?P<url>crunchyroll\.(?:com|fr)/(?:[^/]*/[^/?&]*?|media/\?id=)(?P<video_id>[0-9]+))(?:[/?&]|$)'
+    _NETRC_MACHINE = 'crunchyroll'
     _TESTS = [{
         'url': 'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513',
         'info_dict': {
diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index f7b467b0a..51796f3a4 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -12,6 +12,7 @@ from ..utils import remove_end
 
 class GDCVaultIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?gdcvault\.com/play/(?P<id>\d+)/(?P<name>(\w|-)+)'
+    _NETRC_MACHINE = 'gdcvault'
     _TESTS = [
         {
             'url': 'http://www.gdcvault.com/play/1019721/Doki-Doki-Universe-Sweet-Simple',
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
index 1f02bef44..cfd3b14f4 100644
--- a/youtube_dl/extractor/lynda.py
+++ b/youtube_dl/extractor/lynda.py
@@ -19,6 +19,7 @@ class LyndaBaseIE(InfoExtractor):
     _LOGIN_URL = 'https://www.lynda.com/login/login.aspx'
     _SUCCESSFUL_LOGIN_REGEX = r'isLoggedIn: true'
     _ACCOUNT_CREDENTIALS_HINT = 'Use --username and --password options to provide lynda.com account credentials.'
+    _NETRC_MACHINE = 'lynda'
 
     def _real_initialize(self):
         self._login()
diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 4b0ce54df..8af136147 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -23,6 +23,7 @@ class TwitchBaseIE(InfoExtractor):
     _API_BASE = 'https://api.twitch.tv'
     _USHER_BASE = 'http://usher.twitch.tv'
     _LOGIN_URL = 'https://secure.twitch.tv/user/login'
+    _NETRC_MACHINE = 'twitch'
 
     def _handle_error(self, response):
         if not isinstance(response, dict):

From 2461f79d2ad9eee44644f6187e366125a29aa70f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Mar 2015 13:56:06 +0100
Subject: [PATCH 1764/1937] [utils] Correct per-request proxy handling

---
 youtube_dl/YoutubeDL.py      |  3 ++-
 youtube_dl/extractor/letv.py | 10 ++--------
 youtube_dl/utils.py          | 15 +++++++++++++--
 3 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 915963d96..df2aebb59 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1771,7 +1771,8 @@ class YoutubeDL(object):
         https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
         ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
         opener = compat_urllib_request.build_opener(
-            https_handler, proxy_handler, cookie_processor, ydlh)
+            proxy_handler, https_handler, cookie_processor, ydlh)
+
         # Delete the default user-agent header, which would otherwise apply in
         # cases where our custom HTTP handler doesn't come into play
         # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index fd5fd260e..85eee141b 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -40,9 +40,6 @@ class LetvIE(InfoExtractor):
             'title': '美人天下01',
             'description': 'md5:f88573d9d7225ada1359eaf0dbf8bcda',
         },
-        'expected_warnings': [
-            'publish time'
-        ]
     }, {
         'note': 'This video is available only in Mainland China, thus a proxy is needed',
         'url': 'http://www.letv.com/ptv/vplay/1118082.html',
@@ -53,11 +50,8 @@ class LetvIE(InfoExtractor):
             'title': '与龙共舞 完整版',
             'description': 'md5:7506a5eeb1722bb9d4068f85024e3986',
         },
-        'expected_warnings': [
-            'publish time'
-        ],
         'params': {
-            'cn_verification_proxy': 'proxy.uku.im:8888'
+            'cn_verification_proxy': 'http://proxy.uku.im:8888'
         },
     }]
 
@@ -95,7 +89,7 @@ class LetvIE(InfoExtractor):
             'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
         )
         play_json_req.add_header(
-            'Ytdl-Request-Proxy',
+            'Ytdl-request-proxy',
             self._downloader.params.get('cn_verification_proxy'))
         play_json = self._download_json(
             play_json_req,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b568288fa..7426e2a1f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1771,10 +1771,21 @@ def match_filter_func(filter_str):
 
 
 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
+    def __init__(self, proxies=None):
+        # Set default handlers
+        for type in ('http', 'https'):
+            setattr(self, '%s_open' % type,
+                    lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
+                        meth(r, proxy, type))
+        return compat_urllib_request.ProxyHandler.__init__(self, proxies)
+
     def proxy_open(self, req, proxy, type):
-        req_proxy = req.headers.get('Ytdl-Request-Proxy')
+        req_proxy = req.headers.get('Ytdl-request-proxy')
         if req_proxy is not None:
             proxy = req_proxy
-            del req.headers['Ytdl-Request-Proxy']
+            del req.headers['Ytdl-request-proxy']
+
+        if proxy == '__noproxy__':
+            return None  # No Proxy
         return compat_urllib_request.ProxyHandler.proxy_open(
             self, req, proxy, type)

From 7fde87c77da41f24faf34b37f5d05c029ac23d4f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 3 Mar 2015 13:59:38 +0100
Subject: [PATCH 1765/1937] release 2015.03.03.1

---
 README.md             | 2 ++
 youtube_dl/version.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d05549125..5b9dd2cea 100644
--- a/README.md
+++ b/README.md
@@ -67,6 +67,8 @@ which means you can modify it, redistribute it or use it however you like.
     --source-address IP              Client-side IP address to bind to (experimental)
     -4, --force-ipv4                 Make all connections via IPv4 (experimental)
     -6, --force-ipv6                 Make all connections via IPv6 (experimental)
+    --cn-verification-proxy URL      Use this proxy to verify the IP address for some Chinese sites. The default proxy specified by --proxy (or none, if the options is
+                                     not present) is used for the actual downloading. (experimental)
 
 ## Video Selection:
     --playlist-start NUMBER          playlist video to start at (default is 1)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 3a49e52c9..252933993 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.03.03'
+__version__ = '2015.03.03.1'

From e0d0572b731b79393a5c5121acf4a43017160936 Mon Sep 17 00:00:00 2001
From: chaos33 <919946h43k48>
Date: Tue, 3 Mar 2015 22:53:05 +0800
Subject: [PATCH 1766/1937] fix youporn extractor's json search regex

---
 youtube_dl/extractor/youporn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 107c9ac36..a278386d4 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -47,7 +47,7 @@ class YouPornIE(InfoExtractor):
 
         # Get JSON parameters
         json_params = self._search_regex(
-            r'var currentVideo = new Video\((.*)\)[,;]',
+            r'var videoJason = (.*)[,;]',
             webpage, 'JSON parameters')
         try:
             params = json.loads(json_params)

From 63fc80005782430180ed0d13ac6ab5ca497d333a Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Tue, 3 Mar 2015 23:17:19 +0800
Subject: [PATCH 1767/1937] [Letv] Fix test_Letv and test_Letv_1 failures in
 python 3

---
 youtube_dl/extractor/letv.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 85eee141b..9ed81a199 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -88,9 +88,10 @@ class LetvIE(InfoExtractor):
         play_json_req = compat_urllib_request.Request(
             'http://api.letv.com/mms/out/video/playJson?' + compat_urllib_parse.urlencode(params)
         )
-        play_json_req.add_header(
-            'Ytdl-request-proxy',
-            self._downloader.params.get('cn_verification_proxy'))
+        cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
+        if cn_verification_proxy:
+            play_json_req.add_header('Ytdl-request-proxy', cn_verification_proxy)
+
         play_json = self._download_json(
             play_json_req,
             media_id, 'playJson data')

From 50c9949d7ae8f76d7a105481e0e692de3448c385 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 3 Mar 2015 21:39:04 +0600
Subject: [PATCH 1768/1937] [youporn] Imrove JSON regex and preserve the old
 one

---
 youtube_dl/extractor/youporn.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index a278386d4..e4c855ee0 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -47,7 +47,8 @@ class YouPornIE(InfoExtractor):
 
         # Get JSON parameters
         json_params = self._search_regex(
-            r'var videoJason = (.*)[,;]',
+            [r'var\s+videoJa?son\s*=\s*({.+?});',
+             r'var\s+currentVideo\s*=\s*new\s+Video\((.+?)\)[,;]'],
             webpage, 'JSON parameters')
         try:
             params = json.loads(json_params)

From 22d362831978ea2b83ea0c21a634237dc2406ab6 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 3 Mar 2015 18:39:23 +0200
Subject: [PATCH 1769/1937] [tvplay] Adapt _VALID_URL and test case to domain
 name change

---
 youtube_dl/extractor/tvplay.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py
index 9a53a3c74..e83e31a31 100644
--- a/youtube_dl/extractor/tvplay.py
+++ b/youtube_dl/extractor/tvplay.py
@@ -16,6 +16,7 @@ class TVPlayIE(InfoExtractor):
     _VALID_URL = r'''(?x)http://(?:www\.)?
         (?:tvplay\.lv/parraides|
            tv3play\.lt/programos|
+           play\.tv3\.lt/programos|
            tv3play\.ee/sisu|
            tv3play\.se/program|
            tv6play\.se/program|
@@ -45,7 +46,7 @@ class TVPlayIE(InfoExtractor):
             },
         },
         {
-            'url': 'http://www.tv3play.lt/programos/moterys-meluoja-geriau/409229?autostart=true',
+            'url': 'http://play.tv3.lt/programos/moterys-meluoja-geriau/409229?autostart=true',
             'info_dict': {
                 'id': '409229',
                 'ext': 'flv',

From dc570c4951d397ded7535393a9f0972c9cd0bc04 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Tue, 3 Mar 2015 18:41:34 +0200
Subject: [PATCH 1770/1937] [lrt] Pass --realtime to rtmpdump

---
 youtube_dl/extractor/lrt.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/lrt.py b/youtube_dl/extractor/lrt.py
index 9c2fbdd96..e3236f7b5 100644
--- a/youtube_dl/extractor/lrt.py
+++ b/youtube_dl/extractor/lrt.py
@@ -52,6 +52,7 @@ class LRTIE(InfoExtractor):
                     'url': data['streamer'],
                     'play_path': 'mp4:%s' % data['file'],
                     'preference': -1,
+                    'rtmp_real_time': True,
                 })
             else:
                 formats.extend(

From 123397317c9906bf53a23b6b426db0b08c70da32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 3 Mar 2015 18:45:56 +0100
Subject: [PATCH 1771/1937] [downloader/http] Remove wrong '_hook_progress'
 call (fixes #5117)

---
 youtube_dl/downloader/http.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 2e3dac825..5e9982799 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -218,12 +218,6 @@ class HttpFD(FileDownloader):
         if tmpfilename != '-':
             stream.close()
 
-        self._hook_progress({
-            'downloaded_bytes': byte_counter,
-            'total_bytes': data_len,
-            'tmpfilename': tmpfilename,
-            'status': 'error',
-        })
         if data_len is not None and byte_counter != data_len:
             raise ContentTooShortError(byte_counter, int(data_len))
         self.try_rename(tmpfilename, filename)

From f28fe66970370b5df649aa457e52cf99f6a49905 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 4 Mar 2015 12:14:38 +0100
Subject: [PATCH 1772/1937] [downloader/http] Add missing fields for
 _hook_progress call

It would fail if you run 'youtube-dl --no-part URL' a second time when the file has already been downloaded.

(Reported in Fedora: https://bugzilla.redhat.com/show_bug.cgi?id=1195779)
---
 youtube_dl/downloader/http.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 5e9982799..4047d7167 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -92,6 +92,8 @@ class HttpFD(FileDownloader):
                             self._hook_progress({
                                 'filename': filename,
                                 'status': 'finished',
+                                'downloaded_bytes': resume_len,
+                                'total_bytes': resume_len,
                             })
                             return True
                         else:

From 12a129ec6dde840c23e8b1e5d7552de93fce7be4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 5 Mar 2015 02:36:53 +0600
Subject: [PATCH 1773/1937] [playwire] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/playwire.py | 78 ++++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 youtube_dl/extractor/playwire.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ffcc7d9ab..5ca534cdf 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -364,6 +364,7 @@ from .planetaplay import PlanetaPlayIE
 from .played import PlayedIE
 from .playfm import PlayFMIE
 from .playvid import PlayvidIE
+from .playwire import PlaywireIE
 from .podomatic import PodomaticIE
 from .pornhd import PornHdIE
 from .pornhub import (
diff --git a/youtube_dl/extractor/playwire.py b/youtube_dl/extractor/playwire.py
new file mode 100644
index 000000000..bdc71017b
--- /dev/null
+++ b/youtube_dl/extractor/playwire.py
@@ -0,0 +1,78 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    xpath_text,
+    float_or_none,
+    int_or_none,
+)
+
+
+class PlaywireIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:config|cdn)\.playwire\.com(?:/v2)?/(?P<publisher_id>\d+)/(?:videos/v2|embed|config)/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://config.playwire.com/14907/videos/v2/3353705/player.json',
+        'md5': 'e6398701e3595888125729eaa2329ed9',
+        'info_dict': {
+            'id': '3353705',
+            'ext': 'mp4',
+            'title': 'S04_RM_UCL_Rus',
+            'thumbnail': 're:^http://.*\.png$',
+            'duration': 145.94,
+        },
+    }, {
+        'url': 'http://cdn.playwire.com/11625/embed/85228.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://config.playwire.com/12421/videos/v2/3389892/zeus.json',
+        'only_matching': True,
+    }, {
+        'url': 'http://cdn.playwire.com/v2/12342/config/1532636.json',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        publisher_id, video_id = mobj.group('publisher_id'), mobj.group('id')
+
+        player = self._download_json(
+            'http://config.playwire.com/%s/videos/v2/%s/zeus.json' % (publisher_id, video_id),
+            video_id)
+
+        title = player['settings']['title']
+        duration = float_or_none(player.get('duration'), 1000)
+
+        content = player['content']
+        thumbnail = content.get('poster')
+        src = content['media']['f4m']
+
+        f4m = self._download_xml(src, video_id)
+        base_url = xpath_text(f4m, './{http://ns.adobe.com/f4m/1.0}baseURL', 'base url', fatal=True)
+        formats = []
+        for media in f4m.findall('./{http://ns.adobe.com/f4m/1.0}media'):
+            media_url = media.get('url')
+            if not media_url:
+                continue
+            tbr = int_or_none(media.get('bitrate'))
+            width = int_or_none(media.get('width'))
+            height = int_or_none(media.get('height'))
+            f = {
+                'url': '%s/%s' % (base_url, media.attrib['url']),
+                'tbr': tbr,
+                'width': width,
+                'height': height,
+            }
+            if not (tbr or width or height):
+                f['quality'] = 1 if '-hd.' in media_url else 0
+            formats.append(f)
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
+        }

From c2ebea6580481a89c35f9447898b17241ad71ac0 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 5 Mar 2015 14:45:38 +0200
Subject: [PATCH 1774/1937] [extremetube] Fix extraction (Closes #5127)

---
 youtube_dl/extractor/extremetube.py | 30 ++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/extremetube.py b/youtube_dl/extractor/extremetube.py
index 36ba33128..c826a5404 100644
--- a/youtube_dl/extractor/extremetube.py
+++ b/youtube_dl/extractor/extremetube.py
@@ -4,11 +4,11 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urllib_parse_urlparse,
+    compat_parse_qs,
     compat_urllib_request,
-    compat_urllib_parse,
 )
 from ..utils import (
+    qualities,
     str_to_int,
 )
 
@@ -17,7 +17,7 @@ class ExtremeTubeIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?P<url>extremetube\.com/.*?video/.+?(?P<id>[0-9]+))(?:[/?&]|$)'
     _TESTS = [{
         'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
-        'md5': '1fb9228f5e3332ec8c057d6ac36f33e0',
+        'md5': '344d0c6d50e2f16b06e49ca011d8ac69',
         'info_dict': {
             'id': '652431',
             'ext': 'mp4',
@@ -49,19 +49,27 @@ class ExtremeTubeIE(InfoExtractor):
             r'Views:\s*</strong>\s*<span>([\d,\.]+)</span>',
             webpage, 'view count', fatal=False))
 
-        video_url = compat_urllib_parse.unquote(self._html_search_regex(
-            r'video_url=(.+?)&amp;', webpage, 'video_url'))
-        path = compat_urllib_parse_urlparse(video_url).path
-        format = path.split('/')[5].split('_')[:2]
-        format = "-".join(format)
+        flash_vars = compat_parse_qs(self._search_regex(
+            r'<param[^>]+?name="flashvars"[^>]+?value="([^"]+)"', webpage, 'flash vars'))
+
+        formats = []
+        quality = qualities(['180p', '240p', '360p', '480p', '720p', '1080p'])
+        for k, vals in flash_vars.items():
+            m = re.match(r'quality_(?P<quality>[0-9]+p)$', k)
+            if m is not None:
+                formats.append({
+                    'format_id': m.group('quality'),
+                    'quality': quality(m.group('quality')),
+                    'url': vals[0],
+                })
+
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': video_title,
+            'formats': formats,
             'uploader': uploader,
             'view_count': view_count,
-            'url': video_url,
-            'format': format,
-            'format_id': format,
             'age_limit': 18,
         }

From 5ee6fc974e617ce2f8d9d62c416091a1daa6d802 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 6 Mar 2015 02:43:05 +0800
Subject: [PATCH 1775/1937] [sohu] Fix info extractor and add tests

---
 youtube_dl/extractor/sohu.py | 46 +++++++++++++++++++++++++++++-------
 1 file changed, 38 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index c04791997..ef7ec51df 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -4,22 +4,42 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from .common import compat_str
+from ..utils import compat_str
+from ..compat import compat_urllib_request
 
 
 class SohuIE(InfoExtractor):
     _VALID_URL = r'https?://(?P<mytv>my\.)?tv\.sohu\.com/.+?/(?(mytv)|n)(?P<id>\d+)\.shtml.*?'
 
-    _TEST = {
+    _TESTS = [{
+        'note': 'This video is available only in Mainland China',
         'url': 'http://tv.sohu.com/20130724/n382479172.shtml#super',
-        'md5': 'bde8d9a6ffd82c63a1eefaef4eeefec7',
+        'md5': '29175c8cadd8b5cc4055001e85d6b372',
         'info_dict': {
             'id': '382479172',
             'ext': 'mp4',
             'title': 'MV：Far East Movement《The Illest》',
         },
-        'skip': 'Only available from China',
-    }
+        'params': {
+            'cn_verification_proxy': 'proxy.uku.im:8888'
+        }
+    }, {
+        'url': 'http://tv.sohu.com/20150305/n409385080.shtml',
+        'md5': '699060e75cf58858dd47fb9c03c42cfb',
+        'info_dict': {
+            'id': '409385080',
+            'ext': 'mp4',
+            'title': '《2015湖南卫视羊年元宵晚会》唐嫣《花好月圆》',
+        }
+    }, {
+        'url': 'http://my.tv.sohu.com/us/232799889/78693464.shtml',
+        'md5': '9bf34be48f2f4dadcb226c74127e203c',
+        'info_dict': {
+            'id': '78693464',
+            'ext': 'mp4',
+            'title': '【爱范品】第31期：MWC见不到的奇葩手机',
+        }
+    }]
 
     def _real_extract(self, url):
 
@@ -29,9 +49,14 @@ class SohuIE(InfoExtractor):
             else:
                 base_data_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid='
 
-            return self._download_json(
-                base_data_url + vid_id, video_id,
-                'Downloading JSON data for %s' % vid_id)
+            req = compat_urllib_request.Request(base_data_url + vid_id)
+
+            cn_verification_proxy = self._downloader.params.get('cn_verification_proxy')
+            if cn_verification_proxy:
+                req.add_header('Ytdl-request-proxy', cn_verification_proxy)
+
+            return self._download_json(req, video_id,
+                                       'Downloading JSON data for %s' % vid_id)
 
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -77,6 +102,11 @@ class SohuIE(InfoExtractor):
                     % (format_id, i + 1, part_count))
 
                 part_info = part_str.split('|')
+
+                # Sanitize URL to prevent download failure
+                if part_info[0][-1] == '/' and su[i][0] == '/':
+                    su[i] = su[i][1:]
+
                 video_url = '%s%s?key=%s' % (part_info[0], su[i], part_info[3])
 
                 formats.append({

From 5c7495a19429e3b27c003a4bd5bb96ed1e3a4932 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 6 Mar 2015 02:48:27 +0800
Subject: [PATCH 1776/1937] [sohu] Correct wrong imports

---
 youtube_dl/extractor/sohu.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index ef7ec51df..335e84fab 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -4,8 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import compat_str
-from ..compat import compat_urllib_request
+from ..compat import (
+    compat_str,
+    compat_urllib_request
+)
 
 
 class SohuIE(InfoExtractor):

From 9609f02e3ca87e56fc5571e41040ddbcf2a79a73 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 5 Mar 2015 22:34:56 +0100
Subject: [PATCH 1777/1937] [vidme] Modernize

---
 youtube_dl/extractor/vidme.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 5c89824c1..339c3d897 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -1,7 +1,5 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     int_or_none,
@@ -28,12 +26,11 @@ class VidmeIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
+        video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
 
-        video_url = self._html_search_regex(r'<source src="([^"]+)"', webpage, 'video URL')
+        video_url = self._html_search_regex(
+            r'<source src="([^"]+)"', webpage, 'video URL')
 
         title = self._og_search_title(webpage)
         description = self._og_search_description(webpage, default='')

From 55969016e96fded28b97b2ef3bbf66efa83d6afb Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Fri, 6 Mar 2015 12:43:49 +0800
Subject: [PATCH 1778/1937] [utils] Add a function to sanitize consecutive
 slashes in URLs

---
 test/test_utils.py           | 16 ++++++++++++++++
 youtube_dl/extractor/sohu.py |  8 +++-----
 youtube_dl/utils.py          | 15 +++++++++++++++
 3 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 64fad58ad..e02069c4d 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -54,6 +54,7 @@ from youtube_dl.utils import (
     xpath_with_ns,
     render_table,
     match_str,
+    url_sanitize_consecutive_slashes,
 )
 
 
@@ -501,6 +502,21 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
             'like_count > 100 & dislike_count <? 50 & description',
             {'like_count': 190, 'dislike_count': 10}))
 
+    def test_url_sanitize_consecutive_slashes(self):
+        self.assertEqual(url_sanitize_consecutive_slashes(
+            'http://hostname/foo//bar/filename.html'),
+            'http://hostname/foo/bar/filename.html')
+        self.assertEqual(url_sanitize_consecutive_slashes(
+            'http://hostname//foo/bar/filename.html'),
+            'http://hostname/foo/bar/filename.html')
+        self.assertEqual(url_sanitize_consecutive_slashes(
+            'http://hostname//'), 'http://hostname/')
+        self.assertEqual(url_sanitize_consecutive_slashes(
+            'http://hostname/foo/bar/filename.html'),
+            'http://hostname/foo/bar/filename.html')
+        self.assertEqual(url_sanitize_consecutive_slashes(
+            'http://hostname/'), 'http://hostname/')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 335e84fab..5adc734d9 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,6 +8,7 @@ from ..compat import (
     compat_str,
     compat_urllib_request
 )
+from ..utils import url_sanitize_consecutive_slashes
 
 
 class SohuIE(InfoExtractor):
@@ -105,11 +106,8 @@ class SohuIE(InfoExtractor):
 
                 part_info = part_str.split('|')
 
-                # Sanitize URL to prevent download failure
-                if part_info[0][-1] == '/' and su[i][0] == '/':
-                    su[i] = su[i][1:]
-
-                video_url = '%s%s?key=%s' % (part_info[0], su[i], part_info[3])
+                video_url = url_sanitize_consecutive_slashes(
+                    '%s%s?key=%s' % (part_info[0], su[i], part_info[3]))
 
                 formats.append({
                     'url': video_url,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7426e2a1f..ef14f9a36 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1789,3 +1789,18 @@ class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
             return None  # No Proxy
         return compat_urllib_request.ProxyHandler.proxy_open(
             self, req, proxy, type)
+
+
+def url_sanitize_consecutive_slashes(url):
+    """Sanitize URLs with consecutive slashes
+
+    For example, transform both
+        http://hostname/foo//bar/filename.html
+    and
+        http://hostname//foo/bar/filename.html
+    into
+        http://hostname/foo/bar/filename.html
+    """
+    parsed_url = list(compat_urlparse.urlparse(url))
+    parsed_url[2] = re.sub(r'/{2,}', '/', parsed_url[2])
+    return compat_urlparse.urlunparse(parsed_url)

From 7feddd9fc7a4a482edbc9a07d6295d418989bd67 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 6 Mar 2015 10:44:24 +0100
Subject: [PATCH 1779/1937] [travis] Declare 3.2 (Fixes #5144)

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index fb34299fc..511bee64c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,6 +2,7 @@ language: python
 python:
   - "2.6"
   - "2.7"
+  - "3.2"
   - "3.3"
   - "3.4"
 before_install:

From a0bb7c5593b08339f34dc43d9bc1b199ddc3ff7d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 6 Mar 2015 10:49:42 +0100
Subject: [PATCH 1780/1937] [extractor/common] Improve m3u format IDs (#5143)

---
 youtube_dl/extractor/common.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index cf39c0c21..8ff76342f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -883,8 +883,12 @@ class InfoExtractor(object):
                     formats.append({'url': format_url(line)})
                     continue
                 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
+                format_id = last_media.get('NAME')
+                if not format_id:
+                    format_id = '-'.join(filter(None, [
+                        m3u8_id, 'm3u8-%d' % (tbr if tbr else len(formats))]))
                 f = {
-                    'format_id': '-'.join(filter(None, [m3u8_id, 'm3u8-%d' % (tbr if tbr else len(formats))])),
+                    'format_id': format_id,
                     'url': format_url(line.strip()),
                     'tbr': tbr,
                     'ext': ext,

From d0e958c71c8c561d1c3917d114370fcb85972181 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 6 Mar 2015 10:53:49 +0100
Subject: [PATCH 1781/1937] [twitch:vod] Prefer source stream (Fixes #5143)

---
 youtube_dl/extractor/twitch.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 8af136147..b058891bd 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -85,6 +85,14 @@ class TwitchBaseIE(InfoExtractor):
             raise ExtractorError(
                 'Unable to login: %s' % m.group('msg').strip(), expected=True)
 
+    def _prefer_source(self, formats):
+        try:
+            source = next(f for f in formats if f['format_id'] == 'Source')
+            source['preference'] = 10
+        except StopIteration:
+            pass  # No Source stream present
+        self._sort_formats(formats)
+
 
 class TwitchItemBaseIE(TwitchBaseIE):
     def _download_info(self, item, item_id):
@@ -209,6 +217,7 @@ class TwitchVodIE(TwitchItemBaseIE):
             '%s/vod/%s?nauth=%s&nauthsig=%s'
             % (self._USHER_BASE, item_id, access_token['token'], access_token['sig']),
             item_id, 'mp4')
+        self._prefer_source(formats)
         info['formats'] = formats
         return info
 
@@ -357,13 +366,7 @@ class TwitchStreamIE(TwitchBaseIE):
             '%s/api/channel/hls/%s.m3u8?%s'
             % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query).encode('utf-8')),
             channel_id, 'mp4')
-
-        # prefer the 'source' stream, the others are limited to 30 fps
-        def _sort_source(f):
-            if f.get('m3u8_media') is not None and f['m3u8_media'].get('NAME') == 'Source':
-                return 1
-            return 0
-        formats = sorted(formats, key=_sort_source)
+        self._prefer_source(formats)
 
         view_count = stream.get('viewers')
         timestamp = parse_iso8601(stream.get('created_at'))

From 8dc9d361c22754e6b6b52136d496d754dea65b46 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Mar 2015 22:52:50 +0600
Subject: [PATCH 1782/1937] [extractor/common] Fix format_id when `last_media`
 is None and always include `m3u8_id` if present

The rationale behind `m3u8_id` was to resolve duplicates when processing several m3u8 playlists within the same media that give equal resulting `format_id`'s,
e.g. `youtube-dl http://www.rts.ch/play/tv/passe-moi-les-jumelles/video/la-fee-des-bois-mustang-les-chemins-du-vent?id=3854925 -F`
---
 youtube_dl/extractor/common.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8ff76342f..06bce12e0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -883,12 +883,13 @@ class InfoExtractor(object):
                     formats.append({'url': format_url(line)})
                     continue
                 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
-                format_id = last_media.get('NAME')
-                if not format_id:
-                    format_id = '-'.join(filter(None, [
-                        m3u8_id, 'm3u8-%d' % (tbr if tbr else len(formats))]))
+                format_id = []
+                if m3u8_id:
+                    format_id.append(m3u8_id)
+                last_media_name = last_media.get('NAME') if last_media else None
+                format_id.append(last_media_name if last_media_name else '%d' % (tbr if tbr else len(formats)))
                 f = {
-                    'format_id': format_id,
+                    'format_id': '-'.join(format_id),
                     'url': format_url(line.strip()),
                     'tbr': tbr,
                     'ext': ext,

From bd05aa4e24774a9aa9f68a8379900e6b756aae11 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sat, 7 Mar 2015 00:53:52 +0800
Subject: [PATCH 1783/1937] [Yam] Add an error detection and update test cases

---
 youtube_dl/extractor/yam.py | 34 ++++++++++++++++++++++++++++------
 1 file changed, 28 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
index b294767c5..19ad74d04 100644
--- a/youtube_dl/extractor/yam.py
+++ b/youtube_dl/extractor/yam.py
@@ -8,6 +8,7 @@ from ..compat import compat_urlparse
 from ..utils import (
     float_or_none,
     month_by_abbreviation,
+    ExtractorError,
 )
 
 
@@ -28,23 +29,44 @@ class YamIE(InfoExtractor):
         }
     }, {
         # An external video hosted on YouTube
-        'url': 'http://mymedia.yam.com/m/3598173',
-        'md5': '0238ceec479c654e8c2f1223755bf3e9',
+        'url': 'http://mymedia.yam.com/m/3599430',
+        'md5': '03127cf10d8f35d120a9e8e52e3b17c6',
         'info_dict': {
-            'id': 'pJ2Deys283c',
+            'id': 'CNpEoQlrIgA',
             'ext': 'mp4',
-            'upload_date': '20150202',
+            'upload_date': '20150306',
             'uploader': '新莊社大瑜伽社',
-            'description': 'md5:f5cc72f0baf259a70fb731654b0d2eff',
+            'description': 'md5:11e2e405311633ace874f2e6226c8b17',
             'uploader_id': '2323agoy',
-            'title': '外婆的澎湖灣KTV-潘安邦',
+            'title': '20090412陽明山二子坪-1',
         }
+    }, {
+        'url': 'http://mymedia.yam.com/m/3598173',
+        'info_dict': {
+            'id': '3598173',
+            'ext': 'mp4',
+        },
+        'skip': 'cause Yam system error',
+    }, {
+        'url': 'http://mymedia.yam.com/m/3599437',
+        'info_dict': {
+            'id': '3599437',
+            'ext': 'mp4',
+        },
+        'skip': 'invalid YouTube URL',
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         page = self._download_webpage(url, video_id)
 
+        # Check for errors
+        system_msg = self._html_search_regex(
+            r'系統訊息(?:<br>|\n|\r)*([^<>]+)<br>', page, 'system message',
+            default=None)
+        if system_msg:
+            raise ExtractorError(system_msg, expected=True)
+
         # Is it hosted externally on YouTube?
         youtube_url = self._html_search_regex(
             r'<embed src="(http://www.youtube.com/[^"]+)"',

From f207019ce51e18f7151dc9ab332cc8d0933021b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Mar 2015 22:53:53 +0600
Subject: [PATCH 1784/1937] [extractor/common] Remove 'm3u8' from quality
 selection URL

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 06bce12e0..f9e8e2bad 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -839,7 +839,7 @@ class InfoExtractor(object):
                               m3u8_id=None):
 
         formats = [{
-            'format_id': '-'.join(filter(None, [m3u8_id, 'm3u8-meta'])),
+            'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
             'url': m3u8_url,
             'ext': ext,
             'protocol': 'm3u8',

From 3c6f245083db85d7ba9ae5444561339dc1b02671 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Mar 2015 18:16:56 +0100
Subject: [PATCH 1785/1937] [vimeo] Fix upload date extraction

---
 youtube_dl/extractor/vimeo.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 8f540f578..3771769e1 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -20,6 +20,7 @@ from ..utils import (
     RegexNotFoundError,
     smuggle_url,
     std_headers,
+    unified_strdate,
     unsmuggle_url,
     urlencode_postdata,
 )
@@ -140,6 +141,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'description': 'md5:8678b246399b070816b12313e8b4eb5c',
                 'uploader_id': 'atencio',
                 'uploader': 'Peter Atencio',
+                'upload_date': '20130927',
                 'duration': 187,
             },
         },
@@ -323,9 +325,9 @@ class VimeoIE(VimeoBaseInfoExtractor):
 
         # Extract upload date
         video_upload_date = None
-        mobj = re.search(r'<meta itemprop="dateCreated" content="(\d{4})-(\d{2})-(\d{2})T', webpage)
+        mobj = re.search(r'<time[^>]+datetime="([^"]+)"', webpage)
         if mobj is not None:
-            video_upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3)
+            video_upload_date = unified_strdate(mobj.group(1))
 
         try:
             view_count = int(self._search_regex(r'UserPlays:(\d+)', webpage, 'view count'))

From 9c85b5376d3b784d1bfdead7bc473b12d7349614 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Mar 2015 19:08:27 +0100
Subject: [PATCH 1786/1937] [vimeo] Fix and use '_verify_video_password'
 (#5001)

It only supports verifying the password over https now.

Use it instead of manually setting the 'password' cookie because it allows to check if the password is correct.
---
 youtube_dl/extractor/vimeo.py | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 3771769e1..61f2d1ee8 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -4,7 +4,6 @@ from __future__ import unicode_literals
 import json
 import re
 import itertools
-import hashlib
 
 from .common import InfoExtractor
 from ..compat import (
@@ -178,17 +177,15 @@ class VimeoIE(VimeoBaseInfoExtractor):
         password = self._downloader.params.get('videopassword', None)
         if password is None:
             raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
-        token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
-        data = compat_urllib_parse.urlencode({
+        token = self._search_regex(r'xsrft = \'(.*?)\'', webpage, 'login token')
+        data = urlencode_postdata({
             'password': password,
             'token': token,
         })
-        # I didn't manage to use the password with https
-        if url.startswith('https'):
-            pass_url = url.replace('https', 'http')
-        else:
-            pass_url = url
-        password_request = compat_urllib_request.Request(pass_url + '/password', data)
+        if url.startswith('http://'):
+            # vimeo only supports https now, but the user can give an http url
+            url = url.replace('http://', 'https://')
+        password_request = compat_urllib_request.Request(url + '/password', data)
         password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
         password_request.add_header('Cookie', 'xsrft=%s' % token)
         return self._download_webpage(
@@ -227,11 +224,6 @@ class VimeoIE(VimeoBaseInfoExtractor):
         if mobj.group('pro') or mobj.group('player'):
             url = 'http://player.vimeo.com/video/' + video_id
 
-        password = self._downloader.params.get('videopassword', None)
-        if password:
-            headers['Cookie'] = '%s_password=%s' % (
-                video_id, hashlib.md5(password.encode('utf-8')).hexdigest())
-
         # Retrieve video webpage to extract further information
         request = compat_urllib_request.Request(url, None, headers)
         try:

From d1508cd68d0a3e470860dc7f225509b2e422d34b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Mar 2015 22:16:26 +0100
Subject: [PATCH 1787/1937] [vimeo:album] Fix password protected videos

Since it only uses https now, don't recognize http urls.
---
 youtube_dl/extractor/vimeo.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 61f2d1ee8..0d1c92633 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -429,10 +429,10 @@ class VimeoChannelIE(InfoExtractor):
             name="([^"]+)"\s+
             value="([^"]*)"
             ''', login_form))
-        token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
+        token = self._search_regex(r'xsrft = \'(.*?)\'', webpage, 'login token')
         fields['token'] = token
         fields['password'] = password
-        post = compat_urllib_parse.urlencode(fields)
+        post = urlencode_postdata(fields)
         password_path = self._search_regex(
             r'action="([^"]+)"', login_form, 'password URL')
         password_url = compat_urlparse.urljoin(page_url, password_path)
@@ -494,10 +494,10 @@ class VimeoUserIE(VimeoChannelIE):
 
 class VimeoAlbumIE(VimeoChannelIE):
     IE_NAME = 'vimeo:album'
-    _VALID_URL = r'https?://vimeo\.com/album/(?P<id>\d+)'
+    _VALID_URL = r'https://vimeo\.com/album/(?P<id>\d+)'
     _TITLE_RE = r'<header id="page_header">\n\s*<h1>(.*?)</h1>'
     _TESTS = [{
-        'url': 'http://vimeo.com/album/2632481',
+        'url': 'https://vimeo.com/album/2632481',
         'info_dict': {
             'id': '2632481',
             'title': 'Staff Favorites: November 2013',
@@ -521,7 +521,7 @@ class VimeoAlbumIE(VimeoChannelIE):
 
     def _real_extract(self, url):
         album_id = self._match_id(url)
-        return self._extract_videos(album_id, 'http://vimeo.com/album/%s' % album_id)
+        return self._extract_videos(album_id, 'https://vimeo.com/album/%s' % album_id)
 
 
 class VimeoGroupsIE(VimeoAlbumIE):

From 61e00a97756de178db3e2a55b4292273532c9420 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Mar 2015 22:39:05 +0100
Subject: [PATCH 1788/1937] [vimeo] Use https for player.vimeo.com urls (closes
 #5147)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 0d1c92633..b84a83ba6 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -222,7 +222,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         video_id = mobj.group('id')
         orig_url = url
         if mobj.group('pro') or mobj.group('player'):
-            url = 'http://player.vimeo.com/video/' + video_id
+            url = 'https://player.vimeo.com/video/' + video_id
 
         # Retrieve video webpage to extract further information
         request = compat_urllib_request.Request(url, None, headers)

From 23ba76bc0eae4931c1adbd00de45a9ed41848283 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 6 Mar 2015 22:45:05 +0100
Subject: [PATCH 1789/1937] [dailymotion] Replace test

It has been removed.
---
 youtube_dl/extractor/dailymotion.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 42b20a46d..4f67c3aac 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -46,13 +46,13 @@ class DailymotionIE(DailymotionBaseInfoExtractor):
 
     _TESTS = [
         {
-            'url': 'http://www.dailymotion.com/video/x33vw9_tutoriel-de-youtubeur-dl-des-video_tech',
-            'md5': '392c4b85a60a90dc4792da41ce3144eb',
+            'url': 'https://www.dailymotion.com/video/x2iuewm_steam-machine-models-pricing-listed-on-steam-store-ign-news_videogames',
+            'md5': '2137c41a8e78554bb09225b8eb322406',
             'info_dict': {
-                'id': 'x33vw9',
+                'id': 'x2iuewm',
                 'ext': 'mp4',
-                'uploader': 'Amphora Alex and Van .',
-                'title': 'Tutoriel de Youtubeur"DL DES VIDEO DE YOUTUBE"',
+                'uploader': 'IGN',
+                'title': 'Steam Machine Models, Pricing Listed on Steam Store - IGN News',
             }
         },
         # Vevo video

From a172d96292abda037b23b1f2b14dd2c6e65b56bb Mon Sep 17 00:00:00 2001
From: bonfy <foreverbonfy@163.com>
Date: Sat, 7 Mar 2015 14:05:56 +0800
Subject: [PATCH 1790/1937] [douyutv] Add new extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/douyutv.py  | 59 ++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 youtube_dl/extractor/douyutv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ffcc7d9ab..7d0798176 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -106,6 +106,7 @@ from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
 from .dotsub import DotsubIE
+from .douyutv import DouyutvIE
 from .dreisat import DreiSatIE
 from .drbonanza import DRBonanzaIE
 from .drtuber import DrTuberIE
diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
new file mode 100644
index 000000000..e9b92eb3b
--- /dev/null
+++ b/youtube_dl/extractor/douyutv.py
@@ -0,0 +1,59 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import (
+    ExtractorError,
+)
+
+class DouyutvIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?douyutv\.com/(?P<id>[A-Za-z0-9]+)'
+
+    '''
+    show_status: 1 直播中 ，2 没有直播
+    '''
+
+    _TEST = {
+        'url': 'http://www.douyutv.com/iseven',
+        'info_dict': {
+            'id': 'iseven',
+            'title': '清晨醒脑！T-ara根本停不下来！',
+            'ext': 'flv',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'is_live': True,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        info_url = 'http://www.douyutv.com/api/client/room/' + video_id
+
+        config = self._download_json(info_url, video_id)
+
+        error_code = config.get('error')
+        show_status = config['data'].get('show_status')
+        if error_code is not 0:
+            raise ExtractorError('Server reported error %i' % error_code,
+                                 expected=True)
+
+        if show_status == '2':
+            raise ExtractorError('The live show has not yet started',
+                                 expected=True)
+
+        title = config['data'].get('room_name')
+        rtmp_url = config['data'].get('rtmp_url')
+        rtmp_live = config['data'].get('rtmp_live')
+        thumbnail = config['data'].get('room_src') 
+
+        url = rtmp_url+'/'+rtmp_live
+
+        return {
+            'id': video_id,
+            'title': title,
+            'ext':'flv',
+            'url': url,
+            'thumbnail': thumbnail,
+            'is_live': True,
+            # TODO more properties (see youtube_dl/extractor/common.py)
+        }
\ No newline at end of file

From 14137b5781305d3c9db523b14283128335f62135 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Mar 2015 17:31:03 +0600
Subject: [PATCH 1791/1937] [orf:iptv] Add extractor (Closes #5140)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/orf.py      | 94 ++++++++++++++++++++++++++++++++
 2 files changed, 95 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5ca534cdf..f72d06fc4 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -354,6 +354,7 @@ from .orf import (
     ORFTVthekIE,
     ORFOE1IE,
     ORFFM4IE,
+    ORFIPTVIE,
 )
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 4e293392b..ca1a5bb3c 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -11,6 +11,11 @@ from ..utils import (
     HEADRequest,
     unified_strdate,
     ExtractorError,
+    strip_jsonp,
+    int_or_none,
+    float_or_none,
+    determine_ext,
+    remove_end,
 )
 
 
@@ -197,3 +202,92 @@ class ORFFM4IE(InfoExtractor):
             'description': data['subtitle'],
             'entries': entries
         }
+
+
+class ORFIPTVIE(InfoExtractor):
+    IE_NAME = 'orf:iptv'
+    IE_DESC = 'iptv.ORF.at'
+    _VALID_URL = r'http://iptv\.orf\.at/(?:#/)?stories/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://iptv.orf.at/stories/2267952',
+        'md5': '26ffa4bab6dbce1eee78bbc7021016cd',
+        'info_dict': {
+            'id': '339775',
+            'ext': 'flv',
+            'title': 'Kreml-Kritiker Nawalny wieder frei',
+            'description': 'md5:6f24e7f546d364dacd0e616a9e409236',
+            'duration': 84.729,
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'upload_date': '20150306',
+        },
+    }
+
+    def _real_extract(self, url):
+        story_id = self._match_id(url)
+
+        webpage = self._download_webpage(
+            'http://iptv.orf.at/stories/%s' % story_id, story_id)
+
+        video_id = self._search_regex(
+            r'data-video(?:id)?="(\d+)"', webpage, 'video id')
+
+        data = self._download_json(
+            'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
+            video_id)[0]
+
+        duration = float_or_none(data['duration'], 1000)
+
+        video = data['sources']['default']
+        load_balancer_url = video['loadBalancerUrl']
+        abr = int_or_none(video.get('audioBitrate'))
+        vbr = int_or_none(video.get('bitrate'))
+        fps = int_or_none(video.get('videoFps'))
+        width = int_or_none(video.get('videoWidth'))
+        height = int_or_none(video.get('videoHeight'))
+        thumbnail = video.get('preview')
+
+        rendition = self._download_json(
+            load_balancer_url, video_id, transform_source=strip_jsonp)
+
+        f = {
+            'abr': abr,
+            'vbr': vbr,
+            'fps': fps,
+            'width': width,
+            'height': height,
+        }
+
+        formats = []
+        for format_id, format_url in rendition['redirect'].items():
+            if format_id == 'rtmp':
+                ff = f.copy()
+                ff.update({
+                    'url': format_url,
+                    'format_id': format_id,
+                })
+                formats.append(ff)
+            elif determine_ext(format_url) == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    format_url, video_id, f4m_id=format_id))
+            elif determine_ext(format_url) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', m3u8_id=format_id))
+            else:
+                continue
+        self._sort_formats(formats)
+
+        title = remove_end(self._og_search_title(webpage), ' - iptv.ORF.at')
+        description = self._og_search_description(webpage)
+        upload_date = unified_strdate(self._html_search_meta(
+            'dc.date', webpage, 'upload date'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'formats': formats,
+        }

From 98998cded6e179b35055177f75fc921d99ee5938 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Mar 2015 18:59:06 +0600
Subject: [PATCH 1792/1937] [youtube:search_url] Fix extraction (Closes #5155)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3690f8021..27c8c4453 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1532,7 +1532,7 @@ class YoutubeSearchURLIE(InfoExtractor):
 
         webpage = self._download_webpage(url, query)
         result_code = self._search_regex(
-            r'(?s)<ol class="item-section"(.*?)</ol>', webpage, 'result HTML')
+            r'(?s)<ol[^>]+class="item-section"(.*?)</ol>', webpage, 'result HTML')
 
         part_codes = re.findall(
             r'(?s)<h3 class="yt-lockup-title">(.*?)</h3>', result_code)

From 0bf79ac455ff5dc52b61f26c254f83e7c041d75c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Mar 2015 22:16:23 +0600
Subject: [PATCH 1793/1937] [eagleplatform] Add extractor

---
 youtube_dl/extractor/__init__.py      |  1 +
 youtube_dl/extractor/eagleplatform.py | 99 +++++++++++++++++++++++++++
 2 files changed, 100 insertions(+)
 create mode 100644 youtube_dl/extractor/eagleplatform.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f72d06fc4..b489d5770 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -116,6 +116,7 @@ from .defense import DefenseGouvFrIE
 from .discovery import DiscoveryIE
 from .divxstage import DivxStageIE
 from .dropbox import DropboxIE
+from .eagleplatform import EaglePlatformIE
 from .ebaumsworld import EbaumsWorldIE
 from .echomsk import EchoMskIE
 from .ehow import EHowIE
diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
new file mode 100644
index 000000000..654f746fe
--- /dev/null
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -0,0 +1,99 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+)
+
+
+class EaglePlatformIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+                    (?:
+                        eagleplatform:(?P<custom_host>[^/]+):|
+                        https?://(?P<host>.+?\.media\.eagleplatform\.com)/index/player\?.*\brecord_id=
+                    )
+                    (?P<id>\d+)
+                '''
+    _TESTS = [{
+        # http://lenta.ru/news/2015/03/06/navalny/
+        'url': 'http://lentaru.media.eagleplatform.com/index/player?player=new&record_id=227304&player_template_id=5201',
+        'md5': '0b7994faa2bd5c0f69a3db6db28d078d',
+        'info_dict': {
+            'id': '227304',
+            'ext': 'mp4',
+            'title': 'Навальный вышел на свободу',
+            'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 87,
+            'view_count': int,
+            'age_limit': 0,
+        },
+    }, {
+        # http://muz-tv.ru/play/7129/
+        # http://media.clipyou.ru/index/player?record_id=12820&width=730&height=415&autoplay=true
+        'url': 'eagleplatform:media.clipyou.ru:12820',
+        'md5': '6c2ebeab03b739597ce8d86339d5a905',
+        'info_dict': {
+            'id': '12820',
+            'ext': 'mp4',
+            'title': "'O Sole Mio",
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 216,
+            'view_count': int,
+        },
+    }]
+
+    def _handle_error(self, response):
+        print(response['status'])
+        status = int_or_none(response.get('status', 200))
+        if status != 200:
+            raise ExtractorError(' '.join(response['errors']), expected=True)
+
+    def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'):
+        response = super(EaglePlatformIE, self)._download_json(url_or_request, video_id, note)
+        self._handle_error(response)
+        return response
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        host, video_id = mobj.group('custom_host') or mobj.group('host'), mobj.group('id')
+
+        player_data = self._download_json(
+            'http://%s/api/player_data?id=%s' % (host, video_id), video_id)
+
+        media = player_data['data']['playlist']['viewports'][0]['medialist'][0]
+
+        title = media['title']
+        description = media.get('description')
+        thumbnail = media.get('snapshot')
+        duration = int_or_none(media.get('duration'))
+        view_count = int_or_none(media.get('views'))
+
+        age_restriction = media.get('age_restriction')
+        age_limit = None
+        if age_restriction:
+            age_limit = 0 if age_restriction == 'allow_all' else 18
+
+        m3u8_data = self._download_json(
+            media['sources']['secure_m3u8']['auto'],
+            video_id, 'Downloading m3u8 JSON')
+
+        formats = self._extract_m3u8_formats(
+            m3u8_data['data'][0], video_id,
+            'mp4', entry_protocol='m3u8_native')
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'view_count': view_count,
+            'age_limit': age_limit,
+            'formats': formats,
+        }

From 135c9c42bf272c667f38a0a42e97e381fe53df91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Mar 2015 22:22:57 +0600
Subject: [PATCH 1794/1937] [eagleplatform] Add support for embeds

---
 youtube_dl/extractor/generic.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 5dc53685c..a12b29da1 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -570,6 +570,20 @@ class GenericIE(InfoExtractor):
                 'title': 'John Carlson Postgame 2/25/15',
             },
         },
+        # Eagle.Platform embed (generic URL)
+        {
+            'url': 'http://lenta.ru/news/2015/03/06/navalny/',
+            'info_dict': {
+                'id': '227304',
+                'ext': 'mp4',
+                'title': 'Навальный вышел на свободу',
+                'description': 'md5:d97861ac9ae77377f3f20eaf9d04b4f5',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 87,
+                'view_count': int,
+                'age_limit': 0,
+            },
+        },
         # RSS feed with enclosure
         {
             'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
@@ -1155,6 +1169,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result('kaltura:%(partner_id)s:%(id)s' % mobj.groupdict(), 'Kaltura')
 
+        # Look for Eagle.Platform embeds
+        mobj = re.search(
+            r'<iframe[^>]+src="(?P<url>https?://.+?\.media\.eagleplatform\.com/index/player\?.+?)"', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'), 'EaglePlatform')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From d47ae7f620e3b7be2387da2a61b618a3730d932a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Mar 2015 22:34:44 +0600
Subject: [PATCH 1795/1937] [eagleplatform] Add support for ClipYou embeds

---
 youtube_dl/extractor/generic.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a12b29da1..013198b0d 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -584,6 +584,18 @@ class GenericIE(InfoExtractor):
                 'age_limit': 0,
             },
         },
+        # ClipYou (Eagle.Platform) embed (custom URL)
+        {
+            'url': 'http://muz-tv.ru/play/7129/',
+            'info_dict': {
+                'id': '12820',
+                'ext': 'mp4',
+                'title': "'O Sole Mio",
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 216,
+                'view_count': int,
+            },
+        },
         # RSS feed with enclosure
         {
             'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
@@ -1175,6 +1187,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'EaglePlatform')
 
+        # Look for ClipYou (uses Eagle.Platform) embeds
+        mobj = re.search(
+            r'<iframe[^>]+src="https?://(?P<host>media\.clipyou\.ru)/index/player\?.*\brecord_id=(?P<id>\d+).*"', webpage)
+        if mobj is not None:
+            return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From 8a48223a7ba5a3d1e47a36c30dd925e11183ae63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 7 Mar 2015 22:35:36 +0600
Subject: [PATCH 1796/1937] [eagleplatform] Remove debug output

---
 youtube_dl/extractor/eagleplatform.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/eagleplatform.py b/youtube_dl/extractor/eagleplatform.py
index 654f746fe..7173371ee 100644
--- a/youtube_dl/extractor/eagleplatform.py
+++ b/youtube_dl/extractor/eagleplatform.py
@@ -48,7 +48,6 @@ class EaglePlatformIE(InfoExtractor):
     }]
 
     def _handle_error(self, response):
-        print(response['status'])
         status = int_or_none(response.get('status', 200))
         if status != 200:
             raise ExtractorError(' '.join(response['errors']), expected=True)

From 7dcad95d4faa91adfaa4d87d411c4bc55ab000e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 01:13:23 +0600
Subject: [PATCH 1797/1937] [YoutubeDL] Sanitize outtmpl as it may contain
 forbidden characters

---
 youtube_dl/YoutubeDL.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index df2aebb59..bae52e9c7 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -553,16 +553,20 @@ class YoutubeDL(object):
                 elif template_dict.get('width'):
                     template_dict['resolution'] = '?x%d' % template_dict['width']
 
+            restrict_filenames = self.params.get('restrictfilenames')
+
             sanitize = lambda k, v: sanitize_filename(
                 compat_str(v),
-                restricted=self.params.get('restrictfilenames'),
+                restricted=restrict_filenames,
                 is_id=(k == 'id'))
             template_dict = dict((k, sanitize(k, v))
                                  for k, v in template_dict.items()
                                  if v is not None)
             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 
-            outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
+            outtmpl = sanitize_filename(
+                self.params.get('outtmpl', DEFAULT_OUTTMPL),
+                restricted=restrict_filenames)
             tmpl = compat_expanduser(outtmpl)
             filename = tmpl % template_dict
             # Temporary fix for #4787

From ab205b9dc8dcee37786dac597f2c3760c89b191c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 7 Mar 2015 22:18:16 +0100
Subject: [PATCH 1798/1937] Revert "[YoutubeDL] Sanitize outtmpl as it may
 contain forbidden characters"

This reverts commit 7dcad95d4faa91adfaa4d87d411c4bc55ab000e4.

The output template is most definitly allowed to contain forbidden characters; otherwise -o /foo/bar/vid.mp4 wouldn't work.
---
 youtube_dl/YoutubeDL.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index bae52e9c7..df2aebb59 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -553,20 +553,16 @@ class YoutubeDL(object):
                 elif template_dict.get('width'):
                     template_dict['resolution'] = '?x%d' % template_dict['width']
 
-            restrict_filenames = self.params.get('restrictfilenames')
-
             sanitize = lambda k, v: sanitize_filename(
                 compat_str(v),
-                restricted=restrict_filenames,
+                restricted=self.params.get('restrictfilenames'),
                 is_id=(k == 'id'))
             template_dict = dict((k, sanitize(k, v))
                                  for k, v in template_dict.items()
                                  if v is not None)
             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 
-            outtmpl = sanitize_filename(
-                self.params.get('outtmpl', DEFAULT_OUTTMPL),
-                restricted=restrict_filenames)
+            outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
             tmpl = compat_expanduser(outtmpl)
             filename = tmpl % template_dict
             # Temporary fix for #4787

From d34e79492d1d9d9babf85af3737b90c4fe55eb42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 16:54:11 +0600
Subject: [PATCH 1799/1937] [twitch] Fix live streams (Closes #5158)

---
 youtube_dl/extractor/twitch.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index b058891bd..cbdaf9c7a 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -358,13 +358,12 @@ class TwitchStreamIE(TwitchBaseIE):
             'p': random.randint(1000000, 10000000),
             'player': 'twitchweb',
             'segment_preference': '4',
-            'sig': access_token['sig'],
-            'token': access_token['token'],
+            'sig': access_token['sig'].encode('utf-8'),
+            'token': access_token['token'].encode('utf-8'),
         }
-
         formats = self._extract_m3u8_formats(
             '%s/api/channel/hls/%s.m3u8?%s'
-            % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query).encode('utf-8')),
+            % (self._USHER_BASE, channel_id, compat_urllib_parse.urlencode(query)),
             channel_id, 'mp4')
         self._prefer_source(formats)
 

From 1132eae56d0f693b5dc55529d5cfadf32b32700d Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 8 Mar 2015 13:54:01 +0200
Subject: [PATCH 1800/1937] [gazeta] Add new extractor (Closes #4222)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/gazeta.py   | 35 ++++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+)
 create mode 100644 youtube_dl/extractor/gazeta.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b489d5770..e2475a634 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -175,6 +175,7 @@ from .gameone import (
 from .gamespot import GameSpotIE
 from .gamestar import GameStarIE
 from .gametrailers import GametrailersIE
+from .gazeta import GazetaIE
 from .gdcvault import GDCVaultIE
 from .generic import GenericIE
 from .giantbomb import GiantBombIE
diff --git a/youtube_dl/extractor/gazeta.py b/youtube_dl/extractor/gazeta.py
new file mode 100644
index 000000000..3f9e58061
--- /dev/null
+++ b/youtube_dl/extractor/gazeta.py
@@ -0,0 +1,35 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class GazetaIE(InfoExtractor):
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?gazeta\.ru/(?:(?P<category>[^/]*)/)?video/(?:main/)?(?P<id>[A-Za-z0-9-_]+)\.s?html)'
+    _TEST = {
+        'url': 'http://www.gazeta.ru/video/main/zadaite_vopros_vladislavu_yurevichu.shtml',
+        'md5': 'd49c9bdc6e5a7888f27475dc215ee789',
+        'info_dict': {
+            'id': '205566',
+            'ext': 'mp4',
+            'title': '«70–80 процентов гражданских в Донецке на грани голода»',
+            'description': 'md5:38617526050bd17b234728e7f9620a71',
+            'thumbnail': 're:^https?://.*\.jpg',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+
+        display_id = mobj.group('id')
+        embed_url = '%s?p=embed' % mobj.group('url')
+        embed_page = self._download_webpage(
+            embed_url, display_id, 'Downloading embed page')
+
+        video_id = self._search_regex(
+            r'<div[^>]*?class="eagleplayer"[^>]*?data-id="([^"]+)"', embed_page, 'video id')
+
+        return self.url_result(
+            'eagleplatform:gazeta.media.eagleplatform.com:%s' % video_id, 'EaglePlatform')

From 28778d6bae2ff2a0ad57d2c5a694e22ac4ead749 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 18:03:12 +0600
Subject: [PATCH 1801/1937] [pladform] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/pladform.py | 86 ++++++++++++++++++++++++++++++++
 2 files changed, 87 insertions(+)
 create mode 100644 youtube_dl/extractor/pladform.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e2475a634..14172ca56 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -364,6 +364,7 @@ from .pbs import PBSIE
 from .phoenix import PhoenixIE
 from .photobucket import PhotobucketIE
 from .planetaplay import PlanetaPlayIE
+from .pladform import PladformIE
 from .played import PlayedIE
 from .playfm import PlayFMIE
 from .playvid import PlayvidIE
diff --git a/youtube_dl/extractor/pladform.py b/youtube_dl/extractor/pladform.py
new file mode 100644
index 000000000..926e368a2
--- /dev/null
+++ b/youtube_dl/extractor/pladform.py
@@ -0,0 +1,86 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    xpath_text,
+)
+
+
+class PladformIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?:
+                                out\.pladform\.ru/player|
+                                static\.pladform\.ru/player\.swf
+                            )
+                            \?.*\bvideoid=|
+                            video\.pladform\.ru/catalog/video/videoid/
+                        )
+                        (?P<id>\d+)
+                    '''
+    _TESTS = [{
+        # http://muz-tv.ru/kinozal/view/7400/
+        'url': 'http://out.pladform.ru/player?pl=24822&videoid=100183293',
+        'md5': '61f37b575dd27f1bb2e1854777fe31f4',
+        'info_dict': {
+            'id': '100183293',
+            'ext': 'mp4',
+            'title': 'Тайны перевала Дятлова • Тайна перевала Дятлова 1 серия 2 часть',
+            'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
+            'thumbnail': 're:^https?://.*\.jpg$',
+            'duration': 694,
+            'age_limit': 0,
+        },
+    }, {
+        'url': 'http://static.pladform.ru/player.swf?pl=21469&videoid=100183293&vkcid=0',
+        'only_matching': True,
+    }, {
+        'url': 'http://video.pladform.ru/catalog/video/videoid/100183293/vkcid/0',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        video = self._download_xml(
+            'http://out.pladform.ru/getVideo?pl=1&videoid=%s' % video_id,
+            video_id)
+
+        if video.tag == 'error':
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, video.text),
+                expected=True)
+
+        formats = [{
+            'url': src.text,
+            'format_id': src.get('quality'),
+        } for src in video.findall('./src')]
+        self._sort_formats(formats)
+
+        webpage = self._download_webpage(
+            'http://video.pladform.ru/catalog/video/videoid/%s' % video_id,
+            video_id)
+
+        title = self._og_search_title(webpage, fatal=False) or xpath_text(
+            video, './/title', 'title', fatal=True)
+        description = self._search_regex(
+            r'</h3>\s*<p>([^<]+)</p>', webpage, 'description', fatal=False)
+        thumbnail = self._og_search_thumbnail(webpage) or xpath_text(
+            video, './/cover', 'cover')
+
+        duration = int_or_none(xpath_text(video, './/time', 'duration'))
+        age_limit = int_or_none(xpath_text(video, './/age18', 'age limit'))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'age_limit': age_limit,
+            'formats': formats,
+        }

From f8388757269de8a5e87b9a507db55021b3090980 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 18:07:10 +0600
Subject: [PATCH 1802/1937] [pladform] Add support for embeds

---
 youtube_dl/extractor/generic.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 013198b0d..4e6927b08 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -596,6 +596,19 @@ class GenericIE(InfoExtractor):
                 'view_count': int,
             },
         },
+        # Pladform embed
+        {
+            'url': 'http://muz-tv.ru/kinozal/view/7400/',
+            'info_dict': {
+                'id': '100183293',
+                'ext': 'mp4',
+                'title': 'Тайны перевала Дятлова • Тайна перевала Дятлова 1 серия 2 часть',
+                'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
+                'thumbnail': 're:^https?://.*\.jpg$',
+                'duration': 694,
+                'age_limit': 0,
+            },
+        },
         # RSS feed with enclosure
         {
             'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
@@ -1193,6 +1206,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform')
 
+        # Look for Pladform embeds
+        mobj = re.search(
+            r'<iframe[^>]+src="(?P<url>https?://out\.pladform\.ru/player\?.+?)"', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'), 'Pladform')
+
         def check_video(vurl):
             if YoutubeIE.suitable(vurl):
                 return True

From 11101076a1b0eb36dd56b1b5aa1b7559f2d230b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 18:09:47 +0600
Subject: [PATCH 1803/1937] [pladform] Fix format quality sorting

---
 youtube_dl/extractor/pladform.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/pladform.py b/youtube_dl/extractor/pladform.py
index 926e368a2..abde34b94 100644
--- a/youtube_dl/extractor/pladform.py
+++ b/youtube_dl/extractor/pladform.py
@@ -6,6 +6,7 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     xpath_text,
+    qualities,
 )
 
 
@@ -55,9 +56,12 @@ class PladformIE(InfoExtractor):
                 '%s returned error: %s' % (self.IE_NAME, video.text),
                 expected=True)
 
+        quality = qualities(('ld', 'sd', 'hd'))
+
         formats = [{
             'url': src.text,
             'format_id': src.get('quality'),
+            'quality': quality(src.get('quality')),
         } for src in video.findall('./src')]
         self._sort_formats(formats)
 

From 24993e3b396407af9ccdf8dd893588df31aff8af Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 8 Mar 2015 14:08:45 +0200
Subject: [PATCH 1804/1937] [vidme] Fix view_count extraction and remove
 comment_count extraction (Fixes #5133)

Comment counts seem to no longer be listed on vid.me
---
 youtube_dl/extractor/vidme.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py
index 339c3d897..bd953fb4c 100644
--- a/youtube_dl/extractor/vidme.py
+++ b/youtube_dl/extractor/vidme.py
@@ -41,13 +41,10 @@ class VidmeIE(InfoExtractor):
         duration = float_or_none(self._html_search_regex(
             r'data-duration="([^"]+)"', webpage, 'duration', fatal=False))
         view_count = str_to_int(self._html_search_regex(
-            r'<span class="video_views">\s*([\d,\.]+)\s*plays?', webpage, 'view count', fatal=False))
+            r'<(?:li|span) class="video_views">\s*([\d,\.]+)\s*plays?', webpage, 'view count', fatal=False))
         like_count = str_to_int(self._html_search_regex(
             r'class="score js-video-vote-score"[^>]+data-score="([\d,\.\s]+)">',
             webpage, 'like count', fatal=False))
-        comment_count = str_to_int(self._html_search_regex(
-            r'class="js-comment-count"[^>]+data-count="([\d,\.\s]+)">',
-            webpage, 'comment count', fatal=False))
 
         return {
             'id': video_id,
@@ -61,5 +58,4 @@ class VidmeIE(InfoExtractor):
             'duration': duration,
             'view_count': view_count,
             'like_count': like_count,
-            'comment_count': comment_count,
         }

From 8b910bda0cdba859c1971a410aa5d2a51c6e4918 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 8 Mar 2015 14:28:53 +0200
Subject: [PATCH 1805/1937] [teamcoco] Fix extraction

---
 youtube_dl/extractor/teamcoco.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 5793dbc10..7cb06f351 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -53,10 +53,10 @@ class TeamcocoIE(InfoExtractor):
         embed = self._download_webpage(
             embed_url, video_id, 'Downloading embed page')
 
-        encoded_data = self._search_regex(
-            r'"preload"\s*:\s*"([^"]+)"', embed, 'encoded data')
+        player_data = self._parse_json(self._search_regex(
+            r'Y\.Ginger\.Module\.Player\((\{.*?\})\);', embed, 'player data'), video_id)
         data = self._parse_json(
-            base64.b64decode(encoded_data.encode('ascii')).decode('utf-8'), video_id)
+            base64.b64decode(player_data['preload'].encode('ascii')).decode('utf-8'), video_id)
 
         formats = []
         get_quality = qualities(['500k', '480p', '1000k', '720p', '1080p'])

From bdf6eee0aeed8df586569982eaaac04eecc0062d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 19:17:54 +0600
Subject: [PATCH 1806/1937] [gazeta] Extend _VALID_URL

---
 youtube_dl/extractor/gazeta.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/gazeta.py b/youtube_dl/extractor/gazeta.py
index 3f9e58061..ea32b621c 100644
--- a/youtube_dl/extractor/gazeta.py
+++ b/youtube_dl/extractor/gazeta.py
@@ -7,8 +7,8 @@ from .common import InfoExtractor
 
 
 class GazetaIE(InfoExtractor):
-    _VALID_URL = r'(?P<url>https?://(?:www\.)?gazeta\.ru/(?:(?P<category>[^/]*)/)?video/(?:main/)?(?P<id>[A-Za-z0-9-_]+)\.s?html)'
-    _TEST = {
+    _VALID_URL = r'(?P<url>https?://(?:www\.)?gazeta\.ru/(?:[^/]+/)?video/(?:(?:main|\d{4}/\d{2}/\d{2})/)?(?P<id>[A-Za-z0-9-_.]+)\.s?html)'
+    _TESTS = [{
         'url': 'http://www.gazeta.ru/video/main/zadaite_vopros_vladislavu_yurevichu.shtml',
         'md5': 'd49c9bdc6e5a7888f27475dc215ee789',
         'info_dict': {
@@ -18,7 +18,10 @@ class GazetaIE(InfoExtractor):
             'description': 'md5:38617526050bd17b234728e7f9620a71',
             'thumbnail': 're:^https?://.*\.jpg',
         },
-    }
+    }, {
+        'url': 'http://www.gazeta.ru/lifestyle/video/2015/03/08/master-klass_krasivoi_byt._delaem_vesennii_makiyazh.shtml',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From a2aaf4dbc6e5f5d345329b5a845111851453b6a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 20:55:22 +0600
Subject: [PATCH 1807/1937] [utils] Add sanitize_path

---
 test/test_utils.py  | 21 +++++++++++++++++++++
 youtube_dl/utils.py | 18 ++++++++++++++++++
 2 files changed, 39 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 64fad58ad..5ebb8d498 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -38,6 +38,7 @@ from youtube_dl.utils import (
     parse_iso8601,
     read_batch_urls,
     sanitize_filename,
+    sanitize_path,
     shell_quote,
     smuggle_url,
     str_to_int,
@@ -131,6 +132,26 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw')
         self.assertEqual(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI')
 
+    def test_sanitize_path(self):
+        if sys.platform != 'win32':
+            return
+
+        self.assertEqual(sanitize_path('abc'), 'abc')
+        self.assertEqual(sanitize_path('abc/def'), 'abc\\def')
+        self.assertEqual(sanitize_path('abc\\def'), 'abc\\def')
+        self.assertEqual(sanitize_path('abc|def'), 'abc#def')
+        self.assertEqual(sanitize_path('<>:"|?*'), '#######')
+        self.assertEqual(sanitize_path('C:/abc/def'), 'C:\\abc\\def')
+        self.assertEqual(sanitize_path('C?:/abc/def'), 'C##\\abc\\def')
+
+        self.assertEqual(sanitize_path('\\\\?\\UNC\\ComputerName\\abc'), '\\\\?\\UNC\\ComputerName\\abc')
+        self.assertEqual(sanitize_path('\\\\?\\UNC/ComputerName/abc'), '\\\\?\\UNC\\ComputerName\\abc')
+
+        self.assertEqual(sanitize_path('\\\\?\\C:\\abc'), '\\\\?\\C:\\abc')
+        self.assertEqual(sanitize_path('\\\\?\\C:/abc'), '\\\\?\\C:\\abc')
+        self.assertEqual(sanitize_path('\\\\?\\C:\\ab?c\\de:f'), '\\\\?\\C:\\ab#c\\de#f')
+        self.assertEqual(sanitize_path('\\\\?\\C:\\abc'), '\\\\?\\C:\\abc')
+
     def test_ordered_set(self):
         self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
         self.assertEqual(orderedSet([]), [])
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7426e2a1f..0f49d602e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -311,6 +311,24 @@ def sanitize_filename(s, restricted=False, is_id=False):
     return result
 
 
+def sanitize_path(s):
+    """Sanitizes and normalizes path on Windows"""
+    if sys.platform != 'win32':
+        return s
+    drive, _ = os.path.splitdrive(s)
+    unc, _ = os.path.splitunc(s)
+    unc_or_drive = unc or drive
+    norm_path = os.path.normpath(remove_start(s, unc_or_drive)).split(os.path.sep)
+    if unc_or_drive:
+        norm_path.pop(0)
+    sanitized_path = [
+        re.sub('[/<>:"\\|\\\\?\\*]', '#', path_part)
+        for path_part in norm_path]
+    if unc_or_drive:
+        sanitized_path.insert(0, unc_or_drive + os.path.sep)
+    return os.path.join(*sanitized_path)
+
+
 def orderedSet(iterable):
     """ Remove all duplicates from the input iterable """
     res = []

From d55de57b67bceca5d9116ddcc2ada2fa1957d89d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 20:56:28 +0600
Subject: [PATCH 1808/1937] [utils] Fix sanitize_open

---
 youtube_dl/utils.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 0f49d602e..e511232ca 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -252,15 +252,12 @@ def sanitize_open(filename, open_mode):
             raise
 
         # In case of error, try to remove win32 forbidden chars
-        alt_filename = os.path.join(
-            re.sub('[/<>:"\\|\\\\?\\*]', '#', path_part)
-            for path_part in os.path.split(filename)
-        )
+        alt_filename = sanitize_path(filename)
         if alt_filename == filename:
             raise
         else:
             # An exception here should be caught in the caller
-            stream = open(encodeFilename(filename), open_mode)
+            stream = open(encodeFilename(alt_filename), open_mode)
             return (stream, alt_filename)
 
 

From 1bb5c511a551ba7c5d3179d2d3a124f0977e74b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 20:57:30 +0600
Subject: [PATCH 1809/1937] [YoutubeDL] Sanitize outtmpl as path

---
 youtube_dl/YoutubeDL.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index bae52e9c7..4b9151163 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -61,6 +61,7 @@ from .utils import (
     render_table,
     SameFileError,
     sanitize_filename,
+    sanitize_path,
     std_headers,
     subtitles_filename,
     takewhile_inclusive,
@@ -553,20 +554,16 @@ class YoutubeDL(object):
                 elif template_dict.get('width'):
                     template_dict['resolution'] = '?x%d' % template_dict['width']
 
-            restrict_filenames = self.params.get('restrictfilenames')
-
             sanitize = lambda k, v: sanitize_filename(
                 compat_str(v),
-                restricted=restrict_filenames,
+                restricted=self.params.get('restrictfilenames'),
                 is_id=(k == 'id'))
             template_dict = dict((k, sanitize(k, v))
                                  for k, v in template_dict.items()
                                  if v is not None)
             template_dict = collections.defaultdict(lambda: 'NA', template_dict)
 
-            outtmpl = sanitize_filename(
-                self.params.get('outtmpl', DEFAULT_OUTTMPL),
-                restricted=restrict_filenames)
+            outtmpl = sanitize_path(self.params.get('outtmpl', DEFAULT_OUTTMPL))
             tmpl = compat_expanduser(outtmpl)
             filename = tmpl % template_dict
             # Temporary fix for #4787

From f18ef2d14463a13d80e967d1b18ece6a076f60fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 22:08:48 +0600
Subject: [PATCH 1810/1937] [utils] Disallow trailing dot in sanitize_path for
 a path part

---
 test/test_utils.py  | 11 +++++++++++
 youtube_dl/utils.py |  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 5ebb8d498..28bda654e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -152,6 +152,17 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(sanitize_path('\\\\?\\C:\\ab?c\\de:f'), '\\\\?\\C:\\ab#c\\de#f')
         self.assertEqual(sanitize_path('\\\\?\\C:\\abc'), '\\\\?\\C:\\abc')
 
+        self.assertEqual(
+            sanitize_path('youtube/%(uploader)s/%(autonumber)s-%(title)s-%(upload_date)s.%(ext)s'),
+            'youtube\\%(uploader)s\\%(autonumber)s-%(title)s-%(upload_date)s.%(ext)s')
+
+        self.assertEqual(
+            sanitize_path('youtube/TheWreckingYard ./00001-Not bad, Especially for Free! (1987 Yamaha 700)-20141116.mp4.part'),
+            'youtube\\TheWreckingYard #\\00001-Not bad, Especially for Free! (1987 Yamaha 700)-20141116.mp4.part')
+        self.assertEqual(sanitize_path('abc/def...'), 'abc\\def..#')
+        self.assertEqual(sanitize_path('abc.../def'), 'abc..#\\def')
+        self.assertEqual(sanitize_path('abc.../def...'), 'abc..#\\def..#')
+
     def test_ordered_set(self):
         self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
         self.assertEqual(orderedSet([]), [])
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e511232ca..d5597d514 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -319,7 +319,7 @@ def sanitize_path(s):
     if unc_or_drive:
         norm_path.pop(0)
     sanitized_path = [
-        re.sub('[/<>:"\\|\\\\?\\*]', '#', path_part)
+        re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part)
         for path_part in norm_path]
     if unc_or_drive:
         sanitized_path.insert(0, unc_or_drive + os.path.sep)

From e5a11a2293069b1acfa5eb5d2694ad4082dd9755 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 8 Mar 2015 22:09:42 +0600
Subject: [PATCH 1811/1937] [YoutubeDL] Sanitize path before creating
 non-existent paths (Closes #4324)

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4b9151163..bce7587fd 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1262,7 +1262,7 @@ class YoutubeDL(object):
             return
 
         try:
-            dn = os.path.dirname(encodeFilename(filename))
+            dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
             if dn and not os.path.exists(dn):
                 os.makedirs(dn)
         except (OSError, IOError) as err:

From 43d6280d0a03335ec5143383f15e2ca9a49f4046 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 8 Mar 2015 18:25:11 +0100
Subject: [PATCH 1812/1937] [downloader/f4m] Fix use of base64 in python 3.2
 (fixes #5132)

b64decode needs a byte string, but on 3.4 it also accepts strings.
---
 youtube_dl/downloader/f4m.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 3dc796faa..4ab000d67 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -281,7 +281,7 @@ class F4mFD(FileDownloader):
             boot_info = self._get_bootstrap_from_url(bootstrap_url)
         else:
             bootstrap_url = None
-            bootstrap = base64.b64decode(node.text)
+            bootstrap = base64.b64decode(node.text.encode('ascii'))
             boot_info = read_bootstrap_info(bootstrap)
         return (boot_info, bootstrap_url)
 
@@ -308,7 +308,7 @@ class F4mFD(FileDownloader):
         live = boot_info['live']
         metadata_node = media.find(_add_ns('metadata'))
         if metadata_node is not None:
-            metadata = base64.b64decode(metadata_node.text)
+            metadata = base64.b64decode(metadata_node.text.encode('ascii'))
         else:
             metadata = None
 

From 8bba753ccaab38a837efd5d334eff8f1b1c48fad Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 8 Mar 2015 18:37:43 +0100
Subject: [PATCH 1813/1937] [options] Rename --dump-intermediate-pages to
 --dump-pages for consistence with --write-pages

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index a4ca8adc4..eefe008d5 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -563,7 +563,7 @@ def parseOpts(overrideArguments=None):
         action='store_true', dest='verbose', default=False,
         help='print various debugging information')
     verbosity.add_option(
-        '--dump-intermediate-pages',
+        '--dump-pages', '--dump-intermediate-pages',
         action='store_true', dest='dump_intermediate_pages', default=False,
         help='print downloaded pages to debug problems (very verbose)')
     verbosity.add_option(

From cc08b11d16efd125d765cc1a69ac795a592f012c Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Sun, 8 Mar 2015 21:32:42 +0200
Subject: [PATCH 1814/1937] [adultswim] Improve video_info extraction (Fixes
 #5152)

Look for video_info inside `slugged_video`, if slug is not found among collections.
Also, simplify a bit.
---
 youtube_dl/extractor/adultswim.py | 44 ++++++++++++++++++++++---------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py
index 34b8b0115..39335b827 100644
--- a/youtube_dl/extractor/adultswim.py
+++ b/youtube_dl/extractor/adultswim.py
@@ -2,13 +2,12 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    xpath_text,
     float_or_none,
+    xpath_text,
 )
 
 
@@ -60,6 +59,24 @@ class AdultSwimIE(InfoExtractor):
             'title': 'American Dad - Putting Francine Out of Business',
             'description': 'Stan hatches a plan to get Francine out of the real estate business.Watch more American Dad on [adult swim].'
         },
+    }, {
+        'url': 'http://www.adultswim.com/videos/tim-and-eric-awesome-show-great-job/dr-steve-brule-for-your-wine/',
+        'playlist': [
+            {
+                'md5': '3e346a2ab0087d687a05e1e7f3b3e529',
+                'info_dict': {
+                    'id': 'sY3cMUR_TbuE4YmdjzbIcQ-0',
+                    'ext': 'flv',
+                    'title': 'Tim and Eric Awesome Show Great Job! - Dr. Steve Brule, For Your Wine',
+                    'description': 'Dr. Brule reports live from Wine Country with a special report on wines.  \r\nWatch Tim and Eric Awesome Show Great Job! episode #20, "Embarrassed" on Adult Swim.\r\n\r\n',
+                },
+            }
+        ],
+        'info_dict': {
+            'id': 'sY3cMUR_TbuE4YmdjzbIcQ',
+            'title': 'Tim and Eric Awesome Show Great Job! - Dr. Steve Brule, For Your Wine',
+            'description': 'Dr. Brule reports live from Wine Country with a special report on wines.  \r\nWatch Tim and Eric Awesome Show Great Job! episode #20, "Embarrassed" on Adult Swim.\r\n\r\n',
+        },
     }]
 
     @staticmethod
@@ -80,6 +97,7 @@ class AdultSwimIE(InfoExtractor):
             for video in collection.get('videos'):
                 if video.get('slug') == slug:
                     return collection, video
+        return None, None
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -90,28 +108,30 @@ class AdultSwimIE(InfoExtractor):
         webpage = self._download_webpage(url, episode_path)
 
         # Extract the value of `bootstrappedData` from the Javascript in the page.
-        bootstrappedDataJS = self._search_regex(r'var bootstrappedData = ({.*});', webpage, episode_path)
-
-        try:
-            bootstrappedData = json.loads(bootstrappedDataJS)
-        except ValueError as ve:
-            errmsg = '%s: Failed to parse JSON ' % episode_path
-            raise ExtractorError(errmsg, cause=ve)
+        bootstrapped_data = self._parse_json(self._search_regex(
+            r'var bootstrappedData = ({.*});', webpage, 'bootstraped data'), episode_path)
 
         # Downloading videos from a /videos/playlist/ URL needs to be handled differently.
         # NOTE: We are only downloading one video (the current one) not the playlist
         if is_playlist:
-            collections = bootstrappedData['playlists']['collections']
+            collections = bootstrapped_data['playlists']['collections']
             collection = self.find_collection_by_linkURL(collections, show_path)
             video_info = self.find_video_info(collection, episode_path)
 
             show_title = video_info['showTitle']
             segment_ids = [video_info['videoPlaybackID']]
         else:
-            collections = bootstrappedData['show']['collections']
+            collections = bootstrapped_data['show']['collections']
             collection, video_info = self.find_collection_containing_video(collections, episode_path)
 
-            show = bootstrappedData['show']
+            # Video wasn't found in the collections, let's try `slugged_video`.
+            if video_info is None:
+                if bootstrapped_data.get('slugged_video', {}).get('slug') == episode_path:
+                    video_info = bootstrapped_data['slugged_video']
+                else:
+                    raise ExtractorError('Unable to find video info')
+
+            show = bootstrapped_data['show']
             show_title = show['title']
             segment_ids = [clip['videoPlaybackID'] for clip in video_info['clips']]
 

From dd7831fe94a0fb8270e7fa3699677c7476a5cd83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Mar 2015 04:55:35 +0600
Subject: [PATCH 1815/1937] [breakcom] Process only play purpose media formats
 (Closes #5164)

---
 youtube_dl/extractor/breakcom.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/breakcom.py b/youtube_dl/extractor/breakcom.py
index 4bcc897c9..809287d14 100644
--- a/youtube_dl/extractor/breakcom.py
+++ b/youtube_dl/extractor/breakcom.py
@@ -41,7 +41,7 @@ class BreakIE(InfoExtractor):
             'tbr': media['bitRate'],
             'width': media['width'],
             'height': media['height'],
-        } for media in info['media']]
+        } for media in info['media'] if media.get('mediaPurpose') == 'play']
 
         if not formats:
             formats.append({

From d475b3384cd372d89c35b9b1c26499f1e9cc1915 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Mar 2015 03:00:01 +0100
Subject: [PATCH 1816/1937] [README] Better bug reporting instructions

Also address private emails which I get more and more these days.
---
 README.md | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5b9dd2cea..1a5a90c2e 100644
--- a/README.md
+++ b/README.md
@@ -404,6 +404,18 @@ A note on the service that they don't host the infringing content, but just link
 
 Support requests for services that **do** purchase the rights to distribute their content are perfectly fine though. If in doubt, you can simply include a source that mentions the legitimate purchase of content.
 
+### How can I speed up work on my issue?
+
+(Also known as: Help, my important issue not being solved!) The youtube-dl core developer team is quite small. While we do our best to solve as many issues as possible, sometimes that can take quite a while. To speed up your issue, here's what you can do:
+
+First of all, please do report the issue [at our issue tracker](https://yt-dl.org/bugs). That allows us to coordinate all efforts by users and developers, and serves as a unified point. Unfortunately, the youtube-dl project has grown too large to use personal email as an effective communication channel.
+
+Please read the [bug reporting instructions](#bugs) below. A lot of bugs lack all the necessary information. If you can, offer proxy, VPN, or shell access to the youtube-dl developers. If you are able to, test the issue from multiple computers in multiple countries to exclude local censorship or misconfiguration issues.
+
+If nobody is interested in solving your issue, you are welcome to take matters into your own hands and submit a pull request (or coerce/pay somebody else to do so).
+
+Feel free to bump the issue from time to time by writing a small comment ("Issue is still present in youtube-dl version ...from France, but fixed from Belgium"), but please not more than once a month. Please do not declare your issue as `important` or `urgent`.
+
 ### How can I detect whether a given URL is supported by youtube-dl?
 
 For one, have a look at the [list of supported sites](docs/supportedsites.md). Note that it can sometimes happen that the site changes its URL scheme (say, from http://example.com/video/1234567 to http://example.com/v/1234567 ) and youtube-dl reports an URL of a service in that list as unsupported. In that case, simply report a bug.
@@ -572,7 +584,9 @@ If your report is shorter than two lines, it is almost certainly missing some of
 
 For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the -v flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 
-Site support requests **must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
+If your server has multiple IPs or you suspect censorship, adding --call-home may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
+
+**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
 
 ###  Are you using the latest version?
 

From f848215dfce232dfc6618c1cc49da44c3f7df75a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 9 Mar 2015 03:02:03 +0100
Subject: [PATCH 1817/1937] release 2015.03.09

---
 CONTRIBUTING.md        | 4 +++-
 README.md              | 2 +-
 docs/supportedsites.md | 5 +++++
 youtube_dl/version.py  | 2 +-
 4 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 351229f21..588b15bde 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -18,7 +18,9 @@ If your report is shorter than two lines, it is almost certainly missing some of
 
 For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the -v flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information.
 
-Site support requests **must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
+If your server has multiple IPs or you suspect censorship, adding --call-home may be a good idea to get more diagnostics. If the error is `ERROR: Unable to extract ...` and you cannot reproduce it from multiple countries, add `--dump-pages` (warning: this will yield a rather large output, redirect it to the file `log.txt` by adding `>log.txt 2>&1` to your command-line) or upload the `.dump` files you get when you add `--write-pages` [somewhere](https://gist.github.com/).
+
+**Site support requests must contain an example URL**. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL.
 
 ###  Are you using the latest version?
 
diff --git a/README.md b/README.md
index 1a5a90c2e..f41e7ecee 100644
--- a/README.md
+++ b/README.md
@@ -167,7 +167,7 @@ which means you can modify it, redistribute it or use it however you like.
     --no-progress                    do not print progress bar
     --console-title                  display progress in console titlebar
     -v, --verbose                    print various debugging information
-    --dump-intermediate-pages        print downloaded pages to debug problems (very verbose)
+    --dump-pages                     print downloaded pages to debug problems (very verbose)
     --write-pages                    Write downloaded intermediary pages to files in the current directory to debug problems
     --print-traffic                  Display sent and read HTTP traffic
     -C, --call-home                  Contact the youtube-dl server for debugging.
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 062cb3d62..80a696ee3 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -117,6 +117,7 @@
  - **DRTV**
  - **Dump**
  - **dvtv**: http://video.aktualne.cz/
+ - **EaglePlatform**
  - **EbaumsWorld**
  - **EchoMsk**
  - **eHow**
@@ -161,6 +162,7 @@
  - **GameSpot**
  - **GameStar**
  - **Gametrailers**
+ - **Gazeta**
  - **GDCVault**
  - **generic**: Generic downloader that works on some sites
  - **GiantBomb**
@@ -315,6 +317,7 @@
  - **Ooyala**
  - **OpenFilm**
  - **orf:fm4**: radio FM4
+ - **orf:iptv**: iptv.ORF.at
  - **orf:oe1**: Radio Österreich 1
  - **orf:tvthek**: ORF TVthek
  - **parliamentlive.tv**: UK parliament videos
@@ -322,10 +325,12 @@
  - **PBS**
  - **Phoenix**
  - **Photobucket**
+ - **Pladform**
  - **PlanetaPlay**
  - **play.fm**
  - **played.to**
  - **Playvid**
+ - **Playwire**
  - **plus.google**: Google Plus
  - **pluzz.francetv.fr**
  - **podomatic**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 252933993..1f0c88a4d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.03.03.1'
+__version__ = '2015.03.09'

From 4c6039385466e4be3a0e20459e75886052653de7 Mon Sep 17 00:00:00 2001
From: Mamay Alexander <alexander@mamay.su>
Date: Mon, 9 Mar 2015 19:06:49 +0600
Subject: [PATCH 1818/1937] [YandexMusic] Add new extractor

---
 youtube_dl/extractor/__init__.py |   5 ++
 youtube_dl/extractor/yamusic.py  | 104 +++++++++++++++++++++++++++++++
 2 files changed, 109 insertions(+)
 create mode 100644 youtube_dl/extractor/yamusic.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 14172ca56..2db938516 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -611,6 +611,11 @@ from .yahoo import (
     YahooSearchIE,
 )
 from .yam import YamIE
+from .yamusic import (
+    YandexMusicTrackIE,
+    YandexMusicAlbumIE,
+    YandexMusicPlaylistIE,
+)
 from .yesjapan import YesJapanIE
 from .ynet import YnetIE
 from .youjizz import YouJizzIE
diff --git a/youtube_dl/extractor/yamusic.py b/youtube_dl/extractor/yamusic.py
new file mode 100644
index 000000000..5af6df89d
--- /dev/null
+++ b/youtube_dl/extractor/yamusic.py
@@ -0,0 +1,104 @@
+# coding=utf-8
+from __future__ import unicode_literals
+
+import re
+import hashlib
+import time
+
+from .common import InfoExtractor
+
+class YandexMusicAlbumIE(InfoExtractor):
+    _VALID_URL = r'http://music.yandex.ru/album/(?P<id>\d+)'
+
+    def _get_track_url(self, storage_dir, track_id):
+        data = self._download_json('http://music.yandex.ru/api/v1.5/handlers/api-jsonp.jsx?requestId=2&nc=%d&action=getTrackSrc&p=download-info/%s/2.mp3' % (time.time(), storage_dir), track_id)
+
+        hsh = hashlib.md5()
+        hsh.update('XGRlBW9FXlekgbPrRHuSiA' + data['path'][1:] + data['s'])
+        hash = hsh.hexdigest()
+        storage = storage_dir.split('.')
+
+        return 'http://%s/get-mp3/%s/%s?track-id=%s&from=service-10-track&similarities-experiment=default' % (data['host'], hash, data['ts'] + data['path'], storage[1])
+
+    def _get_album_id_and_data(self, url):
+        matched = re.match(self._VALID_URL, url)
+        id = matched.group('id')
+
+        webpage = self._download_webpage(url, id)
+        data = self._parse_json(
+            self._search_regex(
+                r'var\s+Mu\s+=\s+(.+?);\s+<\/script>', webpage, 'player'),
+            id)
+        return id, data['pageData']
+
+    def _real_extract(self, url):
+
+        id, data = self._get_album_id_and_data(url)
+
+        entries = []
+
+        for track in data['volumes'][0]:
+            entries.append({
+                'id': track['id'],
+                'ext': 'mp3',
+                'url': self._get_track_url(track['storageDir'], track['id']),
+                'title': track['artists'][0]['name'] + ' - ' + track['title'],
+            })
+
+        return {
+            '_type': 'playlist',
+            'entries': entries,
+            'id': id,
+            'title': data['title'],
+        }
+
+class YandexMusicPlaylistIE(YandexMusicAlbumIE):
+    _VALID_URL = r'http://music.yandex.ru/users/(?P<user_name>[^/]+)/playlists/(?P<id>\d+)'
+
+    def _real_extract(self, url):
+        id, data = self._get_album_id_and_data(url)
+        data = data['playlist']
+
+        entries = []
+
+        for track in data['tracks']:
+            entries.append({
+                'id': track['id'],
+                'ext': 'mp3',
+                'url': self._get_track_url(track['storageDir'], track['id']),
+                'title': track['artists'][0]['name'] + ' - ' + track['title'],
+            })
+
+        return {
+            '_type': 'playlist',
+            'entries': entries,
+            'id': id,
+            'title': data['title'],
+        }
+
+class YandexMusicTrackIE(YandexMusicAlbumIE):
+    _VALID_URL = r'http://music.yandex.ru/album/(?P<album_id>\d+)/track/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://music.yandex.ru/album/540508/track/4878838',
+        'info_dict': {
+            'id': '4878838',
+            'ext': 'mp3',
+            'title': 'Carlo Ambrosio - Gypsy Eyes 1',
+        }
+    }
+
+    def _real_extract(self, url):
+
+        id, data = self._get_album_id_and_data(url)
+
+        for track in data['volumes'][0]:
+            if track['id'] == id:
+                track_url = self._get_track_url(track['storageDir'], id)
+                break
+
+        return {
+            'id': id,
+            'ext': 'mp3',
+            'url': track_url,
+            'title': track['artists'][0]['name'] + ' - ' + track['title'],
+        }

From 47fe42e1ab3e182aacbc0c749b9a4f17f1b9dad2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Mar 2015 21:43:46 +0600
Subject: [PATCH 1819/1937] [yamusic] Improve, simplify, fix python3 issues and
 add tests

---
 youtube_dl/extractor/yamusic.py | 168 ++++++++++++++++++--------------
 1 file changed, 96 insertions(+), 72 deletions(-)

diff --git a/youtube_dl/extractor/yamusic.py b/youtube_dl/extractor/yamusic.py
index 5af6df89d..e25c0be44 100644
--- a/youtube_dl/extractor/yamusic.py
+++ b/youtube_dl/extractor/yamusic.py
@@ -3,102 +3,126 @@ from __future__ import unicode_literals
 
 import re
 import hashlib
-import time
 
 from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    float_or_none,
+)
 
-class YandexMusicAlbumIE(InfoExtractor):
-    _VALID_URL = r'http://music.yandex.ru/album/(?P<id>\d+)'
 
+class YandexMusicBaseIE(InfoExtractor):
     def _get_track_url(self, storage_dir, track_id):
-        data = self._download_json('http://music.yandex.ru/api/v1.5/handlers/api-jsonp.jsx?requestId=2&nc=%d&action=getTrackSrc&p=download-info/%s/2.mp3' % (time.time(), storage_dir), track_id)
+        data = self._download_json(
+            'http://music.yandex.ru/api/v1.5/handlers/api-jsonp.jsx?action=getTrackSrc&p=download-info/%s'
+            % storage_dir,
+            track_id, 'Downloading track location JSON')
 
-        hsh = hashlib.md5()
-        hsh.update('XGRlBW9FXlekgbPrRHuSiA' + data['path'][1:] + data['s'])
-        hash = hsh.hexdigest()
+        key = hashlib.md5(('XGRlBW9FXlekgbPrRHuSiA' + data['path'][1:] + data['s']).encode('utf-8')).hexdigest()
         storage = storage_dir.split('.')
 
-        return 'http://%s/get-mp3/%s/%s?track-id=%s&from=service-10-track&similarities-experiment=default' % (data['host'], hash, data['ts'] + data['path'], storage[1])
-
-    def _get_album_id_and_data(self, url):
-        matched = re.match(self._VALID_URL, url)
-        id = matched.group('id')
-
-        webpage = self._download_webpage(url, id)
-        data = self._parse_json(
-            self._search_regex(
-                r'var\s+Mu\s+=\s+(.+?);\s+<\/script>', webpage, 'player'),
-            id)
-        return id, data['pageData']
-
-    def _real_extract(self, url):
-
-        id, data = self._get_album_id_and_data(url)
-
-        entries = []
-
-        for track in data['volumes'][0]:
-            entries.append({
-                'id': track['id'],
-                'ext': 'mp3',
-                'url': self._get_track_url(track['storageDir'], track['id']),
-                'title': track['artists'][0]['name'] + ' - ' + track['title'],
-            })
+        return ('http://%s/get-mp3/%s/%s?track-id=%s&from=service-10-track&similarities-experiment=default'
+                % (data['host'], key, data['ts'] + data['path'], storage[1]))
 
+    def _get_track_info(self, track):
         return {
-            '_type': 'playlist',
-            'entries': entries,
-            'id': id,
-            'title': data['title'],
+            'id': track['id'],
+            'ext': 'mp3',
+            'url': self._get_track_url(track['storageDir'], track['id']),
+            'title': '%s - %s' % (track['artists'][0]['name'], track['title']),
+            'filesize': int_or_none(track.get('fileSize')),
+            'duration': float_or_none(track.get('durationMs'), 1000),
         }
 
-class YandexMusicPlaylistIE(YandexMusicAlbumIE):
-    _VALID_URL = r'http://music.yandex.ru/users/(?P<user_name>[^/]+)/playlists/(?P<id>\d+)'
 
-    def _real_extract(self, url):
-        id, data = self._get_album_id_and_data(url)
-        data = data['playlist']
+class YandexMusicTrackIE(YandexMusicBaseIE):
+    IE_NAME = 'yandexmusic:track'
+    IE_DESC = 'Яндекс.Музыка - Трек'
+    _VALID_URL = r'https?://music\.yandex\.ru/album/(?P<album_id>\d+)/track/(?P<id>\d+)'
 
-        entries = []
-
-        for track in data['tracks']:
-            entries.append({
-                'id': track['id'],
-                'ext': 'mp3',
-                'url': self._get_track_url(track['storageDir'], track['id']),
-                'title': track['artists'][0]['name'] + ' - ' + track['title'],
-            })
-
-        return {
-            '_type': 'playlist',
-            'entries': entries,
-            'id': id,
-            'title': data['title'],
-        }
-
-class YandexMusicTrackIE(YandexMusicAlbumIE):
-    _VALID_URL = r'http://music.yandex.ru/album/(?P<album_id>\d+)/track/(?P<id>\d+)'
     _TEST = {
         'url': 'http://music.yandex.ru/album/540508/track/4878838',
+        'md5': 'f496818aa2f60b6c0062980d2e00dc20',
         'info_dict': {
             'id': '4878838',
             'ext': 'mp3',
             'title': 'Carlo Ambrosio - Gypsy Eyes 1',
+            'filesize': 4628061,
+            'duration': 193.04,
         }
     }
 
     def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        album_id, track_id = mobj.group('album_id'), mobj.group('id')
 
-        id, data = self._get_album_id_and_data(url)
+        track = self._download_json(
+            'http://music.yandex.ru/handlers/track.jsx?track=%s:%s' % (track_id, album_id),
+            track_id, 'Downloading track JSON')['track']
 
-        for track in data['volumes'][0]:
-            if track['id'] == id:
-                track_url = self._get_track_url(track['storageDir'], id)
-                break
+        return self._get_track_info(track)
+
+
+class YandexMusicAlbumIE(YandexMusicBaseIE):
+    IE_NAME = 'yandexmusic:album'
+    IE_DESC = 'Яндекс.Музыка - Альбом'
+    _VALID_URL = r'https?://music\.yandex\.ru/album/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://music.yandex.ru/album/540508',
+        'info_dict': {
+            'id': '540508',
+            'title': 'Carlo Ambrosio - Gypsy Soul (2009)',
+        },
+        'playlist_count': 50,
+    }
+
+    def _real_extract(self, url):
+        album_id = self._match_id(url)
+
+        album = self._download_json(
+            'http://music.yandex.ru/handlers/album.jsx?album=%s' % album_id,
+            album_id, 'Downloading album JSON')
+
+        entries = [self._get_track_info(track) for track in album['volumes'][0]]
+
+        title = '%s - %s' % (album['artists'][0]['name'], album['title'])
+        year = album.get('year')
+        if year:
+            title += ' (%s)' % year
+
+        return self.playlist_result(entries, compat_str(album['id']), title)
+
+
+class YandexMusicPlaylistIE(YandexMusicBaseIE):
+    IE_NAME = 'yandexmusic:playlist'
+    IE_DESC = 'Яндекс.Музыка - Плейлист'
+    _VALID_URL = r'https?://music\.yandex\.ru/users/[^/]+/playlists/(?P<id>\d+)'
+
+    _TEST = {
+        'url': 'http://music.yandex.ru/users/music.partners/playlists/1245',
+        'info_dict': {
+            'id': '1245',
+            'title': 'Что слушают Enter Shikari',
+            'description': 'md5:3b9f27b0efbe53f2ee1e844d07155cc9',
+        },
+        'playlist_count': 6,
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        playlist = self._parse_json(
+            self._search_regex(
+                r'var\s+Mu\s*=\s*({.+?});\s*</script>', webpage, 'player'),
+            playlist_id)['pageData']['playlist']
+
+        entries = [self._get_track_info(track) for track in playlist['tracks']]
+
+        return self.playlist_result(
+            entries, compat_str(playlist_id),
+            playlist['title'], playlist.get('description'))
 
-        return {
-            'id': id,
-            'ext': 'mp3',
-            'url': track_url,
-            'title': track['artists'][0]['name'] + ' - ' + track['title'],
-        }

From d1e2e8f583524e837d452d37fa7175a55ebc80ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Mar 2015 21:44:59 +0600
Subject: [PATCH 1820/1937] [yamusic] Rename to yandexmusic

---
 youtube_dl/extractor/__init__.py                    | 2 +-
 youtube_dl/extractor/{yamusic.py => yandexmusic.py} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename youtube_dl/extractor/{yamusic.py => yandexmusic.py} (100%)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2db938516..20bc73dce 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -611,7 +611,7 @@ from .yahoo import (
     YahooSearchIE,
 )
 from .yam import YamIE
-from .yamusic import (
+from .yandexmusic import (
     YandexMusicTrackIE,
     YandexMusicAlbumIE,
     YandexMusicPlaylistIE,
diff --git a/youtube_dl/extractor/yamusic.py b/youtube_dl/extractor/yandexmusic.py
similarity index 100%
rename from youtube_dl/extractor/yamusic.py
rename to youtube_dl/extractor/yandexmusic.py

From 41b2194f86c9440d2d095db3eaec90784703fa7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 9 Mar 2015 21:46:31 +0600
Subject: [PATCH 1821/1937] Credit @MamayAlexander for yandexmusic (#5168)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 4674a5af3..421df69a6 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -113,3 +113,4 @@ Robin de Rooij
 Ryan Schmidt
 Leslie P. Polzer
 Duncan Keall
+Alexander Mamay

From dd77f14c641da26429ba32d2464563bf04e1741c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 9 Mar 2015 18:07:31 +0100
Subject: [PATCH 1822/1937] [yandexmusic] PEP8: remove blank line at the end of
 file

---
 youtube_dl/extractor/yandexmusic.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index e25c0be44..1ee8d1bc0 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -125,4 +125,3 @@ class YandexMusicPlaylistIE(YandexMusicBaseIE):
         return self.playlist_result(
             entries, compat_str(playlist_id),
             playlist['title'], playlist.get('description'))
-

From 937daef4a7725aeecc4b2ce0caa29c6f1aaf0b87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 9 Mar 2015 18:12:41 +0100
Subject: [PATCH 1823/1937] [niconico] Use '_match_id'

---
 youtube_dl/extractor/niconico.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 4c1890416..7fb4e57df 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -41,7 +41,7 @@ class NiconicoIE(InfoExtractor):
         },
     }
 
-    _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/((?:[a-z]{2})?[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
     _NETRC_MACHINE = 'niconico'
     # Determine whether the downloader used authentication to download video
     _AUTHENTICATED = False
@@ -76,8 +76,7 @@ class NiconicoIE(InfoExtractor):
         return True
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group(1)
+        video_id = self._match_id(url)
 
         # Get video webpage. We are not actually interested in it, but need
         # the cookies in order to be able to download the info webpage

From f5d8f58a170c9585e0913dbe1e50e0ed05afb698 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 9 Mar 2015 18:17:22 +0100
Subject: [PATCH 1824/1937] [yandexmusic:album] Improve _VALID_URL to avoid
 matching tracks urls

---
 youtube_dl/extractor/yandexmusic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 1ee8d1bc0..b47aecb15 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -67,7 +67,7 @@ class YandexMusicTrackIE(YandexMusicBaseIE):
 class YandexMusicAlbumIE(YandexMusicBaseIE):
     IE_NAME = 'yandexmusic:album'
     IE_DESC = 'Яндекс.Музыка - Альбом'
-    _VALID_URL = r'https?://music\.yandex\.ru/album/(?P<id>\d+)'
+    _VALID_URL = r'https?://music\.yandex\.ru/album/(?P<id>\d+)/?(\?|$)'
 
     _TEST = {
         'url': 'http://music.yandex.ru/album/540508',

From 2ebfeacabc7b74d03fa7cb096d8b9d1ecbdbb6e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 10 Mar 2015 00:50:11 +0600
Subject: [PATCH 1825/1937] [utils] Keep dot and dotdot unmodified (Closes
 #5171)

---
 test/test_utils.py  | 5 +++++
 youtube_dl/utils.py | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 28bda654e..8f790bf0a 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -163,6 +163,11 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(sanitize_path('abc.../def'), 'abc..#\\def')
         self.assertEqual(sanitize_path('abc.../def...'), 'abc..#\\def..#')
 
+        self.assertEqual(sanitize_path('../abc'), '..\\abc')
+        self.assertEqual(sanitize_path('../../abc'), '..\\..\\abc')
+        self.assertEqual(sanitize_path('./abc'), 'abc')
+        self.assertEqual(sanitize_path('./../abc'), '..\\abc')
+
     def test_ordered_set(self):
         self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
         self.assertEqual(orderedSet([]), [])
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d5597d514..c3135effc 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -319,7 +319,7 @@ def sanitize_path(s):
     if unc_or_drive:
         norm_path.pop(0)
     sanitized_path = [
-        re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part)
+        path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part)
         for path_part in norm_path]
     if unc_or_drive:
         sanitized_path.insert(0, unc_or_drive + os.path.sep)

From 614a7e1e230e095d9a11b59b20f4ff7462be8b21 Mon Sep 17 00:00:00 2001
From: "PishPosh.McGee" <pishposhmcgee@users.noreply.github.com>
Date: Tue, 10 Mar 2015 15:22:46 -0500
Subject: [PATCH 1826/1937] Added subtitles for FunnyOrDie

---
 youtube_dl/extractor/funnyordie.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index a49fc1151..f832ac5c3 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -50,7 +50,6 @@ class FunnyOrDieIE(InfoExtractor):
         bitrates.sort()
 
         formats = []
-
         for bitrate in bitrates:
             for link in links:
                 formats.append({
@@ -59,6 +58,16 @@ class FunnyOrDieIE(InfoExtractor):
                     'vbr': bitrate,
                 })
 
+        subtitles={}
+        subtitle_matches=re.findall(r'<track kind="captions" src="([^"]+)" srclang="([^"]+)"', webpage)
+        for match in subtitle_matches:
+          (suburl,sublang)=match
+          if not sublang in subtitles.keys():
+            subtitles[sublang]=[]
+          subext=suburl.split('/')[-1]
+          print subext
+          subtitles[sublang].append({'url': suburl,'ext': subext})
+
         post_json = self._search_regex(
             r'fb_post\s*=\s*(\{.*?\});', webpage, 'post details')
         post = json.loads(post_json)
@@ -69,4 +78,5 @@ class FunnyOrDieIE(InfoExtractor):
             'description': post.get('description'),
             'thumbnail': post.get('picture'),
             'formats': formats,
+            'subtitles': subtitles,
         }

From 1593194c63acd8dc8aa125edbde443680fabd2f7 Mon Sep 17 00:00:00 2001
From: pishposhmcgee <pishposh.mcgee@gmail.com>
Date: Tue, 10 Mar 2015 15:35:35 -0500
Subject: [PATCH 1827/1937] Update funnyordie.py

---
 youtube_dl/extractor/funnyordie.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index f832ac5c3..0a22d99d2 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -65,8 +65,7 @@ class FunnyOrDieIE(InfoExtractor):
           if not sublang in subtitles.keys():
             subtitles[sublang]=[]
           subext=suburl.split('/')[-1]
-          print subext
-          subtitles[sublang].append({'url': suburl,'ext': subext})
+          subtitles[sublang].append({'url': 'http://www.funnyordie.com'+suburl,'ext': subext})
 
         post_json = self._search_regex(
             r'fb_post\s*=\s*(\{.*?\});', webpage, 'post details')

From 32aaeca775c03ce784b8a30ea987c03a25a00e98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Mar 2015 20:34:32 +0600
Subject: [PATCH 1828/1937] [npo] Improve smooth stream skipping and set low
 preference for streams other than hds ans hls (Closes #5175)

---
 youtube_dl/extractor/npo.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 9c01eb0af..557dffa46 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -219,7 +219,8 @@ class NPOLiveIE(NPOBaseIE):
         if streams:
             for stream in streams:
                 stream_type = stream.get('type').lower()
-                if stream_type == 'ss':
+                # smooth streaming is not supported
+                if stream_type in ['ss', 'ms']:
                     continue
                 stream_info = self._download_json(
                     'http://ida.omroep.nl/aapi/?stream=%s&token=%s&type=jsonp'
@@ -242,6 +243,7 @@ class NPOLiveIE(NPOBaseIE):
                 else:
                     formats.append({
                         'url': stream_url,
+                        'preference': -10,
                     })
 
         self._sort_formats(formats)

From c792b5011fe56af654d99378c1e77837475d7b69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Mar 2015 21:15:36 +0600
Subject: [PATCH 1829/1937] [ssa] Add extractor (Closes #5169)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/ssa.py      | 58 ++++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+)
 create mode 100644 youtube_dl/extractor/ssa.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 20bc73dce..7adcc4dbf 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -460,6 +460,7 @@ from .sport5 import Sport5IE
 from .sportbox import SportBoxIE
 from .sportdeutschland import SportDeutschlandIE
 from .srmediathek import SRMediathekIE
+from .ssa import SSAIE
 from .stanfordoc import StanfordOpenClassroomIE
 from .steam import SteamIE
 from .streamcloud import StreamcloudIE
diff --git a/youtube_dl/extractor/ssa.py b/youtube_dl/extractor/ssa.py
new file mode 100644
index 000000000..13101c714
--- /dev/null
+++ b/youtube_dl/extractor/ssa.py
@@ -0,0 +1,58 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    unescapeHTML,
+    parse_duration,
+)
+
+
+class SSAIE(InfoExtractor):
+    _VALID_URL = r'http://ssa\.nls\.uk/film/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://ssa.nls.uk/film/3561',
+        'info_dict': {
+            'id': '3561',
+            'ext': 'flv',
+            'title': 'SHETLAND WOOL',
+            'description': 'md5:c5afca6871ad59b4271e7704fe50ab04',
+            'duration': 900,
+            'thumbnail': 're:^https?://.*\.jpg$',
+        },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        streamer = self._search_regex(
+            r"'streamer'\s*,\S*'(rtmp[^']+)'", webpage, 'streamer')
+        play_path = self._search_regex(
+            r"'file'\s*,\s*'([^']+)'", webpage, 'file').rpartition('.')[0]
+
+        def search_field(field_name, fatal=False):
+            return self._search_regex(
+                r'<span\s+class="field_title">%s:</span>\s*<span\s+class="field_content">([^<]+)</span>' % field_name,
+                webpage, 'title', fatal=fatal)
+
+        title = unescapeHTML(search_field('Title', fatal=True)).strip('()[]')
+        description = unescapeHTML(search_field('Description'))
+        duration = parse_duration(search_field('Running time'))
+        thumbnail = self._search_regex(
+            r"'image'\s*,\s*'([^']+)'", webpage, 'thumbnails', fatal=False)
+
+        return {
+            'id': video_id,
+            'url': streamer,
+            'play_path': play_path,
+            'ext': 'flv',
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'thumbnail': thumbnail,
+        }

From a3fbd188241b6151b024f6e3ea21c0edc60fae09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Mar 2015 21:56:22 +0600
Subject: [PATCH 1830/1937] [funnyordie] Simplify subtitles

---
 youtube_dl/extractor/funnyordie.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index 0a22d99d2..dd87257c4 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -58,14 +58,12 @@ class FunnyOrDieIE(InfoExtractor):
                     'vbr': bitrate,
                 })
 
-        subtitles={}
-        subtitle_matches=re.findall(r'<track kind="captions" src="([^"]+)" srclang="([^"]+)"', webpage)
-        for match in subtitle_matches:
-          (suburl,sublang)=match
-          if not sublang in subtitles.keys():
-            subtitles[sublang]=[]
-          subext=suburl.split('/')[-1]
-          subtitles[sublang].append({'url': 'http://www.funnyordie.com'+suburl,'ext': subext})
+        subtitles = {}
+        for src, src_lang in re.findall(r'<track kind="captions" src="([^"]+)" srclang="([^"]+)"', webpage):
+            subtitles[src_lang] = [{
+                'ext': src.split('/')[-1],
+                'url': 'http://www.funnyordie.com%s' % src,
+            }]
 
         post_json = self._search_regex(
             r'fb_post\s*=\s*(\{.*?\});', webpage, 'post details')

From 7e195d0e92772b2ea3c748e319574a81f807e807 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 11 Mar 2015 22:00:37 +0600
Subject: [PATCH 1831/1937] [funnyordie] Add subtitles test

---
 test/test_subtitles.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 3f2d8a2ba..891ee620b 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -26,6 +26,7 @@ from youtube_dl.extractor import (
     VikiIE,
     ThePlatformIE,
     RTVEALaCartaIE,
+    FunnyOrDieIE,
 )
 
 
@@ -320,5 +321,17 @@ class TestRtveSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['es']), '69e70cae2d40574fb7316f31d6eb7fca')
 
 
+class TestFunnyOrDieSubtitles(BaseTestSubtitles):
+    url = 'http://www.funnyordie.com/videos/224829ff6d/judd-apatow-will-direct-your-vine'
+    IE = FunnyOrDieIE
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['en']))
+        self.assertEqual(md5(subtitles['en']), 'c5593c193eacd353596c11c2d4f9ecc4')
+
+
 if __name__ == '__main__':
     unittest.main()

From 3a77719c5ac60f7e634a7a8def82acb08ccc9ffc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 11 Mar 2015 17:38:35 +0100
Subject: [PATCH 1832/1937] Don't accept '-1' as format, 'all' is clearer

---
 youtube_dl/YoutubeDL.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index bce7587fd..be5b3c1ab 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1086,8 +1086,7 @@ class YoutubeDL(object):
         if req_format is None:
             req_format = 'best'
         formats_to_download = []
-        # The -1 is for supporting YoutubeIE
-        if req_format in ('-1', 'all'):
+        if req_format == 'all':
             formats_to_download = formats
         else:
             for rfstr in req_format.split(','):

From 71705fa70d3f8b410a25a9c1fb01c33dd5bb6026 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 12 Mar 2015 21:56:56 +0600
Subject: [PATCH 1833/1937] [footyroom] Add extractor (Closes #5000)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/footyroom.py | 41 +++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)
 create mode 100644 youtube_dl/extractor/footyroom.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7adcc4dbf..7f9523c2b 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -151,6 +151,7 @@ from .fktv import (
 )
 from .flickr import FlickrIE
 from .folketinget import FolketingetIE
+from .footyroom import FootyRoomIE
 from .fourtube import FourTubeIE
 from .foxgay import FoxgayIE
 from .foxnews import FoxNewsIE
diff --git a/youtube_dl/extractor/footyroom.py b/youtube_dl/extractor/footyroom.py
new file mode 100644
index 000000000..2b4691ae8
--- /dev/null
+++ b/youtube_dl/extractor/footyroom.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class FootyRoomIE(InfoExtractor):
+    _VALID_URL = r'http://footyroom\.com/(?P<id>[^/]+)'
+    _TEST = {
+        'url': 'http://footyroom.com/schalke-04-0-2-real-madrid-2015-02/',
+        'info_dict': {
+            'id': 'schalke-04-0-2-real-madrid-2015-02',
+            'title': 'Schalke 04 0 – 2 Real Madrid',
+        },
+        'playlist_count': 3,
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        playlist = self._parse_json(
+            self._search_regex(
+                r'VideoSelector\.load\((\[.+?\])\);', webpage, 'video selector'),
+            playlist_id)
+
+        playlist_title = self._og_search_title(webpage)
+
+        entries = []
+        for video in playlist:
+            payload = video.get('payload')
+            if not payload:
+                continue
+            playwire_url = self._search_regex(
+                r'data-config="([^"]+)"', payload,
+                'playwire url', default=None)
+            if playwire_url:
+                entries.append(self.url_result(playwire_url, 'Playwire'))
+
+        return self.playlist_result(entries, playlist_id, playlist_title)

From 1138491631c43c2f758d933675131f0bd8249920 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 12 Mar 2015 21:59:46 +0600
Subject: [PATCH 1834/1937] [yam] Skip test

---
 youtube_dl/extractor/yam.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/yam.py b/youtube_dl/extractor/yam.py
index 19ad74d04..19f8762ae 100644
--- a/youtube_dl/extractor/yam.py
+++ b/youtube_dl/extractor/yam.py
@@ -39,7 +39,8 @@ class YamIE(InfoExtractor):
             'description': 'md5:11e2e405311633ace874f2e6226c8b17',
             'uploader_id': '2323agoy',
             'title': '20090412陽明山二子坪-1',
-        }
+        },
+        'skip': 'Video does not exist',
     }, {
         'url': 'http://mymedia.yam.com/m/3598173',
         'info_dict': {

From d7d79106c7463b455ecc967c5f68b744c1375147 Mon Sep 17 00:00:00 2001
From: Leonardo Amaral <contato@leonardoamaral.com.br>
Date: Thu, 12 Mar 2015 14:23:42 -0300
Subject: [PATCH 1835/1937] * Change globo.py flash ver to 17.0.0.132 - Chrome
 42.0.2311.22

---
 youtube_dl/extractor/globo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py
index 29638a194..8a95793ca 100644
--- a/youtube_dl/extractor/globo.py
+++ b/youtube_dl/extractor/globo.py
@@ -20,7 +20,7 @@ class GloboIE(InfoExtractor):
     _VALID_URL = 'https?://.+?\.globo\.com/(?P<id>.+)'
 
     _API_URL_TEMPLATE = 'http://api.globovideos.com/videos/%s/playlist'
-    _SECURITY_URL_TEMPLATE = 'http://security.video.globo.com/videos/%s/hash?player=flash&version=2.9.9.50&resource_id=%s'
+    _SECURITY_URL_TEMPLATE = 'http://security.video.globo.com/videos/%s/hash?player=flash&version=17.0.0.132&resource_id=%s'
 
     _VIDEOID_REGEXES = [
         r'\bdata-video-id="(\d+)"',

From b84037013edf25866772ba12db3b0c55f94a3e12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 12 Mar 2015 18:45:00 +0100
Subject: [PATCH 1836/1937] [vimeo] Fix login (#3886)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index b84a83ba6..fc720e7e7 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -38,7 +38,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         self.report_login()
         login_url = 'https://vimeo.com/log_in'
         webpage = self._download_webpage(login_url, None, False)
-        token = self._search_regex(r'xsrft: \'(.*?)\'', webpage, 'login token')
+        token = self._search_regex(r'xsrft = \'(.*?)\'', webpage, 'login token')
         data = urlencode_postdata({
             'email': username,
             'password': password,

From 3946864c8a752c068b188126013ddb747b53385d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 12 Mar 2015 19:08:16 +0100
Subject: [PATCH 1837/1937] [vimeo] Use https for all vimeo.com urls

Unfortunately vimeopro.com doesn't support it yet.
---
 test/test_all_urls.py         | 10 +++++-----
 youtube_dl/extractor/vimeo.py | 36 +++++++++++++++++------------------
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index e66264b4b..6ae168b7f 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -104,11 +104,11 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertMatch(':tds', ['ComedyCentralShows'])
 
     def test_vimeo_matching(self):
-        self.assertMatch('http://vimeo.com/channels/tributes', ['vimeo:channel'])
-        self.assertMatch('http://vimeo.com/channels/31259', ['vimeo:channel'])
-        self.assertMatch('http://vimeo.com/channels/31259/53576664', ['vimeo'])
-        self.assertMatch('http://vimeo.com/user7108434', ['vimeo:user'])
-        self.assertMatch('http://vimeo.com/user7108434/videos', ['vimeo:user'])
+        self.assertMatch('https://vimeo.com/channels/tributes', ['vimeo:channel'])
+        self.assertMatch('https://vimeo.com/channels/31259', ['vimeo:channel'])
+        self.assertMatch('https://vimeo.com/channels/31259/53576664', ['vimeo'])
+        self.assertMatch('https://vimeo.com/user7108434', ['vimeo:user'])
+        self.assertMatch('https://vimeo.com/user7108434/videos', ['vimeo:user'])
         self.assertMatch('https://vimeo.com/user21297594/review/75524534/3c257a1b5d', ['vimeo:review'])
 
     # https://github.com/rg3/youtube-dl/issues/1930
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index fc720e7e7..bd09652cd 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -373,7 +373,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
             for tt in text_tracks:
                 subtitles[tt['lang']] = [{
                     'ext': 'vtt',
-                    'url': 'http://vimeo.com' + tt['url'],
+                    'url': 'https://vimeo.com' + tt['url'],
                 }]
 
         return {
@@ -396,11 +396,11 @@ class VimeoIE(VimeoBaseInfoExtractor):
 
 class VimeoChannelIE(InfoExtractor):
     IE_NAME = 'vimeo:channel'
-    _VALID_URL = r'https?://vimeo\.com/channels/(?P<id>[^/?#]+)/?(?:$|[?#])'
+    _VALID_URL = r'https://vimeo\.com/channels/(?P<id>[^/?#]+)/?(?:$|[?#])'
     _MORE_PAGES_INDICATOR = r'<a.+?rel="next"'
     _TITLE_RE = r'<link rel="alternate"[^>]+?title="(.*?)"'
     _TESTS = [{
-        'url': 'http://vimeo.com/channels/tributes',
+        'url': 'https://vimeo.com/channels/tributes',
         'info_dict': {
             'id': 'tributes',
             'title': 'Vimeo Tributes',
@@ -459,7 +459,7 @@ class VimeoChannelIE(InfoExtractor):
             if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:
                 break
 
-        entries = [self.url_result('http://vimeo.com/%s' % video_id, 'Vimeo')
+        entries = [self.url_result('https://vimeo.com/%s' % video_id, 'Vimeo')
                    for video_id in video_ids]
         return {'_type': 'playlist',
                 'id': list_id,
@@ -470,15 +470,15 @@ class VimeoChannelIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         channel_id = mobj.group('id')
-        return self._extract_videos(channel_id, 'http://vimeo.com/channels/%s' % channel_id)
+        return self._extract_videos(channel_id, 'https://vimeo.com/channels/%s' % channel_id)
 
 
 class VimeoUserIE(VimeoChannelIE):
     IE_NAME = 'vimeo:user'
-    _VALID_URL = r'https?://vimeo\.com/(?![0-9]+(?:$|[?#/]))(?P<name>[^/]+)(?:/videos|[#?]|$)'
+    _VALID_URL = r'https://vimeo\.com/(?![0-9]+(?:$|[?#/]))(?P<name>[^/]+)(?:/videos|[#?]|$)'
     _TITLE_RE = r'<a[^>]+?class="user">([^<>]+?)</a>'
     _TESTS = [{
-        'url': 'http://vimeo.com/nkistudio/videos',
+        'url': 'https://vimeo.com/nkistudio/videos',
         'info_dict': {
             'title': 'Nki',
             'id': 'nkistudio',
@@ -489,7 +489,7 @@ class VimeoUserIE(VimeoChannelIE):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         name = mobj.group('name')
-        return self._extract_videos(name, 'http://vimeo.com/%s' % name)
+        return self._extract_videos(name, 'https://vimeo.com/%s' % name)
 
 
 class VimeoAlbumIE(VimeoChannelIE):
@@ -526,9 +526,9 @@ class VimeoAlbumIE(VimeoChannelIE):
 
 class VimeoGroupsIE(VimeoAlbumIE):
     IE_NAME = 'vimeo:group'
-    _VALID_URL = r'(?:https?://)?vimeo\.com/groups/(?P<name>[^/]+)'
+    _VALID_URL = r'https://vimeo\.com/groups/(?P<name>[^/]+)'
     _TESTS = [{
-        'url': 'http://vimeo.com/groups/rolexawards',
+        'url': 'https://vimeo.com/groups/rolexawards',
         'info_dict': {
             'id': 'rolexawards',
             'title': 'Rolex Awards for Enterprise',
@@ -542,13 +542,13 @@ class VimeoGroupsIE(VimeoAlbumIE):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         name = mobj.group('name')
-        return self._extract_videos(name, 'http://vimeo.com/groups/%s' % name)
+        return self._extract_videos(name, 'https://vimeo.com/groups/%s' % name)
 
 
 class VimeoReviewIE(InfoExtractor):
     IE_NAME = 'vimeo:review'
     IE_DESC = 'Review pages on vimeo'
-    _VALID_URL = r'https?://vimeo\.com/[^/]+/review/(?P<id>[^/]+)'
+    _VALID_URL = r'https://vimeo\.com/[^/]+/review/(?P<id>[^/]+)'
     _TESTS = [{
         'url': 'https://vimeo.com/user21297594/review/75524534/3c257a1b5d',
         'md5': 'c507a72f780cacc12b2248bb4006d253',
@@ -560,7 +560,7 @@ class VimeoReviewIE(InfoExtractor):
         }
     }, {
         'note': 'video player needs Referer',
-        'url': 'http://vimeo.com/user22258446/review/91613211/13f927e053',
+        'url': 'https://vimeo.com/user22258446/review/91613211/13f927e053',
         'md5': '6295fdab8f4bf6a002d058b2c6dce276',
         'info_dict': {
             'id': '91613211',
@@ -582,11 +582,11 @@ class VimeoReviewIE(InfoExtractor):
 class VimeoWatchLaterIE(VimeoBaseInfoExtractor, VimeoChannelIE):
     IE_NAME = 'vimeo:watchlater'
     IE_DESC = 'Vimeo watch later list, "vimeowatchlater" keyword (requires authentication)'
-    _VALID_URL = r'https?://vimeo\.com/home/watchlater|:vimeowatchlater'
+    _VALID_URL = r'https://vimeo\.com/home/watchlater|:vimeowatchlater'
     _LOGIN_REQUIRED = True
     _TITLE_RE = r'href="/home/watchlater".*?>(.*?)<'
     _TESTS = [{
-        'url': 'http://vimeo.com/home/watchlater',
+        'url': 'https://vimeo.com/home/watchlater',
         'only_matching': True,
     }]
 
@@ -606,7 +606,7 @@ class VimeoWatchLaterIE(VimeoBaseInfoExtractor, VimeoChannelIE):
 
 
 class VimeoLikesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vimeo\.com/user(?P<id>[0-9]+)/likes/?(?:$|[?#]|sort:)'
+    _VALID_URL = r'https://(?:www\.)?vimeo\.com/user(?P<id>[0-9]+)/likes/?(?:$|[?#]|sort:)'
     IE_NAME = 'vimeo:likes'
     IE_DESC = 'Vimeo user likes'
     _TEST = {
@@ -634,8 +634,8 @@ class VimeoLikesIE(InfoExtractor):
         description = self._html_search_meta('description', webpage)
 
         def _get_page(idx):
-            page_url = '%s//vimeo.com/user%s/likes/page:%d/sort:date' % (
-                self.http_scheme(), user_id, idx + 1)
+            page_url = 'https://vimeo.com/user%s/likes/page:%d/sort:date' % (
+                user_id, idx + 1)
             webpage = self._download_webpage(
                 page_url, user_id,
                 note='Downloading page %d/%d' % (idx + 1, page_count))

From 11984c7467184100ca4a61ae939a8c260480f42c Mon Sep 17 00:00:00 2001
From: "Devin J. Pohly" <djpohly@gmail.com>
Date: Thu, 12 Mar 2015 15:43:13 -0400
Subject: [PATCH 1838/1937] [BeatportPro] Add new extractor

This extractor is for Beatport's 2-minute, low-quality track previews
only.  To obtain an entire track, you obviously have to purchase and
download it normally through the Beatport store!

Possible future improvements:
- Playlists for albums or other track-list pages
- User login to play from My Beatport, Hold Bin, or Cart
---
 youtube_dl/extractor/__init__.py    |   1 +
 youtube_dl/extractor/beatportpro.py | 101 ++++++++++++++++++++++++++++
 2 files changed, 102 insertions(+)
 create mode 100644 youtube_dl/extractor/beatportpro.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7f9523c2b..ac765fdb8 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -37,6 +37,7 @@ from .bandcamp import BandcampIE, BandcampAlbumIE
 from .bbccouk import BBCCoUkIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
+from .beatportpro import BeatportProIE
 from .bet import BetIE
 from .bild import BildIE
 from .bilibili import BiliBiliIE
diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py
new file mode 100644
index 000000000..21048b732
--- /dev/null
+++ b/youtube_dl/extractor/beatportpro.py
@@ -0,0 +1,101 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+import re
+import json
+
+
+class BeatportProIE(InfoExtractor):
+    _VALID_URL = r'https?://pro\.beatport\.com/track/.*/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'https://pro.beatport.com/track/synesthesia-original-mix/5379371',
+        'md5': 'b3c34d8639a2f6a7f734382358478887',
+        'info_dict': {
+            'id': 5379371,
+            'display-id': 'synesthesia-original-mix',
+            'ext': 'mp4',
+            'title': 'Froxic - Synesthesia (Original Mix)',
+        },
+    }, {
+        'url': 'https://pro.beatport.com/track/love-and-war-original-mix/3756896',
+        'md5': 'e44c3025dfa38c6577fbaeb43da43514',
+        'info_dict': {
+            'id': 3756896,
+            'display-id': 'love-and-war-original-mix',
+            'ext': 'mp3',
+            'title': 'Wolfgang Gartner - Love & War (Original Mix)',
+        },
+    }, {
+        'url': 'https://pro.beatport.com/track/birds-original-mix/4991738',
+        'md5': 'a1fd8e8046de3950fd039304c186c05f',
+        'info_dict': {
+            'id': 4991738,
+            'display-id': 'birds-original-mix',
+            'ext': 'mp4',
+            'title': "Tos, Middle Milk, Mumblin' Johnsson - Birds (Original Mix)",
+        }
+    }]
+
+    def _real_extract(self, url):
+        track_id = self._match_id(url)
+        webpage = self._download_webpage(url, track_id)
+
+        # Extract "Playables" JSON information from the page
+        playables = self._search_regex(r'window\.Playables = ({.*?});', webpage,
+                                       'playables info', flags=re.DOTALL)
+        playables = json.loads(playables)
+
+        # Find first track with matching ID (always the first one listed?)
+        track = next(filter(lambda t: t['id'] == int(track_id), playables['tracks']))
+
+        # Construct title from artist(s), track name, and mix name
+        title = ', '.join((a['name'] for a in track['artists'])) + ' - ' + track['name']
+        if track['mix']:
+            title += ' (' + track['mix'] + ')'
+
+        # Get format information
+        formats = []
+        for ext, info in track['preview'].items():
+            if info['url'] is None:
+                continue
+            fmt = {
+                'url': info['url'],
+                'ext': ext,
+                'format_id': ext,
+                'vcodec': 'none',
+            }
+            if ext == 'mp3':
+                fmt['preference'] = 0
+                fmt['acodec'] = 'mp3'
+                fmt['abr'] = 96
+                fmt['asr'] = 44100
+            elif ext == 'mp4':
+                fmt['preference'] = 1
+                fmt['acodec'] = 'aac'
+                fmt['abr'] = 96
+                fmt['asr'] = 44100
+            formats += [fmt]
+        formats.sort(key=lambda f: f['preference'])
+
+        # Get album art as thumbnails
+        imgs = []
+        for name, info in track['images'].items():
+            if name == 'dynamic' or info['url'] is None:
+                continue
+            img = {
+                'id': name,
+                'url': info['url'],
+                'height': info['height'],
+                'width': info['width'],
+            }
+            imgs += [img]
+
+        return {
+            'id': track['id'],
+            'display-id': track['slug'],
+            'title': title,
+            'formats': formats,
+            'thumbnails': imgs,
+        }

From 65c5e044c7ab6d3140d30c98abda07785f2974c6 Mon Sep 17 00:00:00 2001
From: "Devin J. Pohly" <djpohly@gmail.com>
Date: Thu, 12 Mar 2015 16:42:55 -0400
Subject: [PATCH 1839/1937] fix python2

---
 youtube_dl/extractor/beatportpro.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py
index 21048b732..c3c70fb33 100644
--- a/youtube_dl/extractor/beatportpro.py
+++ b/youtube_dl/extractor/beatportpro.py
@@ -48,7 +48,7 @@ class BeatportProIE(InfoExtractor):
         playables = json.loads(playables)
 
         # Find first track with matching ID (always the first one listed?)
-        track = next(filter(lambda t: t['id'] == int(track_id), playables['tracks']))
+        track = next(t for t in playables['tracks'] if t['id'] == int(track_id))
 
         # Construct title from artist(s), track name, and mix name
         title = ', '.join((a['name'] for a in track['artists'])) + ' - ' + track['name']

From 054b99a33079dcc4755e46aaf588424b4bb12020 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 12 Mar 2015 22:33:59 +0100
Subject: [PATCH 1840/1937] [jeuxvideo] Fix extraction (fixes #5190)

---
 youtube_dl/extractor/jeuxvideo.py | 36 ++++++++++++++++---------------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py
index 8094cc2e4..d0720ff56 100644
--- a/youtube_dl/extractor/jeuxvideo.py
+++ b/youtube_dl/extractor/jeuxvideo.py
@@ -2,7 +2,6 @@
 
 from __future__ import unicode_literals
 
-import json
 import re
 
 from .common import InfoExtractor
@@ -15,10 +14,10 @@ class JeuxVideoIE(InfoExtractor):
         'url': 'http://www.jeuxvideo.com/reportages-videos-jeux/0004/00046170/tearaway-playstation-vita-gc-2013-tearaway-nous-presente-ses-papiers-d-identite-00115182.htm',
         'md5': '046e491afb32a8aaac1f44dd4ddd54ee',
         'info_dict': {
-            'id': '5182',
+            'id': '114765',
             'ext': 'mp4',
-            'title': 'GC 2013 : Tearaway nous présente ses papiers d\'identité',
-            'description': 'Lorsque les développeurs de LittleBigPlanet proposent un nouveau titre, on ne peut que s\'attendre à un résultat original et fort attrayant.\n',
+            'title': 'Tearaway : GC 2013 : Tearaway nous présente ses papiers d\'identité',
+            'description': 'Lorsque les développeurs de LittleBigPlanet proposent un nouveau titre, on ne peut que s\'attendre à un résultat original et fort attrayant.',
         },
     }
 
@@ -26,26 +25,29 @@ class JeuxVideoIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         title = mobj.group(1)
         webpage = self._download_webpage(url, title)
-        xml_link = self._html_search_regex(
-            r'<param name="flashvars" value="config=(.*?)" />',
+        title = self._html_search_meta('name', webpage)
+        config_url = self._html_search_regex(
+            r'data-src="(/contenu/medias/video.php.*?)"',
             webpage, 'config URL')
+        config_url = 'http://www.jeuxvideo.com' + config_url
 
         video_id = self._search_regex(
-            r'http://www\.jeuxvideo\.com/config/\w+/\d+/(.*?)/\d+_player\.xml',
-            xml_link, 'video ID')
+            r'id=(\d+)',
+            config_url, 'video ID')
 
-        config = self._download_xml(
-            xml_link, title, 'Downloading XML config')
-        info_json = config.find('format.json').text
-        info = json.loads(info_json)['versions'][0]
+        config = self._download_json(
+            config_url, title, 'Downloading JSON config')
 
-        video_url = 'http://video720.jeuxvideo.com/' + info['file']
+        formats = [{
+            'url': source['file'],
+            'format_id': source['label'],
+            'resolution': source['label'],
+        } for source in reversed(config['sources'])]
 
         return {
             'id': video_id,
-            'title': config.find('titre_video').text,
-            'ext': 'mp4',
-            'url': video_url,
+            'title': title,
+            'formats': formats,
             'description': self._og_search_description(webpage),
-            'thumbnail': config.find('image').text,
+            'thumbnail': config.get('image'),
         }

From 486dd09e0bd8032a3234363231c2b592f725cad6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 13 Mar 2015 08:40:20 +0100
Subject: [PATCH 1841/1937] [YoutubeDL] Check for bytes instead of unicode
 output templates (#5192)

Also adapt the embedding examples for those poor souls still using 2.x.
---
 README.md               | 2 ++
 youtube_dl/YoutubeDL.py | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/README.md b/README.md
index f41e7ecee..f4fbf0034 100644
--- a/README.md
+++ b/README.md
@@ -515,6 +515,7 @@ youtube-dl makes the best effort to be a good command-line program, and thus sho
 From a Python program, you can embed youtube-dl in a more powerful fashion, like this:
 
 ```python
+from __future__ import unicode_literals
 import youtube_dl
 
 ydl_opts = {}
@@ -527,6 +528,7 @@ Most likely, you'll want to use various options. For a list of what can be done,
 Here's a more complete example of a program that outputs only errors (and a short message after the download is finished), and downloads/converts the video to an mp3 file:
 
 ```python
+from __future__ import unicode_literals
 import youtube_dl
 
 
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index be5b3c1ab..e779fc9a8 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -323,6 +323,11 @@ class YoutubeDL(object):
                 'Set the LC_ALL environment variable to fix this.')
             self.params['restrictfilenames'] = True
 
+        if isinstance(params.get('outtmpl'), bytes):
+            self.report_warning(
+                'Parameter outtmpl is bytes, but should be a unicode string. '
+                'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
+
         if '%(stitle)s' in self.params.get('outtmpl', ''):
             self.report_warning('%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
 

From fb7e68833cc67c23428462bf6c1306cbce207369 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 20:51:44 +0600
Subject: [PATCH 1842/1937] [kanalplay] Add extractor (Closes #5188)

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/kanalplay.py | 72 +++++++++++++++++++++++++++++++
 2 files changed, 73 insertions(+)
 create mode 100644 youtube_dl/extractor/kanalplay.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7f9523c2b..5dc56e330 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -231,6 +231,7 @@ from .jove import JoveIE
 from .jukebox import JukeboxIE
 from .jpopsukitv import JpopsukiIE
 from .kaltura import KalturaIE
+from .kanalplay import KanalPlayIE
 from .kankan import KankanIE
 from .karaoketv import KaraoketvIE
 from .keezmovies import KeezMoviesIE
diff --git a/youtube_dl/extractor/kanalplay.py b/youtube_dl/extractor/kanalplay.py
new file mode 100644
index 000000000..869757ec0
--- /dev/null
+++ b/youtube_dl/extractor/kanalplay.py
@@ -0,0 +1,72 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    float_or_none,
+)
+
+
+class KanalPlayIE(InfoExtractor):
+    IE_DESC = 'Kanal 5/9/11 Play'
+    _VALID_URL = r'https?://(?:www\.)?kanal(?P<channel_id>5|9|11)play\.se/(?:#!/)?(?:play/)?program/\d+/video/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'http://www.kanal5play.se/#!/play/program/3060212363/video/3270012277',
+        'md5': '',
+        'info_dict': {
+            'id': '2609989',
+            'ext': 'flv',
+            'title': 'Saknar både dusch och avlopp',
+            'description': 'md5:',
+            'duration': 2636.36,
+        },
+    }, {
+        'url': 'http://www.kanal9play.se/#!/play/program/335032/video/246042',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.kanal11play.se/#!/play/program/232835958/video/367135199',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        channel_id = mobj.group('channel_id')
+
+        video = self._download_json(
+            'http://www.kanal%splay.se/api/getVideo?format=FLASH&videoId=%s' % (channel_id, video_id),
+            video_id)
+
+        reasons_for_no_streams = video.get('reasonsForNoStreams')
+        if reasons_for_no_streams:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, '\n'.join(reasons_for_no_streams)),
+                expected=True)
+
+        title = video['title']
+        description = video.get('description')
+        duration = float_or_none(video.get('length'), 1000)
+        thumbnail = video.get('posterUrl')
+
+        stream_base_url = video['streamBaseUrl']
+
+        formats = [{
+            'url': stream_base_url,
+            'play_path': stream['source'],
+            'ext': 'flv',
+            'tbr': float_or_none(stream.get('bitrate'), 1000),
+            'rtmp_real_time': True,
+        } for stream in video['streams']]
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats,
+        }

From 4a34f69ea6213ffcf5785b30a126b439ceef77e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 21:38:28 +0600
Subject: [PATCH 1843/1937] [extractor/common] Add subtitles timecode formatter

---
 youtube_dl/extractor/common.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f9e8e2bad..e5245ec3f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1062,6 +1062,9 @@ class InfoExtractor(object):
     def _get_automatic_captions(self, *args, **kwargs):
         raise NotImplementedError("This method must be implemented by subclasses")
 
+    def _subtitles_timecode(self, seconds):
+        return '%02d:%02d:%02d.%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
+
 
 class SearchInfoExtractor(InfoExtractor):
     """

From 3f4327520c316043912ca9a8cd6f70d8121c3aa1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 21:39:29 +0600
Subject: [PATCH 1844/1937] [kanalplay] Extract subtitles

---
 youtube_dl/extractor/kanalplay.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/youtube_dl/extractor/kanalplay.py b/youtube_dl/extractor/kanalplay.py
index 869757ec0..772d5aca4 100644
--- a/youtube_dl/extractor/kanalplay.py
+++ b/youtube_dl/extractor/kanalplay.py
@@ -31,6 +31,22 @@ class KanalPlayIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _fix_subtitles(self, subs):
+        return '\r\n\r\n'.join(
+            '%s\r\n%s --> %s\r\n%s'
+            % (
+                num,
+                self._subtitles_timecode(item['startMillis'] / 1000.0),
+                self._subtitles_timecode(item['endMillis'] / 1000.0),
+                item['text'],
+            ) for num, item in enumerate(subs, 1))
+
+    def _get_subtitles(self, channel_id, video_id):
+        subs = self._download_json(
+            'http://www.kanal%splay.se/api/subtitles/%s' % (channel_id, video_id),
+            video_id, 'Downloading subtitles JSON', fatal=False)
+        return {'se': [{'ext': 'srt', 'data': self._fix_subtitles(subs)}]} if subs else {}
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -62,6 +78,10 @@ class KanalPlayIE(InfoExtractor):
         } for stream in video['streams']]
         self._sort_formats(formats)
 
+        subtitles = {}
+        if video.get('hasSubtitle'):
+            subtitles = self.extract_subtitles(channel_id, video_id)
+
         return {
             'id': video_id,
             'title': title,
@@ -69,4 +89,5 @@ class KanalPlayIE(InfoExtractor):
             'thumbnail': thumbnail,
             'duration': duration,
             'formats': formats,
+            'subtitles': subtitles,
         }

From 545315a9856743541f7f7b9b0ab42f2af9dd05c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 21:40:34 +0600
Subject: [PATCH 1845/1937] [nrk] Use generic subtitles timecode formatter

---
 youtube_dl/extractor/nrk.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 1e4cfa2e7..bff36f9d3 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -149,9 +149,6 @@ class NRKTVIE(InfoExtractor):
         }
     ]
 
-    def _seconds2str(self, s):
-        return '%02d:%02d:%02d.%03d' % (s / 3600, (s % 3600) / 60, s % 60, (s % 1) * 1000)
-
     def _debug_print(self, txt):
         if self._downloader.params.get('verbose', False):
             self.to_screen('[debug] %s' % txt)
@@ -168,8 +165,8 @@ class NRKTVIE(InfoExtractor):
         for pos, p in enumerate(ps):
             begin = parse_duration(p.get('begin'))
             duration = parse_duration(p.get('dur'))
-            starttime = self._seconds2str(begin)
-            endtime = self._seconds2str(begin + duration)
+            starttime = self._subtitles_timecode(begin)
+            endtime = self._subtitles_timecode(begin + duration)
             srt += '%s\r\n%s --> %s\r\n%s\r\n\r\n' % (compat_str(pos), starttime, endtime, p.text)
         return {lang: [
             {'ext': 'ttml', 'url': url},

From 1b53778175e43e2bf2cb71885a760d96727ee837 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 21:51:49 +0600
Subject: [PATCH 1846/1937] [beatenpro] Use generic format sort

---
 youtube_dl/extractor/beatportpro.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py
index c3c70fb33..bc201572e 100644
--- a/youtube_dl/extractor/beatportpro.py
+++ b/youtube_dl/extractor/beatportpro.py
@@ -77,7 +77,7 @@ class BeatportProIE(InfoExtractor):
                 fmt['abr'] = 96
                 fmt['asr'] = 44100
             formats += [fmt]
-        formats.sort(key=lambda f: f['preference'])
+        self._sort_formats(formats)
 
         # Get album art as thumbnails
         imgs = []

From 517bcca29925548f9b9b121beec1391ef3ecedec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 22:01:15 +0600
Subject: [PATCH 1847/1937] [beatenpro] Simplify and improve

---
 youtube_dl/extractor/beatportpro.py | 34 ++++++++++++++---------------
 1 file changed, 16 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py
index bc201572e..69657cbde 100644
--- a/youtube_dl/extractor/beatportpro.py
+++ b/youtube_dl/extractor/beatportpro.py
@@ -1,14 +1,15 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
-
 import re
 import json
 
+from .common import InfoExtractor
+from ..utils import int_or_none
+
 
 class BeatportProIE(InfoExtractor):
-    _VALID_URL = r'https?://pro\.beatport\.com/track/.*/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://pro\.beatport\.com/track/.+/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'https://pro.beatport.com/track/synesthesia-original-mix/5379371',
         'md5': 'b3c34d8639a2f6a7f734382358478887',
@@ -42,20 +43,17 @@ class BeatportProIE(InfoExtractor):
         track_id = self._match_id(url)
         webpage = self._download_webpage(url, track_id)
 
-        # Extract "Playables" JSON information from the page
-        playables = self._search_regex(r'window\.Playables = ({.*?});', webpage,
-                                       'playables info', flags=re.DOTALL)
+        playables = self._search_regex(
+            r'window\.Playables\s*=\s*({.*?});', webpage,
+            'playables info', flags=re.DOTALL)
         playables = json.loads(playables)
 
-        # Find first track with matching ID (always the first one listed?)
         track = next(t for t in playables['tracks'] if t['id'] == int(track_id))
 
-        # Construct title from artist(s), track name, and mix name
         title = ', '.join((a['name'] for a in track['artists'])) + ' - ' + track['name']
         if track['mix']:
             title += ' (' + track['mix'] + ')'
 
-        # Get format information
         formats = []
         for ext, info in track['preview'].items():
             if info['url'] is None:
@@ -76,26 +74,26 @@ class BeatportProIE(InfoExtractor):
                 fmt['acodec'] = 'aac'
                 fmt['abr'] = 96
                 fmt['asr'] = 44100
-            formats += [fmt]
+            formats.append(fmt)
         self._sort_formats(formats)
 
-        # Get album art as thumbnails
-        imgs = []
+        images = []
         for name, info in track['images'].items():
-            if name == 'dynamic' or info['url'] is None:
+            image_url = info.get('url')
+            if name == 'dynamic' or not image_url:
                 continue
             img = {
                 'id': name,
-                'url': info['url'],
-                'height': info['height'],
-                'width': info['width'],
+                'url': image_url,
+                'height': int_or_none(info.get('height')),
+                'width': int_or_none(info.get('width')),
             }
-            imgs += [img]
+            images.append(img)
 
         return {
             'id': track['id'],
             'display-id': track['slug'],
             'title': title,
             'formats': formats,
-            'thumbnails': imgs,
+            'thumbnails': images,
         }

From ba1d4c04883cafb55e40734776d9d8ba2ef85582 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 22:03:58 +0600
Subject: [PATCH 1848/1937] [beatenpro] Improve display_id

---
 youtube_dl/extractor/beatportpro.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py
index 69657cbde..5c072b131 100644
--- a/youtube_dl/extractor/beatportpro.py
+++ b/youtube_dl/extractor/beatportpro.py
@@ -9,7 +9,7 @@ from ..utils import int_or_none
 
 
 class BeatportProIE(InfoExtractor):
-    _VALID_URL = r'https?://pro\.beatport\.com/track/.+/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://pro\.beatport\.com/track/(?P<display_id>[^/]+)/(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'https://pro.beatport.com/track/synesthesia-original-mix/5379371',
         'md5': 'b3c34d8639a2f6a7f734382358478887',
@@ -40,8 +40,11 @@ class BeatportProIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        track_id = self._match_id(url)
-        webpage = self._download_webpage(url, track_id)
+        mobj = re.match(self._VALID_URL, url)
+        track_id = mobj.group('id')
+        display_id = mobj.group('display_id')
+
+        webpage = self._download_webpage(url, display_id)
 
         playables = self._search_regex(
             r'window\.Playables\s*=\s*({.*?});', webpage,
@@ -92,7 +95,7 @@ class BeatportProIE(InfoExtractor):
 
         return {
             'id': track['id'],
-            'display-id': track['slug'],
+            'display_id': track.get('slug') or display_id,
             'title': title,
             'formats': formats,
             'thumbnails': images,

From fcd877013e4a8f654c7778019055b57031492889 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 22:11:56 +0600
Subject: [PATCH 1849/1937] [beatenpro] Simplify

---
 youtube_dl/extractor/beatportpro.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py
index 5c072b131..12a7faa4f 100644
--- a/youtube_dl/extractor/beatportpro.py
+++ b/youtube_dl/extractor/beatportpro.py
@@ -2,9 +2,9 @@
 from __future__ import unicode_literals
 
 import re
-import json
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import int_or_none
 
 
@@ -46,10 +46,11 @@ class BeatportProIE(InfoExtractor):
 
         webpage = self._download_webpage(url, display_id)
 
-        playables = self._search_regex(
-            r'window\.Playables\s*=\s*({.*?});', webpage,
-            'playables info', flags=re.DOTALL)
-        playables = json.loads(playables)
+        playables = self._parse_json(
+            self._search_regex(
+                r'window\.Playables\s*=\s*({.+?});', webpage,
+                'playables info', flags=re.DOTALL),
+            track_id)
 
         track = next(t for t in playables['tracks'] if t['id'] == int(track_id))
 
@@ -59,7 +60,7 @@ class BeatportProIE(InfoExtractor):
 
         formats = []
         for ext, info in track['preview'].items():
-            if info['url'] is None:
+            if not info['url']:
                 continue
             fmt = {
                 'url': info['url'],
@@ -85,16 +86,16 @@ class BeatportProIE(InfoExtractor):
             image_url = info.get('url')
             if name == 'dynamic' or not image_url:
                 continue
-            img = {
+            image = {
                 'id': name,
                 'url': image_url,
                 'height': int_or_none(info.get('height')),
                 'width': int_or_none(info.get('width')),
             }
-            images.append(img)
+            images.append(image)
 
         return {
-            'id': track['id'],
+            'id': compat_str(track.get('id')) or track_id,
             'display_id': track.get('slug') or display_id,
             'title': title,
             'formats': formats,

From bba3fc7960c8cd6f0752c31c55ada804ba7e2ae2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 22:13:50 +0600
Subject: [PATCH 1850/1937] [beatenpro] Fix tests

---
 youtube_dl/extractor/beatportpro.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/beatportpro.py b/youtube_dl/extractor/beatportpro.py
index 12a7faa4f..3c7775d3e 100644
--- a/youtube_dl/extractor/beatportpro.py
+++ b/youtube_dl/extractor/beatportpro.py
@@ -14,8 +14,8 @@ class BeatportProIE(InfoExtractor):
         'url': 'https://pro.beatport.com/track/synesthesia-original-mix/5379371',
         'md5': 'b3c34d8639a2f6a7f734382358478887',
         'info_dict': {
-            'id': 5379371,
-            'display-id': 'synesthesia-original-mix',
+            'id': '5379371',
+            'display_id': 'synesthesia-original-mix',
             'ext': 'mp4',
             'title': 'Froxic - Synesthesia (Original Mix)',
         },
@@ -23,8 +23,8 @@ class BeatportProIE(InfoExtractor):
         'url': 'https://pro.beatport.com/track/love-and-war-original-mix/3756896',
         'md5': 'e44c3025dfa38c6577fbaeb43da43514',
         'info_dict': {
-            'id': 3756896,
-            'display-id': 'love-and-war-original-mix',
+            'id': '3756896',
+            'display_id': 'love-and-war-original-mix',
             'ext': 'mp3',
             'title': 'Wolfgang Gartner - Love & War (Original Mix)',
         },
@@ -32,8 +32,8 @@ class BeatportProIE(InfoExtractor):
         'url': 'https://pro.beatport.com/track/birds-original-mix/4991738',
         'md5': 'a1fd8e8046de3950fd039304c186c05f',
         'info_dict': {
-            'id': 4991738,
-            'display-id': 'birds-original-mix',
+            'id': '4991738',
+            'display_id': 'birds-original-mix',
             'ext': 'mp4',
             'title': "Tos, Middle Milk, Mumblin' Johnsson - Birds (Original Mix)",
         }

From 28c6411e4959e342ec8bc016eb8ca5dbcbdc5d7b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 13 Mar 2015 22:14:51 +0600
Subject: [PATCH 1851/1937] Credit @djpohly for BeatportPro (#5189)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 421df69a6..c10f03b98 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -114,3 +114,4 @@ Ryan Schmidt
 Leslie P. Polzer
 Duncan Keall
 Alexander Mamay
+Devin J. Pohly

From 13598940e3e019ce19eaf87ae872ea52b7cb8740 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Mar 2015 01:27:21 +0600
Subject: [PATCH 1852/1937] [kanalplay] Fix test

---
 youtube_dl/extractor/kanalplay.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/kanalplay.py b/youtube_dl/extractor/kanalplay.py
index 772d5aca4..2bb078036 100644
--- a/youtube_dl/extractor/kanalplay.py
+++ b/youtube_dl/extractor/kanalplay.py
@@ -15,14 +15,17 @@ class KanalPlayIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?kanal(?P<channel_id>5|9|11)play\.se/(?:#!/)?(?:play/)?program/\d+/video/(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.kanal5play.se/#!/play/program/3060212363/video/3270012277',
-        'md5': '',
         'info_dict': {
-            'id': '2609989',
+            'id': '3270012277',
             'ext': 'flv',
             'title': 'Saknar både dusch och avlopp',
-            'description': 'md5:',
+            'description': 'md5:6023a95832a06059832ae93bc3c7efb7',
             'duration': 2636.36,
         },
+        'params': {
+            # rtmp download
+            'skip_download': True,
+        }
     }, {
         'url': 'http://www.kanal9play.se/#!/play/program/335032/video/246042',
         'only_matching': True,

From 3647136f24b332be81f9866ae5e2d596ca5c884a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Mar 2015 02:12:11 +0600
Subject: [PATCH 1853/1937] [viewster] Add extractor

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/viewster.py | 53 ++++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)
 create mode 100644 youtube_dl/extractor/viewster.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 4682996cd..ad133603f 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -559,6 +559,7 @@ from .videoweed import VideoWeedIE
 from .vidme import VidmeIE
 from .vidzi import VidziIE
 from .vier import VierIE, VierVideosIE
+from .viewster import ViewsterIE
 from .vimeo import (
     VimeoIE,
     VimeoAlbumIE,
diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
new file mode 100644
index 000000000..d7864d6a6
--- /dev/null
+++ b/youtube_dl/extractor/viewster.py
@@ -0,0 +1,53 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_urllib_request
+
+
+class ViewsterIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?viewster\.com/movie/(?P<id>\d+-\d+-\d+)'
+    _TEST = {
+        'url': 'http://www.viewster.com/movie/1293-19341-000/hout-wood/',
+        'md5': '8f9d94b282d80c42b378dffdbb11caf3',
+        'info_dict': {
+            'id': '1293-19341-000',
+            'ext': 'flv',
+            'title': "'Hout' (Wood)",
+            'description': 'md5:925733185a9242ef96f436937683f33b',
+        },
+    }
+
+    _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        request = compat_urllib_request.Request(
+            'http://api.live.viewster.com/api/v1/movielink?movieid=%s&action=movierent&paymethod=fre&price=0&currency=&language=en&subtitlelanguage=x&ischromecast=false' % video_id)
+        request.add_header('Accept', self._ACCEPT_HEADER)
+
+        movie_link = self._download_json(
+            request, video_id, 'Downloading movie link JSON')
+
+        formats = self._extract_f4m_formats(
+            movie_link['url'] + '&hdcore=3.2.0&plugin=flowplayer-3.2.0.1', video_id)
+        self._sort_formats(formats)
+
+        request = compat_urllib_request.Request(
+            'http://api.live.viewster.com/api/v1/movie/%s' % video_id)
+        request.add_header('Accept', self._ACCEPT_HEADER)
+
+        movie = self._download_json(
+            request, video_id, 'Downloading movie metadata JSON')
+
+        title = movie.get('title') or movie['original_title']
+        description = movie.get('synopsis')
+        thumbnail = movie.get('large_artwork') or movie.get('artwork')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'formats': formats,
+        }

From 7be5a62ed71d574132b648864ce3b741635604f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Mar 2015 03:18:04 +0600
Subject: [PATCH 1854/1937] [viewster] Improve extraction

---
 youtube_dl/extractor/viewster.py | 122 +++++++++++++++++++++++++------
 1 file changed, 99 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/viewster.py b/youtube_dl/extractor/viewster.py
index d7864d6a6..1742e66f4 100644
--- a/youtube_dl/extractor/viewster.py
+++ b/youtube_dl/extractor/viewster.py
@@ -6,33 +6,75 @@ from ..compat import compat_urllib_request
 
 class ViewsterIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?viewster\.com/movie/(?P<id>\d+-\d+-\d+)'
-    _TEST = {
+    _TESTS = [{
+        # movielink, paymethod=fre
         'url': 'http://www.viewster.com/movie/1293-19341-000/hout-wood/',
-        'md5': '8f9d94b282d80c42b378dffdbb11caf3',
+        'playlist': [{
+            'md5': '8f9d94b282d80c42b378dffdbb11caf3',
+            'info_dict': {
+                'id': '1293-19341-000-movie',
+                'ext': 'flv',
+                'title': "'Hout' (Wood) - Movie",
+            },
+        }],
         'info_dict': {
             'id': '1293-19341-000',
-            'ext': 'flv',
             'title': "'Hout' (Wood)",
             'description': 'md5:925733185a9242ef96f436937683f33b',
-        },
-    }
+        }
+    }, {
+        # movielink, paymethod=adv
+        'url': 'http://www.viewster.com/movie/1140-11855-000/the-listening-project/',
+        'playlist': [{
+            'md5': '77a005453ca7396cbe3d35c9bea30aef',
+            'info_dict': {
+                'id': '1140-11855-000-movie',
+                'ext': 'flv',
+                'title': "THE LISTENING PROJECT - Movie",
+            },
+        }],
+        'info_dict': {
+            'id': '1140-11855-000',
+            'title': "THE LISTENING PROJECT",
+            'description': 'md5:714421ae9957e112e672551094bf3b08',
+        }
+    }, {
+        # direct links, no movielink
+        'url': 'http://www.viewster.com/movie/1198-56411-000/sinister/',
+        'playlist': [{
+            'md5': '0307b7eac6bfb21ab0577a71f6eebd8f',
+            'info_dict': {
+                'id': '1198-56411-000-trailer',
+                'ext': 'mp4',
+                'title': "Sinister - Trailer",
+            },
+        }, {
+            'md5': '80b9ee3ad69fb368f104cb5d9732ae95',
+            'info_dict': {
+                'id': '1198-56411-000-behind-scenes',
+                'ext': 'mp4',
+                'title': "Sinister - Behind Scenes",
+            },
+        }, {
+            'md5': '3b3ea897ecaa91fca57a8a94ac1b15c5',
+            'info_dict': {
+                'id': '1198-56411-000-scene-from-movie',
+                'ext': 'mp4',
+                'title': "Sinister - Scene from movie",
+            },
+        }],
+        'info_dict': {
+            'id': '1198-56411-000',
+            'title': "Sinister",
+            'description': 'md5:014c40b0488848de9683566a42e33372',
+        }
+    }]
 
     _ACCEPT_HEADER = 'application/json, text/javascript, */*; q=0.01'
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        request = compat_urllib_request.Request(
-            'http://api.live.viewster.com/api/v1/movielink?movieid=%s&action=movierent&paymethod=fre&price=0&currency=&language=en&subtitlelanguage=x&ischromecast=false' % video_id)
-        request.add_header('Accept', self._ACCEPT_HEADER)
-
-        movie_link = self._download_json(
-            request, video_id, 'Downloading movie link JSON')
-
-        formats = self._extract_f4m_formats(
-            movie_link['url'] + '&hdcore=3.2.0&plugin=flowplayer-3.2.0.1', video_id)
-        self._sort_formats(formats)
-
         request = compat_urllib_request.Request(
             'http://api.live.viewster.com/api/v1/movie/%s' % video_id)
         request.add_header('Accept', self._ACCEPT_HEADER)
@@ -44,10 +86,44 @@ class ViewsterIE(InfoExtractor):
         description = movie.get('synopsis')
         thumbnail = movie.get('large_artwork') or movie.get('artwork')
 
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
-            'formats': formats,
-        }
+        entries = []
+        for clip in movie['play_list']:
+            entry = None
+
+            # movielink api
+            link_request = clip.get('link_request')
+            if link_request:
+                request = compat_urllib_request.Request(
+                    'http://api.live.viewster.com/api/v1/movielink?movieid=%(movieid)s&action=%(action)s&paymethod=%(paymethod)s&price=%(price)s&currency=%(currency)s&language=%(language)s&subtitlelanguage=%(subtitlelanguage)s&ischromecast=%(ischromecast)s'
+                    % link_request)
+                request.add_header('Accept', self._ACCEPT_HEADER)
+
+                movie_link = self._download_json(
+                    request, video_id, 'Downloading movie link JSON', fatal=False)
+
+                if movie_link:
+                    formats = self._extract_f4m_formats(
+                        movie_link['url'] + '&hdcore=3.2.0&plugin=flowplayer-3.2.0.1', video_id)
+                    self._sort_formats(formats)
+                    entry = {
+                        'formats': formats,
+                    }
+
+            # direct link
+            clip_url = clip.get('clip_data', {}).get('url')
+            if clip_url:
+                entry = {
+                    'url': clip_url,
+                    'ext': 'mp4',
+                }
+
+            if entry:
+                entry.update({
+                    'id': '%s-%s' % (video_id, clip['canonical_title']),
+                    'title': '%s - %s' % (title, clip['title']),
+                })
+                entries.append(entry)
+
+        playlist = self.playlist_result(entries, video_id, title, description)
+        playlist['thumbnail'] = thumbnail
+        return playlist

From 29171bc2d2f07a4eebb3b353fab989d7652a2083 Mon Sep 17 00:00:00 2001
From: Mamay Alexander <alexander@mamay.su>
Date: Sat, 14 Mar 2015 13:56:04 +0600
Subject: [PATCH 1855/1937] [yandexmusic] Site mirrors

---
 youtube_dl/extractor/yandexmusic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index b47aecb15..f4c0f5702 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -39,7 +39,7 @@ class YandexMusicBaseIE(InfoExtractor):
 class YandexMusicTrackIE(YandexMusicBaseIE):
     IE_NAME = 'yandexmusic:track'
     IE_DESC = 'Яндекс.Музыка - Трек'
-    _VALID_URL = r'https?://music\.yandex\.ru/album/(?P<album_id>\d+)/track/(?P<id>\d+)'
+    _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/album/(?P<album_id>\d+)/track/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://music.yandex.ru/album/540508/track/4878838',
@@ -67,7 +67,7 @@ class YandexMusicTrackIE(YandexMusicBaseIE):
 class YandexMusicAlbumIE(YandexMusicBaseIE):
     IE_NAME = 'yandexmusic:album'
     IE_DESC = 'Яндекс.Музыка - Альбом'
-    _VALID_URL = r'https?://music\.yandex\.ru/album/(?P<id>\d+)/?(\?|$)'
+    _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/album/(?P<id>\d+)/?(\?|$)'
 
     _TEST = {
         'url': 'http://music.yandex.ru/album/540508',
@@ -98,7 +98,7 @@ class YandexMusicAlbumIE(YandexMusicBaseIE):
 class YandexMusicPlaylistIE(YandexMusicBaseIE):
     IE_NAME = 'yandexmusic:playlist'
     IE_DESC = 'Яндекс.Музыка - Плейлист'
-    _VALID_URL = r'https?://music\.yandex\.ru/users/[^/]+/playlists/(?P<id>\d+)'
+    _VALID_URL = r'https?://music\.yandex\.(?:ru|kz|ua|by)/users/[^/]+/playlists/(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://music.yandex.ru/users/music.partners/playlists/1245',

From 85741b998660bfb1564ca2c02e233db42059be05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Mar 2015 15:52:06 +0600
Subject: [PATCH 1856/1937] [8tracks] Use predefined avg duration when duration
 is negative (Closes #5200)

---
 youtube_dl/extractor/eighttracks.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index fb5dbbe2b..869ff72d6 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -117,6 +117,9 @@ class EightTracksIE(InfoExtractor):
         track_count = data['tracks_count']
         duration = data['duration']
         avg_song_duration = float(duration) / track_count
+        # duration is sometimes negative, use predefined avg duration
+        if avg_song_duration <= 0:
+            avg_song_duration = 300
         first_url = 'http://8tracks.com/sets/%s/play?player=sm&mix_id=%s&format=jsonh' % (session, mix_id)
         next_url = first_url
         entries = []

From 05be67e77dcf5c2a87d0549f6ae0f805ce38f758 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Mar 2015 15:54:23 +0600
Subject: [PATCH 1857/1937] [8tracks] Improve extraction

---
 youtube_dl/extractor/eighttracks.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index 869ff72d6..49ec2be4c 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -108,9 +108,10 @@ class EightTracksIE(InfoExtractor):
 
         webpage = self._download_webpage(url, playlist_id)
 
-        json_like = self._search_regex(
-            r"(?s)PAGE.mix = (.*?);\n", webpage, 'trax information')
-        data = json.loads(json_like)
+        data = self._parse_json(
+            self._search_regex(
+                r"(?s)PAGE\.mix\s*=\s*({.+?});\n", webpage, 'trax information'),
+            playlist_id)
 
         session = str(random.randint(0, 1000000000))
         mix_id = data['id']

From a7e01c438d2f7eb28a44056a1feb1b48f51e1096 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 14 Mar 2015 15:55:21 +0600
Subject: [PATCH 1858/1937] [8tracks] Modernize

---
 youtube_dl/extractor/eighttracks.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index 49ec2be4c..c84d16142 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -103,8 +103,7 @@ class EightTracksIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        playlist_id = mobj.group('id')
+        playlist_id = self._match_id(url)
 
         webpage = self._download_webpage(url, playlist_id)
 

From 9202b1b7878e2763a4406ebe2d59962202f6004b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Mar 2015 12:04:49 +0100
Subject: [PATCH 1859/1937] [eighttracks] Remove unused import

---
 youtube_dl/extractor/eighttracks.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index c84d16142..0b61ea0ba 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 
 import json
 import random
-import re
 
 from .common import InfoExtractor
 from ..compat import (

From 082b1155a36dc9b51424151f80860e52ee30b55e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Mar 2015 12:06:01 +0100
Subject: [PATCH 1860/1937] [livestream] Extract all videos in events (fixes
 #5198)

The webpage only contains the most recent ones, but if you scroll down more will appear.
---
 youtube_dl/extractor/livestream.py | 41 ++++++++++++++++++++++++++----
 1 file changed, 36 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 3642089f7..2467f8bdd 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import re
 import json
+import itertools
 
 from .common import InfoExtractor
 from ..compat import (
@@ -40,6 +41,13 @@ class LivestreamIE(InfoExtractor):
             'id': '2245590',
         },
         'playlist_mincount': 4,
+    }, {
+        'url': 'http://new.livestream.com/chess24/tatasteelchess',
+        'info_dict': {
+            'title': 'Tata Steel Chess',
+            'id': '3705884',
+        },
+        'playlist_mincount': 60,
     }, {
         'url': 'https://new.livestream.com/accounts/362/events/3557232/videos/67864563/player?autoPlay=false&height=360&mute=false&width=640',
         'only_matching': True,
@@ -117,6 +125,30 @@ class LivestreamIE(InfoExtractor):
             'view_count': video_data.get('views'),
         }
 
+    def _extract_event(self, info):
+        event_id = compat_str(info['id'])
+        account = compat_str(info['owner_account_id'])
+        root_url = (
+            'https://new.livestream.com/api/accounts/{account}/events/{event}/'
+            'feed.json'.format(account=account, event=event_id))
+
+        def _extract_videos():
+            last_video = None
+            for i in itertools.count(1):
+                if last_video is None:
+                    info_url = root_url
+                else:
+                    info_url = '{root}?&id={id}&newer=-1&type=video'.format(
+                        root=root_url, id=last_video)
+                videos_info = self._download_json(info_url, event_id, 'Downloading page {0}'.format(i))['data']
+                videos_info = [v['data'] for v in videos_info if v['type'] == 'video']
+                if not videos_info:
+                    break
+                for v in videos_info:
+                    yield self._extract_video_info(v)
+                last_video = videos_info[-1]['id']
+        return self.playlist_result(_extract_videos(), event_id, info['full_name'])
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -144,14 +176,13 @@ class LivestreamIE(InfoExtractor):
                 result = result and compat_str(vdata['data']['id']) == vid
             return result
 
-        videos = [self._extract_video_info(video_data['data'])
-                  for video_data in info['feed']['data']
-                  if is_relevant(video_data, video_id)]
         if video_id is None:
             # This is an event page:
-            return self.playlist_result(
-                videos, '%s' % info['id'], info['full_name'])
+            return self._extract_event(info)
         else:
+            videos = [self._extract_video_info(video_data['data'])
+                      for video_data in info['feed']['data']
+                      if is_relevant(video_data, video_id)]
             if not videos:
                 raise ExtractorError('Cannot find video %s' % video_id)
             return videos[0]

From cd65491c306f644d7bb3c7ad98795a3f8660be49 Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 15 Mar 2015 00:59:49 +0800
Subject: [PATCH 1861/1937] [Sohu] Add a multiplart video test case

---
 youtube_dl/extractor/sohu.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 5adc734d9..7db5b2f13 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -42,6 +42,37 @@ class SohuIE(InfoExtractor):
             'ext': 'mp4',
             'title': '【爱范品】第31期：MWC见不到的奇葩手机',
         }
+    }, {
+        'note': 'Multipart video',
+        'url': 'http://my.tv.sohu.com/pl/8384802/78910339.shtml',
+        'info_dict': {
+            'id': '78910339',
+        },
+        'playlist': [{
+            'md5': 'bdbfb8f39924725e6589c146bc1883ad',
+            'info_dict': {
+                'id': '78910339_part1',
+                'ext': 'mp4',
+                'duration': 294,
+                'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
+            }
+        }, {
+            'md5': '3e1f46aaeb95354fd10e7fca9fc1804e',
+            'info_dict': {
+                'id': '78910339_part2',
+                'ext': 'mp4',
+                'duration': 300,
+                'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
+            }
+        }, {
+            'md5': '8407e634175fdac706766481b9443450',
+            'info_dict': {
+                'id': '78910339_part3',
+                'ext': 'mp4',
+                'duration': 150,
+                'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
+            }
+        }]
     }]
 
     def _real_extract(self, url):

From 2cb434e53ee861c8bcbd538455be107085f444ae Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Sun, 15 Mar 2015 01:05:01 +0800
Subject: [PATCH 1862/1937] [Sohu] Fix title extraction

---
 youtube_dl/extractor/sohu.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 7db5b2f13..ea5cc06b9 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -73,6 +73,17 @@ class SohuIE(InfoExtractor):
                 'title': '【神探苍实战秘籍】第13期 战争之影 赫卡里姆',
             }
         }]
+    }, {
+        'info': 'Video with title containing dash',
+        'url': 'http://my.tv.sohu.com/us/249884221/78932792.shtml',
+        'info_dict': {
+            'id': '78932792',
+            'ext': 'mp4',
+            'title': 'youtube-dl testing video',
+        },
+        'params': {
+            'skip_download': True
+        }
     }]
 
     def _real_extract(self, url):
@@ -97,10 +108,8 @@ class SohuIE(InfoExtractor):
         mytv = mobj.group('mytv') is not None
 
         webpage = self._download_webpage(url, video_id)
-        raw_title = self._html_search_regex(
-            r'(?s)<title>(.+?)</title>',
-            webpage, 'video title')
-        title = raw_title.partition('-')[0].strip()
+
+        title = self._og_search_title(webpage)
 
         vid = self._html_search_regex(
             r'var vid ?= ?["\'](\d+)["\']',

From e7db87f7000143341505cff812d1fa0371ac901e Mon Sep 17 00:00:00 2001
From: phiresky <phiresky@users.noreply.github.com>
Date: Wed, 4 Mar 2015 22:33:56 +0100
Subject: [PATCH 1863/1937] Add metadata from title parser

(Closes #5125)
---
 youtube_dl/__init__.py                        |  5 ++
 youtube_dl/options.py                         |  9 ++++
 youtube_dl/postprocessor/__init__.py          |  2 +
 youtube_dl/postprocessor/ffmpeg.py            | 10 +++-
 youtube_dl/postprocessor/metadatafromtitle.py | 48 +++++++++++++++++++
 5 files changed, 72 insertions(+), 2 deletions(-)
 create mode 100644 youtube_dl/postprocessor/metadatafromtitle.py

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index a08ddd670..852b2fc3d 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -213,6 +213,11 @@ def _real_main(argv=None):
     # PostProcessors
     postprocessors = []
     # Add the metadata pp first, the other pps will copy it
+    if opts.metafromtitle:
+        postprocessors.append({
+            'key': 'MetadataFromTitle',
+            'titleformat': opts.metafromtitle
+        })
     if opts.addmetadata:
         postprocessors.append({'key': 'FFmpegMetadata'})
     if opts.extractaudio:
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index eefe008d5..be9402fdb 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -735,6 +735,15 @@ def parseOpts(overrideArguments=None):
         '--add-metadata',
         action='store_true', dest='addmetadata', default=False,
         help='write metadata to the video file')
+    postproc.add_option(
+        '--metadata-from-title',
+        metavar='FORMAT', dest='metafromtitle',
+        help='parse additional metadata like song title / artist from the video title. \n'
+             'The format syntax is the same as --output, '
+             'the parsed parameters replace existing values.\n'
+             'Additional templates: %(songtitle), %(album), %(artist). \n'
+             'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like '
+             '"Coldplay - Paradise"')
     postproc.add_option(
         '--xattrs',
         action='store_true', dest='xattrs', default=False,
diff --git a/youtube_dl/postprocessor/__init__.py b/youtube_dl/postprocessor/__init__.py
index 708df3dd4..f39acadce 100644
--- a/youtube_dl/postprocessor/__init__.py
+++ b/youtube_dl/postprocessor/__init__.py
@@ -15,6 +15,7 @@ from .ffmpeg import (
 )
 from .xattrpp import XAttrMetadataPP
 from .execafterdownload import ExecAfterDownloadPP
+from .metadatafromtitle import MetadataFromTitlePP
 
 
 def get_postprocessor(key):
@@ -34,5 +35,6 @@ __all__ = [
     'FFmpegPostProcessor',
     'FFmpegSubtitlesConvertorPP',
     'FFmpegVideoConvertorPP',
+    'MetadataFromTitlePP',
     'XAttrMetadataPP',
 ]
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 30094c2f3..a17113cbf 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -541,11 +541,15 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
 class FFmpegMetadataPP(FFmpegPostProcessor):
     def run(self, info):
         metadata = {}
-        if info.get('title') is not None:
+        if info.get('songtitle') is not None:
+            metadata['title'] = info['songtitle']
+        elif info.get('title') is not None:
             metadata['title'] = info['title']
         if info.get('upload_date') is not None:
             metadata['date'] = info['upload_date']
-        if info.get('uploader') is not None:
+        if info.get('artist') is not None:
+            metadata['artist'] = info['artist']
+        elif info.get('uploader') is not None:
             metadata['artist'] = info['uploader']
         elif info.get('uploader_id') is not None:
             metadata['artist'] = info['uploader_id']
@@ -554,6 +558,8 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
             metadata['comment'] = info['description']
         if info.get('webpage_url') is not None:
             metadata['purl'] = info['webpage_url']
+        if info.get('album') is not None:
+            metadata['album'] = info['album']
 
         if not metadata:
             self._downloader.to_screen('[ffmpeg] There isn\'t any metadata to add')
diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
new file mode 100644
index 000000000..4c9d3aafe
--- /dev/null
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+
+import re
+
+from .common import PostProcessor
+from ..utils import PostProcessingError
+
+
+class MetadataFromTitlePPError(PostProcessingError):
+    pass
+
+
+class MetadataFromTitlePP(PostProcessor):
+    def __init__(self, downloader, titleformat):
+        self._titleformat = titleformat
+        self._titleregex = self.fmtToRegex(titleformat)
+
+    def fmtToRegex(self, fmt):
+        """
+        Converts a string like
+           '%(title)s - %(artist)s'
+        to a regex like
+           '(?P<title>.+)\ \-\ (?P<artist>.+)'
+        and a list of the named groups [title, artist]
+        """
+        lastpos = 0
+        regex = ""
+        groups = []
+        # replace %(..)s with regex group and escape other string parts
+        for match in re.finditer(r'%\((\w+)\)s', fmt):
+            regex += re.escape(fmt[lastpos:match.start()])
+            regex += r'(?P<' + match.group(1) + '>.+)'
+            lastpos = match.end()
+        if lastpos < len(fmt):
+            regex += re.escape(fmt[lastpos:len(fmt)])
+        return regex
+
+    def run(self, info):
+        title = info['title']
+        match = re.match(self._titleregex, title)
+        if match is None:
+            raise MetadataFromTitlePPError('Could not interpret title of video as "%s"' % self._titleformat)
+        for attribute, value in match.groupdict().items():
+            value = match.group(attribute)
+            info[attribute] = value
+            self._downloader.to_screen('[fromtitle] parsed ' + attribute + ': ' + value)
+
+        return True, info

From 88cf6fb3685c4e012c9f574cbc5f1836c42fc06d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Mar 2015 19:55:42 +0100
Subject: [PATCH 1864/1937] [metadatafromtitle] Some improvements and cleanup

* Remove the 'songtitle' field, 'title' can be used instead.
* Remove newlines in the help text, for consistency with other options.
* Add 'from __future__ import unicode_literals'.
* Call '__init__' from the parent class.
* Add test for the format_to_regex method
---
 test/test_postprocessors.py                   | 17 +++++++++++++++++
 youtube_dl/options.py                         |  6 +++---
 youtube_dl/postprocessor/ffmpeg.py            |  4 +---
 youtube_dl/postprocessor/metadatafromtitle.py |  9 ++++-----
 4 files changed, 25 insertions(+), 11 deletions(-)
 create mode 100644 test/test_postprocessors.py

diff --git a/test/test_postprocessors.py b/test/test_postprocessors.py
new file mode 100644
index 000000000..addb69d6f
--- /dev/null
+++ b/test/test_postprocessors.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.postprocessor import MetadataFromTitlePP
+
+
+class TestMetadataFromTitle(unittest.TestCase):
+    def test_format_to_regex(self):
+        pp = MetadataFromTitlePP(None, '%(title)s - %(artist)s')
+        self.assertEqual(pp._titleregex, '(?P<title>.+)\ \-\ (?P<artist>.+)')
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index be9402fdb..4e6e47d6f 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -738,10 +738,10 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--metadata-from-title',
         metavar='FORMAT', dest='metafromtitle',
-        help='parse additional metadata like song title / artist from the video title. \n'
+        help='parse additional metadata like song title / artist from the video title. '
              'The format syntax is the same as --output, '
-             'the parsed parameters replace existing values.\n'
-             'Additional templates: %(songtitle), %(album), %(artist). \n'
+             'the parsed parameters replace existing values. '
+             'Additional templates: %(album), %(artist). '
              'Example: --metadata-from-title "%(artist)s - %(title)s" matches a title like '
              '"Coldplay - Paradise"')
     postproc.add_option(
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index a17113cbf..b6f51cfd5 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -541,9 +541,7 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor):
 class FFmpegMetadataPP(FFmpegPostProcessor):
     def run(self, info):
         metadata = {}
-        if info.get('songtitle') is not None:
-            metadata['title'] = info['songtitle']
-        elif info.get('title') is not None:
+        if info.get('title') is not None:
             metadata['title'] = info['title']
         if info.get('upload_date') is not None:
             metadata['date'] = info['upload_date']
diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
index 4c9d3aafe..5019433d3 100644
--- a/youtube_dl/postprocessor/metadatafromtitle.py
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
 
 import re
 
@@ -12,20 +12,19 @@ class MetadataFromTitlePPError(PostProcessingError):
 
 class MetadataFromTitlePP(PostProcessor):
     def __init__(self, downloader, titleformat):
+        super(MetadataFromTitlePP, self).__init__(downloader)
         self._titleformat = titleformat
-        self._titleregex = self.fmtToRegex(titleformat)
+        self._titleregex = self.format_to_regex(titleformat)
 
-    def fmtToRegex(self, fmt):
+    def format_to_regex(self, fmt):
         """
         Converts a string like
            '%(title)s - %(artist)s'
         to a regex like
            '(?P<title>.+)\ \-\ (?P<artist>.+)'
-        and a list of the named groups [title, artist]
         """
         lastpos = 0
         regex = ""
-        groups = []
         # replace %(..)s with regex group and escape other string parts
         for match in re.finditer(r'%\((\w+)\)s', fmt):
             regex += re.escape(fmt[lastpos:match.start()])

From 4d1652484f4a81b8d014159e680f7e20feae85e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Mar 2015 20:25:37 +0100
Subject: [PATCH 1865/1937] [test/unicode_literals] Don't look into the .git
 and .tox directories

The .tox directory contains python code that we can't control
---
 test/test_unicode_literals.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/test/test_unicode_literals.py b/test/test_unicode_literals.py
index 7f816698e..6c1b7ec91 100644
--- a/test/test_unicode_literals.py
+++ b/test/test_unicode_literals.py
@@ -17,13 +17,22 @@ IGNORED_FILES = [
     'buildserver.py',
 ]
 
+IGNORED_DIRS = [
+    '.git',
+    '.tox',
+]
 
 from test.helper import assertRegexpMatches
 
 
 class TestUnicodeLiterals(unittest.TestCase):
     def test_all_files(self):
-        for dirpath, _, filenames in os.walk(rootDir):
+        for dirpath, dirnames, filenames in os.walk(rootDir):
+            for ignore_dir in IGNORED_DIRS:
+                if ignore_dir in dirnames:
+                    # If we remove the directory from dirnames os.walk won't
+                    # recurse into it
+                    dirnames.remove(ignore_dir)
             for basename in filenames:
                 if not basename.endswith('.py'):
                     continue

From 8508557e77362bf1e5bcc919e89e6b4aa5d101ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Mar 2015 20:51:42 +0100
Subject: [PATCH 1866/1937] [test/YoutubeDL] Use valid urls

It failed on python 3.4 when building the http_headers field
---
 test/test_YoutubeDL.py | 46 ++++++++++++++++++++++--------------------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 055e42555..db8a47d2d 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -15,6 +15,8 @@ from youtube_dl import YoutubeDL
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.postprocessor.common import PostProcessor
 
+TEST_URL = 'http://localhost/sample.mp4'
+
 
 class YDL(FakeYDL):
     def __init__(self, *args, **kwargs):
@@ -46,8 +48,8 @@ class TestFormatSelection(unittest.TestCase):
         ydl = YDL()
         ydl.params['prefer_free_formats'] = True
         formats = [
-            {'ext': 'webm', 'height': 460, 'url': 'x'},
-            {'ext': 'mp4', 'height': 460, 'url': 'y'},
+            {'ext': 'webm', 'height': 460, 'url': TEST_URL},
+            {'ext': 'mp4', 'height': 460, 'url': TEST_URL},
         ]
         info_dict = _make_result(formats)
         yie = YoutubeIE(ydl)
@@ -60,8 +62,8 @@ class TestFormatSelection(unittest.TestCase):
         ydl = YDL()
         ydl.params['prefer_free_formats'] = True
         formats = [
-            {'ext': 'webm', 'height': 720, 'url': 'a'},
-            {'ext': 'mp4', 'height': 1080, 'url': 'b'},
+            {'ext': 'webm', 'height': 720, 'url': TEST_URL},
+            {'ext': 'mp4', 'height': 1080, 'url': TEST_URL},
         ]
         info_dict['formats'] = formats
         yie = YoutubeIE(ydl)
@@ -74,9 +76,9 @@ class TestFormatSelection(unittest.TestCase):
         ydl = YDL()
         ydl.params['prefer_free_formats'] = False
         formats = [
-            {'ext': 'webm', 'height': 720, 'url': '_'},
-            {'ext': 'mp4', 'height': 720, 'url': '_'},
-            {'ext': 'flv', 'height': 720, 'url': '_'},
+            {'ext': 'webm', 'height': 720, 'url': TEST_URL},
+            {'ext': 'mp4', 'height': 720, 'url': TEST_URL},
+            {'ext': 'flv', 'height': 720, 'url': TEST_URL},
         ]
         info_dict['formats'] = formats
         yie = YoutubeIE(ydl)
@@ -88,8 +90,8 @@ class TestFormatSelection(unittest.TestCase):
         ydl = YDL()
         ydl.params['prefer_free_formats'] = False
         formats = [
-            {'ext': 'flv', 'height': 720, 'url': '_'},
-            {'ext': 'webm', 'height': 720, 'url': '_'},
+            {'ext': 'flv', 'height': 720, 'url': TEST_URL},
+            {'ext': 'webm', 'height': 720, 'url': TEST_URL},
         ]
         info_dict['formats'] = formats
         yie = YoutubeIE(ydl)
@@ -133,10 +135,10 @@ class TestFormatSelection(unittest.TestCase):
 
     def test_format_selection(self):
         formats = [
-            {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': '_'},
-            {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': '_'},
-            {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': '_'},
-            {'format_id': '2', 'ext': 'flv', 'preference': 4, 'url': '_'},
+            {'format_id': '35', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
+            {'format_id': '45', 'ext': 'webm', 'preference': 2, 'url': TEST_URL},
+            {'format_id': '47', 'ext': 'webm', 'preference': 3, 'url': TEST_URL},
+            {'format_id': '2', 'ext': 'flv', 'preference': 4, 'url': TEST_URL},
         ]
         info_dict = _make_result(formats)
 
@@ -167,10 +169,10 @@ class TestFormatSelection(unittest.TestCase):
 
     def test_format_selection_audio(self):
         formats = [
-            {'format_id': 'audio-low', 'ext': 'webm', 'preference': 1, 'vcodec': 'none', 'url': '_'},
-            {'format_id': 'audio-mid', 'ext': 'webm', 'preference': 2, 'vcodec': 'none', 'url': '_'},
-            {'format_id': 'audio-high', 'ext': 'flv', 'preference': 3, 'vcodec': 'none', 'url': '_'},
-            {'format_id': 'vid', 'ext': 'mp4', 'preference': 4, 'url': '_'},
+            {'format_id': 'audio-low', 'ext': 'webm', 'preference': 1, 'vcodec': 'none', 'url': TEST_URL},
+            {'format_id': 'audio-mid', 'ext': 'webm', 'preference': 2, 'vcodec': 'none', 'url': TEST_URL},
+            {'format_id': 'audio-high', 'ext': 'flv', 'preference': 3, 'vcodec': 'none', 'url': TEST_URL},
+            {'format_id': 'vid', 'ext': 'mp4', 'preference': 4, 'url': TEST_URL},
         ]
         info_dict = _make_result(formats)
 
@@ -185,8 +187,8 @@ class TestFormatSelection(unittest.TestCase):
         self.assertEqual(downloaded['format_id'], 'audio-low')
 
         formats = [
-            {'format_id': 'vid-low', 'ext': 'mp4', 'preference': 1, 'url': '_'},
-            {'format_id': 'vid-high', 'ext': 'mp4', 'preference': 2, 'url': '_'},
+            {'format_id': 'vid-low', 'ext': 'mp4', 'preference': 1, 'url': TEST_URL},
+            {'format_id': 'vid-high', 'ext': 'mp4', 'preference': 2, 'url': TEST_URL},
         ]
         info_dict = _make_result(formats)
 
@@ -228,9 +230,9 @@ class TestFormatSelection(unittest.TestCase):
 
     def test_format_selection_video(self):
         formats = [
-            {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': '_'},
-            {'format_id': 'dash-video-high', 'ext': 'mp4', 'preference': 2, 'acodec': 'none', 'url': '_'},
-            {'format_id': 'vid', 'ext': 'mp4', 'preference': 3, 'url': '_'},
+            {'format_id': 'dash-video-low', 'ext': 'mp4', 'preference': 1, 'acodec': 'none', 'url': TEST_URL},
+            {'format_id': 'dash-video-high', 'ext': 'mp4', 'preference': 2, 'acodec': 'none', 'url': TEST_URL},
+            {'format_id': 'vid', 'ext': 'mp4', 'preference': 3, 'url': TEST_URL},
         ]
         info_dict = _make_result(formats)
 

From 873383e9bd2496a31870e6456007463c1f873b33 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Mar 2015 21:41:15 +0100
Subject: [PATCH 1867/1937] tox.ini: Run the same command as 'make offlinetest'
 by default

---
 tox.ini | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index ed01e3386..6fe7fc8f5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,5 +4,8 @@ envlist = py26,py27,py33
 deps =
    nose
    coverage
-commands = nosetests --verbose {posargs:test}  # --with-coverage --cover-package=youtube_dl --cover-html
+defaultargs = test --exclude test_download.py --exclude test_age_restriction.py
+    --exclude test_subtitles.py --exclude test_write_annotations.py
+    --exclude test_youtube_lists.py
+commands = nosetests --verbose {posargs:{[testenv]defaultargs}}  # --with-coverage --cover-package=youtube_dl --cover-html
                                                # test.test_download:TestDownload.test_NowVideo

From 7c42327e0e2f62a64771769e17cda33ea2a4a10d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 14 Mar 2015 21:41:56 +0100
Subject: [PATCH 1868/1937] tox.ini: Add python 3.4

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 6fe7fc8f5..00c6e00e3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py26,py27,py33
+envlist = py26,py27,py33,py34
 [testenv]
 deps =
    nose

From 8f4cc22455f1c08f8e95f867f40c9f378bfe7a49 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Mar 2015 10:08:14 +0600
Subject: [PATCH 1869/1937] [aftenposten] Adapt to new URL format

---
 youtube_dl/extractor/aftenposten.py | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/aftenposten.py b/youtube_dl/extractor/aftenposten.py
index 2b257ede7..e15c015fb 100644
--- a/youtube_dl/extractor/aftenposten.py
+++ b/youtube_dl/extractor/aftenposten.py
@@ -14,10 +14,10 @@ from ..utils import (
 
 
 class AftenpostenIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?aftenposten\.no/webtv/([^/]+/)*(?P<id>[^/]+)-\d+\.html'
+    _VALID_URL = r'https?://(?:www\.)?aftenposten\.no/webtv/(?:#!/)?video/(?P<id>\d+)'
 
     _TEST = {
-        'url': 'http://www.aftenposten.no/webtv/serier-og-programmer/sweatshopenglish/TRAILER-SWEATSHOP---I-cant-take-any-more-7800835.html?paging=&section=webtv_serierogprogrammer_sweatshop_sweatshopenglish',
+        'url': 'http://www.aftenposten.no/webtv/#!/video/21039/trailer-sweatshop-i-can-t-take-any-more',
         'md5': 'fd828cd29774a729bf4d4425fe192972',
         'info_dict': {
             'id': '21039',
@@ -30,12 +30,7 @@ class AftenpostenIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, display_id)
-
-        video_id = self._html_search_regex(
-            r'data-xs-id="(\d+)"', webpage, 'video id')
+        video_id = self._match_id(url)
 
         data = self._download_xml(
             'http://frontend.xstream.dk/ap/feed/video/?platform=web&id=%s' % video_id, video_id)

From 613b2d9dc6eaf515898516c654f79cee8951a269 Mon Sep 17 00:00:00 2001
From: Kang Hyojun <admire9@gmail.com>
Date: Sun, 15 Mar 2015 20:18:23 +0900
Subject: [PATCH 1870/1937] Fix mistyped docstring indent

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e779fc9a8..5a83bc956 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -635,7 +635,7 @@ class YoutubeDL(object):
         Returns a list with a dictionary for each video we find.
         If 'download', also downloads the videos.
         extra_info is a dict containing the extra values to add to each result
-         '''
+        '''
 
         if ie_key:
             ies = [self.get_info_extractor(ie_key)]

From ec1b9577ba31b7b078eec370680f84909b2e70e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 15 Mar 2015 22:42:13 +0600
Subject: [PATCH 1871/1937] [cloudy] Fix key extraction (Closes #5211)

---
 youtube_dl/extractor/cloudy.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cloudy.py b/youtube_dl/extractor/cloudy.py
index abf8cc280..0fa720ee8 100644
--- a/youtube_dl/extractor/cloudy.py
+++ b/youtube_dl/extractor/cloudy.py
@@ -105,6 +105,7 @@ class CloudyIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         file_key = self._search_regex(
-            r'filekey\s*=\s*"([^"]+)"', webpage, 'file_key')
+            [r'key\s*:\s*"([^"]+)"', r'filekey\s*=\s*"([^"]+)"'],
+            webpage, 'file_key')
 
         return self._extract_video(video_host, video_id, file_key)

From 39aa42ffbbe4df95ded3e70d4126808898a43f24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 16 Mar 2015 00:21:38 +0600
Subject: [PATCH 1872/1937] [ard] Capture and output time restricted videos
 (Closes #5213)

---
 youtube_dl/extractor/ard.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 783b53e23..6a35ea463 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -50,6 +50,9 @@ class ARDMediathekIE(InfoExtractor):
         if '>Der gewünschte Beitrag ist nicht mehr verfügbar.<' in webpage:
             raise ExtractorError('Video %s is no longer available' % video_id, expected=True)
 
+        if 'Diese Sendung ist für Jugendliche unter 12 Jahren nicht geeignet. Der Clip ist deshalb nur von 20 bis 6 Uhr verfügbar.' in webpage:
+            raise ExtractorError('This program is only suitable for those aged 12 and older. Video %s is therefore only available between 20 pm and 6 am.' % video_id, expected=True)
+
         if re.search(r'[\?&]rss($|[=&])', url):
             doc = parse_xml(webpage)
             if doc.tag == 'rss':

From 1de4ac1385bf0bedd753f41496055c53a14c20ca Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 15 Mar 2015 19:38:50 +0100
Subject: [PATCH 1873/1937] release 2015.03.15

---
 README.md              | 3 +++
 docs/supportedsites.md | 8 ++++++++
 youtube_dl/version.py  | 2 +-
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index f4fbf0034..4f9fc8174 100644
--- a/README.md
+++ b/README.md
@@ -228,6 +228,9 @@ which means you can modify it, redistribute it or use it however you like.
     --embed-subs                     embed subtitles in the video (only for mp4 videos)
     --embed-thumbnail                embed thumbnail in the audio as cover art
     --add-metadata                   write metadata to the video file
+    --metadata-from-title FORMAT     parse additional metadata like song title / artist from the video title. The format syntax is the same as --output, the parsed
+                                     parameters replace existing values. Additional templates: %(album), %(artist). Example: --metadata-from-title "%(artist)s -
+                                     %(title)s" matches a title like "Coldplay - Paradise"
     --xattrs                         write metadata to the video file's xattrs (using dublin core and xdg standards)
     --fixup POLICY                   Automatically correct known faults of the file. One of never (do nothing), warn (only emit a warning), detect_or_warn(the default;
                                      fix file if we can, warn otherwise)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 80a696ee3..d6a1e67c6 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -47,6 +47,7 @@
  - **Bandcamp**
  - **Bandcamp:album**
  - **bbc.co.uk**: BBC iPlayer
+ - **BeatportPro**
  - **Beeg**
  - **BehindKink**
  - **Bet**
@@ -145,6 +146,7 @@
  - **Firstpost**
  - **Flickr**
  - **Folketinget**: Folketinget (ft.dk; Danish parliament)
+ - **FootyRoom**
  - **Foxgay**
  - **FoxNews**
  - **france2.fr:generation-quoi**
@@ -213,6 +215,7 @@
  - **jpopsuki.tv**
  - **Jukebox**
  - **Kaltura**
+ - **KanalPlay**: Kanal 5/9/11 Play
  - **Kankan**
  - **Karaoketv**
  - **keek**
@@ -414,6 +417,7 @@
  - **SportBox**
  - **SportDeutschland**
  - **SRMediathek**: Saarländischer Rundfunk
+ - **SSA**
  - **stanfordoc**: Stanford Open ClassRoom
  - **Steam**
  - **streamcloud.eu**
@@ -510,6 +514,7 @@
  - **Vidzi**
  - **vier**
  - **vier:videos**
+ - **Viewster**
  - **viki**
  - **vimeo**
  - **vimeo:album**
@@ -556,6 +561,9 @@
  - **XXXYMovies**
  - **Yahoo**: Yahoo screen and movies
  - **Yam**
+ - **yandexmusic:album**: Яндекс.Музыка - Альбом
+ - **yandexmusic:playlist**: Яндекс.Музыка - Плейлист
+ - **yandexmusic:track**: Яндекс.Музыка - Трек
  - **YesJapan**
  - **Ynet**
  - **YouJizz**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1f0c88a4d..7ed07c375 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.03.09'
+__version__ = '2015.03.15'

From d5b559393b793f21f6566321fa08a6bbbe6f524e Mon Sep 17 00:00:00 2001
From: Eduardo Ferro <eduardo.ferro.aldama@gmail.com>
Date: Sun, 15 Mar 2015 21:45:14 +0100
Subject: [PATCH 1874/1937] [rtve] Add new extractor for rtve infantil

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/rtve.py     | 43 ++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ad133603f..1bb3e1a1c 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -405,7 +405,7 @@ from .rtlnow import RTLnowIE
 from .rtl2 import RTL2IE
 from .rtp import RTPIE
 from .rts import RTSIE
-from .rtve import RTVEALaCartaIE, RTVELiveIE
+from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE
 from .ruhd import RUHDIE
 from .rutube import (
     RutubeIE,
diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index b42442d12..b701a322a 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -126,6 +126,49 @@ class RTVEALaCartaIE(InfoExtractor):
             (s['lang'], [{'ext': 'vtt', 'url': s['src']}])
             for s in subs)
 
+class RTVEInfantilIE(InfoExtractor):
+    IE_NAME = 'rtve.es:alacarta'
+    IE_DESC = 'RTVE a la carta'
+    _VALID_URL = r'https?://(?:www\.)?rtve\.es/infantil/serie/(?P<show>[^/]*)/video/(?P<short_tittle>[^/]*)/(?P<id>[0-9]+)/'
+
+    _TESTS = [{
+        'url': 'http://www.rtve.es/infantil/serie/cleo/video/maneras-vivir/3040283/',
+        'md5': '915319587b33720b8e0357caaa6617e6',
+        'info_dict': {
+            'id': '3040283',
+            'ext': 'mp4',
+            'title': 'Maneras de vivir',
+            'thumbnail': 'http://www.rtve.es/resources/jpg/6/5/1426182947956.JPG',
+            'duration': 357.958,
+        },
+    },]
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        short_tittle = mobj.group('short_tittle')
+        info = self._download_json(
+            'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
+            video_id)['page']['items'][0]
+
+        webpage = self._download_webpage(url, video_id)
+        vidplayer_id = self._search_regex(
+            r' id="vidplayer([0-9]+)"', webpage, 'internal video ID')
+
+        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % vidplayer_id
+        png = self._download_webpage(png_url, video_id, 'Downloading url information')
+        video_url = _decrypt_url(png)
+
+        return {
+            'id': video_id,
+            'ext': 'mp4',
+            'title': info['title'],
+            'url': video_url,
+            'thumbnail': info.get('image'),
+            'duration': float_or_none(info.get('duration'), scale=1000),
+        }
+
+
 
 class RTVELiveIE(InfoExtractor):
     IE_NAME = 'rtve.es:live'

From b68eedba23cbb4cc419e3568055bfb86c5975054 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 15 Mar 2015 22:17:40 +0100
Subject: [PATCH 1875/1937] [rtve.es:infantil] Minor fixes (closes #5214)

---
 youtube_dl/extractor/rtve.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index b701a322a..13f071077 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -126,10 +126,11 @@ class RTVEALaCartaIE(InfoExtractor):
             (s['lang'], [{'ext': 'vtt', 'url': s['src']}])
             for s in subs)
 
+
 class RTVEInfantilIE(InfoExtractor):
-    IE_NAME = 'rtve.es:alacarta'
-    IE_DESC = 'RTVE a la carta'
-    _VALID_URL = r'https?://(?:www\.)?rtve\.es/infantil/serie/(?P<show>[^/]*)/video/(?P<short_tittle>[^/]*)/(?P<id>[0-9]+)/'
+    IE_NAME = 'rtve.es:infantil'
+    IE_DESC = 'RTVE infantil'
+    _VALID_URL = r'https?://(?:www\.)?rtve\.es/infantil/serie/(?P<show>[^/]*)/video/(?P<short_title>[^/]*)/(?P<id>[0-9]+)/'
 
     _TESTS = [{
         'url': 'http://www.rtve.es/infantil/serie/cleo/video/maneras-vivir/3040283/',
@@ -141,12 +142,10 @@ class RTVEInfantilIE(InfoExtractor):
             'thumbnail': 'http://www.rtve.es/resources/jpg/6/5/1426182947956.JPG',
             'duration': 357.958,
         },
-    },]
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        short_tittle = mobj.group('short_tittle')
+        video_id = self._match_id(url)
         info = self._download_json(
             'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
             video_id)['page']['items'][0]
@@ -169,7 +168,6 @@ class RTVEInfantilIE(InfoExtractor):
         }
 
 
-
 class RTVELiveIE(InfoExtractor):
     IE_NAME = 'rtve.es:live'
     IE_DESC = 'RTVE.es live streams'

From 90183a46d85758c59814187d9c777e4237a66afe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 15 Mar 2015 22:49:03 +0100
Subject: [PATCH 1876/1937] Credit @eferro for the rtve.es:infantil extractor
 (#5214)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index c10f03b98..872da6071 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -115,3 +115,4 @@ Leslie P. Polzer
 Duncan Keall
 Alexander Mamay
 Devin J. Pohly
+Eduardo Ferro Aldama

From 2e90dff2c2ecade8afb444b086fbc0ad6d2c812d Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Mon, 16 Mar 2015 20:05:02 +0100
Subject: [PATCH 1877/1937] The Daily Show Podcast support

---
 youtube_dl/extractor/__init__.py      |  3 +-
 youtube_dl/extractor/comedycentral.py | 21 ++++++++++++++
 youtube_dl/extractor/libsyn.py        | 41 +++++++++++++++++++++++++++
 3 files changed, 64 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/libsyn.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1bb3e1a1c..e94779d40 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -84,7 +84,7 @@ from .cnn import (
 )
 from .collegehumor import CollegeHumorIE
 from .collegerama import CollegeRamaIE
-from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
+from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE, TheDailyShowPodcastIE
 from .comcarcoff import ComCarCoffIE
 from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
 from .condenast import CondeNastIE
@@ -250,6 +250,7 @@ from .letv import (
     LetvPlaylistIE
 )
 from .lifenews import LifeNewsIE
+from .libsyn import LibsynIE
 from .liveleak import LiveLeakIE
 from .livestream import (
     LivestreamIE,
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index e5edcc84b..e427b9821 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import re
 
+from .common import InfoExtractor
 from .mtv import MTVServicesInfoExtractor
 from ..compat import (
     compat_str,
@@ -272,3 +273,23 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
             'title': show_name + ' ' + title,
             'description': description,
         }
+
+class TheDailyShowPodcastIE(InfoExtractor):
+    _VALID_URL = r'(?P<scheme>https?:)?//thedailyshow\.cc\.com/podcast/(?P<id>[a-z\-]+)'
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        player_url = self._search_regex(r'<iframe(?:\s+[^>]+)?\s*src="((?:https?:)?//html5-player\.libsyn\.com/embed/episode/id/[0-9]+)', webpage, 'player URL')
+        if player_url.startswith('//'):
+            mobj = re.match(self._VALID_URL, url)
+            scheme = mobj.group('scheme')
+            if not scheme:
+                scheme = 'https:'
+            player_url = scheme + player_url
+
+        return {
+            '_type': 'url_transparent',
+            'url': player_url,
+        }
diff --git a/youtube_dl/extractor/libsyn.py b/youtube_dl/extractor/libsyn.py
new file mode 100644
index 000000000..4b5029f89
--- /dev/null
+++ b/youtube_dl/extractor/libsyn.py
@@ -0,0 +1,41 @@
+# encoding: utf-8
+from .common import InfoExtractor
+from ..utils import (
+    unified_strdate,
+)
+
+class LibsynIE(InfoExtractor):
+    _VALID_URL = r'(?:https?:)?//html5-player\.libsyn\.com/embed/episode/id/(?P<id>[0-9]+)(?:/.*)?'
+
+    def _real_extract(self, url):
+        if url.startswith('//'):
+            url = 'https:' + url
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        podcast_title         = self._search_regex(r'<h2>(.*?)</h2>', webpage, 'show title')
+        podcast_episode_title = self._search_regex(r'<h3>(.*?)</h3>', webpage, 'episode title')
+        podcast_date          = unified_strdate(self._search_regex(r'<div class="release_date">Released: (.*?)</div>', webpage, 'release date'))
+        podcast_description   = self._search_regex(r'<div id="info_text_body">(.*?)</div>', webpage, 'description')
+
+        url0 = self._search_regex(r'var mediaURLLibsyn = "(?P<url0>https?://.*)";', webpage, 'first media URL')
+        url1 = self._search_regex(r'var mediaURL = "(?P<url1>https?://.*)";', webpage, 'second media URL')
+
+        if url0 != url1:
+            formats = [{
+                'url': url0
+            }, {
+                'url': url1
+            }]
+        else:
+            formats = [{
+                'url': url0
+            }]
+
+        return {
+            'id': display_id,
+            'title': podcast_episode_title,
+            'description': podcast_description,
+            'upload_date': podcast_date,
+            'formats': formats,
+        }

From c06a9f8730f41fb24835d482114403f227bc8421 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 19:42:50 +0600
Subject: [PATCH 1878/1937] [arte+7] Check formats (Closes #5224)

---
 youtube_dl/extractor/arte.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 929dd3cc5..8273bd6c9 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -146,6 +146,7 @@ class ArteTVPlus7IE(InfoExtractor):
 
             formats.append(format)
 
+        self._check_formats(formats, video_id)
         self._sort_formats(formats)
 
         info_dict['formats'] = formats

From 576904bce64ca036a7b21b43fa3f8c023e0bcdb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 20:01:31 +0600
Subject: [PATCH 1879/1937] [letv] Clarify download message

---
 youtube_dl/extractor/letv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/letv.py b/youtube_dl/extractor/letv.py
index 9ed81a199..1484ac0d2 100644
--- a/youtube_dl/extractor/letv.py
+++ b/youtube_dl/extractor/letv.py
@@ -94,7 +94,7 @@ class LetvIE(InfoExtractor):
 
         play_json = self._download_json(
             play_json_req,
-            media_id, 'playJson data')
+            media_id, 'Downloading playJson data')
 
         # Check for errors
         playstatus = play_json['playstatus']

From 733be371af58ec63be5faa52ee24cab4dd85d388 Mon Sep 17 00:00:00 2001
From: Jeff Buchbinder <jeff@ourexchange.net>
Date: Thu, 15 Jan 2015 21:28:57 -0500
Subject: [PATCH 1880/1937] Add megavideoz.eu support.

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/megavideozeu.py | 39 ++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 youtube_dl/extractor/megavideozeu.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1bb3e1a1c..5316af2d1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -267,6 +267,7 @@ from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
 from .malemotion import MalemotionIE
 from .mdr import MDRIE
+from .megavideozeu import MegavideozeuIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
diff --git a/youtube_dl/extractor/megavideozeu.py b/youtube_dl/extractor/megavideozeu.py
new file mode 100644
index 000000000..e77b5f734
--- /dev/null
+++ b/youtube_dl/extractor/megavideozeu.py
@@ -0,0 +1,39 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_filesize,
+    unified_strdate,
+)
+
+
+class MegavideozeuIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>.*)(?:.*)'
+
+    def _real_extract(self, url):
+        tmp_video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, tmp_video_id)
+
+        config_php = self._html_search_regex(
+            r'var cnf = \'([^\']+)\'', webpage, 'config.php url')
+
+	configpage = self._download_webpage(config_php, tmp_video_id)
+
+        video_id = self._html_search_regex(
+            r'<mediaid>([^<]+)', configpage, 'video id')
+        video_url = self._html_search_regex(
+            r'<file>([^<]+)', configpage, 'video URL')
+        title = self._html_search_regex(
+            r'<title><!\[CDATA\[([^\]]+)', configpage, 'title')
+        duration = int_or_none(self._html_search_regex(
+            r'<duration>([0-9]+)', configpage, 'duration', fatal=False))
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'duration': duration
+        }

From d41a3fa1b47fc050e206bd16feed02b6a62e7d2d Mon Sep 17 00:00:00 2001
From: Jeff Buchbinder <jeff@ourexchange.net>
Date: Wed, 4 Mar 2015 15:02:13 -0500
Subject: [PATCH 1881/1937] [Primesharetv] Add primeshare.tv extractor, still
 need test data

---
 youtube_dl/extractor/__init__.py     |  1 +
 youtube_dl/extractor/primesharetv.py | 46 ++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 youtube_dl/extractor/primesharetv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5316af2d1..b02812365 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -381,6 +381,7 @@ from .pornhub import (
 )
 from .pornotube import PornotubeIE
 from .pornoxo import PornoXOIE
+from .primesharetv import PrimesharetvIE
 from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
 from .puls4 import Puls4IE
diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
new file mode 100644
index 000000000..967125abc
--- /dev/null
+++ b/youtube_dl/extractor/primesharetv.py
@@ -0,0 +1,46 @@
+# encoding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_filesize,
+    unified_strdate,
+    urlencode_postdata,
+)
+from ..compat import (
+    compat_urllib_request,
+)
+
+class PrimesharetvIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?primeshare\.tv/download/(?P<id>.*)(?:.*)'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+       
+        self._sleep(9, video_id)
+        
+        hashtoken = self._search_regex(r' name="hash" value="(.*?)" ', webpage, 'hash token')
+        data = urlencode_postdata({
+            'hash': hashtoken,
+        })
+        headers = {
+            'Referer': url,
+            'Content-Type': 'application/x-www-form-urlencoded',
+        }
+        video_page_request = compat_urllib_request.Request(url, data, headers=headers)
+        video_page = self._download_webpage(video_page_request, None, False, '')
+
+        video_url = self._html_search_regex(
+            r'url: \'(http://l\.primeshare\.tv[^\']+)\',', video_page, 'video url')
+
+        title = self._html_search_regex(
+            r'<h1>Watch&nbsp;[^\(]+\(([^/)]+)\)&nbsp;', video_page, 'title')
+
+        return {
+            'id': video_id,
+            'url': video_url,
+            'title': title,
+            'ext': 'mp4',
+        }

From af69cab21d48da78ff7d2b54effb9dfd5fcfd1d8 Mon Sep 17 00:00:00 2001
From: Jeff Buchbinder <jeff@ourexchange.net>
Date: Wed, 4 Mar 2015 15:18:06 -0500
Subject: [PATCH 1882/1937] [Primesharetv] Add public domain example video

---
 youtube_dl/extractor/primesharetv.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 967125abc..7c545761b 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -15,6 +15,19 @@ from ..compat import (
 class PrimesharetvIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?primeshare\.tv/download/(?P<id>.*)(?:.*)'
 
+    _TESTS = [
+        {
+            'url': 'http://primeshare.tv/download/238790B611',
+            'md5': 'bb41f9f6c0dd434c729f04ce5b677192',
+            'info_dict': {
+                'id': '238790B611',
+                'ext': 'mp4',
+                "title": "Public Domain - 1960s Commercial - Crest Toothpaste-YKsuFona [...]",
+                "duration": 10,
+            },
+        }
+    ]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -33,7 +46,7 @@ class PrimesharetvIE(InfoExtractor):
         video_page = self._download_webpage(video_page_request, None, False, '')
 
         video_url = self._html_search_regex(
-            r'url: \'(http://l\.primeshare\.tv[^\']+)\',', video_page, 'video url')
+            r'url: \'(http://[a-z0-9]+\.primeshare\.tv:443/file/get/[^\']+)\',', video_page, 'video url')
 
         title = self._html_search_regex(
             r'<h1>Watch&nbsp;[^\(]+\(([^/)]+)\)&nbsp;', video_page, 'title')

From 13047f41359e32bf56bdc8dd555bae6c1065512d Mon Sep 17 00:00:00 2001
From: Jeff Buchbinder <jeff@ourexchange.net>
Date: Wed, 4 Mar 2015 17:38:21 -0500
Subject: [PATCH 1883/1937] [Primesharetv] Handle file not existing properly.

---
 youtube_dl/extractor/primesharetv.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 7c545761b..570fd2210 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -1,8 +1,11 @@
 # encoding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     int_or_none,
     parse_filesize,
     unified_strdate,
@@ -31,10 +34,12 @@ class PrimesharetvIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
+        if re.search(r'<h1>File not exist</h1>', webpage) is not None:
+            raise ExtractorError('The file does not exist', expected=True)
+        hashtoken = self._search_regex(r' name="hash" value="(.*?)" ', webpage, 'hash token')
        
         self._sleep(9, video_id)
         
-        hashtoken = self._search_regex(r' name="hash" value="(.*?)" ', webpage, 'hash token')
         data = urlencode_postdata({
             'hash': hashtoken,
         })
@@ -44,7 +49,6 @@ class PrimesharetvIE(InfoExtractor):
         }
         video_page_request = compat_urllib_request.Request(url, data, headers=headers)
         video_page = self._download_webpage(video_page_request, None, False, '')
-
         video_url = self._html_search_regex(
             r'url: \'(http://[a-z0-9]+\.primeshare\.tv:443/file/get/[^\']+)\',', video_page, 'video url')
 
@@ -57,3 +61,8 @@ class PrimesharetvIE(InfoExtractor):
             'title': title,
             'ext': 'mp4',
         }
+    
+    def _debug_print(self, txt):
+        if self._downloader.params.get('verbose'):
+            self.to_screen('[debug] %s' % txt)
+

From 0499cd866e7e746658f33e6c2f44f7e1e699ad1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 21:06:38 +0600
Subject: [PATCH 1884/1937] [primesharetv] Clean up

---
 youtube_dl/extractor/__init__.py     |  2 +-
 youtube_dl/extractor/primesharetv.py | 83 ++++++++++++++--------------
 2 files changed, 43 insertions(+), 42 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index b02812365..3c5401145 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -381,7 +381,7 @@ from .pornhub import (
 )
 from .pornotube import PornotubeIE
 from .pornoxo import PornoXOIE
-from .primesharetv import PrimesharetvIE
+from .primesharetv import PrimeShareTVIE
 from .promptfile import PromptFileIE
 from .prosiebensat1 import ProSiebenSat1IE
 from .puls4 import Puls4IE
diff --git a/youtube_dl/extractor/primesharetv.py b/youtube_dl/extractor/primesharetv.py
index 570fd2210..01cc3d9ea 100644
--- a/youtube_dl/extractor/primesharetv.py
+++ b/youtube_dl/extractor/primesharetv.py
@@ -1,59 +1,65 @@
-# encoding: utf-8
 from __future__ import unicode_literals
 
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    ExtractorError,
-    int_or_none,
-    parse_filesize,
-    unified_strdate,
-    urlencode_postdata,
-)
 from ..compat import (
+    compat_urllib_parse,
     compat_urllib_request,
 )
+from ..utils import ExtractorError
 
-class PrimesharetvIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?primeshare\.tv/download/(?P<id>.*)(?:.*)'
 
-    _TESTS = [
-        {
-            'url': 'http://primeshare.tv/download/238790B611',
-            'md5': 'bb41f9f6c0dd434c729f04ce5b677192',
-            'info_dict': {
-                'id': '238790B611',
-                'ext': 'mp4',
-                "title": "Public Domain - 1960s Commercial - Crest Toothpaste-YKsuFona [...]",
-                "duration": 10,
-            },
-        }
-    ]
+class PrimeShareTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?primeshare\.tv/download/(?P<id>[\da-zA-Z]+)'
+
+    _TEST = {
+        'url': 'http://primeshare.tv/download/238790B611',
+        'md5': 'b92d9bf5461137c36228009f31533fbc',
+        'info_dict': {
+            'id': '238790B611',
+            'ext': 'mp4',
+            'title': 'Public Domain - 1960s Commercial - Crest Toothpaste-YKsuFona',
+        },
+    }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+
         webpage = self._download_webpage(url, video_id)
-        if re.search(r'<h1>File not exist</h1>', webpage) is not None:
-            raise ExtractorError('The file does not exist', expected=True)
-        hashtoken = self._search_regex(r' name="hash" value="(.*?)" ', webpage, 'hash token')
-       
-        self._sleep(9, video_id)
-        
-        data = urlencode_postdata({
-            'hash': hashtoken,
-        })
+
+        if '>File not exist<' in webpage:
+            raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
+        fields = dict(re.findall(r'''(?x)<input\s+
+            type="hidden"\s+
+            name="([^"]+)"\s+
+            (?:id="[^"]+"\s+)?
+            value="([^"]*)"
+            ''', webpage))
+
         headers = {
             'Referer': url,
             'Content-Type': 'application/x-www-form-urlencoded',
         }
-        video_page_request = compat_urllib_request.Request(url, data, headers=headers)
-        video_page = self._download_webpage(video_page_request, None, False, '')
-        video_url = self._html_search_regex(
-            r'url: \'(http://[a-z0-9]+\.primeshare\.tv:443/file/get/[^\']+)\',', video_page, 'video url')
+
+        wait_time = int(self._search_regex(
+            r'var\s+cWaitTime\s*=\s*(\d+)',
+            webpage, 'wait time', default=7)) + 1
+        self._sleep(wait_time, video_id)
+
+        req = compat_urllib_request.Request(
+            url, compat_urllib_parse.urlencode(fields), headers)
+        video_page = self._download_webpage(
+            req, video_id, 'Downloading video page')
+
+        video_url = self._search_regex(
+            r"url\s*:\s*'([^']+\.primeshare\.tv(?::443)?/file/[^']+)'",
+            video_page, 'video url')
 
         title = self._html_search_regex(
-            r'<h1>Watch&nbsp;[^\(]+\(([^/)]+)\)&nbsp;', video_page, 'title')
+            r'<h1>Watch\s*(?:&nbsp;)?\s*\((.+?)(?:\s*\[\.\.\.\])?\)\s*(?:&nbsp;)?\s*<strong>',
+            video_page, 'title')
 
         return {
             'id': video_id,
@@ -61,8 +67,3 @@ class PrimesharetvIE(InfoExtractor):
             'title': title,
             'ext': 'mp4',
         }
-    
-    def _debug_print(self, txt):
-        if self._downloader.params.get('verbose'):
-            self.to_screen('[debug] %s' % txt)
-

From 219da6bb685765186b7ffb878399c32f44351802 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 21:13:42 +0600
Subject: [PATCH 1885/1937] [megavideoeu] Remove extractor

---
 youtube_dl/extractor/__init__.py     |  1 -
 youtube_dl/extractor/megavideozeu.py | 39 ----------------------------
 2 files changed, 40 deletions(-)
 delete mode 100644 youtube_dl/extractor/megavideozeu.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 3c5401145..bceed92e1 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -267,7 +267,6 @@ from .macgamestore import MacGameStoreIE
 from .mailru import MailRuIE
 from .malemotion import MalemotionIE
 from .mdr import MDRIE
-from .megavideozeu import MegavideozeuIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
diff --git a/youtube_dl/extractor/megavideozeu.py b/youtube_dl/extractor/megavideozeu.py
deleted file mode 100644
index e77b5f734..000000000
--- a/youtube_dl/extractor/megavideozeu.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# encoding: utf-8
-from __future__ import unicode_literals
-
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    parse_filesize,
-    unified_strdate,
-)
-
-
-class MegavideozeuIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?megavideoz\.eu/video/(?P<id>.*)(?:.*)'
-
-    def _real_extract(self, url):
-        tmp_video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, tmp_video_id)
-
-        config_php = self._html_search_regex(
-            r'var cnf = \'([^\']+)\'', webpage, 'config.php url')
-
-	configpage = self._download_webpage(config_php, tmp_video_id)
-
-        video_id = self._html_search_regex(
-            r'<mediaid>([^<]+)', configpage, 'video id')
-        video_url = self._html_search_regex(
-            r'<file>([^<]+)', configpage, 'video URL')
-        title = self._html_search_regex(
-            r'<title><!\[CDATA\[([^\]]+)', configpage, 'title')
-        duration = int_or_none(self._html_search_regex(
-            r'<duration>([0-9]+)', configpage, 'duration', fatal=False))
-
-        return {
-            'id': video_id,
-            'url': video_url,
-            'title': title,
-            'duration': duration
-        }

From 92a4793b3cac36677bed5423b12b3d97d15c6036 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 21:34:22 +0600
Subject: [PATCH 1886/1937] [utils] Place sanitize url function near other
 sanitizing functions

---
 test/test_utils.py  | 37 +++++++++++++++++++++----------------
 youtube_dl/utils.py | 22 +++++++---------------
 2 files changed, 28 insertions(+), 31 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 4f0ffd482..3431ad24e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -39,6 +39,7 @@ from youtube_dl.utils import (
     read_batch_urls,
     sanitize_filename,
     sanitize_path,
+    sanitize_url_path_consecutive_slashes,
     shell_quote,
     smuggle_url,
     str_to_int,
@@ -55,7 +56,6 @@ from youtube_dl.utils import (
     xpath_with_ns,
     render_table,
     match_str,
-    url_sanitize_consecutive_slashes,
 )
 
 
@@ -169,6 +169,26 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(sanitize_path('./abc'), 'abc')
         self.assertEqual(sanitize_path('./../abc'), '..\\abc')
 
+    def test_sanitize_url_path_consecutive_slashes(self):
+        self.assertEqual(
+            sanitize_url_path_consecutive_slashes('http://hostname/foo//bar/filename.html'),
+            'http://hostname/foo/bar/filename.html')
+        self.assertEqual(
+            sanitize_url_path_consecutive_slashes('http://hostname//foo/bar/filename.html'),
+            'http://hostname/foo/bar/filename.html')
+        self.assertEqual(
+            sanitize_url_path_consecutive_slashes('http://hostname//'),
+            'http://hostname/')
+        self.assertEqual(
+            sanitize_url_path_consecutive_slashes('http://hostname/foo/bar/filename.html'),
+            'http://hostname/foo/bar/filename.html')
+        self.assertEqual(
+            sanitize_url_path_consecutive_slashes('http://hostname/'),
+            'http://hostname/')
+        self.assertEqual(
+            sanitize_url_path_consecutive_slashes('http://hostname/abc//'),
+            'http://hostname/abc/')
+
     def test_ordered_set(self):
         self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
         self.assertEqual(orderedSet([]), [])
@@ -539,21 +559,6 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
             'like_count > 100 & dislike_count <? 50 & description',
             {'like_count': 190, 'dislike_count': 10}))
 
-    def test_url_sanitize_consecutive_slashes(self):
-        self.assertEqual(url_sanitize_consecutive_slashes(
-            'http://hostname/foo//bar/filename.html'),
-            'http://hostname/foo/bar/filename.html')
-        self.assertEqual(url_sanitize_consecutive_slashes(
-            'http://hostname//foo/bar/filename.html'),
-            'http://hostname/foo/bar/filename.html')
-        self.assertEqual(url_sanitize_consecutive_slashes(
-            'http://hostname//'), 'http://hostname/')
-        self.assertEqual(url_sanitize_consecutive_slashes(
-            'http://hostname/foo/bar/filename.html'),
-            'http://hostname/foo/bar/filename.html')
-        self.assertEqual(url_sanitize_consecutive_slashes(
-            'http://hostname/'), 'http://hostname/')
-
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e82e3998a..472d4df41 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -326,6 +326,13 @@ def sanitize_path(s):
     return os.path.join(*sanitized_path)
 
 
+def sanitize_url_path_consecutive_slashes(url):
+    """Collapses consecutive slashes in URLs' path"""
+    parsed_url = list(compat_urlparse.urlparse(url))
+    parsed_url[2] = re.sub(r'/{2,}', '/', parsed_url[2])
+    return compat_urlparse.urlunparse(parsed_url)
+
+
 def orderedSet(iterable):
     """ Remove all duplicates from the input iterable """
     res = []
@@ -1804,18 +1811,3 @@ class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
             return None  # No Proxy
         return compat_urllib_request.ProxyHandler.proxy_open(
             self, req, proxy, type)
-
-
-def url_sanitize_consecutive_slashes(url):
-    """Sanitize URLs with consecutive slashes
-
-    For example, transform both
-        http://hostname/foo//bar/filename.html
-    and
-        http://hostname//foo/bar/filename.html
-    into
-        http://hostname/foo/bar/filename.html
-    """
-    parsed_url = list(compat_urlparse.urlparse(url))
-    parsed_url[2] = re.sub(r'/{2,}', '/', parsed_url[2])
-    return compat_urlparse.urlunparse(parsed_url)

From cd459b1d490ca8c0639220a835f5e6bee3e9a80d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 21:39:31 +0600
Subject: [PATCH 1887/1937] [sohu] Fix test's note info

---
 youtube_dl/extractor/sohu.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index ea5cc06b9..11edf616a 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,7 +8,7 @@ from ..compat import (
     compat_str,
     compat_urllib_request
 )
-from ..utils import url_sanitize_consecutive_slashes
+from ..utils import sanitize_url_path_consecutive_slashes
 
 
 class SohuIE(InfoExtractor):
@@ -74,7 +74,7 @@ class SohuIE(InfoExtractor):
             }
         }]
     }, {
-        'info': 'Video with title containing dash',
+        'note': 'Video with title containing dash',
         'url': 'http://my.tv.sohu.com/us/249884221/78932792.shtml',
         'info_dict': {
             'id': '78932792',
@@ -100,8 +100,9 @@ class SohuIE(InfoExtractor):
             if cn_verification_proxy:
                 req.add_header('Ytdl-request-proxy', cn_verification_proxy)
 
-            return self._download_json(req, video_id,
-                                       'Downloading JSON data for %s' % vid_id)
+            return self._download_json(
+                req, video_id,
+                'Downloading JSON data for %s' % vid_id)
 
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
@@ -146,7 +147,7 @@ class SohuIE(InfoExtractor):
 
                 part_info = part_str.split('|')
 
-                video_url = url_sanitize_consecutive_slashes(
+                video_url = sanitize_url_path_consecutive_slashes(
                     '%s%s?key=%s' % (part_info[0], su[i], part_info[3]))
 
                 formats.append({

From 2ca1c5aa9f8d7db08608de0bd99f6c994ab496d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 22:27:33 +0600
Subject: [PATCH 1888/1937] [douyutv] Improve and extract all formats

---
 youtube_dl/extractor/__init__.py |  2 +-
 youtube_dl/extractor/douyutv.py  | 72 ++++++++++++++++++++------------
 2 files changed, 46 insertions(+), 28 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f3901bfc3..73c17aa84 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -107,7 +107,7 @@ from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .dfb import DFBIE
 from .dotsub import DotsubIE
-from .douyutv import DouyutvIE
+from .douyutv import DouyuTVIE
 from .dreisat import DreiSatIE
 from .drbonanza import DRBonanzaIE
 from .drtuber import DrTuberIE
diff --git a/youtube_dl/extractor/douyutv.py b/youtube_dl/extractor/douyutv.py
index e9b92eb3b..d7956e6e4 100644
--- a/youtube_dl/extractor/douyutv.py
+++ b/youtube_dl/extractor/douyutv.py
@@ -2,58 +2,76 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..utils import ExtractorError
 
-from ..utils import (
-    ExtractorError,
-)
 
-class DouyutvIE(InfoExtractor):
+class DouyuTVIE(InfoExtractor):
     _VALID_URL = r'http://(?:www\.)?douyutv\.com/(?P<id>[A-Za-z0-9]+)'
-
-    '''
-    show_status: 1 直播中 ，2 没有直播
-    '''
-
     _TEST = {
         'url': 'http://www.douyutv.com/iseven',
         'info_dict': {
             'id': 'iseven',
-            'title': '清晨醒脑！T-ara根本停不下来！',
             'ext': 'flv',
+            'title': 're:^清晨醒脑！T-ara根本停不下来！ [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'description': 'md5:9e525642c25a0a24302869937cf69d17',
             'thumbnail': 're:^https?://.*\.jpg$',
+            'uploader': '7师傅',
+            'uploader_id': '431925',
             'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        info_url = 'http://www.douyutv.com/api/client/room/' + video_id
 
-        config = self._download_json(info_url, video_id)
+        config = self._download_json(
+            'http://www.douyutv.com/api/client/room/%s' % video_id, video_id)
 
-        error_code = config.get('error')
-        show_status = config['data'].get('show_status')
+        data = config['data']
+
+        error_code = config.get('error', 0)
+        show_status = data.get('show_status')
         if error_code is not 0:
-            raise ExtractorError('Server reported error %i' % error_code,
-                                 expected=True)
+            raise ExtractorError(
+                'Server reported error %i' % error_code, expected=True)
 
+        # 1 = live, 2 = offline
         if show_status == '2':
-            raise ExtractorError('The live show has not yet started',
-                                 expected=True)
+            raise ExtractorError(
+                'Live stream is offline', expected=True)
 
-        title = config['data'].get('room_name')
-        rtmp_url = config['data'].get('rtmp_url')
-        rtmp_live = config['data'].get('rtmp_live')
-        thumbnail = config['data'].get('room_src') 
+        base_url = data['rtmp_url']
+        live_path = data['rtmp_live']
 
-        url = rtmp_url+'/'+rtmp_live
+        title = self._live_title(data['room_name'])
+        description = data.get('show_details')
+        thumbnail = data.get('room_src')
+
+        uploader = data.get('nickname')
+        uploader_id = data.get('owner_uid')
+
+        multi_formats = data.get('rtmp_multi_bitrate')
+        if not isinstance(multi_formats, dict):
+            multi_formats = {}
+        multi_formats['live'] = live_path
+
+        formats = [{
+            'url': '%s/%s' % (base_url, format_path),
+            'format_id': format_id,
+            'preference': 1 if format_id == 'live' else 0,
+        } for format_id, format_path in multi_formats.items()]
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
-            'ext':'flv',
-            'url': url,
+            'description': description,
             'thumbnail': thumbnail,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'formats': formats,
             'is_live': True,
-            # TODO more properties (see youtube_dl/extractor/common.py)
-        }
\ No newline at end of file
+        }

From 98f02fdde25bf00fbccfb211abb574f80819a177 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 22:33:05 +0600
Subject: [PATCH 1889/1937] Credit @jbuchbinder for primesharetv (#5123)

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 872da6071..512469f4c 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -116,3 +116,4 @@ Duncan Keall
 Alexander Mamay
 Devin J. Pohly
 Eduardo Ferro Aldama
+Jeff Buchbinder

From b1337948ebeeacfcf104d513fac5913d3a5818a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 23:13:43 +0600
Subject: [PATCH 1890/1937] [grooveshark] Fix extraction

---
 youtube_dl/extractor/grooveshark.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/grooveshark.py b/youtube_dl/extractor/grooveshark.py
index 848d17beb..36ad4915c 100644
--- a/youtube_dl/extractor/grooveshark.py
+++ b/youtube_dl/extractor/grooveshark.py
@@ -140,9 +140,9 @@ class GroovesharkIE(InfoExtractor):
 
         if webpage is not None:
             o = GroovesharkHtmlParser.extract_object_tags(webpage)
-            return (webpage, [x for x in o if x['attrs']['id'] == 'jsPlayerEmbed'])
+            return webpage, [x for x in o if x['attrs']['id'] == 'jsPlayerEmbed']
 
-        return (webpage, None)
+        return webpage, None
 
     def _real_initialize(self):
         self.ts = int(time.time() * 1000)  # timestamp in millis
@@ -154,7 +154,7 @@ class GroovesharkIE(InfoExtractor):
         swf_referer = None
         if self.do_playerpage_request:
             (_, player_objs) = self._get_playerpage(url)
-            if player_objs is not None:
+            if player_objs:
                 swf_referer = self._build_swf_referer(url, player_objs[0])
                 self.to_screen('SWF Referer: %s' % swf_referer)
 

From 84f810160667f9129e6a8d841fae2d0e2e1fec86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 17 Mar 2015 23:51:40 +0600
Subject: [PATCH 1891/1937] [generic] Follow redirects specified by `Refresh`
 HTTP header

---
 youtube_dl/extractor/generic.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 4e6927b08..0b8d96c27 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1270,8 +1270,14 @@ class GenericIE(InfoExtractor):
         if not found:
             found = re.search(
                 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
-                r'(?:[a-z-]+="[^"]+"\s+)*?content="[0-9]{,2};url=\'?([^\'"]+)',
+                r'(?:[a-z-]+="[^"]+"\s+)*?content="[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)',
                 webpage)
+            if not found:
+                # Look also in Refresh HTTP header
+                refresh_header = head_response.headers.get('Refresh')
+                if refresh_header:
+                    found = re.search(
+                        r'[0-9]{,2};\s*(?:URL|url)=(.+)', refresh_header)
             if found:
                 new_url = found.group(1)
                 self.report_following_redirect(new_url)

From 9ef4f12b534578ae3d3e47815492c90826c03c36 Mon Sep 17 00:00:00 2001
From: felix <m.p.isaev@yandex.com>
Date: Tue, 17 Mar 2015 18:54:36 +0100
Subject: [PATCH 1892/1937] testcases for libsyn and The Daily Show Podcast
 extractors

---
 youtube_dl/extractor/comedycentral.py | 4 ++++
 youtube_dl/extractor/libsyn.py        | 9 +++++++++
 2 files changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index e427b9821..bd3817b56 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -276,6 +276,10 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
 
 class TheDailyShowPodcastIE(InfoExtractor):
     _VALID_URL = r'(?P<scheme>https?:)?//thedailyshow\.cc\.com/podcast/(?P<id>[a-z\-]+)'
+    _TESTS = [{
+        "url": "http://thedailyshow.cc.com/podcast/episodetwelve",
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
diff --git a/youtube_dl/extractor/libsyn.py b/youtube_dl/extractor/libsyn.py
index 4b5029f89..88379f276 100644
--- a/youtube_dl/extractor/libsyn.py
+++ b/youtube_dl/extractor/libsyn.py
@@ -6,6 +6,15 @@ from ..utils import (
 
 class LibsynIE(InfoExtractor):
     _VALID_URL = r'(?:https?:)?//html5-player\.libsyn\.com/embed/episode/id/(?P<id>[0-9]+)(?:/.*)?'
+    _TESTS = [{
+        'url': "http://html5-player.libsyn.com/embed/episode/id/3377616/",
+        'info_dict': {
+            'id': "3377616",
+            'ext': "mp3",
+            'title': "Episode 12: Bassem Youssef: Egypt's Jon Stewart",
+            'description': "<p>Bassem Youssef joins executive producer Steve Bodow and senior producer Sara Taksler for a conversation about how&nbsp;<em style=\"font-family: Tahoma, Geneva, sans-serif; font-size: 12.8000001907349px;\">The Daily Show</em>&nbsp;inspired Bassem to create&nbsp;<em style=\"font-family: Tahoma, Geneva, sans-serif; font-size: 12.8000001907349px;\">Al-Bernameg</em>, his massively popular (and now banned) Egyptian news satire program. Sara discusses her soon-to-be-released documentary,&nbsp;<em style=\"font-family: Tahoma, Geneva, sans-serif; font-size: 12.8000001907349px;\">Tickling Giants</em>, which chronicles how Bassem and his staff risked their safety every day to tell jokes.</p>",
+        },
+    }]
 
     def _real_extract(self, url):
         if url.startswith('//'):

From ed9a25dd612fb06d9cf007a6491ac9982535a8f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Mar 2015 00:05:40 +0600
Subject: [PATCH 1893/1937] [generic] Generalize redirect regex

---
 youtube_dl/extractor/generic.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 0b8d96c27..dc5755d12 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1268,16 +1268,16 @@ class GenericIE(InfoExtractor):
             # HTML5 video
             found = re.findall(r'(?s)<video[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']', webpage)
         if not found:
+            REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
             found = re.search(
                 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
-                r'(?:[a-z-]+="[^"]+"\s+)*?content="[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)',
+                r'(?:[a-z-]+="[^"]+"\s+)*?content="%s' % REDIRECT_REGEX,
                 webpage)
             if not found:
                 # Look also in Refresh HTTP header
                 refresh_header = head_response.headers.get('Refresh')
                 if refresh_header:
-                    found = re.search(
-                        r'[0-9]{,2};\s*(?:URL|url)=(.+)', refresh_header)
+                    found = re.search(REDIRECT_REGEX, refresh_header)
             if found:
                 new_url = found.group(1)
                 self.report_following_redirect(new_url)

From c3c5c31517ce121740666440a71becaba11176ac Mon Sep 17 00:00:00 2001
From: Todoroki <todoroki.auone@gmail.com>
Date: Wed, 18 Mar 2015 22:19:55 +0900
Subject: [PATCH 1894/1937] fix nm video DL issue when logged in

---
 youtube_dl/extractor/niconico.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 7fb4e57df..a48df7f62 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -89,7 +89,7 @@ class NiconicoIE(InfoExtractor):
         if self._AUTHENTICATED:
             # Get flv info
             flv_info_webpage = self._download_webpage(
-                'http://flapi.nicovideo.jp/api/getflv?v=' + video_id,
+                'http://flapi.nicovideo.jp/api/getflv/' + video_id + '?as3=1',
                 video_id, 'Downloading flv info')
         else:
             # Get external player info
@@ -97,7 +97,6 @@ class NiconicoIE(InfoExtractor):
                 'http://ext.nicovideo.jp/thumb_watch/' + video_id, video_id)
             thumb_play_key = self._search_regex(
                 r'\'thumbPlayKey\'\s*:\s*\'(.*?)\'', ext_player_info, 'thumbPlayKey')
-
             # Get flv info
             flv_info_data = compat_urllib_parse.urlencode({
                 'k': thumb_play_key,

From ee580538fa741fc35faa8210e0245e252bf23e40 Mon Sep 17 00:00:00 2001
From: Todoroki <todoroki.auone@gmail.com>
Date: Wed, 18 Mar 2015 22:24:17 +0900
Subject: [PATCH 1895/1937] fix nm video DL issue when logged in

---
 youtube_dl/extractor/niconico.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index a48df7f62..4d57ac703 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -97,6 +97,7 @@ class NiconicoIE(InfoExtractor):
                 'http://ext.nicovideo.jp/thumb_watch/' + video_id, video_id)
             thumb_play_key = self._search_regex(
                 r'\'thumbPlayKey\'\s*:\s*\'(.*?)\'', ext_player_info, 'thumbPlayKey')
+
             # Get flv info
             flv_info_data = compat_urllib_parse.urlencode({
                 'k': thumb_play_key,

From 2c2c06e359dc3843a86585a9a4b6419f03af510e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Mar 2015 20:28:00 +0600
Subject: [PATCH 1896/1937] [krasview] Fix extraction (Closes #5228)

---
 youtube_dl/extractor/krasview.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/krasview.py b/youtube_dl/extractor/krasview.py
index e46954b47..96f95979a 100644
--- a/youtube_dl/extractor/krasview.py
+++ b/youtube_dl/extractor/krasview.py
@@ -40,8 +40,10 @@ class KrasViewIE(InfoExtractor):
         description = self._og_search_description(webpage, default=None)
         thumbnail = flashvars.get('image') or self._og_search_thumbnail(webpage)
         duration = int_or_none(flashvars.get('duration'))
-        width = int_or_none(self._og_search_property('video:width', webpage, 'video width'))
-        height = int_or_none(self._og_search_property('video:height', webpage, 'video height'))
+        width = int_or_none(self._og_search_property(
+            'video:width', webpage, 'video width', default=None))
+        height = int_or_none(self._og_search_property(
+            'video:height', webpage, 'video height', default=None))
 
         return {
             'id': video_id,

From fa8ce2690447fa06a10398efd5b6dfa96c2ed1b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Roman=20Le=20N=C3=A9grate?= <roman.lenegrate@gmail.com>
Date: Sun, 15 Mar 2015 22:32:06 +0100
Subject: [PATCH 1897/1937] [mixcloud] Fix extraction like-count

---
 youtube_dl/extractor/mixcloud.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 1831c6749..7c6c885f9 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -99,8 +99,7 @@ class MixcloudIE(InfoExtractor):
             r'\s+"profile": "([^"]+)",', webpage, 'uploader id', fatal=False)
         description = self._og_search_description(webpage)
         like_count = str_to_int(self._search_regex(
-            [r'<meta itemprop="interactionCount" content="UserLikes:([0-9]+)"',
-             r'/favorites/?">([0-9]+)<'],
+            r'\bbutton-favorite\b.+m-ajax-toggle-count="([^"]+)"',
             webpage, 'like count', fatal=False))
         view_count = str_to_int(self._search_regex(
             [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',

From 7a757b7194b1f1151857503baed3fa51d57397aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 18 Mar 2015 16:50:23 +0100
Subject: [PATCH 1898/1937] [mixcloud] Fix extraction of some metadata

The second test had some wrong info.
I couldn't find the timestamp, so I have removed it.
---
 youtube_dl/extractor/mixcloud.py | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 7c6c885f9..2fe55179a 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -10,7 +10,6 @@ from ..utils import (
     ExtractorError,
     HEADRequest,
     str_to_int,
-    parse_iso8601,
 )
 
 
@@ -27,8 +26,6 @@ class MixcloudIE(InfoExtractor):
             'description': 'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.',
             'uploader': 'Daniel Holbach',
             'uploader_id': 'dholbach',
-            'upload_date': '20111115',
-            'timestamp': 1321359578,
             'thumbnail': 're:https?://.*\.jpg',
             'view_count': int,
             'like_count': int,
@@ -37,12 +34,12 @@ class MixcloudIE(InfoExtractor):
         'url': 'http://www.mixcloud.com/gillespeterson/caribou-7-inch-vinyl-mix-chat/',
         'info_dict': {
             'id': 'gillespeterson-caribou-7-inch-vinyl-mix-chat',
-            'ext': 'm4a',
-            'title': 'Electric Relaxation vol. 3',
+            'ext': 'mp3',
+            'title': 'Caribou 7 inch Vinyl Mix & Chat',
             'description': 'md5:2b8aec6adce69f9d41724647c65875e8',
-            'uploader': 'Daniel Drumz',
+            'uploader': 'Gilles Peterson Worldwide',
             'uploader_id': 'gillespeterson',
-            'thumbnail': 're:https?://.*\.jpg',
+            'thumbnail': 're:https?://.*/images/',
             'view_count': int,
             'like_count': int,
         },
@@ -85,7 +82,7 @@ class MixcloudIE(InfoExtractor):
             raise ExtractorError('Unable to extract track url')
 
         PREFIX = (
-            r'<span class="play-button[^"]*?"'
+            r'm-play-on-spacebar[^>]+'
             r'(?:\s+[a-zA-Z0-9-]+(?:="[^"]+")?)*?\s+')
         title = self._html_search_regex(
             PREFIX + r'm-title="([^"]+)"', webpage, 'title')
@@ -105,9 +102,6 @@ class MixcloudIE(InfoExtractor):
             [r'<meta itemprop="interactionCount" content="UserPlays:([0-9]+)"',
              r'/listeners/?">([0-9,.]+)</a>'],
             webpage, 'play count', fatal=False))
-        timestamp = parse_iso8601(self._search_regex(
-            r'<time itemprop="dateCreated" datetime="([^"]+)">',
-            webpage, 'upload date', default=None))
 
         return {
             'id': track_id,
@@ -117,7 +111,6 @@ class MixcloudIE(InfoExtractor):
             'thumbnail': thumbnail,
             'uploader': uploader,
             'uploader_id': uploader_id,
-            'timestamp': timestamp,
             'view_count': view_count,
             'like_count': like_count,
         }

From aae53774f247edcca7671434c16f7931d5c7b9da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Roman=20Le=20N=C3=A9grate?= <roman.lenegrate@gmail.com>
Date: Mon, 16 Mar 2015 00:20:06 +0100
Subject: [PATCH 1899/1937] [mixcloud] Try preview server first, then further
 numbers

---
 youtube_dl/extractor/mixcloud.py | 49 ++++++++++++++++++++++++++------
 1 file changed, 41 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 2fe55179a..21aea0c55 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import re
+import itertools
 
 from .common import InfoExtractor
 from ..compat import (
@@ -45,20 +46,19 @@ class MixcloudIE(InfoExtractor):
         },
     }]
 
-    def _get_url(self, track_id, template_url):
-        server_count = 30
-        for i in range(server_count):
-            url = template_url % i
+    def _get_url(self, track_id, template_url, server_number):
+        boundaries = (1, 30)
+        for nr in server_numbers(server_number, boundaries):
+            url = template_url % nr
             try:
                 # We only want to know if the request succeed
                 # don't download the whole file
                 self._request_webpage(
                     HEADRequest(url), track_id,
-                    'Checking URL %d/%d ...' % (i + 1, server_count + 1))
+                    'Checking URL %d/%d ...' % (nr, boundaries[-1]))
                 return url
             except ExtractorError:
                 pass
-
         return None
 
     def _real_extract(self, url):
@@ -72,12 +72,13 @@ class MixcloudIE(InfoExtractor):
         preview_url = self._search_regex(
             r'\s(?:data-preview-url|m-preview)="([^"]+)"', webpage, 'preview url')
         song_url = preview_url.replace('/previews/', '/c/originals/')
+        server_number = int(self._search_regex(r'stream(\d+)', song_url, 'server number'))
         template_url = re.sub(r'(stream\d*)', 'stream%d', song_url)
-        final_song_url = self._get_url(track_id, template_url)
+        final_song_url = self._get_url(track_id, template_url, server_number)
         if final_song_url is None:
             self.to_screen('Trying with m4a extension')
             template_url = template_url.replace('.mp3', '.m4a').replace('originals/', 'm4a/64/')
-            final_song_url = self._get_url(track_id, template_url)
+            final_song_url = self._get_url(track_id, template_url, server_number)
         if final_song_url is None:
             raise ExtractorError('Unable to extract track url')
 
@@ -114,3 +115,35 @@ class MixcloudIE(InfoExtractor):
             'view_count': view_count,
             'like_count': like_count,
         }
+
+
+def server_numbers(first, boundaries):
+    """ Server numbers to try in descending order of probable availability.
+    Starting from first (i.e. the number of the server hosting the preview file)
+    and going further and further up to the higher boundary and down to the
+    lower one in an alternating fashion. Namely:
+
+        server_numbers(2, (1, 5))
+
+        # Where the preview server is 2, min number is 1 and max is 5.
+        # Yields: 2, 3, 1, 4, 5
+
+    Why not random numbers or increasing sequences? Since from what I've seen,
+    full length files seem to be hosted on servers whose number is closer to
+    that of the preview; to be confirmed.
+    """
+    zip_longest = getattr(itertools, 'zip_longest', None)
+    if zip_longest is None:
+        # python 2.x
+        zip_longest = itertools.izip_longest
+
+    if len(boundaries) != 2:
+        raise ValueError("boundaries should be a two-element tuple")
+    min, max = boundaries
+    highs = range(first + 1, max + 1)
+    lows = range(first - 1, min - 1, -1)
+    rest = filter(
+        None, itertools.chain.from_iterable(zip_longest(highs, lows)))
+    yield first
+    for n in rest:
+        yield n

From 3073a6d5e9036b0b613f57bc08099862a2af87f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Mar 2015 23:08:18 +0600
Subject: [PATCH 1900/1937] [ultimedia] Add extractor

Sponsored by thankyoumotion.com
---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/ultimedia.py | 91 +++++++++++++++++++++++++++++++
 2 files changed, 92 insertions(+)
 create mode 100644 youtube_dl/extractor/ultimedia.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 73c17aa84..867e7c935 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -534,6 +534,7 @@ from .twitch import (
     TwitchStreamIE,
 )
 from .ubu import UbuIE
+from .ultimedia import UltimediaIE
 from .udemy import (
     UdemyIE,
     UdemyCourseIE
diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
new file mode 100644
index 000000000..97e4445d4
--- /dev/null
+++ b/youtube_dl/extractor/ultimedia.py
@@ -0,0 +1,91 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    qualities,
+    unified_strdate,
+    clean_html,
+)
+
+
+class UltimediaIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ultimedia\.com/default/index/video[^/]+/id/(?P<id>[\d+a-z]+)'
+    _TESTS = [{
+        # news
+        'url': 'https://www.ultimedia.com/default/index/videogeneric/id/s8uk0r',
+        'md5': '276a0e49de58c7e85d32b057837952a2',
+        'info_dict': {
+            'id': 's8uk0r',
+            'ext': 'mp4',
+            'title': 'Loi sur la fin de vie: le texte prévoit un renforcement des directives anticipées',
+            'description': 'md5:3e5c8fd65791487333dda5db8aed32af',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'upload_date': '20150317',
+        },
+    }, {
+        # music
+        'url': 'https://www.ultimedia.com/default/index/videomusic/id/xvpfp8',
+        'md5': '2ea3513813cf230605c7e2ffe7eca61c',
+        'info_dict': {
+            'id': 'xvpfp8',
+            'ext': 'mp4',
+            'title': "Two - C'est la vie (Clip)",
+            'description': 'Two',
+            'thumbnail': 're:^https?://.*\.jpg',
+            'upload_date': '20150224',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        deliver_url = self._search_regex(
+            r'<iframe[^>]+src="(https?://(?:www\.)?ultimedia\.com/deliver/[^"]+)"',
+            webpage, 'deliver URL')
+
+        deliver_page = self._download_webpage(
+            deliver_url, video_id, 'Downloading iframe page')
+
+        player = self._parse_json(
+            self._search_regex(
+                r"jwplayer\('player(?:_temp)?'\)\.setup\(({.+?})\)\.on", deliver_page, 'player'),
+            video_id)
+
+        quality = qualities(['flash', 'html5'])
+
+        formats = [{
+            'url': mode['config']['file'],
+            'format_id': mode.get('type'),
+            'quality': quality(mode.get('type')),
+        } for mode in player['modes']]
+        self._sort_formats(formats)
+
+        thumbnail = player.get('image')
+
+        title = clean_html((
+            self._html_search_regex(
+                r'(?s)<div\s+id="catArticle">.+?</div>(.+?)</h1>',
+                webpage, 'title', default=None)
+            or self._search_regex(
+                r"var\s+nameVideo\s*=\s*'([^']+)'",
+                deliver_page, 'title')))
+
+        description = clean_html(self._html_search_regex(
+            r'(?s)<span>Description</span>(.+?)</p>', webpage,
+            'description', fatal=False))
+
+        upload_date = unified_strdate(self._search_regex(
+            r'Ajouté le\s*<span>([^<]+)', webpage,
+            'upload date', fatal=False))
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': upload_date,
+            'formats': formats,
+        }

From d1dc7e39918ddfc3402a8ffd669e6c84ac971803 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 18 Mar 2015 23:11:48 +0600
Subject: [PATCH 1901/1937] [ultimedia] Fix alphabetic order

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 867e7c935..7eb9b4fbb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -534,11 +534,11 @@ from .twitch import (
     TwitchStreamIE,
 )
 from .ubu import UbuIE
-from .ultimedia import UltimediaIE
 from .udemy import (
     UdemyIE,
     UdemyCourseIE
 )
+from .ultimedia import UltimediaIE
 from .unistra import UnistraIE
 from .urort import UrortIE
 from .ustream import UstreamIE, UstreamChannelIE

From 73900846b16d33f769d35ac945a97a66cc17fd5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Mar 2015 00:53:26 +0600
Subject: [PATCH 1902/1937] [ultimedia] Capture and output unavailable video
 message

---
 youtube_dl/extractor/ultimedia.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
index 97e4445d4..0c1b08d7d 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/ultimedia.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    ExtractorError,
     qualities,
     unified_strdate,
     clean_html,
@@ -49,6 +50,10 @@ class UltimediaIE(InfoExtractor):
         deliver_page = self._download_webpage(
             deliver_url, video_id, 'Downloading iframe page')
 
+        if '>This video is currently not available' in deliver_page:
+            raise ExtractorError(
+                'Video %s is currently not available' % video_id, expected=True)
+
         player = self._parse_json(
             self._search_regex(
                 r"jwplayer\('player(?:_temp)?'\)\.setup\(({.+?})\)\.on", deliver_page, 'player'),

From b30ef07c6ccb982cff623c34e7c5cec5d8eb9bb9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Mar 2015 01:06:39 +0600
Subject: [PATCH 1903/1937] [ultimedia] Handle youtube embeds

---
 youtube_dl/extractor/ultimedia.py | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/ultimedia.py b/youtube_dl/extractor/ultimedia.py
index 0c1b08d7d..06554a1be 100644
--- a/youtube_dl/extractor/ultimedia.py
+++ b/youtube_dl/extractor/ultimedia.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
@@ -60,12 +62,18 @@ class UltimediaIE(InfoExtractor):
             video_id)
 
         quality = qualities(['flash', 'html5'])
-
-        formats = [{
-            'url': mode['config']['file'],
-            'format_id': mode.get('type'),
-            'quality': quality(mode.get('type')),
-        } for mode in player['modes']]
+        formats = []
+        for mode in player['modes']:
+            video_url = mode.get('config', {}).get('file')
+            if not video_url:
+                continue
+            if re.match(r'https?://www\.youtube\.com/.+?', video_url):
+                return self.url_result(video_url, 'Youtube')
+            formats.append({
+                'url': video_url,
+                'format_id': mode.get('type'),
+                'quality': quality(mode.get('type')),
+            })
         self._sort_formats(formats)
 
         thumbnail = player.get('image')

From cbc3cfcab41599f1b52e328878e19d15be1792d4 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 18 Mar 2015 22:02:39 +0100
Subject: [PATCH 1904/1937] release 2015.03.18

---
 docs/supportedsites.md | 4 ++++
 youtube_dl/version.py  | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d6a1e67c6..72b365305 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -112,6 +112,7 @@
  - **Discovery**
  - **divxstage**: DivxStage
  - **Dotsub**
+ - **DouyuTV**
  - **DRBonanza**
  - **Dropbox**
  - **DrTuber**
@@ -342,6 +343,7 @@
  - **PornHubPlaylist**
  - **Pornotube**
  - **PornoXO**
+ - **PrimeShareTV**
  - **PromptFile**
  - **prosiebensat1**: ProSiebenSat.1 Digital
  - **Puls4**
@@ -367,6 +369,7 @@
  - **RTP**
  - **RTS**: RTS.ch
  - **rtve.es:alacarta**: RTVE a la carta
+ - **rtve.es:infantil**: RTVE infantil
  - **rtve.es:live**: RTVE.es live streams
  - **RUHD**
  - **rutube**: Rutube videos
@@ -487,6 +490,7 @@
  - **Ubu**
  - **udemy**
  - **udemy:course**
+ - **Ultimedia**
  - **Unistra**
  - **Urort**: NRK P3 Urørt
  - **ustream**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 7ed07c375..51b4260aa 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.03.15'
+__version__ = '2015.03.18'

From 0ae8bbac2d12a883b8eb7a941c65c7b87987d213 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Mar 2015 21:17:04 +0600
Subject: [PATCH 1905/1937] [nytimes] Support embed URL

---
 youtube_dl/extractor/nytimes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index 56e1cad3b..5b57e95b6 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -7,7 +7,7 @@ from ..utils import parse_iso8601
 
 
 class NYTimesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nytimes\.com/video/(?:[^/]+/)+(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)'
 
     _TEST = {
         'url': 'http://www.nytimes.com/video/opinion/100000002847155/verbatim-what-is-a-photocopier.html?playlistId=100000001150263',

From f3c0c667a6d8d1b58ca198b72a353887ea8c4f38 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Mar 2015 21:23:52 +0600
Subject: [PATCH 1906/1937] [nytimes] Modernize

---
 youtube_dl/extractor/nytimes.py | 31 +++++++++++++++++--------------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index 5b57e95b6..d1cf8f4f3 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -1,9 +1,11 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
-from ..utils import parse_iso8601
+from ..utils import (
+    float_or_none,
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class NYTimesIE(InfoExtractor):
@@ -25,15 +27,15 @@ class NYTimesIE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
 
         video_data = self._download_json(
-            'http://www.nytimes.com/svc/video/api/v2/video/%s' % video_id, video_id, 'Downloading video JSON')
+            'http://www.nytimes.com/svc/video/api/v2/video/%s' % video_id,
+            video_id, 'Downloading video JSON')
 
         title = video_data['headline']
-        description = video_data['summary']
-        duration = video_data['duration'] / 1000.0
+        description = video_data.get('summary')
+        duration = float_or_none(video_data.get('duration'), 1000)
 
         uploader = video_data['byline']
         timestamp = parse_iso8601(video_data['publication_date'][:-8])
@@ -49,11 +51,11 @@ class NYTimesIE(InfoExtractor):
         formats = [
             {
                 'url': video['url'],
-                'format_id': video['type'],
-                'vcodec': video['video_codec'],
-                'width': video['width'],
-                'height': video['height'],
-                'filesize': get_file_size(video['fileSize']),
+                'format_id': video.get('type'),
+                'vcodec': video.get('video_codec'),
+                'width': int_or_none(video.get('width')),
+                'height': int_or_none(video.get('height')),
+                'filesize': get_file_size(video.get('fileSize')),
             } for video in video_data['renditions']
         ]
         self._sort_formats(formats)
@@ -61,7 +63,8 @@ class NYTimesIE(InfoExtractor):
         thumbnails = [
             {
                 'url': 'http://www.nytimes.com/%s' % image['url'],
-                'resolution': '%dx%d' % (image['width'], image['height']),
+                'width': int_or_none(image.get('width')),
+                'height': int_or_none(image.get('height')),
             } for image in video_data['images']
         ]
 

From 3378d67a18d2d06574698168120bf8c7f7b9e172 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 19 Mar 2015 21:26:57 +0600
Subject: [PATCH 1907/1937] [generic] Add support for nytimes embeds (Closes
 #5234)

---
 youtube_dl/extractor/generic.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index dc5755d12..8716e4503 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1006,6 +1006,13 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'))
 
+        # Look for NYTimes player
+        mobj = re.search(
+            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>',
+            webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'))
+
         # Look for Ooyala videos
         mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
                 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or

From eecc0685c976abc8fdcebe24e5c3a76d89268c54 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 19 Mar 2015 19:37:39 +0200
Subject: [PATCH 1908/1937] [videomega] Fix extraction and update test (Fixes
 #5235)

---
 youtube_dl/extractor/videomega.py | 45 ++++++++++---------------------
 1 file changed, 14 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/videomega.py b/youtube_dl/extractor/videomega.py
index 273030316..eb309a7cd 100644
--- a/youtube_dl/extractor/videomega.py
+++ b/youtube_dl/extractor/videomega.py
@@ -4,28 +4,21 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_urllib_parse,
-    compat_urllib_request,
-)
-from ..utils import (
-    ExtractorError,
-    remove_start,
-)
+from ..compat import compat_urllib_request
 
 
 class VideoMegaIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://
         (?:www\.)?videomega\.tv/
-        (?:iframe\.php)?\?ref=(?P<id>[A-Za-z0-9]+)
+        (?:iframe\.php|cdn\.php)?\?ref=(?P<id>[A-Za-z0-9]+)
         '''
     _TEST = {
-        'url': 'http://videomega.tv/?ref=QR0HCUHI1661IHUCH0RQ',
+        'url': 'http://videomega.tv/?ref=4GNA688SU99US886ANG4',
         'md5': 'bf5c2f95c4c917536e80936af7bc51e1',
         'info_dict': {
-            'id': 'QR0HCUHI1661IHUCH0RQ',
+            'id': '4GNA688SU99US886ANG4',
             'ext': 'mp4',
-            'title': 'Big Buck Bunny',
+            'title': 'BigBuckBunny_320x180',
             'thumbnail': 're:^https?://.*\.jpg$',
         }
     }
@@ -33,34 +26,24 @@ class VideoMegaIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        iframe_url = 'http://videomega.tv/iframe.php?ref={0:}'.format(video_id)
+        iframe_url = 'http://videomega.tv/cdn.php?ref=%s' % video_id
         req = compat_urllib_request.Request(iframe_url)
         req.add_header('Referer', url)
         webpage = self._download_webpage(req, video_id)
 
-        try:
-            escaped_data = re.findall(r'unescape\("([^"]+)"\)', webpage)[-1]
-        except IndexError:
-            raise ExtractorError('Unable to extract escaped data')
-
-        playlist = compat_urllib_parse.unquote(escaped_data)
-
+        title = self._html_search_regex(
+            r'<title>(.*?)</title>', webpage, 'title')
+        title = re.sub(
+            r'(?:^[Vv]ideo[Mm]ega\.tv\s-\s?|\s?-\svideomega\.tv$)', '', title)
         thumbnail = self._search_regex(
-            r'image:\s*"([^"]+)"', playlist, 'thumbnail', fatal=False)
-        video_url = self._search_regex(r'file:\s*"([^"]+)"', playlist, 'URL')
-        title = remove_start(self._html_search_regex(
-            r'<title>(.*?)</title>', webpage, 'title'), 'VideoMega.tv - ')
-
-        formats = [{
-            'format_id': 'sd',
-            'url': video_url,
-        }]
-        self._sort_formats(formats)
+            r'<video[^>]+?poster="([^"]+)"', webpage, 'thumbnail', fatal=False)
+        video_url = self._search_regex(
+            r'<source[^>]+?src="([^"]+)"', webpage, 'video URL')
 
         return {
             'id': video_id,
             'title': title,
-            'formats': formats,
+            'url': video_url,
             'thumbnail': thumbnail,
             'http_headers': {
                 'Referer': iframe_url,

From ccf3960eeca3ca8b75da709c8c3019531101f1e7 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Thu, 19 Mar 2015 20:55:05 +0200
Subject: [PATCH 1909/1937] [nytimes] Improve _VALID_URL (Fixes #5238)

---
 youtube_dl/extractor/nytimes.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/nytimes.py b/youtube_dl/extractor/nytimes.py
index d1cf8f4f3..03f0a4de6 100644
--- a/youtube_dl/extractor/nytimes.py
+++ b/youtube_dl/extractor/nytimes.py
@@ -9,9 +9,9 @@ from ..utils import (
 
 
 class NYTimesIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+?|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.nytimes.com/video/opinion/100000002847155/verbatim-what-is-a-photocopier.html?playlistId=100000001150263',
         'md5': '18a525a510f942ada2720db5f31644c0',
         'info_dict': {
@@ -24,7 +24,10 @@ class NYTimesIE(InfoExtractor):
             'uploader': 'Brett Weiner',
             'duration': 419,
         }
-    }
+    }, {
+        'url': 'http://www.nytimes.com/video/travel/100000003550828/36-hours-in-dubai.html',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From 2684871bc10fe2b34f58312b6f75d3237a786732 Mon Sep 17 00:00:00 2001
From: Naglis Jonaitis <njonaitis@gmail.com>
Date: Fri, 20 Mar 2015 01:50:36 +0200
Subject: [PATCH 1910/1937] [vine] Fix formats extraction (Closes #5239)

---
 youtube_dl/extractor/vine.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index 0b58fe0fe..c3187cfeb 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -33,14 +33,13 @@ class VineIE(InfoExtractor):
             r'window\.POST_DATA = { %s: ({.+?}) }' % video_id, webpage, 'vine data'))
 
         formats = [{
-            'url': data['videoLowURL'],
-            'ext': 'mp4',
-            'format_id': 'low',
-        }, {
-            'url': data['videoUrl'],
-            'ext': 'mp4',
-            'format_id': 'standard',
-        }]
+            'format_id': '%(format)s-%(rate)s' % f,
+            'vcodec': f['format'],
+            'quality': f['rate'],
+            'url': f['videoUrl'],
+        } for f in data['videoUrls'] if f.get('rate')]
+
+        self._sort_formats(formats)
 
         return {
             'id': video_id,

From 01218f919b3bb5d85449806ea832b29251dc13ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 20 Mar 2015 14:59:38 +0100
Subject: [PATCH 1911/1937] [test/http] Add test for proxy support

---
 test/test_http.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/test/test_http.py b/test/test_http.py
index bd4d46fef..f2e305b6f 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -8,7 +8,7 @@ import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from youtube_dl import YoutubeDL
-from youtube_dl.compat import compat_http_server
+from youtube_dl.compat import compat_http_server, compat_urllib_request
 import ssl
 import threading
 
@@ -68,5 +68,52 @@ class TestHTTP(unittest.TestCase):
         r = ydl.extract_info('https://localhost:%d/video.html' % self.port)
         self.assertEqual(r['url'], 'https://localhost:%d/vid.mp4' % self.port)
 
+
+def _build_proxy_handler(name):
+    class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
+        proxy_name = name
+
+        def log_message(self, format, *args):
+            pass
+
+        def do_GET(self):
+            self.send_response(200)
+            self.send_header('Content-Type', 'text/plain; charset=utf-8')
+            self.end_headers()
+            self.wfile.write('{self.proxy_name}: {self.path}'.format(self=self).encode('utf-8'))
+    return HTTPTestRequestHandler
+
+
+class TestProxy(unittest.TestCase):
+    def setUp(self):
+        self.proxy = compat_http_server.HTTPServer(
+            ('localhost', 0), _build_proxy_handler('normal'))
+        self.port = self.proxy.socket.getsockname()[1]
+        self.proxy_thread = threading.Thread(target=self.proxy.serve_forever)
+        self.proxy_thread.daemon = True
+        self.proxy_thread.start()
+
+        self.cn_proxy = compat_http_server.HTTPServer(
+            ('localhost', 0), _build_proxy_handler('cn'))
+        self.cn_port = self.cn_proxy.socket.getsockname()[1]
+        self.cn_proxy_thread = threading.Thread(target=self.cn_proxy.serve_forever)
+        self.cn_proxy_thread.daemon = True
+        self.cn_proxy_thread.start()
+
+    def test_proxy(self):
+        cn_proxy = 'localhost:{0}'.format(self.cn_port)
+        ydl = YoutubeDL({
+            'proxy': 'localhost:{0}'.format(self.port),
+            'cn_verification_proxy': cn_proxy,
+        })
+        url = 'http://foo.com/bar'
+        response = ydl.urlopen(url).read().decode('utf-8')
+        self.assertEqual(response, 'normal: {0}'.format(url))
+
+        req = compat_urllib_request.Request(url)
+        req.add_header('Ytdl-request-proxy', cn_proxy)
+        response = ydl.urlopen(req).read().decode('utf-8')
+        self.assertEqual(response, 'cn: {0}'.format(url))
+
 if __name__ == '__main__':
     unittest.main()

From f20bf146e217731016b32af598b177952fa1340a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 20 Mar 2015 15:14:25 +0100
Subject: [PATCH 1912/1937] [test/YoutubeDL] split in two classes

The name was misleading
---
 test/test_YoutubeDL.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index db8a47d2d..2597a1993 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -339,6 +339,8 @@ class TestFormatSelection(unittest.TestCase):
         downloaded = ydl.downloaded_info_dicts[0]
         self.assertEqual(downloaded['format_id'], 'G')
 
+
+class TestYoutubeDL(unittest.TestCase):
     def test_subtitles(self):
         def s_formats(lang, autocaption=False):
             return [{

From 1c9a1457fc154697b327a87d9afe674425740287 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Mar 2015 20:53:14 +0600
Subject: [PATCH 1913/1937] [niconico] Add nm video test

---
 youtube_dl/extractor/niconico.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 4d57ac703..ddec7b338 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -22,7 +22,7 @@ class NiconicoIE(InfoExtractor):
     IE_NAME = 'niconico'
     IE_DESC = 'ニコニコ動画'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.nicovideo.jp/watch/sm22312215',
         'md5': 'd1a75c0823e2f629128c43e1212760f9',
         'info_dict': {
@@ -39,7 +39,24 @@ class NiconicoIE(InfoExtractor):
             'username': 'ydl.niconico@gmail.com',
             'password': 'youtube-dl',
         },
-    }
+    }, {
+        'url': 'http://www.nicovideo.jp/watch/nm14296458',
+        'md5': '8db08e0158457cf852a31519fceea5bc',
+        'info_dict': {
+            'id': 'nm14296458',
+            'ext': 'swf',
+            'title': '【鏡音リン】Dance on media【オリジナル】take2!',
+            'description': 'md5:',
+            'uploader': 'りょうた',
+            'uploader_id': '18822557',
+            'upload_date': '20110429',
+            'duration': 209,
+        },
+        'params': {
+            'username': 'ydl.niconico@gmail.com',
+            'password': 'youtube-dl',
+        },
+    }]
 
     _VALID_URL = r'https?://(?:www\.|secure\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
     _NETRC_MACHINE = 'niconico'

From 1887ecd4d629f44370846d372155512ec29e1f27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 20 Mar 2015 21:45:09 +0600
Subject: [PATCH 1914/1937] [twitch] Fix login

---
 youtube_dl/extractor/twitch.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index cbdaf9c7a..aad2bf222 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -23,6 +23,7 @@ class TwitchBaseIE(InfoExtractor):
     _API_BASE = 'https://api.twitch.tv'
     _USHER_BASE = 'http://usher.twitch.tv'
     _LOGIN_URL = 'https://secure.twitch.tv/user/login'
+    _LOGIN_POST_URL = 'https://secure-login.twitch.tv/login'
     _NETRC_MACHINE = 'twitch'
 
     def _handle_error(self, response):
@@ -67,14 +68,14 @@ class TwitchBaseIE(InfoExtractor):
             'authenticity_token': authenticity_token,
             'redirect_on_login': '',
             'embed_form': 'false',
-            'mp_source_action': '',
+            'mp_source_action': 'login-button',
             'follow': '',
-            'user[login]': username,
-            'user[password]': password,
+            'login': username,
+            'password': password,
         }
 
         request = compat_urllib_request.Request(
-            self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
+            self._LOGIN_POST_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8'))
         request.add_header('Referer', self._LOGIN_URL)
         response = self._download_webpage(
             request, None, 'Logging in as %s' % username)

From 531980d89c37427c663152a525fd795dc0b040e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 20 Mar 2015 17:05:28 +0100
Subject: [PATCH 1915/1937] [test/YoutubeDL] test match_filter

---
 test/test_YoutubeDL.py | 68 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 2597a1993..652519831 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -14,6 +14,7 @@ from test.helper import FakeYDL, assertRegexpMatches
 from youtube_dl import YoutubeDL
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.postprocessor.common import PostProcessor
+from youtube_dl.utils import match_filter_func
 
 TEST_URL = 'http://localhost/sample.mp4'
 
@@ -463,6 +464,73 @@ class TestYoutubeDL(unittest.TestCase):
         self.assertTrue(os.path.exists(audiofile), '%s doesn\'t exist' % audiofile)
         os.unlink(audiofile)
 
+    def test_match_filter(self):
+        class FilterYDL(YDL):
+            def __init__(self, *args, **kwargs):
+                super(FilterYDL, self).__init__(*args, **kwargs)
+                self.params['simulate'] = True
+
+            def process_info(self, info_dict):
+                super(YDL, self).process_info(info_dict)
+
+            def _match_entry(self, info_dict, incomplete):
+                res = super(FilterYDL, self)._match_entry(info_dict, incomplete)
+                if res is None:
+                    self.downloaded_info_dicts.append(info_dict)
+                return res
+
+        first = {
+            'id': '1',
+            'url': TEST_URL,
+            'title': 'one',
+            'extractor': 'TEST',
+            'duration': 30,
+            'filesize': 10 * 1024,
+        }
+        second = {
+            'id': '2',
+            'url': TEST_URL,
+            'title': 'two',
+            'extractor': 'TEST',
+            'duration': 10,
+            'description': 'foo',
+            'filesize': 5 * 1024,
+        }
+        videos = [first, second]
+
+        def get_videos(filter_=None):
+            ydl = FilterYDL({'match_filter': filter_})
+            for v in videos:
+                ydl.process_ie_result(v, download=True)
+            return [v['id'] for v in ydl.downloaded_info_dicts]
+
+        res = get_videos()
+        self.assertEqual(res, ['1', '2'])
+
+        def f(v):
+            if v['id'] == '1':
+                return None
+            else:
+                return 'Video id is not 1'
+        res = get_videos(f)
+        self.assertEqual(res, ['1'])
+
+        f = match_filter_func('duration < 30')
+        res = get_videos(f)
+        self.assertEqual(res, ['2'])
+
+        f = match_filter_func('description = foo')
+        res = get_videos(f)
+        self.assertEqual(res, ['2'])
+
+        f = match_filter_func('description =? foo')
+        res = get_videos(f)
+        self.assertEqual(res, ['1', '2'])
+
+        f = match_filter_func('filesize > 5KiB')
+        res = get_videos(f)
+        self.assertEqual(res, ['1'])
+
 
 if __name__ == '__main__':
     unittest.main()

From a7d9ded45dac709fbb9140ab3525a10a2a6d3c85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Mar 2015 12:07:23 +0100
Subject: [PATCH 1916/1937] [test] Add tests for aes

---
 devscripts/generate_aes_testdata.py | 36 ++++++++++++++++++++++
 test/test_aes.py                    | 47 +++++++++++++++++++++++++++++
 2 files changed, 83 insertions(+)
 create mode 100644 devscripts/generate_aes_testdata.py
 create mode 100644 test/test_aes.py

diff --git a/devscripts/generate_aes_testdata.py b/devscripts/generate_aes_testdata.py
new file mode 100644
index 000000000..ff66449eb
--- /dev/null
+++ b/devscripts/generate_aes_testdata.py
@@ -0,0 +1,36 @@
+from __future__ import unicode_literals
+
+import codecs
+import subprocess
+
+import os
+import sys
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.utils import intlist_to_bytes
+from youtube_dl.aes import aes_encrypt, key_expansion
+
+secret_msg = b'Secret message goes here'
+
+
+def hex_str(int_list):
+    return codecs.encode(intlist_to_bytes(int_list), 'hex')
+
+
+def openssl_encode(algo, key, iv):
+    cmd = ['openssl', 'enc', '-e', '-' + algo, '-K', hex_str(key), '-iv', hex_str(iv)]
+    prog = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    out, _ = prog.communicate(secret_msg)
+    return out
+
+iv = key = [0x20, 0x15] + 14 * [0]
+
+r = openssl_encode('aes-128-cbc', key, iv)
+print('aes_cbc_decrypt')
+print(repr(r))
+
+password = key
+new_key = aes_encrypt(password, key_expansion(password))
+r = openssl_encode('aes-128-ctr', new_key, iv)
+print('aes_decrypt_text')
+print(repr(r))
diff --git a/test/test_aes.py b/test/test_aes.py
new file mode 100644
index 000000000..111b902e1
--- /dev/null
+++ b/test/test_aes.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_decrypt_text
+from youtube_dl.utils import bytes_to_intlist, intlist_to_bytes
+import base64
+
+# the encrypted data can be generate with 'devscripts/generate_aes_testdata.py'
+
+
+class TestAES(unittest.TestCase):
+    def setUp(self):
+        self.key = self.iv = [0x20, 0x15] + 14 * [0]
+        self.secret_msg = b'Secret message goes here'
+
+    def test_encrypt(self):
+        msg = b'message'
+        key = list(range(16))
+        encrypted = aes_encrypt(bytes_to_intlist(msg), key)
+        decrypted = intlist_to_bytes(aes_decrypt(encrypted, key))
+        self.assertEqual(decrypted, msg)
+
+    def test_cbc_decrypt(self):
+        data = bytes_to_intlist(
+            b"\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6'\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd"
+        )
+        decrypted = intlist_to_bytes(aes_cbc_decrypt(data, self.key, self.iv))
+        self.assertEqual(decrypted.rstrip(b'\x08'), self.secret_msg)
+
+    def test_decrypt_text(self):
+        password = intlist_to_bytes(self.key).decode('utf-8')
+        encrypted = base64.b64encode(
+            intlist_to_bytes(self.iv[:8]) +
+            b'\x17\x15\x93\xab\x8d\x80V\xcdV\xe0\t\xcdo\xc2\xa5\xd8ksM\r\xe27N\xae'
+        )
+        decrypted = (aes_decrypt_text(encrypted, password, 16))
+        self.assertEqual(decrypted, self.secret_msg)
+
+if __name__ == '__main__':
+    unittest.main()

From 85698c508651cc047b0ba2181580d99ed057ef0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Mar 2015 12:18:33 +0100
Subject: [PATCH 1917/1937] [crunchyroll] Remove unused class

---
 youtube_dl/extractor/crunchyroll.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index e64b88fbc..6ded723c9 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -23,7 +23,6 @@ from ..utils import (
 )
 from ..aes import (
     aes_cbc_decrypt,
-    inc,
 )
 
 
@@ -102,13 +101,6 @@ class CrunchyrollIE(InfoExtractor):
 
         key = obfuscate_key(id)
 
-        class Counter:
-            __value = iv
-
-            def next_value(self):
-                temp = self.__value
-                self.__value = inc(self.__value)
-                return temp
         decrypted_data = intlist_to_bytes(aes_cbc_decrypt(data, key, iv))
         return zlib.decompress(decrypted_data)
 

From 179d6678b1ce5ecdf22e2ae254f451183d836e75 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Mar 2015 12:34:44 +0100
Subject: [PATCH 1918/1937] Remove the 'stitle' field

A warning has been printed for more than 2 years (since 97cd3afc7525394c46398f1526d412d081c02085)
---
 youtube_dl/YoutubeDL.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5a83bc956..b5ef5e009 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -328,9 +328,6 @@ class YoutubeDL(object):
                 'Parameter outtmpl is bytes, but should be a unicode string. '
                 'Put  from __future__ import unicode_literals  at the top of your code file or consider switching to Python 3.x.')
 
-        if '%(stitle)s' in self.params.get('outtmpl', ''):
-            self.report_warning('%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
-
         self._setup_opener()
 
         if auto_init:
@@ -1218,9 +1215,6 @@ class YoutubeDL(object):
         if len(info_dict['title']) > 200:
             info_dict['title'] = info_dict['title'][:197] + '...'
 
-        # Keep for backwards compatibility
-        info_dict['stitle'] = info_dict['title']
-
         if 'format' not in info_dict:
             info_dict['format'] = info_dict['ext']
 

From 4e6a2286899ab156ad342e3e9003b0c3b239e0ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Mar 2015 18:20:49 +0600
Subject: [PATCH 1919/1937] [nrk] Adapt to new URL format

---
 youtube_dl/extractor/nrk.py | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index bff36f9d3..d17b7ed49 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -14,22 +14,22 @@ from ..utils import (
 
 
 class NRKIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?:video|lyd)/[^/]+/(?P<id>[\dA-F]{16})'
+    _VALID_URL = r'(?:nrk:|http://(?:www\.)?nrk\.no/video/PS\*)(?P<id>\d+)'
 
     _TESTS = [
         {
-            'url': 'http://www.nrk.no/video/dompap_og_andre_fugler_i_piip_show/D0FA54B5C8B6CE59/emne/piipshow/',
-            'md5': 'a6eac35052f3b242bb6bb7f43aed5886',
+            'url': 'http://www.nrk.no/video/PS*150533',
+            'md5': 'bccd850baebefe23b56d708a113229c2',
             'info_dict': {
                 'id': '150533',
                 'ext': 'flv',
                 'title': 'Dompap og andre fugler i Piip-Show',
-                'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f'
+                'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f',
             }
         },
         {
-            'url': 'http://www.nrk.no/lyd/lyd_av_oppleser_for_blinde/AEFDDD5473BA0198/',
-            'md5': '3471f2a51718195164e88f46bf427668',
+            'url': 'http://www.nrk.no/video/PS*154915',
+            'md5': '0b1493ba1aae7d9579a5ad5531bc395a',
             'info_dict': {
                 'id': '154915',
                 'ext': 'flv',
@@ -40,20 +40,18 @@ class NRKIE(InfoExtractor):
     ]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-
-        page = self._download_webpage(url, video_id)
-
-        video_id = self._html_search_regex(r'<div class="nrk-video" data-nrk-id="(\d+)">', page, 'video id')
+        video_id = self._match_id(url)
 
         data = self._download_json(
-            'http://v7.psapi.nrk.no/mediaelement/%s' % video_id, video_id, 'Downloading media JSON')
+            'http://v8.psapi.nrk.no/mediaelement/%s' % video_id,
+            video_id, 'Downloading media JSON')
 
         if data['usageRights']['isGeoBlocked']:
-            raise ExtractorError('NRK har ikke rettig-heter til å vise dette programmet utenfor Norge', expected=True)
+            raise ExtractorError(
+                'NRK har ikke rettig-heter til å vise dette programmet utenfor Norge',
+                expected=True)
 
-        video_url = data['mediaUrl'] + '?hdcore=3.1.1&plugin=aasp-3.1.1.69.124'
+        video_url = data['mediaUrl'] + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81'
 
         images = data.get('images')
         if images:

From 393d9fc6d28704ef7b60acb1dfc785493ecbe0eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Mar 2015 18:21:19 +0600
Subject: [PATCH 1920/1937] [nrk] Extract duration

---
 youtube_dl/extractor/nrk.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index d17b7ed49..117a78aa1 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -25,6 +25,7 @@ class NRKIE(InfoExtractor):
                 'ext': 'flv',
                 'title': 'Dompap og andre fugler i Piip-Show',
                 'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f',
+                'duration': 263,
             }
         },
         {
@@ -35,6 +36,7 @@ class NRKIE(InfoExtractor):
                 'ext': 'flv',
                 'title': 'Slik høres internett ut når du er blind',
                 'description': 'md5:a621f5cc1bd75c8d5104cb048c6b8568',
+                'duration': 20,
             }
         },
     ]
@@ -53,6 +55,8 @@ class NRKIE(InfoExtractor):
 
         video_url = data['mediaUrl'] + '?hdcore=3.5.0&plugin=aasp-3.5.0.151.81'
 
+        duration = parse_duration(data.get('duration'))
+
         images = data.get('images')
         if images:
             thumbnails = images['webImages']
@@ -67,6 +71,7 @@ class NRKIE(InfoExtractor):
             'ext': 'flv',
             'title': data['title'],
             'description': data['description'],
+            'duration': duration,
             'thumbnail': thumbnail,
         }
 

From faa1b5c2920f1d50f01b83e96b5b0f136edc74f7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Mar 2015 18:22:08 +0600
Subject: [PATCH 1921/1937] [nrk:playlist] Add extractor (Closes #5245)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/nrk.py      | 31 +++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 7eb9b4fbb..e25e4a582 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -346,6 +346,7 @@ from .npo import (
 )
 from .nrk import (
     NRKIE,
+    NRKPlaylistIE,
     NRKTVIE,
 )
 from .ntvde import NTVDeIE
diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 117a78aa1..c2a8202dd 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -76,6 +76,37 @@ class NRKIE(InfoExtractor):
         }
 
 
+class NRKPlaylistIE(InfoExtractor):
+    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?:[^/]+/)*(?P<id>[^/]+)'
+
+    _TEST = {
+        'url': 'http://www.nrk.no/troms/gjenopplev-den-historiske-solformorkelsen-1.12270763',
+        'info_dict': {
+            'id': 'gjenopplev-den-historiske-solformorkelsen-1.12270763',
+            'title': 'Gjenopplev den historiske solformørkelsen',
+            'description': 'md5:c2df8ea3bac5654a26fc2834a542feed',
+        },
+        'playlist_mincount': 2,
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, playlist_id)
+
+        entries = [
+            self.url_result('nrk:%s' % video_id, 'NRK')
+            for video_id in re.findall(
+                r'class="[^"]*\brich\b[^"]*"[^>]+data-video-id="(\d+)"', webpage)
+        ]
+
+        playlist_title = self._og_search_title(webpage)
+        playlist_description = self._og_search_description(webpage)
+
+        return self.playlist_result(
+            entries, playlist_id, playlist_title, playlist_description)
+
+
 class NRKTVIE(InfoExtractor):
     _VALID_URL = r'(?P<baseurl>http://tv\.nrk(?:super)?\.no/)(?:serie/[^/]+|program)/(?P<id>[a-zA-Z]{4}\d{8})(?:/\d{2}-\d{2}-\d{4})?(?:#del=(?P<part_id>\d+))?'
 

From c9450c7ab19bbd24a4ff8643370cb4f235b0b380 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Mar 2015 14:00:37 +0100
Subject: [PATCH 1922/1937] [nrk:playlist] Restrict _VALID_URL

It would also match /videos/PS... urls
---
 youtube_dl/extractor/nrk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index c2a8202dd..1355ecea2 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -77,7 +77,7 @@ class NRKIE(InfoExtractor):
 
 
 class NRKPlaylistIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?:[^/]+/)*(?P<id>[^/]+)'
+    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?!video)[^/]+/(?P<id>[^/]+)'
 
     _TEST = {
         'url': 'http://www.nrk.no/troms/gjenopplev-den-historiske-solformorkelsen-1.12270763',

From 5379a2d40db3674fead7c4239afdb65ff7b389c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 21 Mar 2015 14:12:43 +0100
Subject: [PATCH 1923/1937] [test/utils] Test xpath_text

---
 test/test_utils.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index 3431ad24e..a8ab87685 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -24,6 +24,7 @@ from youtube_dl.utils import (
     encodeFilename,
     escape_rfc3986,
     escape_url,
+    ExtractorError,
     find_xpath_attr,
     fix_xml_ampersands,
     InAdvancePagedList,
@@ -54,6 +55,7 @@ from youtube_dl.utils import (
     urlencode_postdata,
     version_tuple,
     xpath_with_ns,
+    xpath_text,
     render_table,
     match_str,
 )
@@ -250,6 +252,17 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(find('media:song/media:author').text, 'The Author')
         self.assertEqual(find('media:song/url').text, 'http://server.com/download.mp3')
 
+    def test_xpath_text(self):
+        testxml = '''<root>
+            <div>
+                <p>Foo</p>
+            </div>
+        </root>'''
+        doc = xml.etree.ElementTree.fromstring(testxml)
+        self.assertEqual(xpath_text(doc, 'div/p'), 'Foo')
+        self.assertTrue(xpath_text(doc, 'div/bar') is None)
+        self.assertRaises(ExtractorError, xpath_text, doc, 'div/bar', fatal=True)
+
     def test_smuggle_url(self):
         data = {"ö": "ö", "abc": [3]}
         url = 'https://foo.bar/baz?x=y#a'

From a09141548aa31db7c7d9457b10f5c84e6e32beba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 21 Mar 2015 20:42:48 +0600
Subject: [PATCH 1924/1937] [nrk:playlist] Relax video id regex and improve
 _VALID_URL

---
 youtube_dl/extractor/nrk.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 1355ecea2..e91d3a248 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -77,17 +77,25 @@ class NRKIE(InfoExtractor):
 
 
 class NRKPlaylistIE(InfoExtractor):
-    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?!video)[^/]+/(?P<id>[^/]+)'
+    _VALID_URL = r'http://(?:www\.)?nrk\.no/(?!video)(?:[^/]+/)+(?P<id>[^/]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.nrk.no/troms/gjenopplev-den-historiske-solformorkelsen-1.12270763',
         'info_dict': {
             'id': 'gjenopplev-den-historiske-solformorkelsen-1.12270763',
             'title': 'Gjenopplev den historiske solformørkelsen',
             'description': 'md5:c2df8ea3bac5654a26fc2834a542feed',
         },
-        'playlist_mincount': 2,
-    }
+        'playlist_count': 2,
+    }, {
+        'url': 'http://www.nrk.no/kultur/bok/rivertonprisen-til-karin-fossum-1.12266449',
+        'info_dict': {
+            'id': 'rivertonprisen-til-karin-fossum-1.12266449',
+            'title': 'Rivertonprisen til Karin Fossum',
+            'description': 'Første kvinne på 15 år til å vinne krimlitteraturprisen.',
+        },
+        'playlist_count': 5,
+    }]
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
@@ -97,7 +105,8 @@ class NRKPlaylistIE(InfoExtractor):
         entries = [
             self.url_result('nrk:%s' % video_id, 'NRK')
             for video_id in re.findall(
-                r'class="[^"]*\brich\b[^"]*"[^>]+data-video-id="(\d+)"', webpage)
+                r'class="[^"]*\brich\b[^"]*"[^>]+data-video-id="([^"]+)"',
+                webpage)
         ]
 
         playlist_title = self._og_search_title(webpage)

From 49aeedb8cb4dcf317c970a58c590d42e37904720 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Mar 2015 08:11:10 +0600
Subject: [PATCH 1925/1937] [libsyn] Improve and simplify

---
 youtube_dl/extractor/libsyn.py | 81 +++++++++++++++++++---------------
 1 file changed, 45 insertions(+), 36 deletions(-)

diff --git a/youtube_dl/extractor/libsyn.py b/youtube_dl/extractor/libsyn.py
index 88379f276..6bf741db8 100644
--- a/youtube_dl/extractor/libsyn.py
+++ b/youtube_dl/extractor/libsyn.py
@@ -1,50 +1,59 @@
-# encoding: utf-8
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
 from .common import InfoExtractor
-from ..utils import (
-    unified_strdate,
-)
+from ..utils import unified_strdate
+
 
 class LibsynIE(InfoExtractor):
-    _VALID_URL = r'(?:https?:)?//html5-player\.libsyn\.com/embed/episode/id/(?P<id>[0-9]+)(?:/.*)?'
-    _TESTS = [{
-        'url': "http://html5-player.libsyn.com/embed/episode/id/3377616/",
+    _VALID_URL = r'https?://html5-player\.libsyn\.com/embed/episode/id/(?P<id>[0-9]+)'
+
+    _TEST = {
+        'url': 'http://html5-player.libsyn.com/embed/episode/id/3377616/',
+        'md5': '443360ee1b58007bc3dcf09b41d093bb',
         'info_dict': {
-            'id': "3377616",
-            'ext': "mp3",
-            'title': "Episode 12: Bassem Youssef: Egypt's Jon Stewart",
-            'description': "<p>Bassem Youssef joins executive producer Steve Bodow and senior producer Sara Taksler for a conversation about how&nbsp;<em style=\"font-family: Tahoma, Geneva, sans-serif; font-size: 12.8000001907349px;\">The Daily Show</em>&nbsp;inspired Bassem to create&nbsp;<em style=\"font-family: Tahoma, Geneva, sans-serif; font-size: 12.8000001907349px;\">Al-Bernameg</em>, his massively popular (and now banned) Egyptian news satire program. Sara discusses her soon-to-be-released documentary,&nbsp;<em style=\"font-family: Tahoma, Geneva, sans-serif; font-size: 12.8000001907349px;\">Tickling Giants</em>, which chronicles how Bassem and his staff risked their safety every day to tell jokes.</p>",
+            'id': '3377616',
+            'ext': 'mp3',
+            'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
+            'description': 'md5:601cb790edd05908957dae8aaa866465',
+            'upload_date': '20150220',
         },
-    }]
+    }
 
     def _real_extract(self, url):
-        if url.startswith('//'):
-            url = 'https:' + url
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
+        video_id = self._match_id(url)
 
-        podcast_title         = self._search_regex(r'<h2>(.*?)</h2>', webpage, 'show title')
-        podcast_episode_title = self._search_regex(r'<h3>(.*?)</h3>', webpage, 'episode title')
-        podcast_date          = unified_strdate(self._search_regex(r'<div class="release_date">Released: (.*?)</div>', webpage, 'release date'))
-        podcast_description   = self._search_regex(r'<div id="info_text_body">(.*?)</div>', webpage, 'description')
+        webpage = self._download_webpage(url, video_id)
 
-        url0 = self._search_regex(r'var mediaURLLibsyn = "(?P<url0>https?://.*)";', webpage, 'first media URL')
-        url1 = self._search_regex(r'var mediaURL = "(?P<url1>https?://.*)";', webpage, 'second media URL')
+        formats = [{
+            'url': media_url,
+        } for media_url in set(re.findall('var\s+mediaURL(?:Libsyn)?\s*=\s*"([^"]+)"', webpage))]
 
-        if url0 != url1:
-            formats = [{
-                'url': url0
-            }, {
-                'url': url1
-            }]
-        else:
-            formats = [{
-                'url': url0
-            }]
+        podcast_title = self._search_regex(
+            r'<h2>([^<]+)</h2>', webpage, 'title')
+        episode_title = self._search_regex(
+            r'<h3>([^<]+)</h3>', webpage, 'title', default=None)
+
+        title = '%s - %s' %(podcast_title, episode_title) if podcast_title else episode_title
+
+        description = self._html_search_regex(
+            r'<div id="info_text_body">(.+?)</div>', webpage,
+            'description', fatal=False)
+
+        thumbnail = self._search_regex(
+            r'<img[^>]+class="info-show-icon"[^>]+src="([^"]+)"',
+            webpage, 'thumbnail', fatal=False)
+
+        release_date = unified_strdate(self._search_regex(
+            r'<div class="release_date">Released: ([^<]+)<', webpage, 'release date', fatal=False))
 
         return {
-            'id': display_id,
-            'title': podcast_episode_title,
-            'description': podcast_description,
-            'upload_date': podcast_date,
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'upload_date': release_date,
             'formats': formats,
         }

From a1d0aa7b882484685a1a02185d0dafd51c545701 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Mar 2015 08:11:47 +0600
Subject: [PATCH 1926/1937] [libsyn] Fix extractor alphabetic order

---
 youtube_dl/extractor/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index a20492fc3..82b75a144 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -250,8 +250,8 @@ from .letv import (
     LetvTvIE,
     LetvPlaylistIE
 )
-from .lifenews import LifeNewsIE
 from .libsyn import LibsynIE
+from .lifenews import LifeNewsIE
 from .liveleak import LiveLeakIE
 from .livestream import (
     LivestreamIE,

From cefdf970ccd8017cd67e949004e5e4c770aacdb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Mar 2015 08:18:13 +0600
Subject: [PATCH 1927/1937] [extractor/generic] Support Libsyn embeds

---
 youtube_dl/extractor/generic.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 8716e4503..84e8f14b2 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -1013,6 +1013,12 @@ class GenericIE(InfoExtractor):
         if mobj is not None:
             return self.url_result(mobj.group('url'))
 
+        # Look for Libsyn player
+        mobj = re.search(
+            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//html5-player\.libsyn\.com/embed/.+?)\1', webpage)
+        if mobj is not None:
+            return self.url_result(mobj.group('url'))
+
         # Look for Ooyala videos
         mobj = (re.search(r'player\.ooyala\.com/[^"?]+\?[^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage) or
                 re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage) or

From 2051acdeb2ed9a0edf3b6b70682699c37d19d851 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Mar 2015 08:20:27 +0600
Subject: [PATCH 1928/1937] [extractor/generic] Add test for Libsyn embed

---
 youtube_dl/extractor/generic.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 84e8f14b2..8a49b0b54 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -527,6 +527,17 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Viddler'],
         },
+        # Libsyn embed
+        {
+            'url': 'http://thedailyshow.cc.com/podcast/episodetwelve',
+            'info_dict': {
+                'id': '3377616',
+                'ext': 'mp3',
+                'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
+                'description': 'md5:601cb790edd05908957dae8aaa866465',
+                'upload_date': '20150220',
+            },
+        },
         # jwplayer YouTube
         {
             'url': 'http://media.nationalarchives.gov.uk/index.php/webinar-using-discovery-national-archives-online-catalogue/',

From cf2e2eb1c0b626f2d5f210ffd14642aceb0358e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Mar 2015 08:23:20 +0600
Subject: [PATCH 1929/1937] [comedycentral] Drop thedailyshow podcast extractor

Generic extractor is just fine for Libsyn embeds
---
 youtube_dl/extractor/__init__.py      |  2 +-
 youtube_dl/extractor/comedycentral.py | 24 ------------------------
 2 files changed, 1 insertion(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 82b75a144..d73826d44 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -84,7 +84,7 @@ from .cnn import (
 )
 from .collegehumor import CollegeHumorIE
 from .collegerama import CollegeRamaIE
-from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE, TheDailyShowPodcastIE
+from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE
 from .comcarcoff import ComCarCoffIE
 from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
 from .condenast import CondeNastIE
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index bd3817b56..648a6f990 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -273,27 +273,3 @@ class ComedyCentralShowsIE(MTVServicesInfoExtractor):
             'title': show_name + ' ' + title,
             'description': description,
         }
-
-class TheDailyShowPodcastIE(InfoExtractor):
-    _VALID_URL = r'(?P<scheme>https?:)?//thedailyshow\.cc\.com/podcast/(?P<id>[a-z\-]+)'
-    _TESTS = [{
-        "url": "http://thedailyshow.cc.com/podcast/episodetwelve",
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-
-        player_url = self._search_regex(r'<iframe(?:\s+[^>]+)?\s*src="((?:https?:)?//html5-player\.libsyn\.com/embed/episode/id/[0-9]+)', webpage, 'player URL')
-        if player_url.startswith('//'):
-            mobj = re.match(self._VALID_URL, url)
-            scheme = mobj.group('scheme')
-            if not scheme:
-                scheme = 'https:'
-            player_url = scheme + player_url
-
-        return {
-            '_type': 'url_transparent',
-            'url': player_url,
-        }

From 1a4123de04d0168ef4a14e6064148eb248d65dc4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 22 Mar 2015 08:23:38 +0600
Subject: [PATCH 1930/1937] [comedycentral] Remove unused import

---
 youtube_dl/extractor/comedycentral.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index 648a6f990..e5edcc84b 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -2,7 +2,6 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
 from .mtv import MTVServicesInfoExtractor
 from ..compat import (
     compat_str,

From 336d19044c84128cc3bdcb62882372243dfa32c8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 22 Mar 2015 11:03:52 +0100
Subject: [PATCH 1931/1937] [lybsyn] pep8: add space around operator

---
 youtube_dl/extractor/libsyn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/libsyn.py b/youtube_dl/extractor/libsyn.py
index 6bf741db8..9ab1416f5 100644
--- a/youtube_dl/extractor/libsyn.py
+++ b/youtube_dl/extractor/libsyn.py
@@ -36,7 +36,7 @@ class LibsynIE(InfoExtractor):
         episode_title = self._search_regex(
             r'<h3>([^<]+)</h3>', webpage, 'title', default=None)
 
-        title = '%s - %s' %(podcast_title, episode_title) if podcast_title else episode_title
+        title = '%s - %s' % (podcast_title, episode_title) if podcast_title else episode_title
 
         description = self._html_search_regex(
             r'<div id="info_text_body">(.+?)</div>', webpage,

From f9544f6e8fe4697a4a48364b6d7a7dc7d93cabf5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 22 Mar 2015 12:09:58 +0100
Subject: [PATCH 1932/1937] [test/aes] Test aes_decrypt_text with 256 bit

---
 devscripts/generate_aes_testdata.py | 8 +++++++-
 test/test_aes.py                    | 8 ++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/devscripts/generate_aes_testdata.py b/devscripts/generate_aes_testdata.py
index ff66449eb..2e389fc8e 100644
--- a/devscripts/generate_aes_testdata.py
+++ b/devscripts/generate_aes_testdata.py
@@ -32,5 +32,11 @@ print(repr(r))
 password = key
 new_key = aes_encrypt(password, key_expansion(password))
 r = openssl_encode('aes-128-ctr', new_key, iv)
-print('aes_decrypt_text')
+print('aes_decrypt_text 16')
+print(repr(r))
+
+password = key + 16 * [0]
+new_key = aes_encrypt(password, key_expansion(password)) * (32 // 16)
+r = openssl_encode('aes-256-ctr', new_key, iv)
+print('aes_decrypt_text 32')
 print(repr(r))
diff --git a/test/test_aes.py b/test/test_aes.py
index 111b902e1..4dc7de7b5 100644
--- a/test/test_aes.py
+++ b/test/test_aes.py
@@ -43,5 +43,13 @@ class TestAES(unittest.TestCase):
         decrypted = (aes_decrypt_text(encrypted, password, 16))
         self.assertEqual(decrypted, self.secret_msg)
 
+        password = intlist_to_bytes(self.key).decode('utf-8')
+        encrypted = base64.b64encode(
+            intlist_to_bytes(self.iv[:8]) +
+            b'\x0b\xe6\xa4\xd9z\x0e\xb8\xb9\xd0\xd4i_\x85\x1d\x99\x98_\xe5\x80\xe7.\xbf\xa5\x83'
+        )
+        decrypted = (aes_decrypt_text(encrypted, password, 32))
+        self.assertEqual(decrypted, self.secret_msg)
+
 if __name__ == '__main__':
     unittest.main()

From 93f787070ff2de48023c5bfeaebd153b3e29137a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 22 Mar 2015 15:39:35 +0100
Subject: [PATCH 1933/1937] [twitch] Only match digits for the video id

Urls can also contain contain a query (for example a timestamp '?t=foo')
---
 youtube_dl/extractor/twitch.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index aad2bf222..94bd6345d 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -149,7 +149,7 @@ class TwitchItemBaseIE(TwitchBaseIE):
 
 class TwitchVideoIE(TwitchItemBaseIE):
     IE_NAME = 'twitch:video'
-    _VALID_URL = r'%s/[^/]+/b/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'%s/[^/]+/b/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
     _ITEM_TYPE = 'video'
     _ITEM_SHORTCUT = 'a'
 
@@ -165,7 +165,7 @@ class TwitchVideoIE(TwitchItemBaseIE):
 
 class TwitchChapterIE(TwitchItemBaseIE):
     IE_NAME = 'twitch:chapter'
-    _VALID_URL = r'%s/[^/]+/c/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'%s/[^/]+/c/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
     _ITEM_TYPE = 'chapter'
     _ITEM_SHORTCUT = 'c'
 
@@ -184,7 +184,7 @@ class TwitchChapterIE(TwitchItemBaseIE):
 
 class TwitchVodIE(TwitchItemBaseIE):
     IE_NAME = 'twitch:vod'
-    _VALID_URL = r'%s/[^/]+/v/(?P<id>[^/]+)' % TwitchBaseIE._VALID_URL_BASE
+    _VALID_URL = r'%s/[^/]+/v/(?P<id>\d+)' % TwitchBaseIE._VALID_URL_BASE
     _ITEM_TYPE = 'vod'
     _ITEM_SHORTCUT = 'v'
 

From 9f790b9901445d532564b3c5c90c0ecfaea4805d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Mar 2015 21:23:57 +0600
Subject: [PATCH 1934/1937] [mlb] Improve _VALID_URL (Closes #5260)

---
 youtube_dl/extractor/mlb.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index 1a241aca7..e369551c2 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 
 class MLBIE(InfoExtractor):
-    _VALID_URL = r'https?://m(?:lb)?\.mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|(?:shared/video/embed/embed\.html|[^/]+/video/play\.jsp)\?.*?\bcontent_id=)(?P<id>n?\d+)'
+    _VALID_URL = r'https?://m(?:lb)?\.(?:[\da-z_-]+\.)?mlb\.com/(?:(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v|(?:shared/video/embed/embed\.html|[^/]+/video/play\.jsp)\?.*?\bcontent_id=)(?P<id>n?\d+)'
     _TESTS = [
         {
             'url': 'http://m.mlb.com/sea/video/topic/51231442/v34698933/nymsea-ackley-robs-a-home-run-with-an-amazing-catch/?c_id=sea',
@@ -80,6 +80,10 @@ class MLBIE(InfoExtractor):
             'url': 'http://mlb.mlb.com/es/video/play.jsp?content_id=36599553',
             'only_matching': True,
         },
+        {
+            'url': 'http://m.cardinals.mlb.com/stl/video/v51175783/atlstl-piscotty-makes-great-sliding-catch-on-line/?partnerId=as_mlb_20150321_42500876&adbid=579409712979910656&adbpl=tw&adbpr=52847728',
+            'only_matching': True,
+        }
     ]
 
     def _real_extract(self, url):

From b0872c19ea6fb5dcc20d695d6faead4af42da364 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 23 Mar 2015 22:15:01 +0600
Subject: [PATCH 1935/1937] [npo] Skip broken URL links (Closes #5266)

---
 youtube_dl/extractor/npo.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py
index 557dffa46..5d8448571 100644
--- a/youtube_dl/extractor/npo.py
+++ b/youtube_dl/extractor/npo.py
@@ -231,7 +231,10 @@ class NPOLiveIE(NPOBaseIE):
                 stream_url = self._download_json(
                     stream_info['stream'], display_id,
                     'Downloading %s URL' % stream_type,
-                    transform_source=strip_jsonp)
+                    'Unable to download %s URL' % stream_type,
+                    transform_source=strip_jsonp, fatal=False)
+                if not stream_url:
+                    continue
                 if stream_type == 'hds':
                     f4m_formats = self._extract_f4m_formats(stream_url, display_id)
                     # f4m downloader downloads only piece of live stream

From f5e2efbbf0a4a1fa02df09a12ce9ec69970ab14c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Mar 2015 16:39:46 +0100
Subject: [PATCH 1936/1937] [options] Handle special characters in argv (Fixes
 #5157)

---
 test/test_execution.py | 9 +++++++++
 youtube_dl/options.py  | 5 +++++
 2 files changed, 14 insertions(+)

diff --git a/test/test_execution.py b/test/test_execution.py
index 60df187de..f31e51558 100644
--- a/test/test_execution.py
+++ b/test/test_execution.py
@@ -1,4 +1,6 @@
 #!/usr/bin/env python
+# coding: utf-8
+
 from __future__ import unicode_literals
 
 import unittest
@@ -27,5 +29,12 @@ class TestExecution(unittest.TestCase):
     def test_main_exec(self):
         subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)
 
+    def test_cmdline_umlauts(self):
+        p = subprocess.Popen(
+            [sys.executable, 'youtube_dl/__main__.py', 'ä', '--version'],
+            cwd=rootDir, stdout=_DEV_NULL, stderr=subprocess.PIPE)
+        _, stderr = p.communicate()
+        self.assertFalse(stderr)
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 4e6e47d6f..35c7e5fb3 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -794,6 +794,11 @@ def parseOpts(overrideArguments=None):
             write_string('[debug] Override config: ' + repr(overrideArguments) + '\n')
     else:
         command_line_conf = sys.argv[1:]
+        # Workaround for Python 2.x, where argv is a byte list
+        if sys.version_info < (3,):
+            command_line_conf = [
+                a.decode('utf-8', 'replace') for a in command_line_conf]
+
         if '--ignore-config' in command_line_conf:
             system_conf = []
             user_conf = []

From 48c971e07336f1e5d33a760e454c52fac83392d5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 24 Mar 2015 16:39:53 +0100
Subject: [PATCH 1937/1937] release 2015.03.24

---
 docs/supportedsites.md | 2 ++
 youtube_dl/version.py  | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 72b365305..baf7b3880 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -231,6 +231,7 @@
  - **Letv**
  - **LetvPlaylist**
  - **LetvTv**
+ - **Libsyn**
  - **lifenews**: LIFE | NEWS
  - **LiveLeak**
  - **livestream**
@@ -310,6 +311,7 @@
  - **npo.nl:radio**
  - **npo.nl:radio:fragment**
  - **NRK**
+ - **NRKPlaylist**
  - **NRKTV**
  - **ntv.ru**
  - **Nuvid**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 51b4260aa..039ceadf2 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2015.03.18'
+__version__ = '2015.03.24'