From ac2547f5ffc30a352207336194e7bbb0435d01a7 Mon Sep 17 00:00:00 2001
From: Alex Van't Hof <alexvh@cs.columbia.edu>
Date: Thu, 31 Oct 2013 01:57:22 -0400
Subject: [PATCH 001/124] [teamcoco] Fix video url extraction for some videos

Video url extraction failed for some videos,
e.g. http://teamcoco.com/video/old-time-baseball

The url extracted was also occasionally suboptimal quality,
e.g. http://teamcoco.com/video/louis-ck-interview-george-w-bush
---
 youtube_dl/extractor/teamcoco.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index c910110ca..76246c7cc 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -3,6 +3,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
+    RegexNotFoundError,
 )
 
 
@@ -11,7 +12,7 @@ class TeamcocoIE(InfoExtractor):
     _TEST = {
         u'url': u'http://teamcoco.com/video/louis-ck-interview-george-w-bush',
         u'file': u'19705.mp4',
-        u'md5': u'27b6f7527da5acf534b15f21b032656e',
+        u'md5': u'cde9ba0fa3506f5f017ce11ead928f9a',
         u'info_dict': {
             u"description": u"Louis C.K. got starstruck by George W. Bush, so what? Part one.", 
             u"title": u"Louis C.K. Interview Pt. 1 11/3/11"
@@ -33,8 +34,21 @@ class TeamcocoIE(InfoExtractor):
         data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id
         data = self._download_webpage(data_url, video_id, 'Downloading data webpage')
 
-        video_url = self._html_search_regex(r'<file [^>]*type="high".*?>(.*?)</file>',
-            data, u'video URL')
+
+        qualities = [ '1080p', '720p', '1000k', '480p', '500k' ]
+        best_quality_idx = len(qualities)+1  # First regex match may not be optimal
+        for idx, quality in enumerate(qualities):
+            regex = r'<file [^>]*type="(?:high|standard)".*?>(.*%s.*)</file>' % quality
+            try:
+                url = self._html_search_regex(regex, data, u'video URL')
+                if idx < best_quality_idx:
+                    video_url = url
+                    best_quality_idx = idx
+            except RegexNotFoundError:
+                # Just catch fatal exc. Don't want the fatal=False warning
+                continue
+        if not video_url:
+            raise RegexNotFoundError(u'Unable to extract video URL')
 
         return [{
             'id':          video_id,

From 31366066bd18cfd32de901264f53f42fe96f55c2 Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Sat, 2 Nov 2013 18:08:16 +0100
Subject: [PATCH 002/124] Add support for live parameter to rtmpdump

---
 youtube_dl/FileDownloader.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index 8ecabab1a..0804dfbe1 100644
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -267,7 +267,7 @@ class FileDownloader(object):
             self.to_screen(u'\r%s[download] 100%% of %s in %s' %
                 (clear_line, data_len_str, self.format_seconds(tot_time)))
 
-    def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url):
+    def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url, live):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
         test = self.params.get('test', False)
@@ -294,6 +294,8 @@ class FileDownloader(object):
             basic_args += ['--tcUrl', url]
         if test:
             basic_args += ['--stop', '1']
+        if live:
+            basic_args += ['--live']
         args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)]
         if self.params.get('verbose', False):
             try:
@@ -411,7 +413,8 @@ class FileDownloader(object):
                                                 info_dict.get('player_url', None),
                                                 info_dict.get('page_url', None),
                                                 info_dict.get('play_path', None),
-                                                info_dict.get('tc_url', None))
+                                                info_dict.get('tc_url', None),
+                                                info_dict.get('live', False))
 
         # Attempt to download using mplayer
         if url.startswith('mms') or url.startswith('rtsp'):

From 0a43ddf3209e13f5e87b07c440e03a45deea3e57 Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Sat, 2 Nov 2013 18:08:35 +0100
Subject: [PATCH 003/124] [CinemassacreIE] Add live paramter to extracted info
 as a workaround

---
 youtube_dl/extractor/cinemassacre.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index 2fe1033f0..79d879ced 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -65,6 +65,7 @@ class CinemassacreIE(InfoExtractor):
             {
                 'url': url,
                 'play_path': 'mp4:' + sd_file,
+                'live': True, # workaround
                 'ext': 'flv',
                 'format': 'sd',
                 'format_id': 'sd',
@@ -72,6 +73,7 @@ class CinemassacreIE(InfoExtractor):
             {
                 'url': url,
                 'play_path': 'mp4:' + hd_file,
+                'live': True, # workaround
                 'ext': 'flv',
                 'format': 'hd',
                 'format_id': 'hd',

From 1f343eaabbb9e0daf67363b7737833cf5e2a3e16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= <iemejia@gmail.com>
Date: Sat, 2 Nov 2013 18:01:05 +0100
Subject: [PATCH 004/124] [subtitles] refactor to support websites with
 subtitle information the webpage.

I added the parameter webpage, so now it's similar to the way automatic
captions are handled. This is an improvement needed for websites like
TED.
---
 youtube_dl/extractor/dailymotion.py |  6 +++---
 youtube_dl/extractor/subtitles.py   | 12 ++++++------
 youtube_dl/extractor/youtube.py     |  2 +-
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 7d8353946..3aef82bcf 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -113,9 +113,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
         video_url = info[max_quality]
 
         # subtitles
-        video_subtitles = self.extract_subtitles(video_id)
+        video_subtitles = self.extract_subtitles(video_id, webpage)
         if self._downloader.params.get('listsubtitles', False):
-            self._list_available_subtitles(video_id)
+            self._list_available_subtitles(video_id, webpage)
             return
 
         return [{
@@ -129,7 +129,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor):
             'thumbnail': info['thumbnail_url']
         }]
 
-    def _get_available_subtitles(self, video_id):
+    def _get_available_subtitles(self, video_id, webpage):
         try:
             sub_list = self._download_webpage(
                 'https://api.dailymotion.com/video/%s/subtitles?fields=id,language,url' % video_id,
diff --git a/youtube_dl/extractor/subtitles.py b/youtube_dl/extractor/subtitles.py
index 90de7de3a..4b4c5235d 100644
--- a/youtube_dl/extractor/subtitles.py
+++ b/youtube_dl/extractor/subtitles.py
@@ -12,9 +12,9 @@ class SubtitlesInfoExtractor(InfoExtractor):
         return any([self._downloader.params.get('writesubtitles', False),
                     self._downloader.params.get('writeautomaticsub')])
 
-    def _list_available_subtitles(self, video_id, webpage=None):
+    def _list_available_subtitles(self, video_id, webpage):
         """ outputs the available subtitles for the video """
-        sub_lang_list = self._get_available_subtitles(video_id)
+        sub_lang_list = self._get_available_subtitles(video_id, webpage)
         auto_captions_list = self._get_available_automatic_caption(video_id, webpage)
         sub_lang = ",".join(list(sub_lang_list.keys()))
         self.to_screen(u'%s: Available subtitles for video: %s' %
@@ -23,7 +23,7 @@ class SubtitlesInfoExtractor(InfoExtractor):
         self.to_screen(u'%s: Available automatic captions for video: %s' %
                        (video_id, auto_lang))
 
-    def extract_subtitles(self, video_id, video_webpage=None):
+    def extract_subtitles(self, video_id, webpage):
         """
         returns {sub_lang: sub} ,{} if subtitles not found or None if the
         subtitles aren't requested.
@@ -32,9 +32,9 @@ class SubtitlesInfoExtractor(InfoExtractor):
             return None
         available_subs_list = {}
         if self._downloader.params.get('writeautomaticsub', False):
-            available_subs_list.update(self._get_available_automatic_caption(video_id, video_webpage))
+            available_subs_list.update(self._get_available_automatic_caption(video_id, webpage))
         if self._downloader.params.get('writesubtitles', False):
-            available_subs_list.update(self._get_available_subtitles(video_id))
+            available_subs_list.update(self._get_available_subtitles(video_id, webpage))
 
         if not available_subs_list:  # error, it didn't get the available subtitles
             return {}
@@ -74,7 +74,7 @@ class SubtitlesInfoExtractor(InfoExtractor):
             return
         return sub
 
-    def _get_available_subtitles(self, video_id):
+    def _get_available_subtitles(self, video_id, webpage):
         """
         returns {sub_lang: url} or {} if not available
         Must be redefined by the subclasses
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4347651d7..d7c9b38f9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1099,7 +1099,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         else:
             raise ExtractorError(u'Unable to decrypt signature, key length %d not supported; retrying might work' % (len(s)))
 
-    def _get_available_subtitles(self, video_id):
+    def _get_available_subtitles(self, video_id, webpage):
         try:
             sub_list = self._download_webpage(
                 'http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,

From a9a3876d55be943a7eaf505cbeb8fb862514db6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= <iemejia@gmail.com>
Date: Sat, 2 Nov 2013 19:48:39 +0100
Subject: [PATCH 005/124] [ted] Added support for subtitle download

---
 test/test_ted_subtitles.py  | 63 +++++++++++++++++++++++++++++++++++++
 youtube_dl/extractor/ted.py | 28 ++++++++++++++---
 2 files changed, 87 insertions(+), 4 deletions(-)
 create mode 100644 test/test_ted_subtitles.py

diff --git a/test/test_ted_subtitles.py b/test/test_ted_subtitles.py
new file mode 100644
index 000000000..3283253ab
--- /dev/null
+++ b/test/test_ted_subtitles.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+
+import sys
+import unittest
+import hashlib
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.extractor import TEDIE
+from youtube_dl.utils import *
+from helper import FakeYDL
+
+md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
+
+class TestTedSubtitles(unittest.TestCase):
+    def setUp(self):
+        self.DL = FakeYDL()
+        self.url = 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html'
+    def getInfoDict(self):
+        IE = TEDIE(self.DL)
+        info_dict = IE.extract(self.url)
+        return info_dict
+    def getSubtitles(self):
+        info_dict = self.getInfoDict()
+        return info_dict[0]['subtitles']
+    def test_no_writesubtitles(self):
+        subtitles = self.getSubtitles()
+        self.assertEqual(subtitles, None)
+    def test_subtitles(self):
+        self.DL.params['writesubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(md5(subtitles['en']), '2154f31ff9b9f89a0aa671537559c21d')
+    def test_subtitles_lang(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['subtitleslangs'] = ['fr']
+        subtitles = self.getSubtitles()
+        self.assertEqual(md5(subtitles['fr']), '7616cbc6df20ec2c1204083c83871cf6')
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(len(subtitles.keys()), 28)
+    def test_list_subtitles(self):
+        self.DL.params['listsubtitles'] = True
+        info_dict = self.getInfoDict()
+        self.assertEqual(info_dict, [None])
+    def test_automatic_captions(self):
+        self.DL.params['writeautomaticsub'] = True
+        self.DL.params['subtitleslang'] = ['en']
+        subtitles = self.getSubtitles()
+        self.assertTrue(len(subtitles.keys()) == 0)
+    def test_multiple_langs(self):
+        self.DL.params['writesubtitles'] = True
+        langs = ['es', 'fr', 'de']
+        self.DL.params['subtitleslangs'] = langs
+        subtitles = self.getSubtitles()
+        for lang in langs:
+            self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index dfa1176a3..239e2a448 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -1,10 +1,9 @@
 import json
 import re
 
-from .common import InfoExtractor
+from .subtitles import SubtitlesInfoExtractor
 
-
-class TEDIE(InfoExtractor):
+class TEDIE(SubtitlesInfoExtractor):
     _VALID_URL=r'''http://www\.ted\.com/
                    (
                         ((?P<type_playlist>playlists)/(?P<playlist_id>\d+)) # We have a playlist
@@ -82,11 +81,21 @@ class TEDIE(InfoExtractor):
             'url': stream['file'],
             'format': stream['id']
             } for stream in info['htmlStreams']]
+
+        video_id = info['id']
+
+        # subtitles
+        video_subtitles = self.extract_subtitles(video_id, webpage)
+        if self._downloader.params.get('listsubtitles', False):
+            self._list_available_subtitles(video_id, webpage)
+            return
+
         info = {
-            'id': info['id'],
+            'id': video_id,
             'title': title,
             'thumbnail': thumbnail,
             'description': desc,
+            'subtitles': video_subtitles,
             'formats': formats,
         }
 
@@ -94,3 +103,14 @@ class TEDIE(InfoExtractor):
         info.update(info['formats'][-1])
 
         return info
+
+    def _get_available_subtitles(self, video_id, webpage):
+        options = self._search_regex(r'(?:<select name="subtitles_language_select" id="subtitles_language_select">)(.*?)(?:</select>)', webpage, 'subtitles_language_select', flags=re.DOTALL)
+        languages = re.findall(r'(?:<option value=")(\S+)"', options)
+        if languages:
+            sub_lang_list = {}
+            for l in languages:
+                url = 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/srt' % (video_id, l)
+                sub_lang_list[l] = url
+            return sub_lang_list
+        return {}

From e7e6b54d8a02f7f76bd56e35bcd87a8723f71642 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 3 Nov 2013 17:48:12 +0100
Subject: [PATCH 006/124] [teamcoco] Parse the xml file and extract all the
 formats

---
 youtube_dl/extractor/teamcoco.py | 47 ++++++++++++++++++++------------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index 76246c7cc..bc48620f0 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -1,9 +1,9 @@
 import re
+import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    RegexNotFoundError,
 )
 
 
@@ -32,29 +32,40 @@ class TeamcocoIE(InfoExtractor):
         self.report_extraction(video_id)
 
         data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id
-        data = self._download_webpage(data_url, video_id, 'Downloading data webpage')
+        data_xml = self._download_webpage(data_url, video_id, 'Downloading data webpage')
+        data = xml.etree.ElementTree.fromstring(data_xml.encode('utf-8'))
 
 
-        qualities = [ '1080p', '720p', '1000k', '480p', '500k' ]
-        best_quality_idx = len(qualities)+1  # First regex match may not be optimal
-        for idx, quality in enumerate(qualities):
-            regex = r'<file [^>]*type="(?:high|standard)".*?>(.*%s.*)</file>' % quality
+        qualities = ['500k', '480p', '1000k', '720p', '1080p']
+        formats = []
+        for file in data.findall('files/file'):
+            if file.attrib.get('playmode') == 'all':
+                # it just duplicates one of the entries
+                break
+            file_url = file.text
+            m_format = re.search(r'(\d+(k|p))\.mp4', file_url)
+            if m_format is not None:
+                format_id = m_format.group(1)
+            else:
+                format_id = file.attrib['bitrate']
+            formats.append({
+                'url': file_url,
+                'ext': 'mp4',
+                'format_id': format_id,
+            })
+        def sort_key(f):
             try:
-                url = self._html_search_regex(regex, data, u'video URL')
-                if idx < best_quality_idx:
-                    video_url = url
-                    best_quality_idx = idx
-            except RegexNotFoundError:
-                # Just catch fatal exc. Don't want the fatal=False warning
-                continue
-        if not video_url:
+                return qualities.index(f['format_id'])
+            except ValueError:
+                return -1
+        formats.sort(key=sort_key)
+        if not formats:
             raise RegexNotFoundError(u'Unable to extract video URL')
 
-        return [{
+        return {
             'id':          video_id,
-            'url':         video_url,
-            'ext':         'mp4',
+            'formats': formats,
             'title':       self._og_search_title(webpage),
             'thumbnail':   self._og_search_thumbnail(webpage),
             'description': self._og_search_description(webpage),
-        }]
+        }

From 19b0668251583d5c7dbc55696746539979b2ba11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 4 Nov 2013 22:20:22 +0100
Subject: [PATCH 007/124] [canal2c] Accept more urls (fixes #1723)

The url only needs to have the 'idVideo' field in the query, in any position.
We have to set the 'void=oui' in the webpage url, so that we get the file name.
---
 youtube_dl/extractor/canalc2.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index e7f4fa9fd..3d8d7f9d2 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 
 class Canalc2IE(InfoExtractor):
     IE_NAME = 'canalc2.tv'
-    _VALID_URL = r'http://.*?\.canalc2\.tv/video\.asp\?idVideo=(\d+)&voir=oui'
+    _VALID_URL = r'http://.*?\.canalc2\.tv/video\.asp\?.*?idVideo=(?P<id>\d+)'
 
     _TEST = {
         u'url': u'http://www.canalc2.tv/video.asp?idVideo=12163&voir=oui',
@@ -18,7 +18,9 @@ class Canalc2IE(InfoExtractor):
     }
 
     def _real_extract(self, url):
-        video_id = re.match(self._VALID_URL, url).group(1)
+        video_id = re.match(self._VALID_URL, url).group('id')
+        # We need to set the voir field for getting the file name
+        url = 'http://www.canalc2.tv/video.asp?idVideo=%s&voir=oui' % video_id
         webpage = self._download_webpage(url, video_id)
         file_name = self._search_regex(
             r"so\.addVariable\('file','(.*?)'\);",

From 2dcf7d8f99604de9bdfa1b1e12a2bd5840d523d2 Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Tue, 5 Nov 2013 02:08:02 +0100
Subject: [PATCH 008/124] [GenericIE] Also detect youtube if src url of iframe
 is embedded in ' instaed of "

---
 youtube_dl/extractor/generic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 2c8fcf5ae..b3fec8e86 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -160,9 +160,9 @@ class GenericIE(InfoExtractor):
 
         # Look for embedded YouTube player
         mobj = re.search(
-            r'<iframe[^>]+?src="(https?://(?:www\.)?youtube.com/embed/.+?)"', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?youtube.com/embed/.+?)\1', webpage)
         if mobj:
-            surl = unescapeHTML(mobj.group(1))
+            surl = unescapeHTML(mobj.group(u'url'))
             return self.url_result(surl, 'Youtube')
 
         # Look for Bandcamp pages with custom domain

From 4ed3e51080dea651b3db8889b0c8f2e40d261075 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= <iemejia@gmail.com>
Date: Tue, 5 Nov 2013 12:00:13 +0100
Subject: [PATCH 009/124] [ted] fixed error in case of no subtitles present

I created a test, but I leave it commented since TED videos get
new subtitles frequently.
---
 test/test_ted_subtitles.py  |  6 ++++++
 youtube_dl/extractor/ted.py | 24 ++++++++++++++++--------
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/test/test_ted_subtitles.py b/test/test_ted_subtitles.py
index 3283253ab..a0dd7eeed 100644
--- a/test/test_ted_subtitles.py
+++ b/test/test_ted_subtitles.py
@@ -51,6 +51,12 @@ class TestTedSubtitles(unittest.TestCase):
         self.DL.params['subtitleslang'] = ['en']
         subtitles = self.getSubtitles()
         self.assertTrue(len(subtitles.keys()) == 0)
+    # def test_nosubtitles(self):
+    #     self.DL.expect_warning(u'video doesn\'t have subtitles')
+    #     self.url = 'http://www.ted.com/talks/rodrigo_canales_the_deadly_genius_of_drug_cartels.html'
+    #     self.DL.params['writesubtitles'] = True
+    #     self.DL.params['allsubtitles'] = True
+    #     subtitles = self.getSubtitles()
     def test_multiple_langs(self):
         self.DL.params['writesubtitles'] = True
         langs = ['es', 'fr', 'de']
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 239e2a448..1b006bc9b 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -3,6 +3,11 @@ import re
 
 from .subtitles import SubtitlesInfoExtractor
 
+from ..utils import (
+    compat_str,
+    RegexNotFoundError,
+)
+
 class TEDIE(SubtitlesInfoExtractor):
     _VALID_URL=r'''http://www\.ted\.com/
                    (
@@ -105,12 +110,15 @@ class TEDIE(SubtitlesInfoExtractor):
         return info
 
     def _get_available_subtitles(self, video_id, webpage):
-        options = self._search_regex(r'(?:<select name="subtitles_language_select" id="subtitles_language_select">)(.*?)(?:</select>)', webpage, 'subtitles_language_select', flags=re.DOTALL)
-        languages = re.findall(r'(?:<option value=")(\S+)"', options)
-        if languages:
-            sub_lang_list = {}
-            for l in languages:
-                url = 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/srt' % (video_id, l)
-                sub_lang_list[l] = url
-            return sub_lang_list
+        try:
+            options = self._search_regex(r'(?:<select name="subtitles_language_select" id="subtitles_language_select">)(.*?)(?:</select>)', webpage, 'subtitles_language_select', flags=re.DOTALL)
+            languages = re.findall(r'(?:<option value=")(\S+)"', options)
+            if languages:
+                sub_lang_list = {}
+                for l in languages:
+                    url = 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/srt' % (video_id, l)
+                    sub_lang_list[l] = url
+                return sub_lang_list
+        except RegexNotFoundError as err:
+            self._downloader.report_warning(u'video doesn\'t have subtitles')
         return {}

From a8eeb0597b11dbc9d1b48f95264cc2815311aa15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marcin=20Cie=C5=9Blak?= <saper@saper.info>
Date: Tue, 5 Nov 2013 23:19:29 +0100
Subject: [PATCH 010/124] Fix AssertionError when og property not found

On tvp.pl some webpages contain OpenGraph
metadata and some don't.

If og property is not found, _og_search_description
fails with

WARNING: unable to extract OpenGraph description; please report this issue on http://yt-dl.org/bug
Traceback (most recent call last):
  File "/usr/home/saper/bin/youtube-dl", line 18, in <module>
    youtube_dl.main()
  File "/usr/home/saper/sw/youtube-dl/youtube_dl/__init__.py", line 766, in main
    _real_main(argv)
  File "/usr/home/saper/sw/youtube-dl/youtube_dl/__init__.py", line 719, in _real_main
    retcode = ydl.download(all_urls)
  File "/usr/home/saper/sw/youtube-dl/youtube_dl/YoutubeDL.py", line 715, in download
    videos = self.extract_info(url)
  File "/usr/home/saper/sw/youtube-dl/youtube_dl/YoutubeDL.py", line 348, in extract_info
    ie_result = ie.extract(url)
  File "/usr/home/saper/sw/youtube-dl/youtube_dl/extractor/common.py", line 125, in extract
    return self._real_extract(url)
  File "/usr/home/saper/sw/youtube-dl/youtube_dl/extractor/tvp.py", line 56, in _real_extract
    info['description'] = self._og_search_description(webpage)
  File "/usr/home/saper/sw/youtube-dl/youtube_dl/extractor/common.py", line 331, in _og_search_description
    return self._og_search_property('description', html, fatal=False, **kargs)
  File "/usr/home/saper/sw/youtube-dl/youtube_dl/extractor/common.py", line 325, in _og_search_property
    return unescapeHTML(escaped)
  File "/usr/home/saper/sw/youtube-dl/youtube_dl/utils.py", line 494, in unescapeHTML
    assert type(s) == type(u'')
AssertionError

The patch allows me to use:

  try:
    info['description'] = self._og_search_description(webpage)
    info['thumbnail'] = self._og_search_thumbnail(webpage)
  except RegexNotFoundError:
    pass
---
 youtube_dl/extractor/common.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e0ccba533..fb2d50a09 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -322,7 +322,9 @@ class InfoExtractor(object):
         if name is None:
             name = 'OpenGraph %s' % prop
         escaped = self._search_regex(self._og_regex(prop), html, name, flags=re.DOTALL, **kargs)
-        return unescapeHTML(escaped)
+        if not escaped is None:
+            return unescapeHTML(escaped)
+        return None
 
     def _og_search_thumbnail(self, html, **kargs):
         return self._og_search_property('image', html, u'thumbnail url', fatal=False, **kargs)

From 5137ebac0b1438d22fe2c007e6172ee65e9311a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marcin=20Cie=C5=9Blak?= <saper@saper.info>
Date: Tue, 5 Nov 2013 23:30:25 +0100
Subject: [PATCH 011/124] [tvp] Telewizja Polska: new extractor for tvp.pl,
 fixes #1719

Thanks-To: mplonski

https://github.com/mplonski/linux/blob/master/tvp-dl.py
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/tvp.py      | 60 ++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 youtube_dl/extractor/tvp.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 888a91cce..78f84cea3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -132,6 +132,7 @@ from .tube8 import Tube8IE
 from .tudou import TudouIE
 from .tumblr import TumblrIE
 from .tutv import TutvIE
+from .tvp import TvpIE
 from .unistra import UnistraIE
 from .ustream import UstreamIE, UstreamChannelIE
 from .vbox7 import Vbox7IE
diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
new file mode 100644
index 000000000..63fb57bbe
--- /dev/null
+++ b/youtube_dl/extractor/tvp.py
@@ -0,0 +1,60 @@
+# encoding: utf-8
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    ExtractorError,
+    RegexNotFoundError,
+)
+
+class TvpIE(InfoExtractor):
+    IE_NAME = u'tvp.pl'
+    _VALID_URL = r'https?://www\.tvp\.pl/.*?wideo/(?P<date>\d+)/(?P<id>\d+)'
+    _INFO_URL = 'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s'
+
+
+    _TEST = {
+        u'url': u'http://www.tvp.pl/warszawa/magazyny/campusnews/wideo/31102013/12878238',
+        u'file': u'31.10.2013-12878238.wmv',
+        u'info_dict': {
+            u'title': u'31.10.2013',
+            u'description': u'31.10.2013',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        webpage = self._download_webpage(url, video_id, "Downloading video webpage")
+        json_params = self._download_webpage(self._INFO_URL % video_id, video_id, "Downloading video metadata")
+
+        try:
+            params = json.loads(json_params)
+        except:
+            raise ExtractorError(u'Invalid JSON')
+
+        self.report_extraction(video_id)
+        try:
+            video_url = params['video_url']
+        except KeyError:
+            raise ExtractorError('Missing JSON parameter: ' + sys.exc_info()[1])
+
+        try:
+            title = self._og_search_title(webpage)
+        except RegexNotFoundError:
+            title = video_id
+        info = {
+            'id': video_id,
+            'title': title,
+            'ext': 'wmv',
+            'url': video_url,
+        }
+        try:
+            info['description'] = self._og_search_description(webpage)
+            info['thumbnail'] = self._og_search_thumbnail(webpage)
+        except RegexNotFoundError:
+            pass
+
+        return info

From 76e67c2cb6bd041a84756ec291edf4028171fdfe Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 6 Nov 2013 14:01:43 +0100
Subject: [PATCH 012/124] Clean up imports

---
 youtube_dl/FileDownloader.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index 8ecabab1a..47f678811 100644
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -4,12 +4,19 @@ import re
 import subprocess
 import sys
 import time
-import traceback
 
 if os.name == 'nt':
     import ctypes
 
-from .utils import *
+from .utils import (
+    compat_urllib_error,
+    compat_urllib_request,
+    ContentTooShortError,
+    determine_ext,
+    encodeFilename,
+    sanitize_open,
+    timeconvert,
+)
 
 
 class FileDownloader(object):
@@ -194,7 +201,7 @@ class FileDownloader(object):
             if old_filename == new_filename:
                 return
             os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
-        except (IOError, OSError) as err:
+        except (IOError, OSError):
             self.report_error(u'unable to rename file')
 
     def try_utime(self, filename, last_modified_hdr):
@@ -251,7 +258,7 @@ class FileDownloader(object):
         """Report file has already been fully downloaded."""
         try:
             self.to_screen(u'[download] %s has already been downloaded' % file_name)
-        except (UnicodeEncodeError) as err:
+        except UnicodeEncodeError:
             self.to_screen(u'[download] The file has already been downloaded')
 
     def report_unable_to_resume(self):

From 50a886b7abdcce61986a2b589ca2366cc0984a58 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 6 Nov 2013 14:02:33 +0100
Subject: [PATCH 013/124] Fix reporting when file size is unkown (Fixes #1731)

---
 youtube_dl/FileDownloader.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index 47f678811..b0f26fc6f 100644
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -557,7 +557,7 @@ class FileDownloader(object):
             # Progress message
             speed = self.calc_speed(start, time.time(), byte_counter - resume_len)
             if data_len is None:
-                self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
+                self.report_progress('Unknown %', data_len_str, speed, 'Unknown ETA')
                 eta = None
             else:
                 percent = self.calc_percent(byte_counter, data_len)

From da54be877a59bfd040570168743501d7fd618278 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 6 Nov 2013 14:02:52 +0100
Subject: [PATCH 014/124] release 2013.11.06

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index cc0f9cb4e..57b4d419c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2013.11.03'
+__version__ = '2013.11.06'

From 9ee2b5f6f2a6e07ab901e47ede59eb5382ebb6ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 3 Nov 2013 19:14:53 +0100
Subject: [PATCH 015/124] tests: don't run the test if any of the extractors
 listed in the 'add_ie' field is marked as not working

---
 test/test_download.py            | 8 +++++++-
 youtube_dl/extractor/generic.py  | 2 ++
 youtube_dl/extractor/mtv.py      | 1 +
 youtube_dl/extractor/slashdot.py | 1 +
 youtube_dl/extractor/weibo.py    | 1 +
 5 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/test/test_download.py b/test/test_download.py
index 73379beb1..16f200809 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -31,6 +31,7 @@ from youtube_dl.utils import (
     ExtractorError,
     UnavailableVideoError,
 )
+from youtube_dl.extractor import get_info_extractor
 
 RETRIES = 3
 
@@ -63,9 +64,10 @@ def generator(test_case):
 
     def test_template(self):
         ie = youtube_dl.extractor.get_info_extractor(test_case['name'])
+        other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])]
         def print_skipping(reason):
             print('Skipping %s: %s' % (test_case['name'], reason))
-        if not ie._WORKING:
+        if not ie.working():
             print_skipping('IE marked as not _WORKING')
             return
         if 'playlist' not in test_case:
@@ -77,6 +79,10 @@ def generator(test_case):
         if 'skip' in test_case:
             print_skipping(test_case['skip'])
             return
+        for other_ie in other_ies:
+            if not other_ie.working():
+                print_skipping(u'test depends on %sIE, marked as not WORKING' % other_ie.ie_key())
+                return
 
         params = get_params(test_case.get('params', {}))
 
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index b3fec8e86..76d369273 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -33,6 +33,7 @@ class GenericIE(InfoExtractor):
         },
         # embedded vimeo video
         {
+            u'add_ie': ['Vimeo'],
             u'url': u'http://skillsmatter.com/podcast/home/move-semanticsperfect-forwarding-and-rvalue-references',
             u'file': u'22444065.mp4',
             u'md5': u'2903896e23df39722c33f015af0666e2',
@@ -44,6 +45,7 @@ class GenericIE(InfoExtractor):
         },
         # bandcamp page with custom domain
         {
+            u'add_ie': ['Bandcamp'],
             u'url': u'http://bronyrock.com/track/the-pony-mash',
             u'file': u'3235767654.mp3',
             u'info_dict': {
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index e96d3952c..24a79ae13 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -26,6 +26,7 @@ class MTVIE(InfoExtractor):
             },
         },
         {
+            u'add_ie': ['Vevo'],
             u'url': u'http://www.mtv.com/videos/taylor-swift/916187/everything-has-changed-ft-ed-sheeran.jhtml',
             u'file': u'USCJY1331283.mp4',
             u'md5': u'73b4e7fcadd88929292fe52c3ced8caf',
diff --git a/youtube_dl/extractor/slashdot.py b/youtube_dl/extractor/slashdot.py
index 2cba53076..f5003c7f9 100644
--- a/youtube_dl/extractor/slashdot.py
+++ b/youtube_dl/extractor/slashdot.py
@@ -7,6 +7,7 @@ class SlashdotIE(InfoExtractor):
     _VALID_URL = r'https?://tv.slashdot.org/video/\?embed=(?P<id>.*?)(&|$)'
 
     _TEST = {
+        u'add_ie': ['Ooyala'],
         u'url': u'http://tv.slashdot.org/video/?embed=JscHMzZDplD0p-yNLOzTfzC3Q3xzJaUz',
         u'file': u'JscHMzZDplD0p-yNLOzTfzC3Q3xzJaUz.mp4',
         u'md5': u'd2222e7a4a4c1541b3e0cf732fb26735',
diff --git a/youtube_dl/extractor/weibo.py b/youtube_dl/extractor/weibo.py
index 0757495bd..fa784ab99 100644
--- a/youtube_dl/extractor/weibo.py
+++ b/youtube_dl/extractor/weibo.py
@@ -13,6 +13,7 @@ class WeiboIE(InfoExtractor):
     _VALID_URL = r'https?://video\.weibo\.com/v/weishipin/t_(?P<id>.+?)\.htm'
 
     _TEST = {
+        u'add_ie': ['Sina'],
         u'url': u'http://video.weibo.com/v/weishipin/t_zjUw2kZ.htm',
         u'file': u'98322879.flv',
         u'info_dict': {

From eeb165e674e07aaa798f69e15f16faa01bc8feaa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Nov 2013 16:40:24 +0100
Subject: [PATCH 016/124] [brightcove] Add the extraction of the url from
 generic

---
 youtube_dl/extractor/brightcove.py | 16 +++++++++++++++-
 youtube_dl/extractor/generic.py    | 20 +++++++++++++++++---
 2 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 0d9b87a34..b3c3dc0fd 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -9,6 +9,7 @@ from ..utils import (
     compat_urllib_parse,
     find_xpath_attr,
     compat_urlparse,
+    compat_str,
 
     ExtractorError,
 )
@@ -71,6 +72,19 @@ class BrightcoveIE(InfoExtractor):
         data = compat_urllib_parse.urlencode(params)
         return cls._FEDERATED_URL_TEMPLATE % data
 
+    @classmethod
+    def _extract_brightcove_url(cls, webpage):
+        """Try to extract the brightcove url from the wepbage, returns None
+        if it can't be found
+        """
+        m_brightcove = re.search(
+            r'<object[^>]+?class=([\'"])[^>]*?BrightcoveExperience.*?\1.+?</object>',
+            webpage, re.DOTALL)
+        if m_brightcove is not None:
+            return cls._build_brighcove_url(m_brightcove.group())
+        else:
+            return None
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         query_str = mobj.group('query')
@@ -109,7 +123,7 @@ class BrightcoveIE(InfoExtractor):
 
     def _extract_video_info(self, video_info):
         info = {
-            'id': video_info['id'],
+            'id': compat_str(video_info['id']),
             'title': video_info['displayName'],
             'description': video_info.get('shortDescription'),
             'thumbnail': video_info.get('videoStillURL') or video_info.get('thumbnailURL'),
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 76d369273..04b7212f4 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -54,6 +54,21 @@ class GenericIE(InfoExtractor):
             },
             u'skip': u'There is a limit of 200 free downloads / month for the test song',
         },
+        # embedded brightcove video
+        {
+            u'add_ie': ['Brightcove'],
+            u'url': u'http://www.scientificamerican.com/article.cfm?id=soap-bubble-physics',
+            u'info_dict': {
+                u'id': u'2365799484001',
+                u'ext': u'mp4',
+                u'title': u'Bubble Simulation',
+                u'description': u'A visualization from a new computer model of foam behavior.',
+                u'uploader': u'Scientific American',
+            },
+            u'params': {
+                u'skip_download': True,
+            },
+        },
     ]
 
     def report_download_webpage(self, video_id):
@@ -146,10 +161,9 @@ class GenericIE(InfoExtractor):
 
         self.report_extraction(video_id)
         # Look for BrightCove:
-        m_brightcove = re.search(r'<object[^>]+?class=([\'"])[^>]*?BrightcoveExperience.*?\1.+?</object>', webpage, re.DOTALL)
-        if m_brightcove is not None:
+        bc_url = BrightcoveIE._extract_brightcove_url(webpage)
+        if bc_url is not None:
             self.to_screen(u'Brightcove video detected.')
-            bc_url = BrightcoveIE._build_brighcove_url(m_brightcove.group())
             return self.url_result(bc_url, 'Brightcove')
 
         # Look for embedded Vimeo player

From fc4a0c2aec42035281771c74575d001f7e8adcd3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Nov 2013 17:25:38 +0100
Subject: [PATCH 017/124] [brightcove] Change the 'videoId' or 'videoID' field
 to '@videoPlayer' (fixes #1697)

It seems to be needed when using the htmlFederated page
---
 youtube_dl/extractor/brightcove.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index b3c3dc0fd..0c6e13b9c 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -42,6 +42,17 @@ class BrightcoveIE(InfoExtractor):
                 u'uploader': u'Oracle',
             },
         },
+        {
+            # From http://mashable.com/2013/10/26/thermoelectric-bracelet-lets-you-control-your-body-temperature/
+            u'url': u'http://c.brightcove.com/services/viewer/federated_f9?&playerID=1265504713001&publisherID=AQ%7E%7E%2CAAABBzUwv1E%7E%2CxP-xFHVUstiMFlNYfvF4G9yFnNaqCw_9&videoID=2750934548001',
+            u'info_dict': {
+                u'id': u'2750934548001',
+                u'ext': u'mp4',
+                u'title': u'This Bracelet Acts as a Personal Thermostat',
+                u'description': u'md5:547b78c64f4112766ccf4e151c20b6a0',
+                u'uploader': u'Mashable',
+            },
+        },
     ]
 
     @classmethod
@@ -86,6 +97,8 @@ class BrightcoveIE(InfoExtractor):
             return None
 
     def _real_extract(self, url):
+        # Change the 'videoId' or 'videoID' field to '@videoPlayer'
+        url = re.sub(r'(?<=[?&])videoI(d|D)', '%40videoPlayer', url)
         mobj = re.match(self._VALID_URL, url)
         query_str = mobj.group('query')
         query = compat_urlparse.parse_qs(query_str)

From 065472936a81fcad263dc2f1b04fdfe4a221eeb9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Nov 2013 17:37:39 +0100
Subject: [PATCH 018/124] Add an extractor for space.com (fixes #1718)

It uses Brightcove, but requires some special process for getting a url with the playerKey field in some videos
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/space.py    | 35 ++++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+)
 create mode 100644 youtube_dl/extractor/space.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 888a91cce..d8626fb39 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -115,6 +115,7 @@ from .slideshare import SlideshareIE
 from .sohu import SohuIE
 from .soundcloud import SoundcloudIE, SoundcloudSetIE, SoundcloudUserIE
 from .southparkstudios import SouthParkStudiosIE
+from .space import SpaceIE
 from .spankwire import SpankwireIE
 from .spiegel import SpiegelIE
 from .stanfordoc import StanfordOpenClassroomIE
diff --git a/youtube_dl/extractor/space.py b/youtube_dl/extractor/space.py
new file mode 100644
index 000000000..0d32a0688
--- /dev/null
+++ b/youtube_dl/extractor/space.py
@@ -0,0 +1,35 @@
+import re
+
+from .common import InfoExtractor
+from .brightcove import BrightcoveIE
+from ..utils import RegexNotFoundError, ExtractorError
+
+
+class SpaceIE(InfoExtractor):
+    _VALID_URL = r'https?://www\.space\.com/\d+-(?P<title>[^/\.\?]*?)-video.html'
+    _TEST = {
+        u'add_ie': ['Brightcove'],
+        u'url': u'http://www.space.com/23373-huge-martian-landforms-detail-revealed-by-european-probe-video.html',
+        u'info_dict': {
+            u'id': u'2780937028001',
+            u'ext': u'mp4',
+            u'title': u'Huge Martian Landforms\' Detail Revealed By European Probe | Video',
+            u'description': u'md5:db81cf7f3122f95ed234b631a6ea1e61',
+            u'uploader': u'TechMedia Networks',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        title = mobj.group('title')
+        webpage = self._download_webpage(url, title)
+        try:
+            # Some videos require the playerKey field, which isn't define in
+            # the BrightcoveExperience object
+            brightcove_url = self._og_search_video_url(webpage)
+        except RegexNotFoundError:
+            # Other videos works fine with the info from the object
+            brightcove_url = BrightcoveIE._extract_brightcove_url(webpage)
+        if brightcove_url is None:
+            raise ExtractorError(u'The webpage does not contain a video', expected=True)
+        return self.url_result(brightcove_url, BrightcoveIE.ie_key())

From b0759f0c19fe223406bf3c7e59222183920eb714 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Nov 2013 19:05:41 +0100
Subject: [PATCH 019/124] [brightcove] Extract all the available formats

---
 youtube_dl/extractor/brightcove.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 0c6e13b9c..f0b79898c 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -146,10 +146,11 @@ class BrightcoveIE(InfoExtractor):
         renditions = video_info.get('renditions')
         if renditions:
             renditions = sorted(renditions, key=lambda r: r['size'])
-            best_format = renditions[-1]
-            info.update({
-                'url': best_format['defaultURL'],
-            })
+            info['formats'] = [{
+                'url': rend['defaultURL'],
+                'height': rend.get('frameHeight'),
+                'width': rend.get('frameWidth'),
+            } for rend in renditions]
         elif video_info.get('FLVFullLengthURL') is not None:
             info.update({
                 'url': video_info['FLVFullLengthURL'],

From 5d7b253ea071e4bf16e99242a84f1b0eab2d6f32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Nov 2013 20:05:28 +0100
Subject: [PATCH 020/124] Add an extractor for eitb.tv (fixes #1608)

The BrighcoveExperience object doesn't contain the video id, the extractor adds it and passes the url to BrightcoveIE.
---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/eitb.py     | 37 ++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+)
 create mode 100644 youtube_dl/extractor/eitb.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index d8626fb39..f9caca4ef 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -38,6 +38,7 @@ from .defense import DefenseGouvFrIE
 from .ebaumsworld import EbaumsWorldIE
 from .ehow import EHowIE
 from .eighttracks import EightTracksIE
+from .eitb import EitbIE
 from .escapist import EscapistIE
 from .exfm import ExfmIE
 from .extremetube import ExtremeTubeIE
diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
new file mode 100644
index 000000000..022d90abc
--- /dev/null
+++ b/youtube_dl/extractor/eitb.py
@@ -0,0 +1,37 @@
+# encoding: utf-8
+import re
+
+from .common import InfoExtractor
+from .brightcove import BrightcoveIE
+from ..utils import ExtractorError
+
+
+class EitbIE(InfoExtractor):
+    IE_NAME = u'eitb.tv'
+    _VALID_URL = r'https?://www\.eitb\.tv/(eu/bideoa|es/video)/[^/]+/(?P<playlist_id>\d+)/(?P<chapter_id>\d+)'
+
+    _TEST = {
+        u'add_ie': ['Brightcove'],
+        u'url': u'http://www.eitb.tv/es/video/60-minutos-60-minutos-2013-2014/2677100210001/2743577154001/lasa-y-zabala-30-anos/',
+        u'md5': u'edf4436247185adee3ea18ce64c47998',
+        u'info_dict': {
+            u'id': u'2743577154001',
+            u'ext': u'mp4',
+            u'title': u'60 minutos (Lasa y Zabala, 30 años)',
+            # All videos from eitb has this description in the brightcove info
+            u'description': u'.',
+            u'uploader': u'Euskal Telebista',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        chapter_id = mobj.group('chapter_id')
+        webpage = self._download_webpage(url, chapter_id)
+        bc_url = BrightcoveIE._extract_brightcove_url(webpage)
+        if bc_url is None:
+            raise ExtractorError(u'Could not extract the Brightcove url')
+        # The BrightcoveExperience object doesn't contain the video id, we set
+        # it manually
+        bc_url += '&%40videoPlayer={}'.format(chapter_id)
+        return self.url_result(bc_url, BrightcoveIE.ie_key())

From 4f045eef8fe63bcb925faaa5108a42d00b83f93a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Nov 2013 21:42:33 +0100
Subject: [PATCH 021/124] [youtube:channel] Fix the extraction

The page don't include the 'load more' button anymore, now we directly get the 'c4_browse_ajax' pages.
---
 youtube_dl/extractor/youtube.py | 31 ++++++++++---------------------
 1 file changed, 10 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 74a381fe2..f745b8b14 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1572,7 +1572,6 @@ class YoutubePlaylistIE(InfoExtractor):
 class YoutubeChannelIE(InfoExtractor):
     IE_DESC = u'YouTube.com channels'
     _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)"
-    _TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
     _MORE_PAGES_INDICATOR = 'yt-uix-load-more'
     _MORE_PAGES_URL = 'http://www.youtube.com/c4_browse_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
     IE_NAME = u'youtube:channel'
@@ -1593,30 +1592,20 @@ class YoutubeChannelIE(InfoExtractor):
         # Download channel page
         channel_id = mobj.group(1)
         video_ids = []
-        pagenum = 1
 
-        url = self._TEMPLATE_URL % (channel_id, pagenum)
-        page = self._download_webpage(url, channel_id,
-                                      u'Downloading page #%s' % pagenum)
+        # Download all channel pages using the json-based channel_ajax query
+        for pagenum in itertools.count(1):
+            url = self._MORE_PAGES_URL % (pagenum, channel_id)
+            page = self._download_webpage(url, channel_id,
+                                          u'Downloading page #%s' % pagenum)
 
-        # Extract video identifiers
-        ids_in_page = self.extract_videos_from_page(page)
-        video_ids.extend(ids_in_page)
+            page = json.loads(page)
 
-        # Download any subsequent channel pages using the json-based channel_ajax query
-        if self._MORE_PAGES_INDICATOR in page:
-            for pagenum in itertools.count(1):
-                url = self._MORE_PAGES_URL % (pagenum, channel_id)
-                page = self._download_webpage(url, channel_id,
-                                              u'Downloading page #%s' % pagenum)
+            ids_in_page = self.extract_videos_from_page(page['content_html'])
+            video_ids.extend(ids_in_page)
 
-                page = json.loads(page)
-
-                ids_in_page = self.extract_videos_from_page(page['content_html'])
-                video_ids.extend(ids_in_page)
-
-                if self._MORE_PAGES_INDICATOR  not in page['load_more_widget_html']:
-                    break
+            if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
+                break
 
         self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
 

From 51040b72edab854aecb436a415e31789a86166ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Nov 2013 22:03:00 +0100
Subject: [PATCH 022/124] [brightcove] Support redirected urls from bcove.me
 (fixes #1732)

'bctid' needs to be changed to '@videoPlayer', and 'bckey' to 'playerKey'.
---
 youtube_dl/extractor/brightcove.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index f0b79898c..0e60271f1 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -97,8 +97,10 @@ class BrightcoveIE(InfoExtractor):
             return None
 
     def _real_extract(self, url):
-        # Change the 'videoId' or 'videoID' field to '@videoPlayer'
-        url = re.sub(r'(?<=[?&])videoI(d|D)', '%40videoPlayer', url)
+        # Change the 'videoId' and others field to '@videoPlayer'
+        url = re.sub(r'(?<=[?&])(videoI(d|D)|bctid)', '%40videoPlayer', url)
+        # Change bckey (used by bcove.me urls) to playerKey
+        url = re.sub(r'(?<=[?&])bckey', 'playerKey', url)
         mobj = re.match(self._VALID_URL, url)
         query_str = mobj.group('query')
         query = compat_urlparse.parse_qs(query_str)

From 672fe94dcb69d6412a91c0996bf5ea1a6aa5ce93 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 6 Nov 2013 22:11:46 +0100
Subject: [PATCH 023/124] release 2013.11.06.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 57b4d419c..ad253e361 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2013.11.06'
+__version__ = '2013.11.06.1'

From b1a80ec1a986219e3b04fb443a2622dacfba15f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Nov 2013 23:45:01 +0100
Subject: [PATCH 024/124] [xnxx] Accept urls that start with 'www' (fixes
 #1734)

---
 youtube_dl/extractor/xnxx.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xnxx.py b/youtube_dl/extractor/xnxx.py
index 8a0eb1afd..1177a4b14 100644
--- a/youtube_dl/extractor/xnxx.py
+++ b/youtube_dl/extractor/xnxx.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class XNXXIE(InfoExtractor):
-    _VALID_URL = r'^(?:https?://)?video\.xnxx\.com/video([0-9]+)/(.*)'
+    _VALID_URL = r'^(?:https?://)?(?:video|www)\.xnxx\.com/video([0-9]+)/(.*)'
     VIDEO_URL_RE = r'flv_url=(.*?)&amp;'
     VIDEO_TITLE_RE = r'<title>(.*?)\s+-\s+XNXX.COM'
     VIDEO_THUMB_RE = r'url_bigthumb=(.*?)&amp;'

From 4ac5306ae794d26400b7ae4513eb1be4f877403a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 7 Nov 2013 08:03:35 +0100
Subject: [PATCH 025/124] Fix the report progress when file_size is unknown
 (#1731)

The report_progress function will accept eta and percent with None value and will set the message to 'Unknow ETA' or 'Unknown %'.
Otherwise the values must be numbers.
---
 youtube_dl/FileDownloader.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index b0f26fc6f..20f57b6bf 100644
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -234,8 +234,14 @@ class FileDownloader(object):
         if self.params.get('noprogress', False):
             return
         clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'')
-        eta_str = self.format_eta(eta)
-        percent_str = self.format_percent(percent)
+        if eta is not None:
+            eta_str = self.format_eta(eta)
+        else:
+            eta_str = 'Unknown ETA'
+        if percent is not None:
+            percent_str = self.format_percent(percent)
+        else:
+            percent_str = 'Unknown %'
         speed_str = self.format_speed(speed)
         if self.params.get('progress_with_newline', False):
             self.to_screen(u'[download] %s of %s at %s ETA %s' %
@@ -557,12 +563,11 @@ class FileDownloader(object):
             # Progress message
             speed = self.calc_speed(start, time.time(), byte_counter - resume_len)
             if data_len is None:
-                self.report_progress('Unknown %', data_len_str, speed, 'Unknown ETA')
-                eta = None
+                eta = percent = None
             else:
                 percent = self.calc_percent(byte_counter, data_len)
                 eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
-                self.report_progress(percent, data_len_str, speed, eta)
+            self.report_progress(percent, data_len_str, speed, eta)
 
             self._hook_progress({
                 'downloaded_bytes': byte_counter,

From 6161d17579687abb4063380ae64c271398463e6b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Thu, 7 Nov 2013 11:06:34 +0100
Subject: [PATCH 026/124] release 2013.11.07

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index ad253e361..84bf0f35c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2013.11.06.1'
+__version__ = '2013.11.07'

From dd5bcdc4c9cad0ee7e2d61343129ee1111048189 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 7 Nov 2013 21:06:48 +0100
Subject: [PATCH 027/124] [brightcove] Set the 'Referer' header if the url has
 the 'linkBaseUrl' parameter (fixes #1553)

---
 youtube_dl/extractor/brightcove.py | 17 +++++++++++++----
 youtube_dl/extractor/generic.py    | 12 +++++++-----
 2 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 0e60271f1..d8c35465a 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -10,10 +10,12 @@ from ..utils import (
     find_xpath_attr,
     compat_urlparse,
     compat_str,
+    compat_urllib_request,
 
     ExtractorError,
 )
 
+
 class BrightcoveIE(InfoExtractor):
     _VALID_URL = r'https?://.*brightcove\.com/(services|viewer).*\?(?P<query>.*)'
     _FEDERATED_URL_TEMPLATE = 'http://c.brightcove.com/services/viewer/htmlFederated?%s'
@@ -80,6 +82,9 @@ class BrightcoveIE(InfoExtractor):
         videoPlayer = find_xpath_attr(object_doc, './param', 'name', '@videoPlayer')
         if videoPlayer is not None:
             params['@videoPlayer'] = videoPlayer.attrib['value']
+        linkBase = find_xpath_attr(object_doc, './param', 'name', 'linkBaseURL')
+        if linkBase is not None:
+            params['linkBaseURL'] = linkBase.attrib['value']
         data = compat_urllib_parse.urlencode(params)
         return cls._FEDERATED_URL_TEMPLATE % data
 
@@ -107,14 +112,18 @@ class BrightcoveIE(InfoExtractor):
 
         videoPlayer = query.get('@videoPlayer')
         if videoPlayer:
-            return self._get_video_info(videoPlayer[0], query_str)
+            return self._get_video_info(videoPlayer[0], query_str, query)
         else:
             player_key = query['playerKey']
             return self._get_playlist_info(player_key[0])
 
-    def _get_video_info(self, video_id, query):
-        request_url = self._FEDERATED_URL_TEMPLATE % query
-        webpage = self._download_webpage(request_url, video_id)
+    def _get_video_info(self, video_id, query_str, query):
+        request_url = self._FEDERATED_URL_TEMPLATE % query_str
+        req = compat_urllib_request.Request(request_url)
+        linkBase = query.get('linkBaseURL')
+        if linkBase is not None:
+            req.add_header('Referer', linkBase[0])
+        webpage = self._download_webpage(req, video_id)
 
         self.report_extraction(video_id)
         info = self._search_regex(r'var experienceJSON = ({.*?});', webpage, 'json')
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 04b7212f4..c7552fddb 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -55,15 +55,17 @@ class GenericIE(InfoExtractor):
             u'skip': u'There is a limit of 200 free downloads / month for the test song',
         },
         # embedded brightcove video
+        # it also tests brightcove videos that need to set the 'Referer' in the
+        # http requests
         {
             u'add_ie': ['Brightcove'],
-            u'url': u'http://www.scientificamerican.com/article.cfm?id=soap-bubble-physics',
+            u'url': u'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
             u'info_dict': {
-                u'id': u'2365799484001',
+                u'id': u'2765128793001',
                 u'ext': u'mp4',
-                u'title': u'Bubble Simulation',
-                u'description': u'A visualization from a new computer model of foam behavior.',
-                u'uploader': u'Scientific American',
+                u'title': u'Le cours de bourse : l’analyse technique',
+                u'description': u'md5:7e9ad046e968cb2d1114004aba466fd9',
+                u'uploader': u'BFM BUSINESS',
             },
             u'params': {
                 u'skip_download': True,

From be07375b66df1a803eab27fee7112214f1f2b392 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Nov 2013 16:40:00 +0100
Subject: [PATCH 028/124] Don't recode the video with m3u8 downloads (fixes
 #1741)

---
 youtube_dl/FileDownloader.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index 20f57b6bf..35fa3ca61 100644
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -379,7 +379,8 @@ class FileDownloader(object):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
-        args = ['ffmpeg', '-y', '-i', url, '-f', 'mp4', tmpfilename]
+        args = ['ffmpeg', '-y', '-i', url, '-f', 'mp4', '-c', 'copy',
+            '-absf', 'aac_adtstoasc', tmpfilename]
         # Check for ffmpeg first
         try:
             subprocess.call(['ffmpeg', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)

From 20aafee7fa17407124eccb9bcc047079acd31fcf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 6 Nov 2013 21:47:02 +0100
Subject: [PATCH 029/124] [kankan] Fix the video url

It now requires two additional parameters, one is a timestamp we get from the getCdnresource_flv page and the other is a key we have to build.
---
 youtube_dl/extractor/kankan.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/kankan.py b/youtube_dl/extractor/kankan.py
index 445d46501..50916f4a6 100644
--- a/youtube_dl/extractor/kankan.py
+++ b/youtube_dl/extractor/kankan.py
@@ -1,8 +1,10 @@
 import re
+import hashlib
 
 from .common import InfoExtractor
 from ..utils import determine_ext
 
+_md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
 
 class KankanIE(InfoExtractor):
     _VALID_URL = r'https?://(?:.*?\.)?kankan\.com/.+?/(?P<id>\d+)\.shtml'
@@ -30,7 +32,10 @@ class KankanIE(InfoExtractor):
                                                  video_id, u'Downloading video url info')
         ip = self._search_regex(r'ip:"(.+?)"', video_info_page, u'video url ip')
         path = self._search_regex(r'path:"(.+?)"', video_info_page, u'video url path')
-        video_url = 'http://%s%s' % (ip, path)
+        param1 = self._search_regex(r'param1:(\d+)', video_info_page, u'param1')
+        param2 = self._search_regex(r'param2:(\d+)', video_info_page, u'param2')
+        key = _md5('xl_mp43651' + param1 + param2)
+        video_url = 'http://%s%s?key=%s&key1=%s' % (ip, path, key, param2)
 
         return {'id': video_id,
                 'title': title,

From 12c167c881875360345b9b542052ac204b7c34a7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Nov 2013 18:06:09 +0100
Subject: [PATCH 030/124] [soundcloud] Allow to download tracks marked as not
 'streamable'

They use the rtmp protocol but if the are marked as 'downloadable' it can use the direct download link.
---
 test/test_playlists.py             |   9 ++
 youtube_dl/extractor/soundcloud.py | 130 +++++++++++------------------
 2 files changed, 59 insertions(+), 80 deletions(-)

diff --git a/test/test_playlists.py b/test/test_playlists.py
index de1e8d88e..706b6bdca 100644
--- a/test/test_playlists.py
+++ b/test/test_playlists.py
@@ -17,6 +17,7 @@ from youtube_dl.extractor import (
     DailymotionUserIE,
     VimeoChannelIE,
     UstreamChannelIE,
+    SoundcloudSetIE,
     SoundcloudUserIE,
     LivestreamIE,
     NHLVideocenterIE,
@@ -61,6 +62,14 @@ class TestPlaylists(unittest.TestCase):
         self.assertEqual(result['id'], u'5124905')
         self.assertTrue(len(result['entries']) >= 11)
 
+    def test_soundcloud_set(self):
+        dl = FakeYDL()
+        ie = SoundcloudSetIE(dl)
+        result = ie.extract('https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep')
+        self.assertIsPlaylist(result)
+        self.assertEqual(result['title'], u'The Royal Concept EP')
+        self.assertTrue(len(result['entries']) >= 6)
+
     def test_soundcloud_user(self):
         dl = FakeYDL()
         ie = SoundcloudUserIE(dl)
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 29cd5617c..4717fbb77 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -29,17 +29,34 @@ class SoundcloudIE(InfoExtractor):
                     )
                     '''
     IE_NAME = u'soundcloud'
-    _TEST = {
-        u'url': u'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy',
-        u'file': u'62986583.mp3',
-        u'md5': u'ebef0a451b909710ed1d7787dddbf0d7',
-        u'info_dict': {
-            u"upload_date": u"20121011", 
-            u"description": u"No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o'd", 
-            u"uploader": u"E.T. ExTerrestrial Music", 
-            u"title": u"Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1"
-        }
-    }
+    _TESTS = [
+        {
+            u'url': u'http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy',
+            u'file': u'62986583.mp3',
+            u'md5': u'ebef0a451b909710ed1d7787dddbf0d7',
+            u'info_dict': {
+                u"upload_date": u"20121011", 
+                u"description": u"No Downloads untill we record the finished version this weekend, i was too pumped n i had to post it , earl is prolly gonna b hella p.o'd", 
+                u"uploader": u"E.T. ExTerrestrial Music", 
+                u"title": u"Lostin Powers - She so Heavy (SneakPreview) Adrian Ackers Blueprint 1"
+            }
+        },
+        # not streamable song
+        {
+            u'url': u'https://soundcloud.com/the-concept-band/goldrushed-mastered?in=the-concept-band/sets/the-royal-concept-ep',
+            u'info_dict': {
+                u'id': u'47127627',
+                u'ext': u'mp3',
+                u'title': u'Goldrushed',
+                u'uploader': u'The Royal Concept',
+                u'upload_date': u'20120521',
+            },
+            u'params': {
+                # rtmp
+                u'skip_download': True,
+            },
+        },
+    ]
 
     _CLIENT_ID = 'b45b1aa10f1ac2941910a7f0d10f8e28'
 
@@ -56,16 +73,16 @@ class SoundcloudIE(InfoExtractor):
         return 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID
 
     def _extract_info_dict(self, info, full_title=None, quiet=False):
-        video_id = info['id']
-        name = full_title or video_id
+        track_id = compat_str(info['id'])
+        name = full_title or track_id
         if quiet == False:
             self.report_extraction(name)
 
         thumbnail = info['artwork_url']
         if thumbnail is not None:
             thumbnail = thumbnail.replace('-large', '-t500x500')
-        return {
-            'id':       info['id'],
+        result = {
+            'id':       track_id,
             'url':      info['stream_url'] + '?client_id=' + self._CLIENT_ID,
             'uploader': info['user']['username'],
             'upload_date': unified_strdate(info['created_at']),
@@ -74,6 +91,21 @@ class SoundcloudIE(InfoExtractor):
             'description': info['description'],
             'thumbnail': thumbnail,
         }
+        if info.get('downloadable', False):
+            result['url'] = 'https://api.soundcloud.com/tracks/{0}/download?client_id={1}'.format(track_id, self._CLIENT_ID)
+        if not info.get('streamable', False):
+            # We have to get the rtmp url
+            stream_json = self._download_webpage(
+                'http://api.soundcloud.com/i1/tracks/{0}/streams?client_id={1}'.format(track_id, self._CLIENT_ID),
+                track_id, u'Downloading track url')
+            rtmp_url = json.loads(stream_json)['rtmp_mp3_128_url']
+            # The url doesn't have an rtmp app, we have to extract the playpath
+            url, path = rtmp_url.split('mp3:', 1)
+            result.update({
+                'url': url,
+                'play_path': 'mp3:' + path,
+            })
+        return result
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url, flags=re.VERBOSE)
@@ -106,70 +138,8 @@ class SoundcloudIE(InfoExtractor):
 class SoundcloudSetIE(SoundcloudIE):
     _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/sets/([\w\d-]+)(?:[?].*)?$'
     IE_NAME = u'soundcloud:set'
-    _TEST = {
-        u"url":"https://soundcloud.com/the-concept-band/sets/the-royal-concept-ep",
-        u"playlist": [
-            {
-                u"file":"30510138.mp3",
-                u"md5":"f9136bf103901728f29e419d2c70f55d",
-                u"info_dict": {
-                    u"upload_date": u"20111213",
-                    u"description": u"The Royal Concept from Stockholm\r\nFilip / Povel / David / Magnus\r\nwww.royalconceptband.com",
-                    u"uploader": u"The Royal Concept",
-                    u"title": u"D-D-Dance"
-                }
-            },
-            {
-                u"file":"47127625.mp3",
-                u"md5":"09b6758a018470570f8fd423c9453dd8",
-                u"info_dict": {
-                    u"upload_date": u"20120521",
-                    u"description": u"The Royal Concept from Stockholm\r\nFilip / Povel / David / Magnus\r\nwww.royalconceptband.com",
-                    u"uploader": u"The Royal Concept",
-                    u"title": u"The Royal Concept - Gimme Twice"
-                }
-            },
-            {
-                u"file":"47127627.mp3",
-                u"md5":"154abd4e418cea19c3b901f1e1306d9c",
-                u"info_dict": {
-                    u"upload_date": u"20120521",
-                    u"uploader": u"The Royal Concept",
-                    u"title": u"Goldrushed"
-                }
-            },
-            {
-                u"file":"47127629.mp3",
-                u"md5":"2f5471edc79ad3f33a683153e96a79c1",
-                u"info_dict": {
-                    u"upload_date": u"20120521",
-                    u"description": u"The Royal Concept from Stockholm\r\nFilip / Povel / David / Magnus\r\nwww.royalconceptband.com",
-                    u"uploader": u"The Royal Concept",
-                    u"title": u"In the End"
-                }
-            },
-            {
-                u"file":"47127631.mp3",
-                u"md5":"f9ba87aa940af7213f98949254f1c6e2",
-                u"info_dict": {
-                    u"upload_date": u"20120521",
-                    u"description": u"The Royal Concept from Stockholm\r\nFilip / David / Povel / Magnus\r\nwww.theroyalconceptband.com",
-                    u"uploader": u"The Royal Concept",
-                    u"title": u"Knocked Up"
-                }
-            },
-            {
-                u"file":"75206121.mp3",
-                u"md5":"f9d1fe9406717e302980c30de4af9353",
-                u"info_dict": {
-                    u"upload_date": u"20130116",
-                    u"description": u"The unreleased track World on Fire premiered on the CW's hit show Arrow (8pm/7pm central).  \r\nAs a gift to our fans we would like to offer you a free download of the track!  ",
-                    u"uploader": u"The Royal Concept",
-                    u"title": u"World On Fire"
-                }
-            }
-        ]
-    }
+    # it's in tests/test_playlists.py
+    _TESTS = []
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
@@ -208,7 +178,7 @@ class SoundcloudUserIE(SoundcloudIE):
     IE_NAME = u'soundcloud:user'
 
     # it's in tests/test_playlists.py
-    _TEST = None
+    _TESTS = []
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)

From c2b6a482d5aadd40972944694307fca3d4d352f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Nov 2013 18:10:11 +0100
Subject: [PATCH 031/124] [brightcove] the format function requires to specify
 the index in python2.6

---
 youtube_dl/extractor/eitb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eitb.py b/youtube_dl/extractor/eitb.py
index 022d90abc..4ba323148 100644
--- a/youtube_dl/extractor/eitb.py
+++ b/youtube_dl/extractor/eitb.py
@@ -33,5 +33,5 @@ class EitbIE(InfoExtractor):
             raise ExtractorError(u'Could not extract the Brightcove url')
         # The BrightcoveExperience object doesn't contain the video id, we set
         # it manually
-        bc_url += '&%40videoPlayer={}'.format(chapter_id)
+        bc_url += '&%40videoPlayer={0}'.format(chapter_id)
         return self.url_result(bc_url, BrightcoveIE.ie_key())

From 81be02d2f97f63fef0ec6216b8dca2b7184b36a9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Nov 2013 18:16:32 +0100
Subject: [PATCH 032/124] [cnn] Accept www.cnn.com urls (fixes #1740)

---
 youtube_dl/extractor/cnn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py
index a79f881cd..34adf6dda 100644
--- a/youtube_dl/extractor/cnn.py
+++ b/youtube_dl/extractor/cnn.py
@@ -6,7 +6,7 @@ from ..utils import determine_ext
 
 
 class CNNIE(InfoExtractor):
-    _VALID_URL = r'''(?x)https?://(edition\.)?cnn\.com/video/(data/.+?|\?)/
+    _VALID_URL = r'''(?x)https?://((edition|www)\.)?cnn\.com/video/(data/.+?|\?)/
         (?P<path>.+?/(?P<title>[^/]+?)(?:\.cnn|(?=&)))'''
 
     _TESTS = [{

From 566d4e0425245e46f211fec8cd38d369e2c8de2c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Nov 2013 19:01:23 +0100
Subject: [PATCH 033/124] [arte] Make sure the format_id is unique (closes
 #1739)

Include the bitrate and use the height instead of the quality field.
---
 youtube_dl/extractor/arte.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index e10c74c11..a636c6be5 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -10,6 +10,7 @@ from ..utils import (
     unified_strdate,
     determine_ext,
     get_element_by_id,
+    compat_str,
 )
 
 # There are different sources of video in arte.tv, the extraction process 
@@ -191,10 +192,13 @@ class ArteTVPlus7IE(InfoExtractor):
         formats = sorted(formats, key=lambda f: re.match(r'VO(F|A)-STM\1', f.get('versionCode', '')) is None)
         # Pick the best quality
         def _format(format_info):
-            quality = format_info['quality']
-            m_quality = re.match(r'\w*? - (\d*)p', quality)
-            if m_quality is not None:
-                quality = m_quality.group(1)
+            quality = ''
+            height = format_info.get('height')
+            if height is not None:
+                quality = compat_str(height)
+            bitrate = format_info.get('bitrate')
+            if bitrate is not None:
+                quality += '-%d' % bitrate
             if format_info.get('versionCode') is not None:
                 format_id = u'%s-%s' % (quality, format_info['versionCode'])
             else:
@@ -203,7 +207,7 @@ class ArteTVPlus7IE(InfoExtractor):
                 'format_id': format_id,
                 'format_note': format_info.get('versionLibelle'),
                 'width': format_info.get('width'),
-                'height': format_info.get('height'),
+                'height': height,
             }
             if format_info['mediaType'] == u'rtmp':
                 info['url'] = format_info['streamer']

From f470c6c812882d3726fe379d21069adac3cb2c69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Nov 2013 19:05:19 +0100
Subject: [PATCH 034/124] [arte] Improve the format sorting

Also use the bitrate.
Prefer normal version and sourds/mal version over original version with subtitles.
---
 youtube_dl/extractor/arte.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index a636c6be5..b35a679e3 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -182,15 +182,22 @@ class ArteTVPlus7IE(InfoExtractor):
                 formats = all_formats
             else:
                 raise ExtractorError(u'The formats list is empty')
-        # We order the formats by quality
+
         if re.match(r'[A-Z]Q', formats[0]['quality']) is not None:
-            sort_key = lambda f: ['HQ', 'MQ', 'EQ', 'SQ'].index(f['quality'])
+            def sort_key(f):
+                return ['HQ', 'MQ', 'EQ', 'SQ'].index(f['quality'])
         else:
-            sort_key = lambda f: int(f.get('height',-1))
+            def sort_key(f):
+                return (
+                    # Sort first by quality
+                    int(f.get('height',-1)),
+                    int(f.get('bitrate',-1)),
+                    # The original version with subtitles has lower relevance
+                    re.match(r'VO-ST(F|A)', f.get('versionCode', '')) is None,
+                    # The version with sourds/mal subtitles has also lower relevance
+                    re.match(r'VO?(F|A)-STM\1', f.get('versionCode', '')) is None,
+                )
         formats = sorted(formats, key=sort_key)
-        # Prefer videos without subtitles in the same language
-        formats = sorted(formats, key=lambda f: re.match(r'VO(F|A)-STM\1', f.get('versionCode', '')) is None)
-        # Pick the best quality
         def _format(format_info):
             quality = ''
             height = format_info.get('height')

From 897d6cc43a25f68c941cda80afaa47acaf7779bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Nov 2013 19:07:34 +0100
Subject: [PATCH 035/124] Improve format listing for long format ids

Now arte.tv videos have quite long ids.
---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 86a6fd043..5253c39e1 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -782,7 +782,7 @@ class YoutubeDL(object):
 
     def list_formats(self, info_dict):
         def line(format):
-            return (u'%-15s%-10s%-12s%s' % (
+            return (u'%-20s%-10s%-12s%s' % (
                 format['format_id'],
                 format['ext'],
                 self.format_resolution(format),

From 2a9e9b210b27778a71d920d4c7a0508c0cd446f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sat, 9 Nov 2013 19:21:30 +0100
Subject: [PATCH 036/124] Fix the documentation of '--autonumber-size' (#1743)

it's '--auto-number' not '--autonumber'
---
 youtube_dl/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 48ffcbf8e..ab7879c5d 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -349,7 +349,7 @@ def parseOpts(overrideArguments=None):
                   'for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .'))
     filesystem.add_option('--autonumber-size',
             dest='autonumber_size', metavar='NUMBER',
-            help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --autonumber option is given')
+            help='Specifies the number of digits in %(autonumber)s when it is present in output filename template or --auto-number option is given')
     filesystem.add_option('--restrict-filenames',
             action='store_true', dest='restrictfilenames',
             help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)

From be6dfd1b4954e32009f788ea55ee06bb82781f4b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 10 Nov 2013 12:09:12 +0100
Subject: [PATCH 037/124] [ted] Return a single info_dict for talks urls

It failed with the --list-subs option
---
 test/test_ted_subtitles.py  | 8 +++-----
 youtube_dl/extractor/ted.py | 2 +-
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/test/test_ted_subtitles.py b/test/test_ted_subtitles.py
index a0dd7eeed..4c164ae99 100644
--- a/test/test_ted_subtitles.py
+++ b/test/test_ted_subtitles.py
@@ -10,9 +10,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from youtube_dl.extractor import TEDIE
 from youtube_dl.utils import *
-from helper import FakeYDL
-
-md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
+from test.helper import FakeYDL, md5
 
 class TestTedSubtitles(unittest.TestCase):
     def setUp(self):
@@ -24,7 +22,7 @@ class TestTedSubtitles(unittest.TestCase):
         return info_dict
     def getSubtitles(self):
         info_dict = self.getInfoDict()
-        return info_dict[0]['subtitles']
+        return info_dict['subtitles']
     def test_no_writesubtitles(self):
         subtitles = self.getSubtitles()
         self.assertEqual(subtitles, None)
@@ -45,7 +43,7 @@ class TestTedSubtitles(unittest.TestCase):
     def test_list_subtitles(self):
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
-        self.assertEqual(info_dict, [None])
+        self.assertEqual(info_dict, None)
     def test_automatic_captions(self):
         self.DL.params['writeautomaticsub'] = True
         self.DL.params['subtitleslang'] = ['en']
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 1b006bc9b..76cfdfb90 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -36,7 +36,7 @@ class TEDIE(SubtitlesInfoExtractor):
     def _real_extract(self, url):
         m=re.match(self._VALID_URL, url, re.VERBOSE)
         if m.group('type_talk'):
-            return [self._talk_info(url)]
+            return self._talk_info(url)
         else :
             playlist_id=m.group('playlist_id')
             name=m.group('name')

From ae84f879d77bab5ea6fd9a4d74b82b46a11c9072 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 10 Nov 2013 12:28:21 +0100
Subject: [PATCH 038/124] Merge all the subtitles test into a single file

They reuse a base class
---
 test/test_dailymotion_subtitles.py |  70 ----------
 test/test_subtitles.py             | 211 +++++++++++++++++++++++++++++
 test/test_ted_subtitles.py         |  67 ---------
 test/test_youtube_subtitles.py     |  95 -------------
 4 files changed, 211 insertions(+), 232 deletions(-)
 delete mode 100644 test/test_dailymotion_subtitles.py
 create mode 100644 test/test_subtitles.py
 delete mode 100644 test/test_ted_subtitles.py
 delete mode 100644 test/test_youtube_subtitles.py

diff --git a/test/test_dailymotion_subtitles.py b/test/test_dailymotion_subtitles.py
deleted file mode 100644
index ba3580ea4..000000000
--- a/test/test_dailymotion_subtitles.py
+++ /dev/null
@@ -1,70 +0,0 @@
-#!/usr/bin/env python
-
-# Allow direct execution
-import os
-import sys
-import unittest
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from test.helper import FakeYDL, global_setup, md5
-global_setup()
-
-
-from youtube_dl.extractor import DailymotionIE
-
-class TestDailymotionSubtitles(unittest.TestCase):
-    def setUp(self):
-        self.DL = FakeYDL()
-        self.url = 'http://www.dailymotion.com/video/xczg00'
-    def getInfoDict(self):
-        IE = DailymotionIE(self.DL)
-        info_dict = IE.extract(self.url)
-        return info_dict
-    def getSubtitles(self):
-        info_dict = self.getInfoDict()
-        return info_dict['subtitles']
-    def test_no_writesubtitles(self):
-        subtitles = self.getSubtitles()
-        self.assertEqual(subtitles, None)
-    def test_subtitles(self):
-        self.DL.params['writesubtitles'] = True
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '976553874490cba125086bbfea3ff76f')
-    def test_subtitles_lang(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['subtitleslangs'] = ['fr']
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['fr']), '594564ec7d588942e384e920e5341792')
-    def test_allsubtitles(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['allsubtitles'] = True
-        subtitles = self.getSubtitles()
-        self.assertEqual(len(subtitles.keys()), 5)
-    def test_list_subtitles(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
-        self.DL.params['listsubtitles'] = True
-        info_dict = self.getInfoDict()
-        self.assertEqual(info_dict, None)
-    def test_automatic_captions(self):
-        self.DL.expect_warning(u'Automatic Captions not supported by this server')
-        self.DL.params['writeautomaticsub'] = True
-        self.DL.params['subtitleslang'] = ['en']
-        subtitles = self.getSubtitles()
-        self.assertTrue(len(subtitles.keys()) == 0)
-    def test_nosubtitles(self):
-        self.DL.expect_warning(u'video doesn\'t have subtitles')
-        self.url = 'http://www.dailymotion.com/video/x12u166_le-zapping-tele-star-du-08-aout-2013_tv'
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['allsubtitles'] = True
-        subtitles = self.getSubtitles()
-        self.assertEqual(len(subtitles), 0)
-    def test_multiple_langs(self):
-        self.DL.params['writesubtitles'] = True
-        langs = ['es', 'fr', 'de']
-        self.DL.params['subtitleslangs'] = langs
-        subtitles = self.getSubtitles()
-        for lang in langs:
-            self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/test/test_subtitles.py b/test/test_subtitles.py
new file mode 100644
index 000000000..06a304879
--- /dev/null
+++ b/test/test_subtitles.py
@@ -0,0 +1,211 @@
+#!/usr/bin/env python
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import FakeYDL, global_setup, md5
+global_setup()
+
+
+from youtube_dl.extractor import (
+    YoutubeIE,
+    DailymotionIE,
+    TEDIE,
+)
+
+
+class BaseTestSubtitles(unittest.TestCase):
+    url = None
+    IE = None
+    def setUp(self):
+        self.DL = FakeYDL()
+        self.ie = self.IE(self.DL)
+
+    def getInfoDict(self):
+        info_dict = self.ie.extract(self.url)
+        return info_dict
+
+    def getSubtitles(self):
+        info_dict = self.getInfoDict()
+        return info_dict['subtitles']
+
+
+class TestYoutubeSubtitles(BaseTestSubtitles):
+    url = 'QRS8MkLhQmM'
+    IE = YoutubeIE
+
+    def getSubtitles(self):
+        info_dict = self.getInfoDict()
+        return info_dict[0]['subtitles']
+
+    def test_youtube_no_writesubtitles(self):
+        self.DL.params['writesubtitles'] = False
+        subtitles = self.getSubtitles()
+        self.assertEqual(subtitles, None)
+
+    def test_youtube_subtitles(self):
+        self.DL.params['writesubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(md5(subtitles['en']), '4cd9278a35ba2305f47354ee13472260')
+
+    def test_youtube_subtitles_lang(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['subtitleslangs'] = ['it']
+        subtitles = self.getSubtitles()
+        self.assertEqual(md5(subtitles['it']), '164a51f16f260476a05b50fe4c2f161d')
+
+    def test_youtube_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(len(subtitles.keys()), 13)
+
+    def test_youtube_subtitles_sbv_format(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['subtitlesformat'] = 'sbv'
+        subtitles = self.getSubtitles()
+        self.assertEqual(md5(subtitles['en']), '13aeaa0c245a8bed9a451cb643e3ad8b')
+
+    def test_youtube_subtitles_vtt_format(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['subtitlesformat'] = 'vtt'
+        subtitles = self.getSubtitles()
+        self.assertEqual(md5(subtitles['en']), '356cdc577fde0c6783b9b822e7206ff7')
+
+    def test_youtube_list_subtitles(self):
+        self.DL.expect_warning(u'Video doesn\'t have automatic captions')
+        self.DL.params['listsubtitles'] = True
+        info_dict = self.getInfoDict()
+        self.assertEqual(info_dict, None)
+
+    def test_youtube_automatic_captions(self):
+        self.url = '8YoUxe5ncPo'
+        self.DL.params['writeautomaticsub'] = True
+        self.DL.params['subtitleslangs'] = ['it']
+        subtitles = self.getSubtitles()
+        self.assertTrue(subtitles['it'] is not None)
+
+    def test_youtube_nosubtitles(self):
+        self.DL.expect_warning(u'video doesn\'t have subtitles')
+        self.url = 'sAjKT8FhjI8'
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(len(subtitles), 0)
+
+    def test_youtube_multiple_langs(self):
+        self.url = 'QRS8MkLhQmM'
+        self.DL.params['writesubtitles'] = True
+        langs = ['it', 'fr', 'de']
+        self.DL.params['subtitleslangs'] = langs
+        subtitles = self.getSubtitles()
+        for lang in langs:
+            self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
+
+
+class TestDailymotionSubtitles(BaseTestSubtitles):
+    url = 'http://www.dailymotion.com/video/xczg00'
+    IE = DailymotionIE
+
+    def test_no_writesubtitles(self):
+        subtitles = self.getSubtitles()
+        self.assertEqual(subtitles, None)
+
+    def test_subtitles(self):
+        self.DL.params['writesubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(md5(subtitles['en']), '976553874490cba125086bbfea3ff76f')
+
+    def test_subtitles_lang(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['subtitleslangs'] = ['fr']
+        subtitles = self.getSubtitles()
+        self.assertEqual(md5(subtitles['fr']), '594564ec7d588942e384e920e5341792')
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(len(subtitles.keys()), 5)
+
+    def test_list_subtitles(self):
+        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.params['listsubtitles'] = True
+        info_dict = self.getInfoDict()
+        self.assertEqual(info_dict, None)
+
+    def test_automatic_captions(self):
+        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.params['writeautomaticsub'] = True
+        self.DL.params['subtitleslang'] = ['en']
+        subtitles = self.getSubtitles()
+        self.assertTrue(len(subtitles.keys()) == 0)
+
+    def test_nosubtitles(self):
+        self.DL.expect_warning(u'video doesn\'t have subtitles')
+        self.url = 'http://www.dailymotion.com/video/x12u166_le-zapping-tele-star-du-08-aout-2013_tv'
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(len(subtitles), 0)
+
+    def test_multiple_langs(self):
+        self.DL.params['writesubtitles'] = True
+        langs = ['es', 'fr', 'de']
+        self.DL.params['subtitleslangs'] = langs
+        subtitles = self.getSubtitles()
+        for lang in langs:
+            self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
+
+
+class TestTedSubtitles(BaseTestSubtitles):
+    url = 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html'
+    IE = TEDIE
+
+    def test_no_writesubtitles(self):
+        subtitles = self.getSubtitles()
+        self.assertEqual(subtitles, None)
+
+    def test_subtitles(self):
+        self.DL.params['writesubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(md5(subtitles['en']), '2154f31ff9b9f89a0aa671537559c21d')
+
+    def test_subtitles_lang(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['subtitleslangs'] = ['fr']
+        subtitles = self.getSubtitles()
+        self.assertEqual(md5(subtitles['fr']), '7616cbc6df20ec2c1204083c83871cf6')
+
+    def test_allsubtitles(self):
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(len(subtitles.keys()), 28)
+
+    def test_list_subtitles(self):
+        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.params['listsubtitles'] = True
+        info_dict = self.getInfoDict()
+        self.assertEqual(info_dict, None)
+
+    def test_automatic_captions(self):
+        self.DL.expect_warning(u'Automatic Captions not supported by this server')
+        self.DL.params['writeautomaticsub'] = True
+        self.DL.params['subtitleslang'] = ['en']
+        subtitles = self.getSubtitles()
+        self.assertTrue(len(subtitles.keys()) == 0)
+
+    def test_multiple_langs(self):
+        self.DL.params['writesubtitles'] = True
+        langs = ['es', 'fr', 'de']
+        self.DL.params['subtitleslangs'] = langs
+        subtitles = self.getSubtitles()
+        for lang in langs:
+            self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/test_ted_subtitles.py b/test/test_ted_subtitles.py
deleted file mode 100644
index 4c164ae99..000000000
--- a/test/test_ted_subtitles.py
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/usr/bin/env python
-
-import sys
-import unittest
-import hashlib
-
-# Allow direct execution
-import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from youtube_dl.extractor import TEDIE
-from youtube_dl.utils import *
-from test.helper import FakeYDL, md5
-
-class TestTedSubtitles(unittest.TestCase):
-    def setUp(self):
-        self.DL = FakeYDL()
-        self.url = 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html'
-    def getInfoDict(self):
-        IE = TEDIE(self.DL)
-        info_dict = IE.extract(self.url)
-        return info_dict
-    def getSubtitles(self):
-        info_dict = self.getInfoDict()
-        return info_dict['subtitles']
-    def test_no_writesubtitles(self):
-        subtitles = self.getSubtitles()
-        self.assertEqual(subtitles, None)
-    def test_subtitles(self):
-        self.DL.params['writesubtitles'] = True
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '2154f31ff9b9f89a0aa671537559c21d')
-    def test_subtitles_lang(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['subtitleslangs'] = ['fr']
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['fr']), '7616cbc6df20ec2c1204083c83871cf6')
-    def test_allsubtitles(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['allsubtitles'] = True
-        subtitles = self.getSubtitles()
-        self.assertEqual(len(subtitles.keys()), 28)
-    def test_list_subtitles(self):
-        self.DL.params['listsubtitles'] = True
-        info_dict = self.getInfoDict()
-        self.assertEqual(info_dict, None)
-    def test_automatic_captions(self):
-        self.DL.params['writeautomaticsub'] = True
-        self.DL.params['subtitleslang'] = ['en']
-        subtitles = self.getSubtitles()
-        self.assertTrue(len(subtitles.keys()) == 0)
-    # def test_nosubtitles(self):
-    #     self.DL.expect_warning(u'video doesn\'t have subtitles')
-    #     self.url = 'http://www.ted.com/talks/rodrigo_canales_the_deadly_genius_of_drug_cartels.html'
-    #     self.DL.params['writesubtitles'] = True
-    #     self.DL.params['allsubtitles'] = True
-    #     subtitles = self.getSubtitles()
-    def test_multiple_langs(self):
-        self.DL.params['writesubtitles'] = True
-        langs = ['es', 'fr', 'de']
-        self.DL.params['subtitleslangs'] = langs
-        subtitles = self.getSubtitles()
-        for lang in langs:
-            self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/test/test_youtube_subtitles.py b/test/test_youtube_subtitles.py
deleted file mode 100644
index 00430a338..000000000
--- a/test/test_youtube_subtitles.py
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/usr/bin/env python
-
-# Allow direct execution
-import os
-import sys
-import unittest
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from test.helper import FakeYDL, global_setup, md5
-global_setup()
-
-
-from youtube_dl.extractor import YoutubeIE
-
-
-class TestYoutubeSubtitles(unittest.TestCase):
-    def setUp(self):
-        self.DL = FakeYDL()
-        self.url = 'QRS8MkLhQmM'
-
-    def getInfoDict(self):
-        IE = YoutubeIE(self.DL)
-        info_dict = IE.extract(self.url)
-        return info_dict
-
-    def getSubtitles(self):
-        info_dict = self.getInfoDict()
-        return info_dict[0]['subtitles']
-
-    def test_youtube_no_writesubtitles(self):
-        self.DL.params['writesubtitles'] = False
-        subtitles = self.getSubtitles()
-        self.assertEqual(subtitles, None)
-
-    def test_youtube_subtitles(self):
-        self.DL.params['writesubtitles'] = True
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '4cd9278a35ba2305f47354ee13472260')
-
-    def test_youtube_subtitles_lang(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['subtitleslangs'] = ['it']
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['it']), '164a51f16f260476a05b50fe4c2f161d')
-
-    def test_youtube_allsubtitles(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['allsubtitles'] = True
-        subtitles = self.getSubtitles()
-        self.assertEqual(len(subtitles.keys()), 13)
-
-    def test_youtube_subtitles_sbv_format(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['subtitlesformat'] = 'sbv'
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '13aeaa0c245a8bed9a451cb643e3ad8b')
-
-    def test_youtube_subtitles_vtt_format(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['subtitlesformat'] = 'vtt'
-        subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '356cdc577fde0c6783b9b822e7206ff7')
-
-    def test_youtube_list_subtitles(self):
-        self.DL.expect_warning(u'Video doesn\'t have automatic captions')
-        self.DL.params['listsubtitles'] = True
-        info_dict = self.getInfoDict()
-        self.assertEqual(info_dict, None)
-
-    def test_youtube_automatic_captions(self):
-        self.url = '8YoUxe5ncPo'
-        self.DL.params['writeautomaticsub'] = True
-        self.DL.params['subtitleslangs'] = ['it']
-        subtitles = self.getSubtitles()
-        self.assertTrue(subtitles['it'] is not None)
-
-    def test_youtube_nosubtitles(self):
-        self.DL.expect_warning(u'video doesn\'t have subtitles')
-        self.url = 'sAjKT8FhjI8'
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['allsubtitles'] = True
-        subtitles = self.getSubtitles()
-        self.assertEqual(len(subtitles), 0)
-
-    def test_youtube_multiple_langs(self):
-        self.url = 'QRS8MkLhQmM'
-        self.DL.params['writesubtitles'] = True
-        langs = ['it', 'fr', 'de']
-        self.DL.params['subtitleslangs'] = langs
-        subtitles = self.getSubtitles()
-        for lang in langs:
-            self.assertTrue(subtitles.get(lang) is not None, u'Subtitles for \'%s\' not extracted' % lang)
-
-if __name__ == '__main__':
-    unittest.main()

From 0ed05a1d2db2335168497eb26b8588f2ee0039d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 10 Nov 2013 12:45:17 +0100
Subject: [PATCH 039/124]  Use the 'rtmp_live'  field for the live parameter of
 rtmpdump

---
 youtube_dl/FileDownloader.py         | 2 +-
 youtube_dl/extractor/cinemassacre.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index 04784f602..d0b3dd8fb 100644
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -428,7 +428,7 @@ class FileDownloader(object):
                                                 info_dict.get('page_url', None),
                                                 info_dict.get('play_path', None),
                                                 info_dict.get('tc_url', None),
-                                                info_dict.get('live', False))
+                                                info_dict.get('rtmp_live', False))
 
         # Attempt to download using mplayer
         if url.startswith('mms') or url.startswith('rtsp'):
diff --git a/youtube_dl/extractor/cinemassacre.py b/youtube_dl/extractor/cinemassacre.py
index d26f0f434..f0d08cebf 100644
--- a/youtube_dl/extractor/cinemassacre.py
+++ b/youtube_dl/extractor/cinemassacre.py
@@ -65,7 +65,7 @@ class CinemassacreIE(InfoExtractor):
             {
                 'url': url,
                 'play_path': 'mp4:' + sd_file,
-                'live': True, # workaround
+                'rtmp_live': True, # workaround
                 'ext': 'flv',
                 'format': 'sd',
                 'format_id': 'sd',
@@ -73,7 +73,7 @@ class CinemassacreIE(InfoExtractor):
             {
                 'url': url,
                 'play_path': 'mp4:' + hd_file,
-                'live': True, # workaround
+                'rtmp_live': True, # workaround
                 'ext': 'flv',
                 'format': 'hd',
                 'format_id': 'hd',

From 801dbbdffd9df57e4051e8fd01efbb6104f99c26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Sun, 10 Nov 2013 16:47:03 +0100
Subject: [PATCH 040/124] Use avconv for downloading with m3u8 manifests if
 it's available (fixes #1735)

---
 youtube_dl/FileDownloader.py | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index d0b3dd8fb..088f59586 100644
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -381,16 +381,20 @@ class FileDownloader(object):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
-        args = ['ffmpeg', '-y', '-i', url, '-f', 'mp4', '-c', 'copy',
-            '-absf', 'aac_adtstoasc', tmpfilename]
-        # Check for ffmpeg first
-        try:
-            subprocess.call(['ffmpeg', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
-        except (OSError, IOError):
-            self.report_error(u'm3u8 download detected but "%s" could not be run' % args[0] )
-            return False
+        args = ['-y', '-i', url, '-f', 'mp4', '-c', 'copy',
+            '-bsf:a', 'aac_adtstoasc', tmpfilename]
 
-        retval = subprocess.call(args)
+        for program in ['avconv', 'ffmpeg']:
+            try:
+                subprocess.call([program, '-version'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
+                break
+            except (OSError, IOError):
+                pass
+        else:
+            self.report_error(u'm3u8 download detected but ffmpeg or avconv could not be found')
+        cmd = [program] + args
+
+        retval = subprocess.call(cmd)
         if retval == 0:
             fsize = os.path.getsize(encodeFilename(tmpfilename))
             self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize))

From d4b7da84c3c20e86b67489a0521731c2c93385f7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 11 Nov 2013 14:21:09 +0100
Subject: [PATCH 041/124] Clarify -c. Do not pass it in if you don't know what
 you're doing

Suggested in #1743
---
 youtube_dl/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index ab7879c5d..1f1db9f67 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -358,7 +358,7 @@ def parseOpts(overrideArguments=None):
     filesystem.add_option('-w', '--no-overwrites',
             action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
     filesystem.add_option('-c', '--continue',
-            action='store_true', dest='continue_dl', help='resume partially downloaded files', default=True)
+            action='store_true', dest='continue_dl', help='force resume of partially downloaded files. By default, youtube-dl will resume downloads if possible.', default=True)
     filesystem.add_option('--no-continue',
             action='store_false', dest='continue_dl',
             help='do not resume partially downloaded files (restart from beginning)')

From 1777d5a952ab8d6728172b0a10afe20f060b432a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 11 Nov 2013 18:28:17 +0100
Subject: [PATCH 042/124] release 2013.11.11

---
 README.md             | 6 ++++--
 youtube_dl/version.py | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index a2b296613..6632e5865 100644
--- a/README.md
+++ b/README.md
@@ -92,12 +92,14 @@ which means you can modify it, redistribute it or use it however you like.
                                ownloads/%(uploader)s/%(title)s-%(id)s.%(ext)s' .
     --autonumber-size NUMBER   Specifies the number of digits in %(autonumber)s
                                when it is present in output filename template or
-                               --autonumber option is given
+                               --auto-number option is given
     --restrict-filenames       Restrict filenames to only ASCII characters, and
                                avoid "&" and spaces in filenames
     -a, --batch-file FILE      file containing URLs to download ('-' for stdin)
     -w, --no-overwrites        do not overwrite files
-    -c, --continue             resume partially downloaded files
+    -c, --continue             force resume of partially downloaded files. By
+                               default, youtube-dl will resume downloads if
+                               possible.
     --no-continue              do not resume partially downloaded files (restart
                                from beginning)
     --cookies FILE             file to read cookies from and dump cookie jar in
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 84bf0f35c..338e7ba1f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2013.11.07'
+__version__ = '2013.11.11'

From eb0a83986642cf660820b168bd83c8770e3e5ce6 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 12 Nov 2013 10:36:23 +0100
Subject: [PATCH 043/124] [common] Simplify og_search_property

---
 youtube_dl/extractor/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index fb2d50a09..9c20d30b4 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -322,9 +322,9 @@ class InfoExtractor(object):
         if name is None:
             name = 'OpenGraph %s' % prop
         escaped = self._search_regex(self._og_regex(prop), html, name, flags=re.DOTALL, **kargs)
-        if not escaped is None:
-            return unescapeHTML(escaped)
-        return None
+        if escaped is None:
+            return None
+        return unescapeHTML(escaped)
 
     def _og_search_thumbnail(self, html, **kargs):
         return self._og_search_property('image', html, u'thumbnail url', fatal=False, **kargs)

From 72b18c5d34d7bf02f83d335b886921ff25c501f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 12 Nov 2013 20:38:13 +0100
Subject: [PATCH 044/124] FFmpegMetadataPP: don't enclose the values with "
 (fixes #1756)

---
 youtube_dl/PostProcessor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/PostProcessor.py b/youtube_dl/PostProcessor.py
index 13b56ede5..69aedf87a 100644
--- a/youtube_dl/PostProcessor.py
+++ b/youtube_dl/PostProcessor.py
@@ -501,7 +501,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
 
         options = ['-c', 'copy']
         for (name, value) in metadata.items():
-            options.extend(['-metadata', '%s="%s"' % (name, value)])
+            options.extend(['-metadata', '%s=%s' % (name, value)])
         options.extend(['-f', ext])
 
         self._downloader.to_screen(u'[ffmpeg] Adding metadata to \'%s\'' % filename)

From 8b8cbd8f6d3b525dcee3bc3df98c1ad0f093231d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 12 Nov 2013 20:50:52 +0100
Subject: [PATCH 045/124] [vine] Fix uploader extraction

---
 youtube_dl/extractor/vine.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py
index c4ec1f06f..651ba317d 100644
--- a/youtube_dl/extractor/vine.py
+++ b/youtube_dl/extractor/vine.py
@@ -27,7 +27,7 @@ class VineIE(InfoExtractor):
         video_url = self._html_search_regex(r'<meta property="twitter:player:stream" content="(.+?)"',
             webpage, u'video URL')
 
-        uploader = self._html_search_regex(r'<div class="user">.*?<h2>(.+?)</h2>',
+        uploader = self._html_search_regex(r'<p class="username">(.*?)</p>',
             webpage, u'uploader', fatal=False, flags=re.DOTALL)
 
         return [{

From 0bd59f3723a22914a538527921815436c6d2f2d1 Mon Sep 17 00:00:00 2001
From: migbac <ml.mathieulouis@gmail.com>
Date: Tue, 12 Nov 2013 23:32:03 +0100
Subject: [PATCH 046/124] Add support for d8.tv

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/d8.py       | 52 ++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 youtube_dl/extractor/d8.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f9caca4ef..d7db840dd 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -25,6 +25,7 @@ from .comedycentral import ComedyCentralIE
 from .condenast import CondeNastIE
 from .criterion import CriterionIE
 from .cspan import CSpanIE
+from .d8 import D8IE
 from .dailymotion import (
     DailymotionIE,
     DailymotionPlaylistIE,
diff --git a/youtube_dl/extractor/d8.py b/youtube_dl/extractor/d8.py
new file mode 100644
index 000000000..5ce483b16
--- /dev/null
+++ b/youtube_dl/extractor/d8.py
@@ -0,0 +1,52 @@
+# encoding: utf-8
+import re
+import xml.etree.ElementTree
+
+from .common import InfoExtractor
+from ..utils import unified_strdate
+
+class D8IE(InfoExtractor):
+    _VALID_URL = r'https?://www\.d8\.tv/.*?/(?P<path>.*)'
+    _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/d8/%s'
+    IE_NAME = u'd8.tv'
+
+    _TEST = {
+        u'url': u'http://www.d8.tv/d8-docs-mags/pid6589-d8-campagne-intime.html',
+        u'file': u'966289.flv',
+        u'info_dict': {
+            u'title': u'Campagne intime - Documentaire exceptionnel',
+            u'description': u'md5:d2643b799fb190846ae09c61e59a859f',
+            u'upload_date': u'20131108',
+        },
+        u'params': {
+            u'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        webpage = self._download_webpage(url, mobj.group('path'))
+        video_id = self._search_regex(r'videoId = "(\d+)";', webpage, u'video id')
+        info_url = self._VIDEO_INFO_TEMPLATE % video_id
+        info_page = self._download_webpage(info_url,video_id, 
+                                           u'Downloading video info')
+
+        self.report_extraction(video_id)
+        doc = xml.etree.ElementTree.fromstring(info_page.encode('utf-8'))
+        video_info = [video for video in doc if video.find('ID').text == video_id][0]
+        infos = video_info.find('INFOS')
+        media = video_info.find('MEDIA')
+        formats = [media.find('VIDEOS/%s' % format)
+            for format in ['BAS_DEBIT', 'HAUT_DEBIT', 'HD']]
+        video_url = [format.text for format in formats if format is not None][-1]
+
+        return {'id': video_id,
+                'title': u'%s - %s' % (infos.find('TITRAGE/TITRE').text,
+                                       infos.find('TITRAGE/SOUS_TITRE').text),
+                'url': video_url,
+                'ext': 'flv',
+                'upload_date': unified_strdate(infos.find('PUBLICATION/DATE').text),
+                'thumbnail': media.find('IMAGES/GRAND').text,
+                'description': infos.find('DESCRIPTION').text,
+                'view_count': int(infos.find('NB_VUES').text),
+                }

From eb9b5bffef9d247bba4f2e8d387ddfbc47ece77b Mon Sep 17 00:00:00 2001
From: Jelle van der Waa <jelle@vdwaa.nl>
Date: Wed, 13 Nov 2013 10:30:41 +0100
Subject: [PATCH 047/124] Add extractor for gamekings.tv

---
 youtube_dl/extractor/__init__.py  |  1 +
 youtube_dl/extractor/gamekings.py | 39 +++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100644 youtube_dl/extractor/gamekings.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index f9caca4ef..67a0ad5cb 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -57,6 +57,7 @@ from .francetv import (
 )
 from .freesound import FreesoundIE
 from .funnyordie import FunnyOrDieIE
+from .gamekings import GamekingsIE
 from .gamespot import GameSpotIE
 from .gametrailers import GametrailersIE
 from .generic import GenericIE
diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
new file mode 100644
index 000000000..eca71ab05
--- /dev/null
+++ b/youtube_dl/extractor/gamekings.py
@@ -0,0 +1,39 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+)
+
+
+class GamekingsIE(InfoExtractor):
+    _VALID_URL = r'http?://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-])'
+    _TEST = {
+        u"url": u"http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/",
+        u'file': u'20130811_PhoenixWright.mp4',
+        u'md5': u'8d42d15381e2dfa81dee86c7956d35ff',
+        u'info_dict': {
+            u"title": u"Phoenix Wright: Ace Attorney &#8211; Dual Destinies Review",
+            u"description": u"Melle en Steven hebben voor de review een week in de rechtbank doorbracht met Phoenix Wright: Ace Attorney - Dual Destinies.",
+        }
+    }
+
+    def _real_extract(self, url):
+
+        mobj = re.match(self._VALID_URL, url)
+        name = mobj.group('name')
+        webpage = self._download_webpage(url, name)
+        gamekings_url = self._og_search_video_url(webpage)
+
+        video = re.search(r'[0-9]+',gamekings_url)
+        video_id = video.group(0)
+
+        # Todo: add medium format 
+        gamekings_url = gamekings_url.replace(video_id,'large/' + video_id)
+
+        return {'id': video_id,
+                'ext': 'mp4',
+                'url': gamekings_url,
+                'title': self._og_search_title(webpage),
+                'description': self._og_search_description(webpage),
+                }

From 384b98cd8f90ef2ac25c1f4f20ba9385adabaca8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 13 Nov 2013 10:50:53 +0100
Subject: [PATCH 048/124] [gamekings] Minor fixes (#1759)

---
 youtube_dl/extractor/gamekings.py | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index eca71ab05..4b4259447 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -7,13 +7,13 @@ from ..utils import (
 
 
 class GamekingsIE(InfoExtractor):
-    _VALID_URL = r'http?://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-])'
+    _VALID_URL = r'http?://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-]+)'
     _TEST = {
         u"url": u"http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/",
-        u'file': u'20130811_PhoenixWright.mp4',
-        u'md5': u'8d42d15381e2dfa81dee86c7956d35ff',
+        u'file': u'20130811.mp4',
+        u'md5': u'17f6088f7d0149ff2b46f2714bdb1954',
         u'info_dict': {
-            u"title": u"Phoenix Wright: Ace Attorney &#8211; Dual Destinies Review",
+            u"title": u"Phoenix Wright: Ace Attorney \u2013 Dual Destinies Review",
             u"description": u"Melle en Steven hebben voor de review een week in de rechtbank doorbracht met Phoenix Wright: Ace Attorney - Dual Destinies.",
         }
     }
@@ -23,17 +23,18 @@ class GamekingsIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         name = mobj.group('name')
         webpage = self._download_webpage(url, name)
-        gamekings_url = self._og_search_video_url(webpage)
+        video_url = self._og_search_video_url(webpage)
 
-        video = re.search(r'[0-9]+',gamekings_url)
+        video = re.search(r'[0-9]+', video_url)
         video_id = video.group(0)
 
-        # Todo: add medium format 
-        gamekings_url = gamekings_url.replace(video_id,'large/' + video_id)
+        # Todo: add medium format
+        video_url = video_url.replace(video_id, 'large/' + video_id)
 
-        return {'id': video_id,
-                'ext': 'mp4',
-                'url': gamekings_url,
-                'title': self._og_search_title(webpage),
-                'description': self._og_search_description(webpage),
-                }
+        return {
+            'id': video_id,
+            'ext': 'mp4',
+            'url': video_url,
+            'title': self._og_search_title(webpage),
+            'description': self._og_search_description(webpage),
+        }

From b5bdc2699a5fead926114f6db6f1178181d75c58 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 13 Nov 2013 10:52:22 +0100
Subject: [PATCH 049/124] Credit @jelly for gamekings extractor (#1759)

---
 youtube_dl/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 1f1db9f67..254fcd39c 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -32,6 +32,7 @@ __authors__  = (
     'Ismael Mejía',
     'Steffan \'Ruirize\' James',
     'Andras Elso',
+    'Jelle van der Waa',
 )
 
 __license__ = 'Public Domain'

From c3a3028f9f7fa79ee7357b65252ff2c9a062bdc8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 13 Nov 2013 11:06:53 +0100
Subject: [PATCH 050/124] [tvp] Minor improvements (#1730)

---
 youtube_dl/extractor/tvp.py | 47 +++++++++++--------------------------
 1 file changed, 14 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index 63fb57bbe..32e0f5037 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -1,23 +1,17 @@
-# encoding: utf-8
-import re
 import json
+import re
 
 from .common import InfoExtractor
-from ..utils import (
-    determine_ext,
-    ExtractorError,
-    RegexNotFoundError,
-)
+
 
 class TvpIE(InfoExtractor):
     IE_NAME = u'tvp.pl'
     _VALID_URL = r'https?://www\.tvp\.pl/.*?wideo/(?P<date>\d+)/(?P<id>\d+)'
-    _INFO_URL = 'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s'
-
 
     _TEST = {
         u'url': u'http://www.tvp.pl/warszawa/magazyny/campusnews/wideo/31102013/12878238',
-        u'file': u'31.10.2013-12878238.wmv',
+        u'md5': u'148408967a6a468953c0a75cbdaf0d7a',
+        u'file': u'12878238.wmv',
         u'info_dict': {
             u'title': u'31.10.2013',
             u'description': u'31.10.2013',
@@ -27,34 +21,21 @@ class TvpIE(InfoExtractor):
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
-        webpage = self._download_webpage(url, video_id, "Downloading video webpage")
-        json_params = self._download_webpage(self._INFO_URL % video_id, video_id, "Downloading video metadata")
-
-        try:
-            params = json.loads(json_params)
-        except:
-            raise ExtractorError(u'Invalid JSON')
+        webpage = self._download_webpage(url, video_id)
+        json_url = 'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s' % video_id
+        json_params = self._download_webpage(
+            json_url, video_id, u"Downloading video metadata")
 
+        params = json.loads(json_params)
         self.report_extraction(video_id)
-        try:
-            video_url = params['video_url']
-        except KeyError:
-            raise ExtractorError('Missing JSON parameter: ' + sys.exc_info()[1])
+        video_url = params['video_url']
 
-        try:
-            title = self._og_search_title(webpage)
-        except RegexNotFoundError:
-            title = video_id
-        info = {
+        title = self._og_search_title(webpage, fatal=True)
+        return {
             'id': video_id,
             'title': title,
             'ext': 'wmv',
             'url': video_url,
+            'description': self._og_search_description(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
         }
-        try:
-            info['description'] = self._og_search_description(webpage)
-            info['thumbnail'] = self._og_search_thumbnail(webpage)
-        except RegexNotFoundError:
-            pass
-
-        return info

From d37936386f0217a411efe96483b97e590b7eceb3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 13 Nov 2013 11:08:07 +0100
Subject: [PATCH 051/124] Credit @saper for tvp IE (#1730)

---
 youtube_dl/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 254fcd39c..4dee487ab 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -33,6 +33,7 @@ __authors__  = (
     'Steffan \'Ruirize\' James',
     'Andras Elso',
     'Jelle van der Waa',
+    'Marcin Cieślak',
 )
 
 __license__ = 'Public Domain'

From 80b9bbce8687f800b79edb36edf8c193dcf26a78 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 13 Nov 2013 11:09:04 +0100
Subject: [PATCH 052/124] release 2013.11.13

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 338e7ba1f..26b91105f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2013.11.11'
+__version__ = '2013.11.13'

From ea7a7af1d46ecb51566db0af3e8779ab2a04b516 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 13 Nov 2013 17:13:06 +0100
Subject: [PATCH 053/124] [gamekings] Fix the test video checksum

---
 youtube_dl/extractor/gamekings.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 4b4259447..6aa657ef4 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -11,7 +11,7 @@ class GamekingsIE(InfoExtractor):
     _TEST = {
         u"url": u"http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/",
         u'file': u'20130811.mp4',
-        u'md5': u'17f6088f7d0149ff2b46f2714bdb1954',
+        u'md5': u'2f32b1f7b80fdc5cb616efb4f387f8a3',
         u'info_dict': {
             u"title": u"Phoenix Wright: Ace Attorney \u2013 Dual Destinies Review",
             u"description": u"Melle en Steven hebben voor de review een week in de rechtbank doorbracht met Phoenix Wright: Ace Attorney - Dual Destinies.",

From ca715127a2f95da30b6700a5e217a2acc904b459 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Wed, 13 Nov 2013 17:06:02 +0100
Subject: [PATCH 054/124] Don't assume the 'subtitlesformat' is set in the
 params dict (fixes #1750)

---
 youtube_dl/YoutubeDL.py         | 2 +-
 youtube_dl/extractor/youtube.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 5253c39e1..f615911de 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -640,7 +640,7 @@ class YoutubeDL(object):
             # subtitles download errors are already managed as troubles in relevant IE
             # that way it will silently go on when used with unsupporting IE
             subtitles = info_dict['subtitles']
-            sub_format = self.params.get('subtitlesformat')
+            sub_format = self.params.get('subtitlesformat', 'srt')
             for sub_lang in subtitles.keys():
                 sub = subtitles[sub_lang]
                 if sub is None:
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c992cba97..56d30f52a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1098,7 +1098,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             params = compat_urllib_parse.urlencode({
                 'lang': lang,
                 'v': video_id,
-                'fmt': self._downloader.params.get('subtitlesformat'),
+                'fmt': self._downloader.params.get('subtitlesformat', 'srt'),
                 'name': l[0].encode('utf-8'),
             })
             url = u'http://www.youtube.com/api/timedtext?' + params
@@ -1111,7 +1111,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
     def _get_available_automatic_caption(self, video_id, webpage):
         """We need the webpage for getting the captions url, pass it as an
            argument to speed up the process."""
-        sub_format = self._downloader.params.get('subtitlesformat')
+        sub_format = self._downloader.params.get('subtitlesformat', 'srt')
         self.to_screen(u'%s: Looking for automatic captions' % video_id)
         mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
         err_msg = u'Couldn\'t find automatic captions for %s' % video_id

From 9771cceb2cfab153d0c2a5b0a7dbc923de90f2c1 Mon Sep 17 00:00:00 2001
From: Janez Troha <dz0ny@users.noreply.github.com>
Date: Wed, 13 Nov 2013 18:34:03 +0100
Subject: [PATCH 055/124] Fix filename extension leaking to json filename

Makes writeinfojson behaving exactly as writethumbnail in case where filename contains mediafile extension.

Case:

video.mp4 converted to music.mp3 would yield music.mp4.info.json instead music.mp3.info.json or music.info.json
---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index f615911de..3160d9712 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -655,7 +655,7 @@ class YoutubeDL(object):
                     return
 
         if self.params.get('writeinfojson', False):
-            infofn = filename + u'.info.json'
+            infofn = os.path.splitext(filename)[0] + u'.info.json'
             self.report_writeinfojson(infofn)
             try:
                 json_info_dict = dict((k, v) for k, v in info_dict.items() if not k in ['urlhandle'])

From 08bc37cdd02bc861a0b5271abb34623f0ebcd66c Mon Sep 17 00:00:00 2001
From: Janez Troha <dz0ny@users.noreply.github.com>
Date: Wed, 13 Nov 2013 18:55:49 +0100
Subject: [PATCH 056/124] Update test_write_info_json.py

---
 test/test_write_info_json.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_write_info_json.py b/test/test_write_info_json.py
index a5b6f6972..30c4859fd 100644
--- a/test/test_write_info_json.py
+++ b/test/test_write_info_json.py
@@ -31,7 +31,7 @@ params = get_params({
 
 
 TEST_ID = 'BaW_jenozKc'
-INFO_JSON_FILE = TEST_ID + '.mp4.info.json'
+INFO_JSON_FILE = TEST_ID + '.info.json'
 DESCRIPTION_FILE = TEST_ID + '.mp4.description'
 EXPECTED_DESCRIPTION = u'''test chars:  "'/\ä↭𝕐
 

From c66d2baa9cb33327a3318f49cbb89f9ac559c978 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 14 Nov 2013 13:16:32 +0100
Subject: [PATCH 057/124] [livestream] Add an extractor for the original
 version of livestream (closes #1764)

The two versions use different systems.
---
 youtube_dl/extractor/__init__.py   |  2 +-
 youtube_dl/extractor/livestream.py | 44 ++++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 0594a3666..ee3173468 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -80,7 +80,7 @@ from .keezmovies import KeezMoviesIE
 from .kickstarter import KickStarterIE
 from .keek import KeekIE
 from .liveleak import LiveLeakIE
-from .livestream import LivestreamIE
+from .livestream import LivestreamIE, LivestreamOriginalIE
 from .metacafe import MetacafeIE
 from .metacritic import MetacriticIE
 from .mit import TechTVMITIE, MITIE
diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 4531fd6ab..1a3e0ae6b 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -1,16 +1,19 @@
 import re
 import json
+import xml.etree.ElementTree
 
 from .common import InfoExtractor
 from ..utils import (
     compat_urllib_parse_urlparse,
     compat_urlparse,
     get_meta_content,
+    xpath_with_ns,
     ExtractorError,
 )
 
 
 class LivestreamIE(InfoExtractor):
+    IE_NAME = u'livestream'
     _VALID_URL = r'http://new.livestream.com/.*?/(?P<event_name>.*?)(/videos/(?P<id>\d+))?/?$'
     _TEST = {
         u'url': u'http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370',
@@ -54,3 +57,44 @@ class LivestreamIE(InfoExtractor):
             info = json.loads(self._download_webpage(api_url, video_id,
                                                      u'Downloading video info'))
             return self._extract_video_info(info)
+
+
+# The original version of Livestream uses a different system
+class LivestreamOriginalIE(InfoExtractor):
+    IE_NAME = u'livestream:original'
+    _VALID_URL = r'https?://www\.livestream\.com/(?P<user>[^/]+)/video\?.*?clipId=(?P<id>.*?)(&|$)'
+    _TEST = {
+        u'url': u'http://www.livestream.com/dealbook/video?clipId=pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
+        u'info_dict': {
+            u'id': u'pla_8aa4a3f1-ba15-46a4-893b-902210e138fb',
+            u'ext': u'flv',
+            u'title': u'Spark 1 (BitCoin) with Cameron Winklevoss & Tyler Winklevoss of Winklevoss Capital',
+        },
+        u'params': {
+            # rtmp
+            u'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        user = mobj.group('user')
+        api_url = 'http://x{0}x.api.channel.livestream.com/2.0/clipdetails?extendedInfo=true&id={1}'.format(user, video_id)
+
+        api_response = self._download_webpage(api_url, video_id)
+        info = xml.etree.ElementTree.fromstring(api_response.encode('utf-8'))
+        item = info.find('channel').find('item')
+        ns = {'media': 'http://search.yahoo.com/mrss'}
+        thumbnail_url = item.find(xpath_with_ns('media:thumbnail', ns)).attrib['url']
+        # Remove the extension and number from the path (like 1.jpg)
+        path = self._search_regex(r'(user-files/.+)_.*?\.jpg$', thumbnail_url, u'path')
+
+        return {
+            'id': video_id,
+            'title': item.find('title').text,
+            'url': 'rtmp://extondemand.livestream.com/ondemand',
+            'play_path': 'mp4:trans/dv15/mogulus-{0}.mp4'.format(path),
+            'ext': 'flv',
+            'thumbnail': thumbnail_url,
+        }

From e3b9ab5e187a590143f7f6110e55d43fb78f15bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Thu, 14 Nov 2013 19:45:39 +0100
Subject: [PATCH 058/124] [soundlcoud] Set the correct extension for the tracks
 (fixes #1766)

Some tracks are not in mp3 format, they can be wav files.
---
 youtube_dl/extractor/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 4717fbb77..83e1f055f 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -87,7 +87,7 @@ class SoundcloudIE(InfoExtractor):
             'uploader': info['user']['username'],
             'upload_date': unified_strdate(info['created_at']),
             'title':    info['title'],
-            'ext':      u'mp3',
+            'ext':      info.get('original_format', u'mp3'),
             'description': info['description'],
             'thumbnail': thumbnail,
         }

From 9f9be844fcc5155ab3e832c8428c8f016bea819b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 15 Nov 2013 01:45:34 +0100
Subject: [PATCH 059/124] [youtube] Fix protocol-independent URLs (Fixes #1768)

---
 youtube_dl/extractor/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c992cba97..ed82e4fc0 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1019,6 +1019,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         """Turn the encrypted s field into a working signature"""
 
         if player_url is not None:
+            if player_url.startswith(u'//'):
+                player_url = u'https:' + player_url
             try:
                 player_id = (player_url, len(s))
                 if player_id not in self._player_cache:

From a25a5cfeecc1c6371ab28d6d458cd066baf7013d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 15 Nov 2013 01:47:15 +0100
Subject: [PATCH 060/124] release 2013.11.15

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 26b91105f..cd9f0b546 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2013.11.13'
+__version__ = '2013.11.15'

From feee2ecfa9fbc6fd34246c7e167ac9542ae7def2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 Nov 2013 11:04:26 +0100
Subject: [PATCH 061/124] Pass the 'download' argument to
 'process_video_result' (fixes #1769)

---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index f615911de..b5c670dd4 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -385,7 +385,7 @@ class YoutubeDL(object):
         result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system
         if result_type == 'video':
             self.add_extra_info(ie_result, extra_info)
-            return self.process_video_result(ie_result)
+            return self.process_video_result(ie_result, download=download)
         elif result_type == 'url':
             # We have to add extra_info to the results because it may be
             # contained in a playlist

From b9643eed7c9b081c91e72257b380ccbd92555254 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 Nov 2013 11:51:45 +0100
Subject: [PATCH 062/124] [youtube:channel] Fix the extraction of autogenerated
 channels

The ajax pages are empty, now it looks directly in the channel's /videos page
---
 youtube_dl/extractor/youtube.py | 37 +++++++++++++++++++++------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 7d7aeb461..8c0e6f252 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1594,20 +1594,31 @@ class YoutubeChannelIE(InfoExtractor):
         # Download channel page
         channel_id = mobj.group(1)
         video_ids = []
+        url = 'https://www.youtube.com/channel/%s/videos' % channel_id
+        channel_page = self._download_webpage(url, channel_id)
+        if re.search(r'channel-header-autogenerated-label', channel_page) is not None:
+            autogenerated = True
+        else:
+            autogenerated = False
 
-        # Download all channel pages using the json-based channel_ajax query
-        for pagenum in itertools.count(1):
-            url = self._MORE_PAGES_URL % (pagenum, channel_id)
-            page = self._download_webpage(url, channel_id,
-                                          u'Downloading page #%s' % pagenum)
-
-            page = json.loads(page)
-
-            ids_in_page = self.extract_videos_from_page(page['content_html'])
-            video_ids.extend(ids_in_page)
-
-            if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
-                break
+        if autogenerated:
+            # The videos are contained in a single page
+            # the ajax pages can't be used, they are empty
+            video_ids = self.extract_videos_from_page(channel_page)
+        else:
+            # Download all channel pages using the json-based channel_ajax query
+            for pagenum in itertools.count(1):
+                url = self._MORE_PAGES_URL % (pagenum, channel_id)
+                page = self._download_webpage(url, channel_id,
+                                              u'Downloading page #%s' % pagenum)
+    
+                page = json.loads(page)
+    
+                ids_in_page = self.extract_videos_from_page(page['content_html'])
+                video_ids.extend(ids_in_page)
+    
+                if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
+                    break
 
         self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
 

From 85d61685f15bdc62709c699e849af512db78089f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 Nov 2013 12:10:22 +0100
Subject: [PATCH 063/124] [tvp] Update the title and the description of the
 test video

---
 youtube_dl/extractor/tvp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index 32e0f5037..76721a986 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -13,8 +13,8 @@ class TvpIE(InfoExtractor):
         u'md5': u'148408967a6a468953c0a75cbdaf0d7a',
         u'file': u'12878238.wmv',
         u'info_dict': {
-            u'title': u'31.10.2013',
-            u'description': u'31.10.2013',
+            u'title': u'31.10.2013 - Odcinek 2',
+            u'description': u'31.10.2013 - Odcinek 2',
         },
     }
 

From ab2d524780736249c8988313db021e83642c24d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 Nov 2013 12:24:54 +0100
Subject: [PATCH 064/124] Improve the OpenGraph regex

* Do not accept '>' between the property and content attributes.
* Recognize the properties if the content attribute is before the property attribute using two regexes (fixes the extraction of the description for SlideshareIE).
---
 youtube_dl/extractor/common.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9c20d30b4..e02176852 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -315,13 +315,17 @@ class InfoExtractor(object):
 
     # Helper functions for extracting OpenGraph info
     @staticmethod
-    def _og_regex(prop):
-        return r'<meta.+?property=[\'"]og:%s[\'"].+?content=(?:"(.+?)"|\'(.+?)\')' % re.escape(prop)
+    def _og_regexes(prop):
+        esc_prop = re.escape(prop)
+        return [
+            r'<meta[^>]+?property=[\'"]og:%s[\'"][^>]+?content=(?:"(.+?)"|\'(.+?)\')' % esc_prop,
+            r'<meta[^>]+?content=(?:"(.+?)"|\'(.+?)\')[^>]+?property=[\'"]og:%s[\'"]' % esc_prop,
+        ]
 
     def _og_search_property(self, prop, html, name=None, **kargs):
         if name is None:
             name = 'OpenGraph %s' % prop
-        escaped = self._search_regex(self._og_regex(prop), html, name, flags=re.DOTALL, **kargs)
+        escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, **kargs)
         if escaped is None:
             return None
         return unescapeHTML(escaped)
@@ -336,8 +340,8 @@ class InfoExtractor(object):
         return self._og_search_property('title', html, **kargs)
 
     def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
-        regexes = [self._og_regex('video')]
-        if secure: regexes.insert(0, self._og_regex('video:secure_url'))
+        regexes = self._og_regexes('video')
+        if secure: regexes = self._og_regexes('video:secure_url') + regexes
         return self._html_search_regex(regexes, html, name, **kargs)
 
     def _rta_search(self, html):

From 78fb87b2837e15124b5855734a951598dfe025fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 Nov 2013 12:54:13 +0100
Subject: [PATCH 065/124] Don't accept '>' inside the content attribute in
 OpenGraph regexes

---
 youtube_dl/extractor/common.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index e02176852..45dd01789 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -316,10 +316,12 @@ class InfoExtractor(object):
     # Helper functions for extracting OpenGraph info
     @staticmethod
     def _og_regexes(prop):
-        esc_prop = re.escape(prop)
+        content_re = r'content=(?:"([^>]+?)"|\'(.+?)\')'
+        property_re = r'property=[\'"]og:%s[\'"]' % re.escape(prop)
+        template = r'<meta[^>]+?%s[^>]+?%s'
         return [
-            r'<meta[^>]+?property=[\'"]og:%s[\'"][^>]+?content=(?:"(.+?)"|\'(.+?)\')' % esc_prop,
-            r'<meta[^>]+?content=(?:"(.+?)"|\'(.+?)\')[^>]+?property=[\'"]og:%s[\'"]' % esc_prop,
+            template % (property_re, content_re),
+            template % (content_re, property_re),
         ]
 
     def _og_search_property(self, prop, html, name=None, **kargs):

From d24ffe1cfadca8fdc37a7b2fb5c2f080c785ad1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Fri, 15 Nov 2013 12:57:59 +0100
Subject: [PATCH 066/124] [rtlnow] Remove the test for nitro

The videos expire.
---
 youtube_dl/extractor/rtlnow.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py
index 9ac7c3be8..2f238de35 100644
--- a/youtube_dl/extractor/rtlnow.py
+++ b/youtube_dl/extractor/rtlnow.py
@@ -62,18 +62,6 @@ class RTLnowIE(InfoExtractor):
             u'skip_download': True,
         },
     },
-    {
-        u'url': u'http://www.rtlnitronow.de/recht-ordnung/stadtpolizei-frankfurt-gerichtsvollzieher-leipzig.php?film_id=129679&player=1&season=1',
-        u'file': u'129679.flv',
-        u'info_dict': {
-            u'upload_date': u'20131016', 
-            u'title': u'Recht & Ordnung - Stadtpolizei Frankfurt/ Gerichtsvollzieher...',
-            u'description': u'Stadtpolizei Frankfurt/ Gerichtsvollzieher Leipzig',
-        },
-        u'params': {
-            u'skip_download': True,
-        },
-    },
     {
         u'url': u'http://www.n-tvnow.de/top-gear/episode-1-2013-01-01-00-00-00.php?film_id=124903&player=1&season=10',
         u'file': u'124903.flv',

From 463a908705674b9411a01b7a696c84348ec0244e Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 15 Nov 2013 14:06:38 +0100
Subject: [PATCH 067/124] [ted] simplify

---
 youtube_dl/extractor/ted.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 76cfdfb90..8001ca5a3 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -85,7 +85,7 @@ class TEDIE(SubtitlesInfoExtractor):
             'ext': 'mp4',
             'url': stream['file'],
             'format': stream['id']
-            } for stream in info['htmlStreams']]
+        } for stream in info['htmlStreams']]
 
         video_id = info['id']
 
@@ -95,7 +95,7 @@ class TEDIE(SubtitlesInfoExtractor):
             self._list_available_subtitles(video_id, webpage)
             return
 
-        info = {
+        return {
             'id': video_id,
             'title': title,
             'thumbnail': thumbnail,
@@ -104,11 +104,6 @@ class TEDIE(SubtitlesInfoExtractor):
             'formats': formats,
         }
 
-        # TODO: Remove when #980 has been merged
-        info.update(info['formats'][-1])
-
-        return info
-
     def _get_available_subtitles(self, video_id, webpage):
         try:
             options = self._search_regex(r'(?:<select name="subtitles_language_select" id="subtitles_language_select">)(.*?)(?:</select>)', webpage, 'subtitles_language_select', flags=re.DOTALL)

From fc2ef392bee9e46564d4106d3b0e5f6b8d71e37b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 15 Nov 2013 14:33:51 +0100
Subject: [PATCH 068/124] [ted] Fix playlists (Fixes #1770)

---
 youtube_dl/extractor/ted.py | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 8001ca5a3..2e497c86e 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -43,26 +43,25 @@ class TEDIE(SubtitlesInfoExtractor):
             self.to_screen(u'Getting info of playlist %s: "%s"' % (playlist_id,name))
             return [self._playlist_videos_info(url,name,playlist_id)]
 
-    def _playlist_videos_info(self,url,name,playlist_id=0):
+
+    def _playlist_videos_info(self, url, name, playlist_id):
         '''Returns the videos of the playlist'''
-        video_RE=r'''
-                     <li\ id="talk_(\d+)"([.\s]*?)data-id="(?P<video_id>\d+)"
-                     ([.\s]*?)data-playlist_item_id="(\d+)"
-                     ([.\s]*?)data-mediaslug="(?P<mediaSlug>.+?)"
-                     '''
-        video_name_RE=r'<p\ class="talk-title"><a href="(?P<talk_url>/talks/(.+).html)">(?P<fullname>.+?)</a></p>'
-        webpage=self._download_webpage(url, playlist_id, 'Downloading playlist webpage')
-        m_videos=re.finditer(video_RE,webpage,re.VERBOSE)
-        m_names=re.finditer(video_name_RE,webpage)
+
+        webpage = self._download_webpage(
+            url, playlist_id, u'Downloading playlist webpage')
+        matches = re.finditer(
+            r'<p\s+class="talk-title[^"]*"><a\s+href="(?P<talk_url>/talks/[^"]+\.html)">[^<]*</a></p>',
+            webpage)
 
         playlist_title = self._html_search_regex(r'div class="headline">\s*?<h1>\s*?<span>(.*?)</span>',
                                                  webpage, 'playlist title')
 
-        playlist_entries = []
-        for m_video, m_name in zip(m_videos,m_names):
-            talk_url='http://www.ted.com%s' % m_name.group('talk_url')
-            playlist_entries.append(self.url_result(talk_url, 'TED'))
-        return self.playlist_result(playlist_entries, playlist_id = playlist_id, playlist_title = playlist_title)
+        playlist_entries = [
+            self.url_result(u'http://www.ted.com' + m.group('talk_url'), 'TED')
+            for m in matches
+        ]
+        return self.playlist_result(
+            playlist_entries, playlist_id=playlist_id, playlist_title=playlist_title)
 
     def _talk_info(self, url, video_id=0):
         """Return the video for the talk in the url"""

From aa13b2dffde73978fe2a169d3f99de7e5e7754cb Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Fri, 15 Nov 2013 14:35:00 +0100
Subject: [PATCH 069/124] release 2013.11.15.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index cd9f0b546..b04238eb5 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2013.11.15'
+__version__ = '2013.11.15.1'

From 91c7271aabdd74c833ef570db59018e2d9f9d803 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 16 Nov 2013 01:08:43 +0100
Subject: [PATCH 070/124] Add automatic generation of format note based on
 bitrate and codecs

---
 youtube_dl/YoutubeDL.py        | 18 +++++++++++++++++-
 youtube_dl/extractor/common.py |  4 ++++
 youtube_dl/extractor/vevo.py   |  7 ++++---
 3 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index b5c670dd4..9c79af1f2 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -781,12 +781,28 @@ class YoutubeDL(object):
         return res
 
     def list_formats(self, info_dict):
+        def format_note(fdict):
+            if fdict.get('format_note') is not None:
+                return fdict['format_note']
+            res = u''
+            if fdict.get('vcodec') is not None:
+                res += fdict['vcodec']
+            if fdict.get('vbr') is not None:
+                res += u'@%4dk' % fdict['vbr']
+            if fdict.get('acodec') is not None:
+                if res:
+                    res += u', '
+                res += fdict['acodec']
+            if fdict.get('abr') is not None:
+                res += u'@%3dk' % fdict['abr']
+            return res
+
         def line(format):
             return (u'%-20s%-10s%-12s%s' % (
                 format['format_id'],
                 format['ext'],
                 self.format_resolution(format),
-                format.get('format_note', ''),
+                format_note(format),
                 )
             )
 
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 45dd01789..f787d0a3c 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -71,6 +71,10 @@ class InfoExtractor(object):
                                 ("3D" or "DASH video")
                     * width     Width of the video, if known
                     * height    Height of the video, if known
+                    * abr       Average audio bitrate in KBit/s
+                    * acodec    Name of the audio codec in use
+                    * vbr       Average video bitrate in KBit/s
+                    * vcodec    Name of the video codec in use
     webpage_url:    The url to the video webpage, if given to youtube-dl it
                     should allow to get the same result again. (It will be set
                     by YoutubeDL if it's missing)
diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py
index 3f6020f74..4378b1780 100644
--- a/youtube_dl/extractor/vevo.py
+++ b/youtube_dl/extractor/vevo.py
@@ -78,12 +78,13 @@ class VevoIE(InfoExtractor):
                 continue
 
             format_url = self._SMIL_BASE_URL + m.group('path')
-            format_note = ('%(vcodec)s@%(vbr)4sk, %(acodec)s@%(abr)3sk' %
-                           m.groupdict())
             formats.append({
                 'url': format_url,
                 'format_id': u'SMIL_' + m.group('cbr'),
-                'format_note': format_note,
+                'vcodec': m.group('vcodec'),
+                'acodec': m.group('acodec'),
+                'vbr': int(m.group('vbr')),
+                'abr': int(m.group('abr')),
                 'ext': m.group('ext'),
                 'width': int(m.group('width')),
                 'height': int(m.group('height')),

From 7150858d49f05a5650a12b5f4694f91dfb9595d3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 16 Nov 2013 01:33:12 +0100
Subject: [PATCH 071/124] [spiegel] Implement format selection

---
 youtube_dl/YoutubeDL.py         | 10 +++++--
 youtube_dl/extractor/spiegel.py | 50 ++++++++++++++++++++++++---------
 2 files changed, 44 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 9c79af1f2..273f7d977 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -786,13 +786,19 @@ class YoutubeDL(object):
                 return fdict['format_note']
             res = u''
             if fdict.get('vcodec') is not None:
-                res += fdict['vcodec']
+                res += u'%-5s' % fdict['vcodec']
+            elif fdict.get('vbr') is not None:
+                res += u'video'
             if fdict.get('vbr') is not None:
                 res += u'@%4dk' % fdict['vbr']
             if fdict.get('acodec') is not None:
                 if res:
                     res += u', '
-                res += fdict['acodec']
+                res += u'%-5s' % fdict['acodec']
+            elif fdict.get('abr') is not None:
+                if res:
+                    res += u', '
+                res += 'audio'
             if fdict.get('abr') is not None:
                 res += u'@%3dk' % fdict['abr']
             return res
diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index 13c86401c..6dc2eda6d 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -2,18 +2,27 @@ import re
 import xml.etree.ElementTree
 
 from .common import InfoExtractor
+from ..utils import determine_ext
 
 
 class SpiegelIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P<videoID>[0-9]+)(?:\.html)?(?:#.*)?$'
-    _TEST = {
+    _TESTS = [{
         u'url': u'http://www.spiegel.de/video/vulkan-tungurahua-in-ecuador-ist-wieder-aktiv-video-1259285.html',
         u'file': u'1259285.mp4',
         u'md5': u'2c2754212136f35fb4b19767d242f66e',
         u'info_dict': {
             u"title": u"Vulkanausbruch in Ecuador: Der \"Feuerschlund\" ist wieder aktiv"
         }
-    }
+    },
+    {
+        u'url': u'http://www.spiegel.de/video/schach-wm-videoanalyse-des-fuenften-spiels-video-1309159.html',
+        u'file': u'1309159.mp4',
+        u'md5': u'f2cdf638d7aa47654e251e1aee360af1',
+        u'info_dict': {
+            u'title': u'Schach-WM in der Videoanalyse: Carlsen nutzt die Fehlgriffe des Titelverteidigers'
+        }
+    }]
 
     def _real_extract(self, url):
         m = re.match(self._VALID_URL, url)
@@ -21,25 +30,38 @@ class SpiegelIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        video_title = self._html_search_regex(r'<div class="module-title">(.*?)</div>',
-            webpage, u'title')
+        video_title = self._html_search_regex(
+            r'<div class="module-title">(.*?)</div>', webpage, u'title')
 
         xml_url = u'http://video2.spiegel.de/flash/' + video_id + u'.xml'
-        xml_code = self._download_webpage(xml_url, video_id,
-                    note=u'Downloading XML', errnote=u'Failed to download XML')
+        xml_code = self._download_webpage(
+            xml_url, video_id,
+            note=u'Downloading XML', errnote=u'Failed to download XML')
 
         idoc = xml.etree.ElementTree.fromstring(xml_code)
-        last_type = idoc[-1]
-        filename = last_type.findall('./filename')[0].text
-        duration = float(last_type.findall('./duration')[0].text)
 
-        video_url = 'http://video2.spiegel.de/flash/' + filename
-        video_ext = filename.rpartition('.')[2]
+        formats = [
+            {
+                'format_id': n.tag.rpartition('type')[2],
+                'url': u'http://video2.spiegel.de/flash/' + n.find('./filename').text,
+                'width': int(n.find('./width').text),
+                'height': int(n.find('./height').text),
+                'abr': int(n.find('./audiobitrate').text),
+                'vbr': int(n.find('./videobitrate').text),
+                'vcodec': n.find('./codec').text,
+                'acodec': 'MP4A',
+            }
+            for n in list(idoc)
+            # Blacklist type 6, it's extremely LQ and not available on the same server
+            if n.tag.startswith('type') and n.tag != 'type6'
+        ]
+        formats.sort(key=lambda f: f['vbr'])
+        duration = float(idoc[0].findall('./duration')[0].text)
+
         info = {
             'id': video_id,
-            'url': video_url,
-            'ext': video_ext,
             'title': video_title,
             'duration': duration,
+            'formats': formats,
         }
-        return [info]
+        return info

From b5349e8721d0580a50519593926726a2ea832c9b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 16 Nov 2013 01:39:45 +0100
Subject: [PATCH 072/124] Fix indentation of (best) and (worst) in
 --list-formats

---
 youtube_dl/YoutubeDL.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 273f7d977..d29e8bec5 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -815,8 +815,8 @@ class YoutubeDL(object):
         formats = info_dict.get('formats', [info_dict])
         formats_s = list(map(line, formats))
         if len(formats) > 1:
-            formats_s[0] += (' ' if formats[0].get('format_note') else '') + '(worst)'
-            formats_s[-1] += (' ' if formats[-1].get('format_note') else '') + '(best)'
+            formats_s[0] += (' ' if format_note(formats[0]) else '') + '(worst)'
+            formats_s[-1] += (' ' if format_note(formats[-1]) else '') + '(best)'
 
         header_line = line({
             'format_id': u'format code', 'ext': u'extension',

From f058e340114cb599e21001cef25ab3d02cfb194f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 16 Nov 2013 01:56:23 +0100
Subject: [PATCH 073/124] [dailymotion] Fix playlists

---
 youtube_dl/extractor/dailymotion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index e87690f9d..71f5e03ee 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -186,7 +186,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor):
             webpage = self._download_webpage(request,
                                              id, u'Downloading page %s' % pagenum)
 
-            playlist_el = get_element_by_attribute(u'class', u'video_list', webpage)
+            playlist_el = get_element_by_attribute(u'class', u'row video_list', webpage)
             video_ids.extend(re.findall(r'data-id="(.+?)"', playlist_el))
 
             if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None:

From ce152341a118c94e442f42db8008de95aebec56b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 16 Nov 2013 01:59:28 +0100
Subject: [PATCH 074/124] [bambuser] Do not test for MD5, seems to be flaky

---
 youtube_dl/extractor/bambuser.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py
index f3b36f473..967568c4a 100644
--- a/youtube_dl/extractor/bambuser.py
+++ b/youtube_dl/extractor/bambuser.py
@@ -15,7 +15,8 @@ class BambuserIE(InfoExtractor):
 
     _TEST = {
         u'url': u'http://bambuser.com/v/4050584',
-        u'md5': u'fba8f7693e48fd4e8641b3fd5539a641',
+        # MD5 seems to be flaky, see https://travis-ci.org/rg3/youtube-dl/jobs/14051016#L388
+        #u'md5': u'fba8f7693e48fd4e8641b3fd5539a641',
         u'info_dict': {
             u'id': u'4050584',
             u'ext': u'flv',

From 52d703d3d1b0d24f6c4c01ef8fcf33bfee78928b Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 16 Nov 2013 02:09:30 +0100
Subject: [PATCH 075/124] [tvp] Skip tests

---
 youtube_dl/extractor/tvp.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py
index 76721a986..bfed9dd04 100644
--- a/youtube_dl/extractor/tvp.py
+++ b/youtube_dl/extractor/tvp.py
@@ -16,6 +16,7 @@ class TvpIE(InfoExtractor):
             u'title': u'31.10.2013 - Odcinek 2',
             u'description': u'31.10.2013 - Odcinek 2',
         },
+        u'skip': u'Download has to use same server IP as extraction. Therefore, a good (load-balancing) DNS resolver will make the download fail.'
     }
 
     def _real_extract(self, url):

From 21ea3e06c9e39ac2cbfa24e7c02ebac936202893 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 16 Nov 2013 02:31:02 +0100
Subject: [PATCH 076/124] [gamekings] remove unnecessary import

---
 youtube_dl/extractor/gamekings.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 6aa657ef4..64715581d 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -1,9 +1,6 @@
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    determine_ext,
-)
 
 
 class GamekingsIE(InfoExtractor):

From eab2724138670159f1946608359f5e5bd5a7e7af Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 16 Nov 2013 02:32:08 +0100
Subject: [PATCH 077/124] [gamekings] Do not test md5 sum, precise file changes
 regularly

---
 youtube_dl/extractor/gamekings.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py
index 64715581d..c91669b0e 100644
--- a/youtube_dl/extractor/gamekings.py
+++ b/youtube_dl/extractor/gamekings.py
@@ -8,7 +8,8 @@ class GamekingsIE(InfoExtractor):
     _TEST = {
         u"url": u"http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/",
         u'file': u'20130811.mp4',
-        u'md5': u'2f32b1f7b80fdc5cb616efb4f387f8a3',
+        # MD5 is flaky, seems to change regularly
+        #u'md5': u'2f32b1f7b80fdc5cb616efb4f387f8a3',
         u'info_dict': {
             u"title": u"Phoenix Wright: Ace Attorney \u2013 Dual Destinies Review",
             u"description": u"Melle en Steven hebben voor de review een week in de rechtbank doorbracht met Phoenix Wright: Ace Attorney - Dual Destinies.",

From d1c252048bf74add82de0b9812b589956f3c8a69 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sat, 16 Nov 2013 10:30:09 +0100
Subject: [PATCH 078/124] [redtube] Do not test md5, seems to vary

---
 youtube_dl/extractor/redtube.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py
index 994778e16..3bbda128e 100644
--- a/youtube_dl/extractor/redtube.py
+++ b/youtube_dl/extractor/redtube.py
@@ -8,7 +8,9 @@ class RedTubeIE(InfoExtractor):
     _TEST = {
         u'url': u'http://www.redtube.com/66418',
         u'file': u'66418.mp4',
-        u'md5': u'7b8c22b5e7098a3e1c09709df1126d2d',
+        # md5 varies from time to time, as in
+        # https://travis-ci.org/rg3/youtube-dl/jobs/14052463#L295
+        #u'md5': u'7b8c22b5e7098a3e1c09709df1126d2d',
         u'info_dict': {
             u"title": u"Sucked on a toilet",
             u"age_limit": 18,

From ba3881dffd241d8719430c31f107156ed8830996 Mon Sep 17 00:00:00 2001
From: Anton Larionov <diffident.cat@gmail.com>
Date: Sat, 16 Nov 2013 18:26:34 +0400
Subject: [PATCH 079/124] Add support for anitube.se (#1417)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/anitube.py  | 59 ++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 youtube_dl/extractor/anitube.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ee3173468..a30de3033 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -1,5 +1,6 @@
 from .appletrailers import AppleTrailersIE
 from .addanime import AddAnimeIE
+from .anitube import AnitubeIE
 from .archiveorg import ArchiveOrgIE
 from .ard import ARDIE
 from .arte import (
diff --git a/youtube_dl/extractor/anitube.py b/youtube_dl/extractor/anitube.py
new file mode 100644
index 000000000..2954966a6
--- /dev/null
+++ b/youtube_dl/extractor/anitube.py
@@ -0,0 +1,59 @@
+import re
+import xml.etree.ElementTree
+
+from .common import InfoExtractor
+
+
+class AnitubeIE(InfoExtractor):
+    IE_NAME = u'anitube.se'
+    _VALID_URL = r'http?://(?:www\.)?anitube\.se/video/(?P<id>\d+)'
+
+    _TEST = {
+        u'url': u'http://www.anitube.se/video/36621',
+        u'md5': u'0c4e4f1051bf50f5982f829f7230f539',
+        u'info_dict': {
+            u'id': u'36621',
+            u'ext': u'mp4',
+            u'title': u'Recorder to Randoseru 01',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+
+        webpage = self._download_webpage(url, video_id)
+
+        key = self._html_search_regex(r'http://www\.anitube\.se/embed/([A-Za-z0-9_-]*)',
+                                      webpage, u'key')
+
+        webpage_config = self._download_webpage('http://www.anitube.se/nuevo/econfig.php?key=%s' % key,
+                                                key)
+
+        config_xml = xml.etree.ElementTree.fromstring(webpage_config.encode('utf-8'))
+
+        video_title = config_xml.find('title').text
+
+
+        formats = []
+
+        video_url = config_xml.find('file')
+        if video_url is not None:
+            formats.append({
+                'format_id': 'sd',
+                'url': video_url.text,
+            })
+
+        video_url = config_xml.find('filehd')
+        if video_url is not None:
+            formats.append({
+                'format_id': 'hd',
+                'url': video_url.text,
+            })
+
+        return {
+            'id': video_id,
+            'title': video_title,
+            'ext': 'mp4',
+            'formats': formats
+        }

From ddf49c63445d236138846d778b18db6ede067fc8 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 17 Nov 2013 11:05:49 +0100
Subject: [PATCH 080/124] [arte] remove two typos

---
 youtube_dl/extractor/arte.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index b35a679e3..44d0b5d70 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -69,7 +69,7 @@ class ArteTvIE(InfoExtractor):
             lang = mobj.group('lang')
             return self._extract_liveweb(url, name, lang)
 
-        if re.search(self._LIVE_URL, video_id) is not None:
+        if re.search(self._LIVE_URL, url) is not None:
             raise ExtractorError(u'Arte live streams are not yet supported, sorry')
             # self.extractLiveStream(url)
             # return
@@ -115,7 +115,7 @@ class ArteTvIE(InfoExtractor):
         event_doc = config_doc.find('event')
         url_node = event_doc.find('video').find('urlHd')
         if url_node is None:
-            url_node = video_doc.find('urlSd')
+            url_node = event_doc.find('urlSd')
 
         return {'id': video_id,
                 'title': event_doc.find('name%s' % lang.capitalize()).text,

From 1d699755e0978f2ec0932ef8d6562394a4799871 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 17 Nov 2013 11:06:16 +0100
Subject: [PATCH 081/124] [youtube] Add view_count (Fixes #1781)

---
 youtube_dl/extractor/youtube.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 8c0e6f252..1aa549740 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1301,6 +1301,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             else:
                 raise ExtractorError(u'"token" parameter not in video info for unknown reason')
 
+        if 'view_count' in video_info:
+            view_count = int(video_info['view_count'][0])
+        else:
+            view_count = None
+
         # Check for "rental" videos
         if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
             raise ExtractorError(u'"rental" videos not supported')
@@ -1489,6 +1494,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 'age_limit':    18 if age_gate else 0,
                 'annotations':  video_annotations,
                 'webpage_url': 'https://www.youtube.com/watch?v=%s' % video_id,
+                'view_count': view_count,
             })
         return results
 

From 1e5b9a95fd2049e024b3ee2f13b4da5c308d2e9c Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 17 Nov 2013 11:39:52 +0100
Subject: [PATCH 082/124] Move console_title to YoutubeDL

---
 youtube_dl/FileDownloader.py | 17 +++--------------
 youtube_dl/YoutubeDL.py      | 13 +++++++++++++
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index 088f59586..e5a542ed5 100644
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -5,9 +5,6 @@ import subprocess
 import sys
 import time
 
-if os.name == 'nt':
-    import ctypes
-
 from .utils import (
     compat_urllib_error,
     compat_urllib_request,
@@ -151,16 +148,8 @@ class FileDownloader(object):
     def to_stderr(self, message):
         self.ydl.to_screen(message)
 
-    def to_cons_title(self, message):
-        """Set console/terminal window title to message."""
-        if not self.params.get('consoletitle', False):
-            return
-        if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
-            # c_wchar_p() might not be necessary if `message` is
-            # already of type unicode()
-            ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
-        elif 'TERM' in os.environ:
-            self.to_screen('\033]0;%s\007' % message, skip_eol=True)
+    def to_console_title(self, message):
+        self.ydl.to_console_title(message)
 
     def trouble(self, *args, **kargs):
         self.ydl.trouble(*args, **kargs)
@@ -249,7 +238,7 @@ class FileDownloader(object):
         else:
             self.to_screen(u'\r%s[download] %s of %s at %s ETA %s' %
                 (clear_line, percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
-        self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
+        self.to_console_title(u'youtube-dl - %s of %s at %s ETA %s' %
                 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
 
     def report_resuming_byte(self, resume_len):
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d29e8bec5..6ea865bd9 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -13,6 +13,9 @@ import sys
 import time
 import traceback
 
+if os.name == 'nt':
+    import ctypes
+
 from .utils import *
 from .extractor import get_info_extractor, gen_extractors
 from .FileDownloader import FileDownloader
@@ -176,6 +179,16 @@ class YoutubeDL(object):
             output = output.encode(preferredencoding())
         sys.stderr.write(output)
 
+    def to_console_title(self, message):
+        if not self.params.get('consoletitle', False):
+            return
+        if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
+            # c_wchar_p() might not be necessary if `message` is
+            # already of type unicode()
+            ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
+        elif 'TERM' in os.environ:
+            self.to_screen('\033]0;%s\007' % message, skip_eol=True)
+
     def fixed_template(self):
         """Checks if the output template is fixed."""
         return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None)

From ce02ed60f27ea27e66c33af745dc7e716377b46f Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 17 Nov 2013 16:47:52 +0100
Subject: [PATCH 083/124] Remove * imports

---
 youtube_dl/YoutubeDL.py | 28 ++++++++++++++++++++++++++--
 youtube_dl/update.py    |  6 +++++-
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6ea865bd9..6e5ae44d3 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -16,7 +16,31 @@ import traceback
 if os.name == 'nt':
     import ctypes
 
-from .utils import *
+from .utils import (
+    compat_http_client,
+    compat_print,
+    compat_str,
+    compat_urllib_error,
+    compat_urllib_request,
+    ContentTooShortError,
+    date_from_str,
+    DateRange,
+    determine_ext,
+    DownloadError,
+    encodeFilename,
+    ExtractorError,
+    locked_file,
+    MaxDownloadsReached,
+    PostProcessingError,
+    preferredencoding,
+    SameFileError,
+    sanitize_filename,
+    subtitles_filename,
+    takewhile_inclusive,
+    UnavailableVideoError,
+    write_json_file,
+    write_string,
+)
 from .extractor import get_info_extractor, gen_extractors
 from .FileDownloader import FileDownloader
 
@@ -267,7 +291,7 @@ class YoutubeDL(object):
         """Report file has already been fully downloaded."""
         try:
             self.to_screen(u'[download] %s has already been downloaded' % file_name)
-        except (UnicodeEncodeError) as err:
+        except UnicodeEncodeError:
             self.to_screen(u'[download] The file has already been downloaded')
 
     def increment_downloads(self):
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 0689a4891..f41b4785a 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -2,11 +2,15 @@ import io
 import json
 import traceback
 import hashlib
+import os
 import subprocess
 import sys
 from zipimport import zipimporter
 
-from .utils import *
+from .utils import (
+    compat_str,
+    compat_urllib_request,
+)
 from .version import __version__
 
 def rsa_verify(message, signature, key):

From 90b6bbc38c7258358e294e7f1bcb3d46cd56ffd9 Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Sun, 17 Nov 2013 17:42:24 +0100
Subject: [PATCH 084/124] [SouthParkStudiosIE] Also detect urls without http://
 or www

---
 youtube_dl/extractor/southparkstudios.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/southparkstudios.py b/youtube_dl/extractor/southparkstudios.py
index b1e96b679..bb0c4b393 100644
--- a/youtube_dl/extractor/southparkstudios.py
+++ b/youtube_dl/extractor/southparkstudios.py
@@ -5,7 +5,7 @@ from .mtv import MTVIE, _media_xml_tag
 
 class SouthParkStudiosIE(MTVIE):
     IE_NAME = u'southparkstudios.com'
-    _VALID_URL = r'https?://www\.southparkstudios\.com/(clips|full-episodes)/(?P<id>.+?)(\?|#|$)'
+    _VALID_URL = r'(https?://)?(www\.)?(?P<url>southparkstudios\.com/(clips|full-episodes)/(?P<id>.+?)(\?|#|$))'
 
     _FEED_URL = 'http://www.southparkstudios.com/feeds/video-player/mrss'
 
@@ -31,6 +31,7 @@ class SouthParkStudiosIE(MTVIE):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
+        url = u'http://www.' + mobj.group(u'url')
         video_id = mobj.group('id')
         webpage = self._download_webpage(url, video_id)
         mgid = self._search_regex(r'swfobject.embedSWF\(".*?(mgid:.*?)"',

From 1672647ade97e070fa67eeff68370910ae715573 Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Sun, 17 Nov 2013 17:43:58 +0100
Subject: [PATCH 085/124] [SouthParkStudiosIE] Move from _TEST to _TESTS

---
 youtube_dl/extractor/southparkstudios.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/southparkstudios.py b/youtube_dl/extractor/southparkstudios.py
index bb0c4b393..a75e328a7 100644
--- a/youtube_dl/extractor/southparkstudios.py
+++ b/youtube_dl/extractor/southparkstudios.py
@@ -9,17 +9,15 @@ class SouthParkStudiosIE(MTVIE):
 
     _FEED_URL = 'http://www.southparkstudios.com/feeds/video-player/mrss'
 
-    _TEST = {
+    # Overwrite MTVIE properties we don't want
+    _TESTS = [{
         u'url': u'http://www.southparkstudios.com/clips/104437/bat-daded#tab=featured',
         u'file': u'a7bff6c2-ed00-11e0-aca6-0026b9414f30.mp4',
         u'info_dict': {
             u'title': u'Bat Daded',
             u'description': u'Randy disqualifies South Park by getting into a fight with Bat Dad.',
         },
-    }
-
-    # Overwrite MTVIE properties we don't want
-    _TESTS = []
+    }]
 
     def _get_thumbnail_url(self, uri, itemdoc):
         search_path = '%s/%s' % (_media_xml_tag('group'), _media_xml_tag('thumbnail'))

From 746f491f82cfddd6fafacbf9978205963d7a214d Mon Sep 17 00:00:00 2001
From: rzhxeo <rzhxeot7z81b4700@mailcatch.com>
Date: Sun, 17 Nov 2013 17:54:47 +0100
Subject: [PATCH 086/124] Add support for southpark.de

---
 youtube_dl/extractor/__init__.py         |  5 ++++-
 youtube_dl/extractor/southparkstudios.py | 14 ++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index ee3173468..2d1e3cdfd 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -116,7 +116,10 @@ from .slashdot import SlashdotIE
 from .slideshare import SlideshareIE
 from .sohu import SohuIE
 from .soundcloud import SoundcloudIE, SoundcloudSetIE, SoundcloudUserIE
-from .southparkstudios import SouthParkStudiosIE
+from .southparkstudios import (
+    SouthParkStudiosIE,
+    SouthparkDeIE,
+)
 from .space import SpaceIE
 from .spankwire import SpankwireIE
 from .spiegel import SpiegelIE
diff --git a/youtube_dl/extractor/southparkstudios.py b/youtube_dl/extractor/southparkstudios.py
index a75e328a7..a711531e6 100644
--- a/youtube_dl/extractor/southparkstudios.py
+++ b/youtube_dl/extractor/southparkstudios.py
@@ -35,3 +35,17 @@ class SouthParkStudiosIE(MTVIE):
         mgid = self._search_regex(r'swfobject.embedSWF\(".*?(mgid:.*?)"',
                                   webpage, u'mgid')
         return self._get_videos_info(mgid)
+
+class SouthparkDeIE(SouthParkStudiosIE):
+    IE_NAME = u'southpark.de'
+    _VALID_URL = r'(https?://)?(www\.)?(?P<url>southpark\.de/(clips|alle-episoden)/(?P<id>.+?)(\?|#|$))'
+    _FEED_URL = 'http://www.southpark.de/feeds/video-player/mrss/'
+
+    _TESTS = [{
+        u'url': u'http://www.southpark.de/clips/uygssh/the-government-wont-respect-my-privacy#tab=featured',
+        u'file': u'85487c96-b3b9-4e39-9127-ad88583d9bf2.mp4',
+        u'info_dict': {
+            u'title': u'The Government Won\'t Respect My Privacy',
+            u'description': u'Cartman explains the benefits of "Shitter" to Stan, Kyle and Craig.',
+        },
+    }]

From bdde425cbe01329d8c24e18cf0492465abb21411 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 17 Nov 2013 21:05:14 +0100
Subject: [PATCH 087/124] Save and restore console title (Fixes #1782)

---
 youtube_dl/YoutubeDL.py | 19 ++++++++
 youtube_dl/__init__.py  | 97 +++++++++++++++++++++--------------------
 2 files changed, 68 insertions(+), 48 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6e5ae44d3..4e28f9120 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -213,6 +213,25 @@ class YoutubeDL(object):
         elif 'TERM' in os.environ:
             self.to_screen('\033]0;%s\007' % message, skip_eol=True)
 
+    def save_console_title(self):
+        if not self.params.get('consoletitle', False):
+            return
+        if 'TERM' in os.environ:
+            self.to_screen('\033[22t')
+
+    def restore_console_title(self):
+        if not self.params.get('consoletitle', False):
+            return
+        if 'TERM' in os.environ:
+            self.to_screen('\033[23t')
+
+    def __enter__(self):
+        self.save_console_title()
+        return self
+
+    def __exit__(self, *args):
+        self.restore_console_title()
+
     def fixed_template(self):
         """Checks if the output template is fixed."""
         return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None)
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 4dee487ab..af4c9c5c4 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -603,8 +603,7 @@ def _real_main(argv=None):
                      u' file! Use "%%(ext)s" instead of %r' %
                      determine_ext(outtmpl, u''))
 
-    # YoutubeDL
-    ydl = YoutubeDL({
+    ydl_opts = {
         'usenetrc': opts.usenetrc,
         'username': opts.username,
         'password': opts.password,
@@ -667,61 +666,63 @@ def _real_main(argv=None):
         'youtube_print_sig_code': opts.youtube_print_sig_code,
         'age_limit': opts.age_limit,
         'download_archive': opts.download_archive,
-        })
+    }
 
-    if opts.verbose:
-        write_string(u'[debug] youtube-dl version ' + __version__ + u'\n')
-        try:
-            sp = subprocess.Popen(
-                ['git', 'rev-parse', '--short', 'HEAD'],
-                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                cwd=os.path.dirname(os.path.abspath(__file__)))
-            out, err = sp.communicate()
-            out = out.decode().strip()
-            if re.match('[0-9a-f]+', out):
-                write_string(u'[debug] Git HEAD: ' + out + u'\n')
-        except:
+    with YoutubeDL(ydl_opts) as ydl:
+        if opts.verbose:
+            write_string(u'[debug] youtube-dl version ' + __version__ + u'\n')
             try:
-                sys.exc_clear()
+                sp = subprocess.Popen(
+                    ['git', 'rev-parse', '--short', 'HEAD'],
+                    stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                    cwd=os.path.dirname(os.path.abspath(__file__)))
+                out, err = sp.communicate()
+                out = out.decode().strip()
+                if re.match('[0-9a-f]+', out):
+                    write_string(u'[debug] Git HEAD: ' + out + u'\n')
             except:
-                pass
-        write_string(u'[debug] Python version %s - %s' %(platform.python_version(), platform_name()) + u'\n')
+                try:
+                    sys.exc_clear()
+                except:
+                    pass
+            write_string(u'[debug] Python version %s - %s' %
+                         (platform.python_version(), platform_name()) + u'\n')
 
-        proxy_map = {}
-        for handler in opener.handlers:
-            if hasattr(handler, 'proxies'):
-                proxy_map.update(handler.proxies)
-        write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n')
+            proxy_map = {}
+            for handler in opener.handlers:
+                if hasattr(handler, 'proxies'):
+                    proxy_map.update(handler.proxies)
+            write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n')
 
-    ydl.add_default_info_extractors()
+        ydl.add_default_info_extractors()
 
-    # PostProcessors
-    # Add the metadata pp first, the other pps will copy it
-    if opts.addmetadata:
-        ydl.add_post_processor(FFmpegMetadataPP())
-    if opts.extractaudio:
-        ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
-    if opts.recodevideo:
-        ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
-    if opts.embedsubtitles:
-        ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
+        # PostProcessors
+        # Add the metadata pp first, the other pps will copy it
+        if opts.addmetadata:
+            ydl.add_post_processor(FFmpegMetadataPP())
+        if opts.extractaudio:
+            ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites))
+        if opts.recodevideo:
+            ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo))
+        if opts.embedsubtitles:
+            ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat))
 
-    # Update version
-    if opts.update_self:
-        update_self(ydl.to_screen, opts.verbose)
+        # Update version
+        if opts.update_self:
+            update_self(ydl.to_screen, opts.verbose)
 
-    # Maybe do nothing
-    if len(all_urls) < 1:
-        if not opts.update_self:
-            parser.error(u'you must provide at least one URL')
-        else:
-            sys.exit()
+        # Maybe do nothing
+        if len(all_urls) < 1:
+            if not opts.update_self:
+                parser.error(u'you must provide at least one URL')
+            else:
+                sys.exit()
 
-    try:
-        retcode = ydl.download(all_urls)
-    except MaxDownloadsReached:
-        ydl.to_screen(u'--max-download limit reached, aborting.')
-        retcode = 101
+        try:
+            retcode = ydl.download(all_urls)
+        except MaxDownloadsReached:
+            ydl.to_screen(u'--max-download limit reached, aborting.')
+            retcode = 101
 
     # Dump cookie jar if requested
     if opts.cookiefile is not None:

From 749febf4d1513328ca8b9b6c16969d8c70ff4555 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 17 Nov 2013 21:12:50 +0100
Subject: [PATCH 088/124] Allow --console-title when --quiet is given (Fixes
 #1783)

---
 youtube_dl/YoutubeDL.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4e28f9120..20eed96ca 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -211,19 +211,19 @@ class YoutubeDL(object):
             # already of type unicode()
             ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
         elif 'TERM' in os.environ:
-            self.to_screen('\033]0;%s\007' % message, skip_eol=True)
+            write_string(u'\033]0;%s\007' % message, self._screen_file)
 
     def save_console_title(self):
         if not self.params.get('consoletitle', False):
             return
         if 'TERM' in os.environ:
-            self.to_screen('\033[22t')
+            write_string(u'\033[22t', self._screen_file)
 
     def restore_console_title(self):
         if not self.params.get('consoletitle', False):
             return
         if 'TERM' in os.environ:
-            self.to_screen('\033[23t')
+            write_string(u'\033[23t', self._screen_file)
 
     def __enter__(self):
         self.save_console_title()

From 63b7b7224a9cbb33856d3e1f8755ba4bdd3a9d58 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 17 Nov 2013 22:11:39 +0100
Subject: [PATCH 089/124] [MTVIE] Try with RTMP URL if download fails

This fixes youtube-dl http://www.southpark.de/clips/155251/cartman-vs-the-dog-whisperer
---
 youtube_dl/extractor/mtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 24a79ae13..3df7f9b85 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -48,7 +48,7 @@ class MTVIE(InfoExtractor):
     def _transform_rtmp_url(rtmp_video_url):
         m = re.match(r'^rtmpe?://.*?/(?P<finalid>gsp\..+?/.*)$', rtmp_video_url)
         if not m:
-            raise ExtractorError(u'Cannot transform RTMP url')
+            return rtmp_video_url
         base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/'
         return base + m.group('finalid')
 

From 73c566695fac926e7e9e6922fe4e6d82c64a1850 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Sun, 17 Nov 2013 22:14:13 +0100
Subject: [PATCH 090/124] release 2013.11.17

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b04238eb5..110058c79 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2013.11.15.1'
+__version__ = '2013.11.17'

From fccd377198e9be2ea95cc761d841704a5f4f5ec5 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 18 Nov 2013 13:05:18 +0100
Subject: [PATCH 091/124] Suppor embed-only videos (Fixes #1746)

---
 youtube_dl/extractor/youtube.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1aa549740..514a11f7f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -139,7 +139,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
 class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
     IE_DESC = u'YouTube.com'
-    _VALID_URL = r"""^
+    _VALID_URL = r"""(?xi)^
                      (
                          (?:https?://)?                                       # http(s):// (optional)
                          (?:(?:(?:(?:\w+\.)?youtube(?:-nocookie)?\.com/|
@@ -363,6 +363,18 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
                 u"uploader_id": u"justintimberlakeVEVO"
             }
         },
+        {
+            u"url":  u"https://www.YouTube.com/watch?v=yZIXLfi8CZQ",
+            u"file":  u"yZIXLfi8CZQ.mp4",
+            u"note": u"Embed-only video (#1746)",
+            u"info_dict": {
+                u"upload_date": u"20120608",
+                u"title": u"Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012",
+                u"description": u"md5:09b78bd971f1e3e289601dfba15ca4f7",
+                u"uploader": u"SET India",
+                u"uploader_id": u"setindia"
+            }
+        },
     ]
 
 
@@ -370,7 +382,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
     def suitable(cls, url):
         """Receives a URL and returns True if suitable for this IE."""
         if YoutubePlaylistIE.suitable(url): return False
-        return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
+        return re.match(cls._VALID_URL, url) is not None
 
     def __init__(self, *args, **kwargs):
         super(YoutubeIE, self).__init__(*args, **kwargs)
@@ -1272,7 +1284,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             # We simulate the access to the video from www.youtube.com/v/{video_id}
             # this can be viewed without login into Youtube
             data = compat_urllib_parse.urlencode({'video_id': video_id,
-                                                  'el': 'embedded',
+                                                  'el': 'player_embedded',
                                                   'gl': 'US',
                                                   'hl': 'en',
                                                   'eurl': 'https://youtube.googleapis.com/v/' + video_id,

From 96b31b65331710ad6d3582abc7ecbaee0efa605a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 18 Nov 2013 13:05:58 +0100
Subject: [PATCH 092/124] Add iPhone to UA (#1746)

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1d9785341..659ac859d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -176,7 +176,7 @@ def compat_ord(c):
 compiled_regex_type = type(re.compile(''))
 
 std_headers = {
-    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0 (Chrome)',
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0 (Chrome) (iPhone)',
     'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
     'Accept-Encoding': 'gzip, deflate',

From 83aa529330245b9221ae99e6769a40531f4ba216 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 18 Nov 2013 13:18:17 +0100
Subject: [PATCH 093/124] Support protocol-independent URLs (#1787)

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 514a11f7f..9aa4c6fab 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -141,7 +141,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
     IE_DESC = u'YouTube.com'
     _VALID_URL = r"""(?xi)^
                      (
-                         (?:https?://)?                                       # http(s):// (optional)
+                         (?:https?://|//)?                                    # http(s):// or protocol-independent URL (optional)
                          (?:(?:(?:(?:\w+\.)?youtube(?:-nocookie)?\.com/|
                             tube\.majestyc\.net/|
                             youtube\.googleapis\.com/)                        # the various hostnames, with wildcard subdomains
@@ -364,7 +364,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
             }
         },
         {
-            u"url":  u"https://www.YouTube.com/watch?v=yZIXLfi8CZQ",
+            u"url":  u"//www.YouTube.com/watch?v=yZIXLfi8CZQ",
             u"file":  u"yZIXLfi8CZQ.mp4",
             u"note": u"Embed-only video (#1746)",
             u"info_dict": {

From 887c6acdf231e6846e0dbb63b644fc7dc3378700 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 18 Nov 2013 13:28:26 +0100
Subject: [PATCH 094/124] Support multiple embedded YouTube URLs (Fixes #1787)

---
 youtube_dl/extractor/generic.py | 31 +++++++++++++++++--------------
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c7552fddb..e1d6a2a01 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -162,6 +162,16 @@ class GenericIE(InfoExtractor):
             raise ExtractorError(u'Failed to download URL: %s' % url)
 
         self.report_extraction(video_id)
+
+        # it's tempting to parse this further, but you would
+        # have to take into account all the variations like
+        #   Video Title - Site Name
+        #   Site Name | Video Title
+        #   Video Title - Tagline | Site Name
+        # and so on and so forth; it's just not practical
+        video_title = self._html_search_regex(r'<title>(.*)</title>',
+            webpage, u'video title', default=u'video', flags=re.DOTALL)
+
         # Look for BrightCove:
         bc_url = BrightcoveIE._extract_brightcove_url(webpage)
         if bc_url is not None:
@@ -177,11 +187,13 @@ class GenericIE(InfoExtractor):
             return self.url_result(surl, 'Vimeo')
 
         # Look for embedded YouTube player
-        mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?youtube.com/embed/.+?)\1', webpage)
-        if mobj:
-            surl = unescapeHTML(mobj.group(u'url'))
-            return self.url_result(surl, 'Youtube')
+        matches = re.findall(
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?youtube.com/embed/.+?)\1', webpage)
+        if matches:
+            urlrs = [self.url_result(unescapeHTML(tuppl[1]), 'Youtube')
+                     for tuppl in matches]
+            return self.playlist_result(
+                urlrs, playlist_id=video_id, playlist_title=video_title)
 
         # Look for Bandcamp pages with custom domain
         mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
@@ -226,15 +238,6 @@ class GenericIE(InfoExtractor):
         video_extension = os.path.splitext(video_id)[1][1:]
         video_id = os.path.splitext(video_id)[0]
 
-        # it's tempting to parse this further, but you would
-        # have to take into account all the variations like
-        #   Video Title - Site Name
-        #   Site Name | Video Title
-        #   Video Title - Tagline | Site Name
-        # and so on and so forth; it's just not practical
-        video_title = self._html_search_regex(r'<title>(.*)</title>',
-            webpage, u'video title', default=u'video', flags=re.DOTALL)
-
         # video uploader is domain name
         video_uploader = self._search_regex(r'(?:https?://)?([^/]*)/.*',
             url, u'video uploader')

From a81b4d5c8fe32611fa79104510d422b208f2a1a2 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 18 Nov 2013 13:30:43 +0100
Subject: [PATCH 095/124] release 2013.11.18

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 110058c79..16e32abe4 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2013.11.17'
+__version__ = '2013.11.18'

From ae8f7871412ae9db40c5060d2d24e5a50f5fe9d0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 18 Nov 2013 13:52:24 +0100
Subject: [PATCH 096/124] Remove iPhone from user agent. This breaks a lot of
 extractors

In the future, it might be worth investigating whether we get better content when we claime to be an iPhone.
---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 659ac859d..1d9785341 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -176,7 +176,7 @@ def compat_ord(c):
 compiled_regex_type = type(re.compile(''))
 
 std_headers = {
-    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0 (Chrome) (iPhone)',
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0 (Chrome)',
     'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
     'Accept-Encoding': 'gzip, deflate',

From 9906d397a05de3c89fb0ba2d60c284c16cb72581 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 18 Nov 2013 13:56:45 +0100
Subject: [PATCH 097/124] [auengine] Simplify

---
 youtube_dl/extractor/auengine.py | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/auengine.py b/youtube_dl/extractor/auengine.py
index 0febbff4f..90dfa9a46 100644
--- a/youtube_dl/extractor/auengine.py
+++ b/youtube_dl/extractor/auengine.py
@@ -1,10 +1,10 @@
-import os.path
 import re
 
 from .common import InfoExtractor
 from ..utils import (
     compat_urllib_parse,
-    compat_urllib_parse_urlparse,
+    determine_ext,
+    ExtractorError,
 )
 
 class AUEngineIE(InfoExtractor):
@@ -25,22 +25,26 @@ class AUEngineIE(InfoExtractor):
         title = self._html_search_regex(r'<title>(?P<title>.+?)</title>',
                 webpage, u'title')
         title = title.strip()
-        links = re.findall(r'[^A-Za-z0-9]?(?:file|url):\s*["\'](http[^\'"&]*)', webpage)
-        links = [compat_urllib_parse.unquote(l) for l in links]
+        links = re.findall(r'\s(?:file|url):\s*["\']([^\'"]+)["\']', webpage)
+        links = map(compat_urllib_parse.unquote, links)
+
+        thumbnail = None
+        video_url = None
         for link in links:
-            root, pathext = os.path.splitext(compat_urllib_parse_urlparse(link).path)
-            if pathext == '.png':
+            if link.endswith('.png'):
                 thumbnail = link
-            elif pathext == '.mp4':
-                url = link
-                ext = pathext
+            elif '/videos/' in link:
+                video_url = link
+        if not video_url:
+            raise ExtractorError(u'Could not find video URL')
+        ext = u'.' + determine_ext(video_url)
         if ext == title[-len(ext):]:
             title = title[:-len(ext)]
-        ext = ext[1:]
-        return [{
+
+        return {
             'id':        video_id,
-            'url':       url,
+            'url':       video_url,
             'ext':       ext,
             'title':     title,
             'thumbnail': thumbnail,
-        }]
+        }

From 9a942a46712330173683f8a0179868ae0c5a9138 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 18 Nov 2013 13:56:53 +0100
Subject: [PATCH 098/124] release 2013.11.18.1

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 16e32abe4..6b3bf39fa 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2013.11.18'
+__version__ = '2013.11.18.1'

From 4113e6ab561e6f2f014fb2548d1c8691352b8304 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 18 Nov 2013 14:36:01 +0100
Subject: [PATCH 099/124] [auengine] Do not return unnecessary ext

---
 youtube_dl/extractor/auengine.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/auengine.py b/youtube_dl/extractor/auengine.py
index 90dfa9a46..95c038003 100644
--- a/youtube_dl/extractor/auengine.py
+++ b/youtube_dl/extractor/auengine.py
@@ -44,7 +44,6 @@ class AUEngineIE(InfoExtractor):
         return {
             'id':        video_id,
             'url':       video_url,
-            'ext':       ext,
             'title':     title,
             'thumbnail': thumbnail,
         }

From efd6c574a2cabc860f54018af726cd291cbec868 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Mon, 18 Nov 2013 16:35:41 +0100
Subject: [PATCH 100/124] Correctly write and restore the console title on the
 stack (fixes #1782)

---
 youtube_dl/YoutubeDL.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 20eed96ca..fd5a30a0c 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -217,13 +217,15 @@ class YoutubeDL(object):
         if not self.params.get('consoletitle', False):
             return
         if 'TERM' in os.environ:
-            write_string(u'\033[22t', self._screen_file)
+            # Save the title on stack
+            write_string(u'\033[22;0t', self._screen_file)
 
     def restore_console_title(self):
         if not self.params.get('consoletitle', False):
             return
         if 'TERM' in os.environ:
-            write_string(u'\033[23t', self._screen_file)
+            # Restore the title from stack
+            write_string(u'\033[23;0t', self._screen_file)
 
     def __enter__(self):
         self.save_console_title()

From cb7dfeeac407ca973b7d838dd3c8f6faa0a65bac Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Mon, 18 Nov 2013 16:42:35 +0100
Subject: [PATCH 101/124] [youtube] only allow domain name to be upper-case
 (#1786)

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9aa4c6fab..41838237c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -139,10 +139,10 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
 class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
     IE_DESC = u'YouTube.com'
-    _VALID_URL = r"""(?xi)^
+    _VALID_URL = r"""(?x)^
                      (
                          (?:https?://|//)?                                    # http(s):// or protocol-independent URL (optional)
-                         (?:(?:(?:(?:\w+\.)?youtube(?:-nocookie)?\.com/|
+                         (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/|
                             tube\.majestyc\.net/|
                             youtube\.googleapis\.com/)                        # the various hostnames, with wildcard subdomains
                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls

From 34b3afc7be059cbc882b901b897ec99eeee25e14 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Tue, 19 Nov 2013 12:41:01 +0100
Subject: [PATCH 102/124] release 2013.11.19

---
 youtube_dl/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 6b3bf39fa..e9ff3f640 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2013.11.18.1'
+__version__ = '2013.11.19'

From 69545c2affb6b126398fd657a12a560a9857dbee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?=
 <jaime.marquinez.ferrandiz@gmail.com>
Date: Tue, 19 Nov 2013 20:43:49 +0100
Subject: [PATCH 103/124] [d8] inherit from CanalplusIE

it reuses the same extraction process
---
 youtube_dl/extractor/canalplus.py |  3 ++-
 youtube_dl/extractor/d8.py        | 36 +++----------------------------
 2 files changed, 5 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 1db9b24cf..bfa2a8b40 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -5,6 +5,7 @@ import xml.etree.ElementTree
 from .common import InfoExtractor
 from ..utils import unified_strdate
 
+
 class CanalplusIE(InfoExtractor):
     _VALID_URL = r'https?://(www\.canalplus\.fr/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>\d+))'
     _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/cplus/%s'
@@ -25,7 +26,7 @@ class CanalplusIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = mobj.groupdict().get('id')
         if video_id is None:
             webpage = self._download_webpage(url, mobj.group('path'))
             video_id = self._search_regex(r'videoId = "(\d+)";', webpage, u'video id')
diff --git a/youtube_dl/extractor/d8.py b/youtube_dl/extractor/d8.py
index 5ce483b16..a56842b16 100644
--- a/youtube_dl/extractor/d8.py
+++ b/youtube_dl/extractor/d8.py
@@ -1,11 +1,8 @@
 # encoding: utf-8
-import re
-import xml.etree.ElementTree
+from .canalplus import CanalplusIE
 
-from .common import InfoExtractor
-from ..utils import unified_strdate
 
-class D8IE(InfoExtractor):
+class D8IE(CanalplusIE):
     _VALID_URL = r'https?://www\.d8\.tv/.*?/(?P<path>.*)'
     _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/d8/%s'
     IE_NAME = u'd8.tv'
@@ -19,34 +16,7 @@ class D8IE(InfoExtractor):
             u'upload_date': u'20131108',
         },
         u'params': {
+            # rtmp
             u'skip_download': True,
         },
     }
-
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        webpage = self._download_webpage(url, mobj.group('path'))
-        video_id = self._search_regex(r'videoId = "(\d+)";', webpage, u'video id')
-        info_url = self._VIDEO_INFO_TEMPLATE % video_id
-        info_page = self._download_webpage(info_url,video_id, 
-                                           u'Downloading video info')
-
-        self.report_extraction(video_id)
-        doc = xml.etree.ElementTree.fromstring(info_page.encode('utf-8'))
-        video_info = [video for video in doc if video.find('ID').text == video_id][0]
-        infos = video_info.find('INFOS')
-        media = video_info.find('MEDIA')
-        formats = [media.find('VIDEOS/%s' % format)
-            for format in ['BAS_DEBIT', 'HAUT_DEBIT', 'HD']]
-        video_url = [format.text for format in formats if format is not None][-1]
-
-        return {'id': video_id,
-                'title': u'%s - %s' % (infos.find('TITRAGE/TITRE').text,
-                                       infos.find('TITRAGE/SOUS_TITRE').text),
-                'url': video_url,
-                'ext': 'flv',
-                'upload_date': unified_strdate(infos.find('PUBLICATION/DATE').text),
-                'thumbnail': media.find('IMAGES/GRAND').text,
-                'description': infos.find('DESCRIPTION').text,
-                'view_count': int(infos.find('NB_VUES').text),
-                }

From 59040888112e7e5a22ef17bacc27ca88d58a5d92 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 06:13:19 +0100
Subject: [PATCH 104/124] Add support for tou.tv (Fixes #1792)

---
 youtube_dl/extractor/__init__.py |  1 +
 youtube_dl/extractor/common.py   | 28 ++++++++++++
 youtube_dl/extractor/toutv.py    | 75 ++++++++++++++++++++++++++++++++
 youtube_dl/utils.py              |  2 +
 4 files changed, 106 insertions(+)
 create mode 100644 youtube_dl/extractor/toutv.py

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 2679d1a8f..b0df1cef7 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -133,6 +133,7 @@ from .techtalks import TechTalksIE
 from .ted import TEDIE
 from .tf1 import TF1IE
 from .thisav import ThisAVIE
+from .toutv import TouTvIE
 from .traileraddict import TrailerAddictIE
 from .trilulilu import TriluliluIE
 from .tube8 import Tube8IE
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f787d0a3c..eb3435c77 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -350,6 +350,17 @@ class InfoExtractor(object):
         if secure: regexes = self._og_regexes('video:secure_url') + regexes
         return self._html_search_regex(regexes, html, name, **kargs)
 
+    def _html_search_meta(self, name, html, display_name=None):
+        if display_name is None:
+            display_name = name
+        return self._html_search_regex(
+            r'''(?ix)<meta(?=[^>]+(?:name|property)=["\']%s["\'])
+                    [^>]+content=["\']([^"\']+)["\']''' % re.escape(name),
+            html, display_name, fatal=False)
+
+    def _dc_search_uploader(self, html):
+        return self._html_search_meta('dc.creator', html, 'uploader')
+
     def _rta_search(self, html):
         # See http://www.rtalabel.org/index.php?content=howtofaq#single
         if re.search(r'(?ix)<meta\s+name="rating"\s+'
@@ -358,6 +369,23 @@ class InfoExtractor(object):
             return 18
         return 0
 
+    def _media_rating_search(self, html):
+        # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
+        rating = self._html_search_meta('rating', html)
+
+        if not rating:
+            return None
+
+        RATING_TABLE = {
+            'safe for kids': 0,
+            'general': 8,
+            '14 years': 14,
+            'mature': 17,
+            'restricted': 19,
+        }
+        return RATING_TABLE.get(rating.lower(), None)
+
+
 
 class SearchInfoExtractor(InfoExtractor):
     """
diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py
new file mode 100644
index 000000000..73ea67da9
--- /dev/null
+++ b/youtube_dl/extractor/toutv.py
@@ -0,0 +1,75 @@
+# coding: utf-8
+import re
+import xml.etree.ElementTree
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    unified_strdate,
+)
+
+
+class TouTvIE(InfoExtractor):
+    IE_NAME = u'tou.tv'
+    _VALID_URL = r'https?://www\.tou\.tv/(?P<id>[a-zA-Z0-9_-]+(?:/(?P<episode>S[0-9]+E[0-9]+)))'
+
+    _TEST = {
+        u'url': u'http://www.tou.tv/30-vies/S04E41',
+        u'file': u'30-vies_S04E41.mp4',
+        u'info_dict': {
+            u'title': u'30 vies Saison 4 / Épisode 41',
+            u'description': u'md5:da363002db82ccbe4dafeb9cab039b09',
+            u'age_limit': 8,
+            u'uploader': u'Groupe des Nouveaux Médias',
+            u'duration': 1296,
+            u'upload_date': u'20131118',
+            u'thumbnail': u'http://static.tou.tv/medias/images/2013-11-18_19_00_00_30VIES_0341_01_L.jpeg',
+        },
+        u'params': {
+            u'skip_download': True,  # Requires rtmpdump
+        },
+        u'xskip': 'Only available in Canada'
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        video_id = mobj.group('id')
+        webpage = self._download_webpage(url, video_id)
+
+        mediaId = self._search_regex(
+            r'"idMedia":\s*"([^"]+)"', webpage, u'media ID')
+
+        # TODO test from de
+        streams_url = u'http://release.theplatform.com/content.select?pid=' + mediaId
+        streams_webpage = self._download_webpage(
+            streams_url, video_id, note=u'Downloading stream list')
+
+        streams_doc = xml.etree.ElementTree.fromstring(
+            streams_webpage.encode('utf-8'))
+        video_url = next(n.text
+                         for n in streams_doc.findall('.//choice/url')
+                         if u'//ad.doubleclick' not in n.text)
+        if video_url.endswith('/Unavailable.flv'):
+            raise ExtractorError(
+                u'Access to this video is blocked from outside of Canada',
+                expected=True)
+
+        duration_str = self._html_search_meta(
+            'video:duration', webpage, u'duration')
+        duration = int(duration_str) if duration_str else None
+        upload_date_str = self._html_search_meta(
+            'video:release_date', webpage, u'upload date')
+        upload_date = unified_strdate(upload_date_str) if upload_date_str else None
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'url': video_url,
+            'description': self._og_search_description(webpage),
+            'uploader': self._dc_search_uploader(webpage),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'age_limit': self._media_rating_search(webpage),
+            'duration': duration,
+            'upload_date': upload_date,
+            'ext': 'mp4',
+        }
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1d9785341..b50c8166f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -734,6 +734,8 @@ def unified_strdate(date_str):
         '%Y/%m/%d %H:%M:%S',
         '%d.%m.%Y %H:%M',
         '%Y-%m-%dT%H:%M:%SZ',
+        '%Y-%m-%dT%H:%M:%S.%fZ',
+        '%Y-%m-%dT%H:%M:%S.%f0Z',
         '%Y-%m-%dT%H:%M:%S',
     ]
     for expression in format_expressions:

From 9d1538182fdc540fa79ec8341385a980cde334e7 Mon Sep 17 00:00:00 2001
From: Mohamedh Fazal <mohamedhfazal@gmail.com>
Date: Tue, 19 Nov 2013 18:59:22 +0500
Subject: [PATCH 105/124] Add an option to dump json information

---
 youtube_dl/YoutubeDL.py | 3 +++
 youtube_dl/__init__.py  | 8 ++++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index fd5a30a0c..d8d55d7d7 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -84,6 +84,7 @@ class YoutubeDL(object):
     forcethumbnail:    Force printing thumbnail URL.
     forcedescription:  Force printing description.
     forcefilename:     Force printing final filename.
+    forcejson:         Force printing json information.
     simulate:          Do not download the video files.
     format:            Video format code.
     format_limit:      Highest quality format to try.
@@ -651,6 +652,8 @@ class YoutubeDL(object):
             compat_print(filename)
         if self.params.get('forceformat', False):
             compat_print(info_dict['format'])
+        if self.params.get('forcejson', False):
+            compat_print(json.dumps(info_dict))
 
         # Do nothing else if in simulate mode
         if self.params.get('simulate', False):
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index af4c9c5c4..7ab99bdd1 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -306,6 +306,9 @@ def parseOpts(overrideArguments=None):
     verbosity.add_option('--get-format',
             action='store_true', dest='getformat',
             help='simulate, quiet but print output format', default=False)
+    verbosity.add_option('-j', '--dump-json',
+            action='store_true', dest='dumpjson',
+            help='simulate, quiet but print json information', default=False)
     verbosity.add_option('--newline',
             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
     verbosity.add_option('--no-progress',
@@ -608,7 +611,7 @@ def _real_main(argv=None):
         'username': opts.username,
         'password': opts.password,
         'videopassword': opts.videopassword,
-        'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
+        'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.dumpjson),
         'forceurl': opts.geturl,
         'forcetitle': opts.gettitle,
         'forceid': opts.getid,
@@ -616,8 +619,9 @@ def _real_main(argv=None):
         'forcedescription': opts.getdescription,
         'forcefilename': opts.getfilename,
         'forceformat': opts.getformat,
+        'forcejson': opts.dumpjson,
         'simulate': opts.simulate,
-        'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
+        'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.dumpjson),
         'format': opts.format,
         'format_limit': opts.format_limit,
         'listformats': opts.listformats,

From 8694c60000d2aed2b37e10da0ffda132249ff10a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 06:18:24 +0100
Subject: [PATCH 106/124] import json for --dump-json

---
 youtube_dl/YoutubeDL.py | 3 ++-
 youtube_dl/__init__.py  | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index d8d55d7d7..bd093add8 100644
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -5,6 +5,7 @@ from __future__ import absolute_import
 
 import errno
 import io
+import json
 import os
 import re
 import shutil
@@ -84,7 +85,7 @@ class YoutubeDL(object):
     forcethumbnail:    Force printing thumbnail URL.
     forcedescription:  Force printing description.
     forcefilename:     Force printing final filename.
-    forcejson:         Force printing json information.
+    forcejson:         Force printing info_dict as JSON.
     simulate:          Do not download the video files.
     format:            Video format code.
     format_limit:      Highest quality format to try.
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 7ab99bdd1..e27bd4544 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -308,7 +308,7 @@ def parseOpts(overrideArguments=None):
             help='simulate, quiet but print output format', default=False)
     verbosity.add_option('-j', '--dump-json',
             action='store_true', dest='dumpjson',
-            help='simulate, quiet but print json information', default=False)
+            help='simulate, quiet but print JSON information', default=False)
     verbosity.add_option('--newline',
             action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False)
     verbosity.add_option('--no-progress',

From cc13cc0251b287ac88883096fcc36a8072e7ab5d Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 06:25:33 +0100
Subject: [PATCH 107/124] [teamcoco] Correct error

---
 youtube_dl/extractor/teamcoco.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py
index bc48620f0..165d9f88b 100644
--- a/youtube_dl/extractor/teamcoco.py
+++ b/youtube_dl/extractor/teamcoco.py
@@ -60,7 +60,7 @@ class TeamcocoIE(InfoExtractor):
                 return -1
         formats.sort(key=sort_key)
         if not formats:
-            raise RegexNotFoundError(u'Unable to extract video URL')
+            raise ExtractorError(u'Unable to extract video URL')
 
         return {
             'id':          video_id,

From f3682997d7a709e7cf74ba83773570d2a99beb12 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 06:27:48 +0100
Subject: [PATCH 108/124] Clean up unused imports and other minor mistakes

---
 youtube_dl/extractor/collegehumor.py | 4 +---
 youtube_dl/extractor/eighttracks.py  | 1 -
 youtube_dl/extractor/facebook.py     | 1 -
 youtube_dl/extractor/fktv.py         | 1 -
 youtube_dl/extractor/gamespot.py     | 2 +-
 youtube_dl/extractor/jeuxvideo.py    | 2 +-
 youtube_dl/extractor/livestream.py   | 2 --
 youtube_dl/extractor/mtv.py          | 1 -
 youtube_dl/extractor/pornhub.py      | 1 -
 youtube_dl/extractor/soundcloud.py   | 1 -
 youtube_dl/extractor/spankwire.py    | 1 -
 youtube_dl/extractor/spiegel.py      | 1 -
 youtube_dl/extractor/ted.py          | 3 +--
 youtube_dl/extractor/tube8.py        | 2 --
 youtube_dl/extractor/xtube.py        | 1 -
 youtube_dl/extractor/zdf.py          | 2 +-
 16 files changed, 5 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/extractor/collegehumor.py b/youtube_dl/extractor/collegehumor.py
index 8d4c93d6d..0c29acfb1 100644
--- a/youtube_dl/extractor/collegehumor.py
+++ b/youtube_dl/extractor/collegehumor.py
@@ -71,10 +71,8 @@ class CollegeHumorIE(InfoExtractor):
 
             adoc = xml.etree.ElementTree.fromstring(manifestXml)
             try:
-                media_node = adoc.findall('./{http://ns.adobe.com/f4m/1.0}media')[0]
-                node_id = media_node.attrib['url']
                 video_id = adoc.findall('./{http://ns.adobe.com/f4m/1.0}id')[0].text
-            except IndexError as err:
+            except IndexError:
                 raise ExtractorError(u'Invalid manifest file')
             url_pr = compat_urllib_parse_urlparse(info['thumbnail'])
             info['url'] = url_pr.scheme + '://' + url_pr.netloc + video_id[:-2].replace('.csmil','').replace(',','')
diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py
index 2cfbcd363..f21ef8853 100644
--- a/youtube_dl/extractor/eighttracks.py
+++ b/youtube_dl/extractor/eighttracks.py
@@ -1,4 +1,3 @@
-import itertools
 import json
 import random
 import re
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index f8bdfc2d3..3b210710e 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -1,5 +1,4 @@
 import json
-import netrc
 import re
 import socket
 
diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py
index 9c89362ef..dba1a8dc2 100644
--- a/youtube_dl/extractor/fktv.py
+++ b/youtube_dl/extractor/fktv.py
@@ -39,7 +39,6 @@ class FKTVIE(InfoExtractor):
         for i, _ in enumerate(files, 1):
             video_id = '%04d%d' % (episode, i)
             video_url = 'http://dl%d.fernsehkritik.tv/fernsehkritik%d%s.flv' % (server, episode, '' if i == 1 else '-%d' % i)
-            video_title = 'Fernsehkritik %d.%d' % (episode, i)
             videos.append({
                 'id': video_id,
                 'url': video_url,
diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 098768361..9645b00c3 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -24,7 +24,7 @@ class GameSpotIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        page_id = video_id = mobj.group('page_id')
+        page_id = mobj.group('page_id')
         webpage = self._download_webpage(url, page_id)
         data_video_json = self._search_regex(r'data-video=\'(.*?)\'', webpage, u'data video')
         data_video = json.loads(unescapeHTML(data_video_json))
diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py
index 6bb54b932..0020c47cf 100644
--- a/youtube_dl/extractor/jeuxvideo.py
+++ b/youtube_dl/extractor/jeuxvideo.py
@@ -22,7 +22,7 @@ class JeuxVideoIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        title = re.match(self._VALID_URL, url).group(1)
+        title = mobj.group(1)
         webpage = self._download_webpage(url, title)
         xml_link = self._html_search_regex(
             r'<param name="flashvars" value="config=(.*?)" />',
diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py
index 1a3e0ae6b..5f548437c 100644
--- a/youtube_dl/extractor/livestream.py
+++ b/youtube_dl/extractor/livestream.py
@@ -6,9 +6,7 @@ from .common import InfoExtractor
 from ..utils import (
     compat_urllib_parse_urlparse,
     compat_urlparse,
-    get_meta_content,
     xpath_with_ns,
-    ExtractorError,
 )
 
 
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 3df7f9b85..04afd6c4c 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -59,7 +59,6 @@ class MTVIE(InfoExtractor):
         if '/error_country_block.swf' in metadataXml:
             raise ExtractorError(u'This video is not available from your country.', expected=True)
         mdoc = xml.etree.ElementTree.fromstring(metadataXml.encode('utf-8'))
-        renditions = mdoc.findall('.//rendition')
 
         formats = []
         for rendition in mdoc.findall('.//rendition'):
diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 75cf4bb9f..8b3471919 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -6,7 +6,6 @@ from ..utils import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urllib_parse,
-    unescapeHTML,
 )
 from ..aes import (
     aes_decrypt_text
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index 83e1f055f..687457e10 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -158,7 +158,6 @@ class SoundcloudSetIE(SoundcloudIE):
         resolv_url = self._resolv_url(url)
         info_json = self._download_webpage(resolv_url, full_title)
 
-        videos = []
         info = json.loads(info_json)
         if 'errors' in info:
             for err in info['errors']:
diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 97f9c268a..794550c81 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -6,7 +6,6 @@ from ..utils import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
     compat_urllib_parse,
-    unescapeHTML,
 )
 from ..aes import (
     aes_decrypt_text
diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py
index 6dc2eda6d..19ce585cf 100644
--- a/youtube_dl/extractor/spiegel.py
+++ b/youtube_dl/extractor/spiegel.py
@@ -2,7 +2,6 @@ import re
 import xml.etree.ElementTree
 
 from .common import InfoExtractor
-from ..utils import determine_ext
 
 
 class SpiegelIE(InfoExtractor):
diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 2e497c86e..4bca62ba0 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -4,7 +4,6 @@ import re
 from .subtitles import SubtitlesInfoExtractor
 
 from ..utils import (
-    compat_str,
     RegexNotFoundError,
 )
 
@@ -113,6 +112,6 @@ class TEDIE(SubtitlesInfoExtractor):
                     url = 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/srt' % (video_id, l)
                     sub_lang_list[l] = url
                 return sub_lang_list
-        except RegexNotFoundError as err:
+        except RegexNotFoundError:
             self._downloader.report_warning(u'video doesn\'t have subtitles')
         return {}
diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py
index d4b7603c7..4d9d41db3 100644
--- a/youtube_dl/extractor/tube8.py
+++ b/youtube_dl/extractor/tube8.py
@@ -5,8 +5,6 @@ from .common import InfoExtractor
 from ..utils import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
-    compat_urllib_parse,
-    unescapeHTML,
 )
 from ..aes import (
     aes_decrypt_text
diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 03ad88bed..e3458d2bd 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -5,7 +5,6 @@ from .common import InfoExtractor
 from ..utils import (
     compat_urllib_parse_urlparse,
     compat_urllib_request,
-    compat_urllib_parse,
 )
 
 class XTubeIE(InfoExtractor):
diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index faed7ff7f..c6a9d06f2 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -53,7 +53,7 @@ class ZDFIE(InfoExtractor):
             video_id,
             u'Get stream URL')
 
-        MMS_STREAM = r'href="(?P<video_url>mms://[^"]*)"'
+        #MMS_STREAM = r'href="(?P<video_url>mms://[^"]*)"'
         RTSP_STREAM = r'(?P<video_url>rtsp://[^"]*.mp4)'
 
         mobj = re.search(self._MEDIA_STREAM, media_link)

From 4eb92208a3bf05d0860e1f138380e8b5cae20c14 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 06:34:48 +0100
Subject: [PATCH 109/124] Adapt test to changed .info.json name

---
 test/test_download.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/test/test_download.py b/test/test_download.py
index 16f200809..fe7f7b8cb 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -103,7 +103,7 @@ def generator(test_case):
                 tc_filename = get_tc_filename(tc)
                 try_rm(tc_filename)
                 try_rm(tc_filename + '.part')
-                try_rm(tc_filename + '.info.json')
+                try_rm(os.path.splitext(tc_filename)[0] + '.info.json')
         try_rm_tcs_files()
         try:
             try_num = 1
@@ -130,11 +130,12 @@ def generator(test_case):
                 if not test_case.get('params', {}).get('skip_download', False):
                     self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
                     self.assertTrue(tc_filename in finished_hook_called)
-                self.assertTrue(os.path.exists(tc_filename + '.info.json'))
+                info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json'
+                self.assertTrue(os.path.exists(info_json_fn))
                 if 'md5' in tc:
                     md5_for_file = _file_md5(tc_filename)
                     self.assertEqual(md5_for_file, tc['md5'])
-                with io.open(tc_filename + '.info.json', encoding='utf-8') as infof:
+                with io.open(info_json_fn, encoding='utf-8') as infof:
                     info_dict = json.load(infof)
                 for (info_field, expected) in tc.get('info_dict', {}).items():
                     if isinstance(expected, compat_str) and expected.startswith('md5:'):

From 8f053519846d8758de4e3ad67960e4161407d334 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 06:35:02 +0100
Subject: [PATCH 110/124] [anitube] Minor fixes (#1776)

---
 youtube_dl/extractor/anitube.py | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/anitube.py b/youtube_dl/extractor/anitube.py
index 2954966a6..679027f4e 100644
--- a/youtube_dl/extractor/anitube.py
+++ b/youtube_dl/extractor/anitube.py
@@ -6,11 +6,12 @@ from .common import InfoExtractor
 
 class AnitubeIE(InfoExtractor):
     IE_NAME = u'anitube.se'
-    _VALID_URL = r'http?://(?:www\.)?anitube\.se/video/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?anitube\.se/video/(?P<id>\d+)'
 
     _TEST = {
         u'url': u'http://www.anitube.se/video/36621',
-        u'md5': u'0c4e4f1051bf50f5982f829f7230f539',
+        u'md5': u'59d0eeae28ea0bc8c05e7af429998d43',
+        u'file': u'36621.mp4',
         u'info_dict': {
             u'id': u'36621',
             u'ext': u'mp4',
@@ -23,27 +24,22 @@ class AnitubeIE(InfoExtractor):
         video_id = mobj.group('id')
 
         webpage = self._download_webpage(url, video_id)
-
         key = self._html_search_regex(r'http://www\.anitube\.se/embed/([A-Za-z0-9_-]*)',
                                       webpage, u'key')
 
         webpage_config = self._download_webpage('http://www.anitube.se/nuevo/econfig.php?key=%s' % key,
                                                 key)
-
         config_xml = xml.etree.ElementTree.fromstring(webpage_config.encode('utf-8'))
 
         video_title = config_xml.find('title').text
 
-
         formats = []
-
         video_url = config_xml.find('file')
         if video_url is not None:
             formats.append({
                 'format_id': 'sd',
                 'url': video_url.text,
             })
-
         video_url = config_xml.find('filehd')
         if video_url is not None:
             formats.append({
@@ -54,6 +50,5 @@ class AnitubeIE(InfoExtractor):
         return {
             'id': video_id,
             'title': video_title,
-            'ext': 'mp4',
             'formats': formats
         }

From 0a120f74b2556f3569d285993c1a0f3dc654f07a Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 06:36:00 +0100
Subject: [PATCH 111/124] Credit @diffycat for anitube

---
 youtube_dl/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index e27bd4544..64ebf4d48 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -34,6 +34,7 @@ __authors__  = (
     'Andras Elso',
     'Jelle van der Waa',
     'Marcin Cieślak',
+    'Anton Larionov',
 )
 
 __license__ = 'Public Domain'

From 100959a6d9fee68d2876e112f4be260ade756c40 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 06:47:50 +0100
Subject: [PATCH 112/124] [escapist] Add support for HD format (Closes #1755)

---
 youtube_dl/extractor/escapist.py | 57 +++++++++++++++++++-------------
 1 file changed, 34 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 3aa2da52c..f9681fd43 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -11,11 +11,11 @@ from ..utils import (
 
 
 class EscapistIE(InfoExtractor):
-    _VALID_URL = r'^(https?://)?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<episode>[^/?]+)[/?]?.*$'
+    _VALID_URL = r'^https?://?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<episode>[^/?]+)[/?]?.*$'
     _TEST = {
         u'url': u'http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate',
         u'file': u'6618-Breaking-Down-Baldurs-Gate.mp4',
-        u'md5': u'c6793dbda81388f4264c1ba18684a74d',
+        u'md5': u'ab3a706c681efca53f0a35f1415cf0d1',
         u'info_dict': {
             u"description": u"Baldur's Gate: Original, Modded or Enhanced Edition? I'll break down what you can expect from the new Baldur's Gate: Enhanced Edition.", 
             u"uploader": u"the-escapist-presents", 
@@ -25,50 +25,61 @@ class EscapistIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        if mobj is None:
-            raise ExtractorError(u'Invalid URL: %s' % url)
         showName = mobj.group('showname')
         videoId = mobj.group('episode')
 
         self.report_extraction(videoId)
         webpage = self._download_webpage(url, videoId)
 
-        videoDesc = self._html_search_regex('<meta name="description" content="([^"]*)"',
+        videoDesc = self._html_search_regex(
+            r'<meta name="description" content="([^"]*)"',
             webpage, u'description', fatal=False)
 
         playerUrl = self._og_search_video_url(webpage, name='player url')
 
-        title = self._html_search_regex('<meta name="title" content="([^"]*)"',
+        title = self._html_search_regex(
+            'r<meta name="title" content="([^"]*)"',
             webpage, u'player url').split(' : ')[-1]
 
         configUrl = self._search_regex('config=(.*)$', playerUrl, u'config url')
         configUrl = compat_urllib_parse.unquote(configUrl)
 
-        configJSON = self._download_webpage(configUrl, videoId,
-                                            u'Downloading configuration',
-                                            u'unable to download configuration')
+        formats = []
 
-        # Technically, it's JavaScript, not JSON
-        configJSON = configJSON.replace("'", '"')
+        def _add_format(name, cfgurl):
+            configJSON = self._download_webpage(
+                cfgurl, videoId,
+                u'Downloading ' + name + ' configuration',
+                u'Unable to download ' + name + ' configuration')
 
+            # Technically, it's JavaScript, not JSON
+            configJSON = configJSON.replace("'", '"')
+
+            try:
+                config = json.loads(configJSON)
+            except (ValueError,) as err:
+                raise ExtractorError(u'Invalid JSON in configuration file: ' + compat_str(err))
+            playlist = config['playlist']
+            formats.append({
+                'url': playlist[1]['url'],
+                'format_id': name,
+            })
+
+        _add_format(u'normal', configUrl)
         try:
-            config = json.loads(configJSON)
-        except (ValueError,) as err:
-            raise ExtractorError(u'Invalid JSON in configuration file: ' + compat_str(err))
+            hq_url = (configUrl +
+                      ('&hq=1' if '?' in configUrl else configUrl + '?hq=1'))
+            try:
+                _add_format(u'hq', hq_url)
+            except ExtractorError:
+                pass  # That's fine, we'll just use normal quality
 
-        playlist = config['playlist']
-        videoUrl = playlist[1]['url']
-
-        info = {
+        return {
             'id': videoId,
-            'url': videoUrl,
+            'formats': formats,
             'uploader': showName,
-            'upload_date': None,
             'title': title,
-            'ext': 'mp4',
             'thumbnail': self._og_search_thumbnail(webpage),
             'description': videoDesc,
             'player_url': playerUrl,
         }
-
-        return [info]

From 15ff3c831e6dbd6b99c932561d342b96676a47f0 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 06:55:07 +0100
Subject: [PATCH 113/124] [escapist] Fix syntax error

---
 youtube_dl/extractor/escapist.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index f9681fd43..03ebb7c48 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -66,13 +66,12 @@ class EscapistIE(InfoExtractor):
             })
 
         _add_format(u'normal', configUrl)
+        hq_url = (configUrl +
+                  ('&hq=1' if '?' in configUrl else configUrl + '?hq=1'))
         try:
-            hq_url = (configUrl +
-                      ('&hq=1' if '?' in configUrl else configUrl + '?hq=1'))
-            try:
-                _add_format(u'hq', hq_url)
-            except ExtractorError:
-                pass  # That's fine, we'll just use normal quality
+            _add_format(u'hq', hq_url)
+        except ExtractorError:
+            pass  # That's fine, we'll just use normal quality
 
         return {
             'id': videoId,

From c5edcde21f772c6c51bea79edc4ac48f5e5dd8d3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 06:56:59 +0100
Subject: [PATCH 114/124] [escapist] upper-case URL

---
 youtube_dl/extractor/escapist.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 03ebb7c48..2396f163b 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -35,13 +35,13 @@ class EscapistIE(InfoExtractor):
             r'<meta name="description" content="([^"]*)"',
             webpage, u'description', fatal=False)
 
-        playerUrl = self._og_search_video_url(webpage, name='player url')
+        playerUrl = self._og_search_video_url(webpage, name='player URL')
 
         title = self._html_search_regex(
             'r<meta name="title" content="([^"]*)"',
             webpage, u'player url').split(' : ')[-1]
 
-        configUrl = self._search_regex('config=(.*)$', playerUrl, u'config url')
+        configUrl = self._search_regex('config=(.*)$', playerUrl, u'config URL')
         configUrl = compat_urllib_parse.unquote(configUrl)
 
         formats = []

From da6a795fdb49cb463dcca6ff579e68b2a2243470 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 07:23:23 +0100
Subject: [PATCH 115/124] [escapist] Fix title search

---
 youtube_dl/extractor/escapist.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py
index 2396f163b..b1242f6bc 100644
--- a/youtube_dl/extractor/escapist.py
+++ b/youtube_dl/extractor/escapist.py
@@ -35,11 +35,11 @@ class EscapistIE(InfoExtractor):
             r'<meta name="description" content="([^"]*)"',
             webpage, u'description', fatal=False)
 
-        playerUrl = self._og_search_video_url(webpage, name='player URL')
+        playerUrl = self._og_search_video_url(webpage, name=u'player URL')
 
         title = self._html_search_regex(
-            'r<meta name="title" content="([^"]*)"',
-            webpage, u'player url').split(' : ')[-1]
+            r'<meta name="title" content="([^"]*)"',
+            webpage, u'title').split(' : ')[-1]
 
         configUrl = self._search_regex('config=(.*)$', playerUrl, u'config URL')
         configUrl = compat_urllib_parse.unquote(configUrl)

From d323bcb1522d3163918eef2c458e33537c773287 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 07:25:17 +0100
Subject: [PATCH 116/124] release 2013.11.20

---
 README.md             | 1 +
 youtube_dl/version.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 6632e5865..580b16004 100644
--- a/README.md
+++ b/README.md
@@ -123,6 +123,7 @@ which means you can modify it, redistribute it or use it however you like.
     --get-description          simulate, quiet but print video description
     --get-filename             simulate, quiet but print output filename
     --get-format               simulate, quiet but print output format
+    -j, --dump-json            simulate, quiet but print JSON information
     --newline                  output progress bar as new lines
     --no-progress              do not print progress bar
     --console-title            display progress in console titlebar
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e9ff3f640..ad3fad818 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,2 +1,2 @@
 
-__version__ = '2013.11.19'
+__version__ = '2013.11.20'

From f99e0f1ed62e6210a52f1a4bddb860a5b09623ee Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 07:37:07 +0100
Subject: [PATCH 117/124] Adapt age restriction tests to new .info.json
 filenames

---
 test/test_age_restriction.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_age_restriction.py b/test/test_age_restriction.py
index d500c6edc..506572e9e 100644
--- a/test/test_age_restriction.py
+++ b/test/test_age_restriction.py
@@ -24,7 +24,7 @@ def _download_restricted(url, filename, age):
     }
     ydl = YoutubeDL(params)
     ydl.add_default_info_extractors()
-    json_filename = filename + '.info.json'
+    json_filename = os.path.splitext(filename)[0] + '.info.json'
     try_rm(json_filename)
     ydl.download([url])
     res = os.path.exists(json_filename)

From 9a98a466b34206f61543b3efdbcba40d58e51052 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 07:37:14 +0100
Subject: [PATCH 118/124] [toutv] really skip test

---
 youtube_dl/extractor/toutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py
index 73ea67da9..732083617 100644
--- a/youtube_dl/extractor/toutv.py
+++ b/youtube_dl/extractor/toutv.py
@@ -28,7 +28,7 @@ class TouTvIE(InfoExtractor):
         u'params': {
             u'skip_download': True,  # Requires rtmpdump
         },
-        u'xskip': 'Only available in Canada'
+        u'skip': 'Only available in Canada'
     }
 
     def _real_extract(self, url):

From c4864091a1f867c4ce74f459c31ac85d96c417ca Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 07:43:21 +0100
Subject: [PATCH 119/124] [videopremium] Support new crazy redirect scheme

---
 youtube_dl/extractor/videopremium.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/videopremium.py b/youtube_dl/extractor/videopremium.py
index 65f39b982..4800415bd 100644
--- a/youtube_dl/extractor/videopremium.py
+++ b/youtube_dl/extractor/videopremium.py
@@ -24,12 +24,16 @@ class VideoPremiumIE(InfoExtractor):
         webpage_url = 'http://videopremium.tv/' + video_id
         webpage = self._download_webpage(webpage_url, video_id)
 
-        self.report_extraction(video_id)
+        if re.match(r"^<html><head><script[^>]*>window.location\s*=", webpage):
+            # Download again, we need a cookie
+            webpage = self._download_webpage(
+                webpage_url, video_id,
+                note=u'Downloading webpage again (with cookie)')
 
-        video_title = self._html_search_regex(r'<h2(?:.*?)>\s*(.+?)\s*<',
-            webpage, u'video title')
+        video_title = self._html_search_regex(
+            r'<h2(?:.*?)>\s*(.+?)\s*<', webpage, u'video title')
 
-        return [{
+        return {
             'id':          video_id,
             'url':         "rtmp://e%d.md.iplay.md/play" % random.randint(1, 16),
             'play_path':   "mp4:%s.f4v" % video_id,
@@ -37,4 +41,4 @@ class VideoPremiumIE(InfoExtractor):
             'player_url':  "http://videopremium.tv/uplayer/uppod.swf",
             'ext':         'f4v',
             'title':       video_title,
-        }]
+        }
\ No newline at end of file

From 0ad97bbc056c83d49bc771ac202a9eb8354686de Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 07:45:32 +0100
Subject: [PATCH 120/124] [spankwire] fix check for description

---
 youtube_dl/extractor/spankwire.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 794550c81..995b46444 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -35,11 +35,12 @@ class SpankwireIE(InfoExtractor):
         webpage = self._download_webpage(req, video_id)
 
         video_title = self._html_search_regex(r'<h1>([^<]+)', webpage, u'title')
-        video_uploader = self._html_search_regex(r'by:\s*<a [^>]*>(.+?)</a>', webpage, u'uploader', fatal=False)
-        thumbnail = self._html_search_regex(r'flashvars\.image_url = "([^"]+)', webpage, u'thumbnail', fatal=False)
-        description = self._html_search_regex(r'>\s*Description:</div>\s*<[^>]*>([^<]+)', webpage, u'description', fatal=False)
-        if len(description) == 0:
-            description = None
+        video_uploader = self._html_search_regex(
+            r'by:\s*<a [^>]*>(.+?)</a>', webpage, u'uploader', fatal=False)
+        thumbnail = self._html_search_regex(
+            r'flashvars\.image_url = "([^"]+)', webpage, u'thumbnail', fatal=False)
+        description = self._html_search_regex(
+            r'>\s*Description:</div>\s*<[^>]+>([^<]+)', webpage, u'description', fatal=False)
 
         video_urls = list(map(compat_urllib_parse.unquote , re.findall(r'flashvars\.quality_[0-9]{3}p = "([^"]+)', webpage)))
         if webpage.find('flashvars\.encrypted = "true"') != -1:

From f2e87ef4fab5f6d000a7bb152b535a748866bca3 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 07:46:44 +0100
Subject: [PATCH 121/124] [anitube] Skip test (on travis)

---
 youtube_dl/extractor/anitube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/anitube.py b/youtube_dl/extractor/anitube.py
index 679027f4e..691d5a844 100644
--- a/youtube_dl/extractor/anitube.py
+++ b/youtube_dl/extractor/anitube.py
@@ -17,6 +17,7 @@ class AnitubeIE(InfoExtractor):
             u'ext': u'mp4',
             u'title': u'Recorder to Randoseru 01',
         },
+        u'skip': u'Blocked in the US',
     }
 
     def _real_extract(self, url):

From ca872a4c0b5a926be9dccec6dc43ebb5b8a6abb7 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 09:23:53 +0100
Subject: [PATCH 122/124] [spankwire] Fix description search

---
 youtube_dl/extractor/spankwire.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py
index 995b46444..9e2ad0d99 100644
--- a/youtube_dl/extractor/spankwire.py
+++ b/youtube_dl/extractor/spankwire.py
@@ -40,7 +40,7 @@ class SpankwireIE(InfoExtractor):
         thumbnail = self._html_search_regex(
             r'flashvars\.image_url = "([^"]+)', webpage, u'thumbnail', fatal=False)
         description = self._html_search_regex(
-            r'>\s*Description:</div>\s*<[^>]+>([^<]+)', webpage, u'description', fatal=False)
+            r'<div\s+id="descriptionContent">([^<]+)<', webpage, u'description', fatal=False)
 
         video_urls = list(map(compat_urllib_parse.unquote , re.findall(r'flashvars\.quality_[0-9]{3}p = "([^"]+)', webpage)))
         if webpage.find('flashvars\.encrypted = "true"') != -1:

From 0190eecc00c45b26eb5ef5444181cd996b139f18 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 09:45:22 +0100
Subject: [PATCH 123/124] [nhl] Make NHLVideocenter IE_DESC fit with other
 descriptions

---
 youtube_dl/extractor/nhl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py
index 224f56ac8..458fe4063 100644
--- a/youtube_dl/extractor/nhl.py
+++ b/youtube_dl/extractor/nhl.py
@@ -72,7 +72,7 @@ class NHLIE(NHLBaseInfoExtractor):
 
 class NHLVideocenterIE(NHLBaseInfoExtractor):
     IE_NAME = u'nhl.com:videocenter'
-    IE_DESC = u'Download the first 12 videos from a videocenter category'
+    IE_DESC = u'NHL videocenter category'
     _VALID_URL = r'https?://video\.(?P<team>[^.]*)\.nhl\.com/videocenter/(console\?.*?catid=(?P<catid>[^&]+))?'
 
     @classmethod

From 9e4f50a8aee0d3f030be203686c8f0479b32b793 Mon Sep 17 00:00:00 2001
From: Philipp Hagemeister <phihag@phihag.de>
Date: Wed, 20 Nov 2013 09:59:03 +0100
Subject: [PATCH 124/124] [sztv] skip test, site is undergoing mid-term
 maintenance

---
 youtube_dl/extractor/sztvhu.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sztvhu.py b/youtube_dl/extractor/sztvhu.py
index 81fa35c4b..c9359fafb 100644
--- a/youtube_dl/extractor/sztvhu.py
+++ b/youtube_dl/extractor/sztvhu.py
@@ -15,7 +15,8 @@ class SztvHuIE(InfoExtractor):
         u'info_dict': {
             u"title": u"Cserkészek népszerűsítették a környezettudatos életmódot a Savaria téren",
             u"description": u'A zöld nap játékos ismeretterjesztő programjait a Magyar Cserkész Szövetség szervezte, akik az ország nyolc városában adják át tudásukat az érdeklődőknek. A PET...',
-        }
+        },
+        u'skip': u'Service temporarily disabled as of 2013-11-20'
     }
 
     def _real_extract(self, url):