From d5a9bb4ea97287e633e891ddd1a416619c9aada9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rog=C3=A9rio=20Brito?= Date: Sat, 19 Oct 2013 14:04:44 -0300 Subject: [PATCH 001/425] extractor: youtube: Swap video dimensions to match standard practice. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While working on this, I thought about simplifying things like changing 480x854 to 480p, and that seemed like a good option, until I realized that people (me included) usually link the concept of some number followed by a p with the video being 16:9. So, we would be losing some information and, as we all know, [explicit is better than implicit][*]. [*]: http://www.python.org/dev/peps/pep-0020/ This closes #1446. Signed-off-by: Rogério Brito --- youtube_dl/extractor/youtube.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index fb7c42830..143fac98a 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -253,21 +253,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): '248': 'webm', } _video_dimensions = { - '5': '240x400', + '5': '400x240', '6': '???', '13': '???', - '17': '144x176', - '18': '360x640', - '22': '720x1280', - '34': '360x640', - '35': '480x854', - '36': '240x320', - '37': '1080x1920', - '38': '3072x4096', - '43': '360x640', - '44': '480x854', - '45': '720x1280', - '46': '1080x1920', + '17': '176x144', + '18': '640x360', + '22': '1280x720', + '34': '640x360', + '35': '854x480', + '36': '320x240', + '37': '1920x1080', + '38': '4096x3072', + '43': '640x360', + '44': '854x480', + '45': '1280x720', + '46': '1920x1080', '82': '360p', '83': '480p', '84': '720p', From 5137ebac0b1438d22fe2c007e6172ee65e9311a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Cie=C5=9Blak?= Date: Tue, 5 Nov 2013 23:30:25 +0100 Subject: [PATCH 002/425] [tvp] Telewizja Polska: new extractor for tvp.pl, fixes #1719 Thanks-To: mplonski https://github.com/mplonski/linux/blob/master/tvp-dl.py --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/tvp.py | 60 ++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 youtube_dl/extractor/tvp.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 888a91cce..78f84cea3 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -132,6 +132,7 @@ from .tube8 import Tube8IE from .tudou import TudouIE from .tumblr import TumblrIE from .tutv import TutvIE +from .tvp import TvpIE from .unistra import UnistraIE from .ustream import UstreamIE, UstreamChannelIE from .vbox7 import Vbox7IE diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py new file mode 100644 index 000000000..63fb57bbe --- /dev/null +++ b/youtube_dl/extractor/tvp.py @@ -0,0 +1,60 @@ +# encoding: utf-8 +import re +import json + +from .common import InfoExtractor +from ..utils import ( + determine_ext, + ExtractorError, + RegexNotFoundError, +) + +class TvpIE(InfoExtractor): + IE_NAME = u'tvp.pl' + _VALID_URL = r'https?://www\.tvp\.pl/.*?wideo/(?P\d+)/(?P\d+)' + _INFO_URL = 'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s' + + + _TEST = { + u'url': u'http://www.tvp.pl/warszawa/magazyny/campusnews/wideo/31102013/12878238', + u'file': u'31.10.2013-12878238.wmv', + u'info_dict': { + u'title': u'31.10.2013', + u'description': u'31.10.2013', + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + webpage = self._download_webpage(url, video_id, "Downloading video webpage") + json_params = self._download_webpage(self._INFO_URL % video_id, video_id, "Downloading video metadata") + + try: + params = json.loads(json_params) + except: + raise ExtractorError(u'Invalid JSON') + + self.report_extraction(video_id) + try: + video_url = params['video_url'] + except KeyError: + raise ExtractorError('Missing JSON parameter: ' + sys.exc_info()[1]) + + try: + title = self._og_search_title(webpage) + except RegexNotFoundError: + title = video_id + info = { + 'id': video_id, + 'title': title, + 'ext': 'wmv', + 'url': video_url, + } + try: + info['description'] = self._og_search_description(webpage) + info['thumbnail'] = self._og_search_thumbnail(webpage) + except RegexNotFoundError: + pass + + return info From 4894fe8c5baec8b1f21ac6fdebe08175abc7f094 Mon Sep 17 00:00:00 2001 From: rzhxeo Date: Tue, 29 Oct 2013 01:05:21 +0100 Subject: [PATCH 003/425] Report download progress of rtmpdump --- youtube_dl/FileDownloader.py | 70 ++++++++++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 7 deletions(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 8ecabab1a..664b78662 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -268,6 +268,61 @@ class FileDownloader(object): (clear_line, data_len_str, self.format_seconds(tot_time))) def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url): + def run_rtmpdump(args): + start = time.time() + resume_percent = None + resume_downloaded_data_len = None + proc = subprocess.Popen(args, stderr=subprocess.PIPE) + cursor_in_new_line = True + proc_stderr_closed = False + while not proc_stderr_closed: + # read line from stderr + line = u'' + while True: + char = proc.stderr.read(1) + if not char: + proc_stderr_closed = True + break + if char in [b'\r', b'\n']: + break + line += char.decode('ascii', 'replace') + if not line: + # proc_stderr_closed is True + continue + mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec \(([0-9]{1,2}\.[0-9])%\)', line) + if mobj: + downloaded_data_len = int(float(mobj.group(1))*1024) + percent = float(mobj.group(2)) + if not resume_percent: + resume_percent = percent + resume_downloaded_data_len = downloaded_data_len + eta = self.calc_eta(start, time.time(), 100-resume_percent, percent-resume_percent) + speed = self.calc_speed(start, time.time(), downloaded_data_len-resume_downloaded_data_len) + data_len = None + if percent > 0: + data_len = int(downloaded_data_len * 100 / percent) + data_len_str = u'~'+self.format_bytes(data_len) + self.report_progress(percent, data_len_str, speed, eta) + cursor_in_new_line = False + self._hook_progress({ + 'downloaded_bytes': downloaded_data_len, + 'total_bytes': data_len, + 'tmpfilename': tmpfilename, + 'filename': filename, + 'status': 'downloading', + 'eta': eta, + 'speed': speed, + }) + elif self.params.get('verbose', False): + if not cursor_in_new_line: + self.to_screen(u'') + cursor_in_new_line = True + self.to_screen(u'[rtmpdump] '+line) + proc.wait() + if not cursor_in_new_line: + self.to_screen(u'') + return proc.returncode + self.report_destination(filename) tmpfilename = self.temp_name(filename) test = self.params.get('test', False) @@ -278,12 +333,11 @@ class FileDownloader(object): except (OSError, IOError): self.report_error(u'RTMP download detected but "rtmpdump" could not be run') return False - verbosity_option = '--verbose' if self.params.get('verbose', False) else '--quiet' # Download using rtmpdump. rtmpdump returns exit code 2 when # the connection was interrumpted and resuming appears to be # possible. This is part of rtmpdump's normal usage, AFAIK. - basic_args = ['rtmpdump', verbosity_option, '-r', url, '-o', tmpfilename] + basic_args = ['rtmpdump', '--verbose', '-r', url, '-o', tmpfilename] if player_url is not None: basic_args += ['--swfVfy', player_url] if page_url is not None: @@ -302,23 +356,25 @@ class FileDownloader(object): except ImportError: shell_quote = repr self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args)) - retval = subprocess.call(args) + + retval = run_rtmpdump(args) + while (retval == 2 or retval == 1) and not test: prevsize = os.path.getsize(encodeFilename(tmpfilename)) - self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True) + self.to_screen(u'[rtmpdump] %s bytes' % prevsize) time.sleep(5.0) # This seems to be needed - retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) + retval = run_rtmpdump(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) cursize = os.path.getsize(encodeFilename(tmpfilename)) if prevsize == cursize and retval == 1: break # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those if prevsize == cursize and retval == 2 and cursize > 1024: - self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.') + self.to_screen(u'[rtmpdump] Could not download the whole video. This can happen for some advertisements.') retval = 0 break if retval == 0 or (test and retval == 2): fsize = os.path.getsize(encodeFilename(tmpfilename)) - self.to_screen(u'\r[rtmpdump] %s bytes' % fsize) + self.to_screen(u'[rtmpdump] %s bytes' % fsize) self.try_rename(tmpfilename, filename) self._hook_progress({ 'downloaded_bytes': fsize, From c8434e83163fc90007eb5b501ea0e827f8b5e127 Mon Sep 17 00:00:00 2001 From: rzhxeo Date: Mon, 4 Nov 2013 03:08:17 +0100 Subject: [PATCH 004/425] Add support for crunchyroll.com --- youtube_dl/aes.py | 144 ++++++++++++++++++++--- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/crunchyroll.py | 171 ++++++++++++++++++++++++++++ 3 files changed, 298 insertions(+), 18 deletions(-) create mode 100644 youtube_dl/extractor/crunchyroll.py diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py index 9a0c93fa6..e9c5e2152 100644 --- a/youtube_dl/aes.py +++ b/youtube_dl/aes.py @@ -1,4 +1,4 @@ -__all__ = ['aes_encrypt', 'key_expansion', 'aes_ctr_decrypt', 'aes_decrypt_text'] +__all__ = ['aes_encrypt', 'key_expansion', 'aes_ctr_decrypt', 'aes_cbc_decrypt', 'aes_decrypt_text'] import base64 from math import ceil @@ -32,6 +32,31 @@ def aes_ctr_decrypt(data, key, counter): return decrypted_data +def aes_cbc_decrypt(data, key, iv): + """ + Decrypt with aes in CBC mode + + @param {int[]} data cipher + @param {int[]} key 16/24/32-Byte cipher key + @param {int[]} iv 16-Byte IV + @returns {int[]} decrypted data + """ + expanded_key = key_expansion(key) + block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES)) + + decrypted_data=[] + previous_cipher_block = iv + for i in range(block_count): + block = data[i*BLOCK_SIZE_BYTES : (i+1)*BLOCK_SIZE_BYTES] + block += [0]*(BLOCK_SIZE_BYTES - len(block)) + + decrypted_block = aes_decrypt(block, expanded_key) + decrypted_data += xor(decrypted_block, previous_cipher_block) + previous_cipher_block = block + decrypted_data = decrypted_data[:len(data)] + + return decrypted_data + def key_expansion(data): """ Generate key schedule @@ -75,7 +100,7 @@ def aes_encrypt(data, expanded_key): @returns {int[]} 16-Byte cipher """ rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1 - + data = xor(data, expanded_key[:BLOCK_SIZE_BYTES]) for i in range(1, rounds+1): data = sub_bytes(data) @@ -83,6 +108,26 @@ def aes_encrypt(data, expanded_key): if i != rounds: data = mix_columns(data) data = xor(data, expanded_key[i*BLOCK_SIZE_BYTES : (i+1)*BLOCK_SIZE_BYTES]) + + return data + +def aes_decrypt(data, expanded_key): + """ + Decrypt one block with aes + + @param {int[]} data 16-Byte cipher + @param {int[]} expanded_key 176/208/240-Byte expanded key + @returns {int[]} 16-Byte state + """ + rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1 + + for i in range(rounds, 0, -1): + data = xor(data, expanded_key[i*BLOCK_SIZE_BYTES : (i+1)*BLOCK_SIZE_BYTES]) + if i != rounds: + data = mix_columns_inv(data) + data = shift_rows_inv(data) + data = sub_bytes_inv(data) + data = xor(data, expanded_key[:BLOCK_SIZE_BYTES]) return data @@ -139,14 +184,69 @@ SBOX = (0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5, 0x30, 0x01, 0x67, 0x2B, 0x70, 0x3E, 0xB5, 0x66, 0x48, 0x03, 0xF6, 0x0E, 0x61, 0x35, 0x57, 0xB9, 0x86, 0xC1, 0x1D, 0x9E, 0xE1, 0xF8, 0x98, 0x11, 0x69, 0xD9, 0x8E, 0x94, 0x9B, 0x1E, 0x87, 0xE9, 0xCE, 0x55, 0x28, 0xDF, 0x8C, 0xA1, 0x89, 0x0D, 0xBF, 0xE6, 0x42, 0x68, 0x41, 0x99, 0x2D, 0x0F, 0xB0, 0x54, 0xBB, 0x16) -MIX_COLUMN_MATRIX = ((2,3,1,1), - (1,2,3,1), - (1,1,2,3), - (3,1,1,2)) +SBOX_INV = (0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, + 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, + 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, + 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, + 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, + 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, + 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, + 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, + 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, + 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, + 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, + 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, + 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, + 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, + 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, + 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d) +MIX_COLUMN_MATRIX = ((0x2,0x3,0x1,0x1), + (0x1,0x2,0x3,0x1), + (0x1,0x1,0x2,0x3), + (0x3,0x1,0x1,0x2)) +MIX_COLUMN_MATRIX_INV = ((0xE,0xB,0xD,0x9), + (0x9,0xE,0xB,0xD), + (0xD,0x9,0xE,0xB), + (0xB,0xD,0x9,0xE)) +RIJNDAEL_EXP_TABLE = (0x01, 0x03, 0x05, 0x0F, 0x11, 0x33, 0x55, 0xFF, 0x1A, 0x2E, 0x72, 0x96, 0xA1, 0xF8, 0x13, 0x35, + 0x5F, 0xE1, 0x38, 0x48, 0xD8, 0x73, 0x95, 0xA4, 0xF7, 0x02, 0x06, 0x0A, 0x1E, 0x22, 0x66, 0xAA, + 0xE5, 0x34, 0x5C, 0xE4, 0x37, 0x59, 0xEB, 0x26, 0x6A, 0xBE, 0xD9, 0x70, 0x90, 0xAB, 0xE6, 0x31, + 0x53, 0xF5, 0x04, 0x0C, 0x14, 0x3C, 0x44, 0xCC, 0x4F, 0xD1, 0x68, 0xB8, 0xD3, 0x6E, 0xB2, 0xCD, + 0x4C, 0xD4, 0x67, 0xA9, 0xE0, 0x3B, 0x4D, 0xD7, 0x62, 0xA6, 0xF1, 0x08, 0x18, 0x28, 0x78, 0x88, + 0x83, 0x9E, 0xB9, 0xD0, 0x6B, 0xBD, 0xDC, 0x7F, 0x81, 0x98, 0xB3, 0xCE, 0x49, 0xDB, 0x76, 0x9A, + 0xB5, 0xC4, 0x57, 0xF9, 0x10, 0x30, 0x50, 0xF0, 0x0B, 0x1D, 0x27, 0x69, 0xBB, 0xD6, 0x61, 0xA3, + 0xFE, 0x19, 0x2B, 0x7D, 0x87, 0x92, 0xAD, 0xEC, 0x2F, 0x71, 0x93, 0xAE, 0xE9, 0x20, 0x60, 0xA0, + 0xFB, 0x16, 0x3A, 0x4E, 0xD2, 0x6D, 0xB7, 0xC2, 0x5D, 0xE7, 0x32, 0x56, 0xFA, 0x15, 0x3F, 0x41, + 0xC3, 0x5E, 0xE2, 0x3D, 0x47, 0xC9, 0x40, 0xC0, 0x5B, 0xED, 0x2C, 0x74, 0x9C, 0xBF, 0xDA, 0x75, + 0x9F, 0xBA, 0xD5, 0x64, 0xAC, 0xEF, 0x2A, 0x7E, 0x82, 0x9D, 0xBC, 0xDF, 0x7A, 0x8E, 0x89, 0x80, + 0x9B, 0xB6, 0xC1, 0x58, 0xE8, 0x23, 0x65, 0xAF, 0xEA, 0x25, 0x6F, 0xB1, 0xC8, 0x43, 0xC5, 0x54, + 0xFC, 0x1F, 0x21, 0x63, 0xA5, 0xF4, 0x07, 0x09, 0x1B, 0x2D, 0x77, 0x99, 0xB0, 0xCB, 0x46, 0xCA, + 0x45, 0xCF, 0x4A, 0xDE, 0x79, 0x8B, 0x86, 0x91, 0xA8, 0xE3, 0x3E, 0x42, 0xC6, 0x51, 0xF3, 0x0E, + 0x12, 0x36, 0x5A, 0xEE, 0x29, 0x7B, 0x8D, 0x8C, 0x8F, 0x8A, 0x85, 0x94, 0xA7, 0xF2, 0x0D, 0x17, + 0x39, 0x4B, 0xDD, 0x7C, 0x84, 0x97, 0xA2, 0xFD, 0x1C, 0x24, 0x6C, 0xB4, 0xC7, 0x52, 0xF6, 0x01) +RIJNDAEL_LOG_TABLE = (0x00, 0x00, 0x19, 0x01, 0x32, 0x02, 0x1a, 0xc6, 0x4b, 0xc7, 0x1b, 0x68, 0x33, 0xee, 0xdf, 0x03, + 0x64, 0x04, 0xe0, 0x0e, 0x34, 0x8d, 0x81, 0xef, 0x4c, 0x71, 0x08, 0xc8, 0xf8, 0x69, 0x1c, 0xc1, + 0x7d, 0xc2, 0x1d, 0xb5, 0xf9, 0xb9, 0x27, 0x6a, 0x4d, 0xe4, 0xa6, 0x72, 0x9a, 0xc9, 0x09, 0x78, + 0x65, 0x2f, 0x8a, 0x05, 0x21, 0x0f, 0xe1, 0x24, 0x12, 0xf0, 0x82, 0x45, 0x35, 0x93, 0xda, 0x8e, + 0x96, 0x8f, 0xdb, 0xbd, 0x36, 0xd0, 0xce, 0x94, 0x13, 0x5c, 0xd2, 0xf1, 0x40, 0x46, 0x83, 0x38, + 0x66, 0xdd, 0xfd, 0x30, 0xbf, 0x06, 0x8b, 0x62, 0xb3, 0x25, 0xe2, 0x98, 0x22, 0x88, 0x91, 0x10, + 0x7e, 0x6e, 0x48, 0xc3, 0xa3, 0xb6, 0x1e, 0x42, 0x3a, 0x6b, 0x28, 0x54, 0xfa, 0x85, 0x3d, 0xba, + 0x2b, 0x79, 0x0a, 0x15, 0x9b, 0x9f, 0x5e, 0xca, 0x4e, 0xd4, 0xac, 0xe5, 0xf3, 0x73, 0xa7, 0x57, + 0xaf, 0x58, 0xa8, 0x50, 0xf4, 0xea, 0xd6, 0x74, 0x4f, 0xae, 0xe9, 0xd5, 0xe7, 0xe6, 0xad, 0xe8, + 0x2c, 0xd7, 0x75, 0x7a, 0xeb, 0x16, 0x0b, 0xf5, 0x59, 0xcb, 0x5f, 0xb0, 0x9c, 0xa9, 0x51, 0xa0, + 0x7f, 0x0c, 0xf6, 0x6f, 0x17, 0xc4, 0x49, 0xec, 0xd8, 0x43, 0x1f, 0x2d, 0xa4, 0x76, 0x7b, 0xb7, + 0xcc, 0xbb, 0x3e, 0x5a, 0xfb, 0x60, 0xb1, 0x86, 0x3b, 0x52, 0xa1, 0x6c, 0xaa, 0x55, 0x29, 0x9d, + 0x97, 0xb2, 0x87, 0x90, 0x61, 0xbe, 0xdc, 0xfc, 0xbc, 0x95, 0xcf, 0xcd, 0x37, 0x3f, 0x5b, 0xd1, + 0x53, 0x39, 0x84, 0x3c, 0x41, 0xa2, 0x6d, 0x47, 0x14, 0x2a, 0x9e, 0x5d, 0x56, 0xf2, 0xd3, 0xab, + 0x44, 0x11, 0x92, 0xd9, 0x23, 0x20, 0x2e, 0x89, 0xb4, 0x7c, 0xb8, 0x26, 0x77, 0x99, 0xe3, 0xa5, + 0x67, 0x4a, 0xed, 0xde, 0xc5, 0x31, 0xfe, 0x18, 0x0d, 0x63, 0x8c, 0x80, 0xc0, 0xf7, 0x70, 0x07) def sub_bytes(data): return [SBOX[x] for x in data] +def sub_bytes_inv(data): + return [SBOX_INV[x] for x in data] + def rotate(data): return data[1:] + [data[0]] @@ -160,30 +260,31 @@ def key_schedule_core(data, rcon_iteration): def xor(data1, data2): return [x^y for x, y in zip(data1, data2)] -def mix_column(data): +def rijndael_mul(a, b): + if(a==0 or b==0): + return 0 + return RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[a] + RIJNDAEL_LOG_TABLE[b]) % 0xFF] + +def mix_column(data, matrix): data_mixed = [] for row in range(4): mixed = 0 for column in range(4): - addend = data[column] - if MIX_COLUMN_MATRIX[row][column] in (2,3): - addend <<= 1 - if addend > 0xff: - addend &= 0xff - addend ^= 0x1b - if MIX_COLUMN_MATRIX[row][column] == 3: - addend ^= data[column] - mixed ^= addend & 0xff + # xor is (+) and (-) + mixed ^= rijndael_mul(data[column], matrix[row][column]) data_mixed.append(mixed) return data_mixed -def mix_columns(data): +def mix_columns(data, matrix=MIX_COLUMN_MATRIX): data_mixed = [] for i in range(4): column = data[i*4 : (i+1)*4] - data_mixed += mix_column(column) + data_mixed += mix_column(column, matrix) return data_mixed +def mix_columns_inv(data): + return mix_columns(data, MIX_COLUMN_MATRIX_INV) + def shift_rows(data): data_shifted = [] for column in range(4): @@ -191,6 +292,13 @@ def shift_rows(data): data_shifted.append( data[((column + row) & 0b11) * 4 + row] ) return data_shifted +def shift_rows_inv(data): + data_shifted = [] + for column in range(4): + for row in range(4): + data_shifted.append( data[((column - row) & 0b11) * 4 + row] ) + return data_shifted + def inc(data): data = data[:] # copy for i in range(len(data)-1,-1,-1): diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index bcf1cce7f..a61e17ea1 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -23,6 +23,7 @@ from .collegehumor import CollegeHumorIE from .comedycentral import ComedyCentralIE from .condenast import CondeNastIE from .criterion import CriterionIE +from .crunchyroll import CrunchyrollIE from .cspan import CSpanIE from .dailymotion import ( DailymotionIE, diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py new file mode 100644 index 000000000..4bd366079 --- /dev/null +++ b/youtube_dl/extractor/crunchyroll.py @@ -0,0 +1,171 @@ +# encoding: utf-8 +import re, base64, zlib +from hashlib import sha1 +from math import pow, sqrt, floor +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + compat_urllib_parse, + compat_urllib_request, + bytes_to_intlist, + intlist_to_bytes, + unified_strdate, + clean_html, +) +from ..aes import ( + aes_cbc_decrypt, + inc, +) + +class CrunchyrollIE(InfoExtractor): + _VALID_URL = r'(?:https?://)?(?:www\.)?(?Pcrunchyroll\.com/[^/]*/[^/?&]*?(?P[0-9]+))(?:[/?&]|$)' + _TESTS = [{ + u'url': u'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513', + u'file': u'645513.flv', + #u'md5': u'b1639fd6ddfaa43788c85f6d1dddd412', + u'info_dict': { + u'title': u'Wanna be the Strongest in the World – Episode 1 – An Idol-Wrestler is Born!', + u'description': u'md5:2d17137920c64f2f49981a7797d275ef', + u'thumbnail': u'http://img1.ak.crunchyroll.com/i/spire1-tmb/20c6b5e10f1a47b10516877d3c039cae1380951166_full.jpg', + u'uploader': u'Yomiuri Telecasting Corporation (YTV)', + u'upload_date': u'20131013', + }, + u'params': { + # rtmp + u'skip_download': True, + }, + }] + + _FORMAT_IDS = { + u'360': (u'60', u'106'), + u'480': (u'61', u'106'), + u'720': (u'62', u'106'), + u'1080': (u'80', u'108'), + } + + def _decrypt_subtitles(self, data, iv, id): + data = bytes_to_intlist(data) + iv = bytes_to_intlist(iv) + id = int(id) + + def obfuscate_key_aux(count, modulo, start): + output = list(start) + for _ in range(count): + output.append(output[-1] + output[-2]) + # cut off start values + output = output[2:] + output = list(map(lambda x: x % modulo + 33, output)) + return output + + def obfuscate_key(key): + num1 = int(floor(pow(2, 25) * sqrt(6.9))) + num2 = (num1 ^ key) << 5 + num3 = key ^ num1 + num4 = num3 ^ (num3 >> 3) ^ num2 + prefix = intlist_to_bytes(obfuscate_key_aux(20, 97, (1, 2))) + shaHash = bytes_to_intlist(sha1(prefix + str(num4).encode(u'ascii')).digest()) + # Extend 160 Bit hash to 256 Bit + return shaHash + [0] * 12 + + key = obfuscate_key(id) + class Counter: + __value = iv + def next_value(self): + temp = self.__value + self.__value = inc(self.__value) + return temp + decrypted_data = intlist_to_bytes(aes_cbc_decrypt(data, key, iv)) + return zlib.decompress(decrypted_data) + + def _convert_subtitles_to_srt(self, subtitles): + i=1 + output = u'' + for start, end, text in re.findall(r']*?start="([^"]+)" [^>]*?end="([^"]+)" [^>]*?text="([^"]+)"[^>]*?>', subtitles): + start = start.replace(u'.', u',') + end = end.replace(u'.', u',') + text = clean_html(text) + text = text.replace(u'\\N', u'\n') + if not text: + continue + output += u'%d\n%s --> %s\n%s\n\n' % (i, start, end, text) + i+=1 + return output + + def _real_extract(self,url): + mobj = re.match(self._VALID_URL, url) + + webpage_url = u'http://www.' + mobj.group('url') + video_id = mobj.group(u'video_id') + webpage = self._download_webpage(webpage_url, video_id) + note_m = self._html_search_regex(r'
(.+?)
', webpage, u'trailer-notice', default=u'') + if note_m: + raise ExtractorError(note_m) + + video_title = self._html_search_regex(r']*>(.+?)', webpage, u'video_title', flags=re.DOTALL) + video_title = re.sub(r' {5} *–? *', u' – ', video_title) + video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, u'video_description', default=u'') + if not video_description: + video_description = None + video_upload_date = self._html_search_regex(r'
Availability for free users:(.+?)
', webpage, u'video_upload_date', fatal=False, flags=re.DOTALL) + if video_upload_date: + video_upload_date = unified_strdate(video_upload_date) + video_uploader = self._html_search_regex(r'
\s*Publisher:(.+?)
', webpage, u'video_uploader', fatal=False, flags=re.DOTALL) + + playerdata_url = compat_urllib_parse.unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, u'playerdata_url')) + playerdata_req = compat_urllib_request.Request(playerdata_url) + playerdata_req.data = compat_urllib_parse.urlencode({u'current_page': webpage_url}) + playerdata_req.add_header(u'Content-Type', u'application/x-www-form-urlencoded') + playerdata = self._download_webpage(playerdata_req, video_id, note=u'Downloading media info') + + stream_id = self._search_regex(r'([^<]+)', playerdata, u'stream_id') + video_thumbnail = self._search_regex(r'([^<]+)', playerdata, u'thumbnail', fatal=False) + + formats = [] + for fmt in re.findall(r'\?p([0-9]{3,4})=1', webpage): + stream_quality, stream_format = self._FORMAT_IDS[fmt] + video_format = fmt+u'p' + streamdata_req = compat_urllib_request.Request(u'http://www.crunchyroll.com/xml/') + # urlencode doesn't work! + streamdata_req.data = u'req=RpcApiVideoEncode%5FGetStreamInfo&video%5Fencode%5Fquality='+stream_quality+u'&media%5Fid='+stream_id+u'&video%5Fformat='+stream_format + streamdata_req.add_header(u'Content-Type', u'application/x-www-form-urlencoded') + streamdata_req.add_header(u'Content-Length', str(len(streamdata_req.data))) + streamdata = self._download_webpage(streamdata_req, video_id, note=u'Downloading media info for '+video_format) + video_url = self._search_regex(r'([^<]+)', streamdata, u'video_url') + video_play_path = self._search_regex(r'([^<]+)', streamdata, u'video_play_path') + formats.append({ + u'url': video_url, + u'play_path': video_play_path, + u'ext': 'flv', + u'format': video_format, + u'format_id': video_format, + }) + + subtitles = {} + for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage): + sub_page = self._download_webpage(u'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id='+sub_id,\ + video_id, note=u'Downloading subtitles for '+sub_name) + id = self._search_regex(r'id=\'([0-9]+)', sub_page, u'subtitle_id', fatal=False) + iv = self._search_regex(r'([^<]+)', sub_page, u'subtitle_iv', fatal=False) + data = self._search_regex(r'([^<]+)', sub_page, u'subtitle_data', fatal=False) + if not id or not iv or not data: + continue + id = int(id) + iv = base64.b64decode(iv) + data = base64.b64decode(data) + + subtitle = self._decrypt_subtitles(data, iv, id).decode(u'utf-8') + lang_code = self._search_regex(r'lang_code=\'([^\']+)', subtitle, u'subtitle_lang_code', fatal=False) + if not lang_code: + continue + subtitles[lang_code] = self._convert_subtitles_to_srt(subtitle) + + return { + u'id': video_id, + u'title': video_title, + u'description': video_description, + u'thumbnail': video_thumbnail, + u'uploader': video_uploader, + u'upload_date': video_upload_date, + u'subtitles': subtitles, + u'formats': formats, + } From eb0a83986642cf660820b168bd83c8770e3e5ce6 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Tue, 12 Nov 2013 10:36:23 +0100 Subject: [PATCH 005/425] [common] Simplify og_search_property --- youtube_dl/extractor/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index fb2d50a09..9c20d30b4 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -322,9 +322,9 @@ class InfoExtractor(object): if name is None: name = 'OpenGraph %s' % prop escaped = self._search_regex(self._og_regex(prop), html, name, flags=re.DOTALL, **kargs) - if not escaped is None: - return unescapeHTML(escaped) - return None + if escaped is None: + return None + return unescapeHTML(escaped) def _og_search_thumbnail(self, html, **kargs): return self._og_search_property('image', html, u'thumbnail url', fatal=False, **kargs) From 72b18c5d34d7bf02f83d335b886921ff25c501f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 12 Nov 2013 20:38:13 +0100 Subject: [PATCH 006/425] FFmpegMetadataPP: don't enclose the values with " (fixes #1756) --- youtube_dl/PostProcessor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/PostProcessor.py b/youtube_dl/PostProcessor.py index 13b56ede5..69aedf87a 100644 --- a/youtube_dl/PostProcessor.py +++ b/youtube_dl/PostProcessor.py @@ -501,7 +501,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor): options = ['-c', 'copy'] for (name, value) in metadata.items(): - options.extend(['-metadata', '%s="%s"' % (name, value)]) + options.extend(['-metadata', '%s=%s' % (name, value)]) options.extend(['-f', ext]) self._downloader.to_screen(u'[ffmpeg] Adding metadata to \'%s\'' % filename) From 8b8cbd8f6d3b525dcee3bc3df98c1ad0f093231d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 12 Nov 2013 20:50:52 +0100 Subject: [PATCH 007/425] [vine] Fix uploader extraction --- youtube_dl/extractor/vine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/vine.py b/youtube_dl/extractor/vine.py index c4ec1f06f..651ba317d 100644 --- a/youtube_dl/extractor/vine.py +++ b/youtube_dl/extractor/vine.py @@ -27,7 +27,7 @@ class VineIE(InfoExtractor): video_url = self._html_search_regex(r'.*?

(.+?)

', + uploader = self._html_search_regex(r'

(.*?)

', webpage, u'uploader', fatal=False, flags=re.DOTALL) return [{ From 0bd59f3723a22914a538527921815436c6d2f2d1 Mon Sep 17 00:00:00 2001 From: migbac Date: Tue, 12 Nov 2013 23:32:03 +0100 Subject: [PATCH 008/425] Add support for d8.tv --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/d8.py | 52 ++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 youtube_dl/extractor/d8.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index f9caca4ef..d7db840dd 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -25,6 +25,7 @@ from .comedycentral import ComedyCentralIE from .condenast import CondeNastIE from .criterion import CriterionIE from .cspan import CSpanIE +from .d8 import D8IE from .dailymotion import ( DailymotionIE, DailymotionPlaylistIE, diff --git a/youtube_dl/extractor/d8.py b/youtube_dl/extractor/d8.py new file mode 100644 index 000000000..5ce483b16 --- /dev/null +++ b/youtube_dl/extractor/d8.py @@ -0,0 +1,52 @@ +# encoding: utf-8 +import re +import xml.etree.ElementTree + +from .common import InfoExtractor +from ..utils import unified_strdate + +class D8IE(InfoExtractor): + _VALID_URL = r'https?://www\.d8\.tv/.*?/(?P.*)' + _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/d8/%s' + IE_NAME = u'd8.tv' + + _TEST = { + u'url': u'http://www.d8.tv/d8-docs-mags/pid6589-d8-campagne-intime.html', + u'file': u'966289.flv', + u'info_dict': { + u'title': u'Campagne intime - Documentaire exceptionnel', + u'description': u'md5:d2643b799fb190846ae09c61e59a859f', + u'upload_date': u'20131108', + }, + u'params': { + u'skip_download': True, + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + webpage = self._download_webpage(url, mobj.group('path')) + video_id = self._search_regex(r'videoId = "(\d+)";', webpage, u'video id') + info_url = self._VIDEO_INFO_TEMPLATE % video_id + info_page = self._download_webpage(info_url,video_id, + u'Downloading video info') + + self.report_extraction(video_id) + doc = xml.etree.ElementTree.fromstring(info_page.encode('utf-8')) + video_info = [video for video in doc if video.find('ID').text == video_id][0] + infos = video_info.find('INFOS') + media = video_info.find('MEDIA') + formats = [media.find('VIDEOS/%s' % format) + for format in ['BAS_DEBIT', 'HAUT_DEBIT', 'HD']] + video_url = [format.text for format in formats if format is not None][-1] + + return {'id': video_id, + 'title': u'%s - %s' % (infos.find('TITRAGE/TITRE').text, + infos.find('TITRAGE/SOUS_TITRE').text), + 'url': video_url, + 'ext': 'flv', + 'upload_date': unified_strdate(infos.find('PUBLICATION/DATE').text), + 'thumbnail': media.find('IMAGES/GRAND').text, + 'description': infos.find('DESCRIPTION').text, + 'view_count': int(infos.find('NB_VUES').text), + } From eb9b5bffef9d247bba4f2e8d387ddfbc47ece77b Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Wed, 13 Nov 2013 10:30:41 +0100 Subject: [PATCH 009/425] Add extractor for gamekings.tv --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/gamekings.py | 39 +++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 youtube_dl/extractor/gamekings.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index f9caca4ef..67a0ad5cb 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -57,6 +57,7 @@ from .francetv import ( ) from .freesound import FreesoundIE from .funnyordie import FunnyOrDieIE +from .gamekings import GamekingsIE from .gamespot import GameSpotIE from .gametrailers import GametrailersIE from .generic import GenericIE diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py new file mode 100644 index 000000000..eca71ab05 --- /dev/null +++ b/youtube_dl/extractor/gamekings.py @@ -0,0 +1,39 @@ +import re + +from .common import InfoExtractor +from ..utils import ( + determine_ext, +) + + +class GamekingsIE(InfoExtractor): + _VALID_URL = r'http?://www\.gamekings\.tv/videos/(?P[0-9a-z\-])' + _TEST = { + u"url": u"http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/", + u'file': u'20130811_PhoenixWright.mp4', + u'md5': u'8d42d15381e2dfa81dee86c7956d35ff', + u'info_dict': { + u"title": u"Phoenix Wright: Ace Attorney – Dual Destinies Review", + u"description": u"Melle en Steven hebben voor de review een week in de rechtbank doorbracht met Phoenix Wright: Ace Attorney - Dual Destinies.", + } + } + + def _real_extract(self, url): + + mobj = re.match(self._VALID_URL, url) + name = mobj.group('name') + webpage = self._download_webpage(url, name) + gamekings_url = self._og_search_video_url(webpage) + + video = re.search(r'[0-9]+',gamekings_url) + video_id = video.group(0) + + # Todo: add medium format + gamekings_url = gamekings_url.replace(video_id,'large/' + video_id) + + return {'id': video_id, + 'ext': 'mp4', + 'url': gamekings_url, + 'title': self._og_search_title(webpage), + 'description': self._og_search_description(webpage), + } From 384b98cd8f90ef2ac25c1f4f20ba9385adabaca8 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 13 Nov 2013 10:50:53 +0100 Subject: [PATCH 010/425] [gamekings] Minor fixes (#1759) --- youtube_dl/extractor/gamekings.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py index eca71ab05..4b4259447 100644 --- a/youtube_dl/extractor/gamekings.py +++ b/youtube_dl/extractor/gamekings.py @@ -7,13 +7,13 @@ from ..utils import ( class GamekingsIE(InfoExtractor): - _VALID_URL = r'http?://www\.gamekings\.tv/videos/(?P[0-9a-z\-])' + _VALID_URL = r'http?://www\.gamekings\.tv/videos/(?P[0-9a-z\-]+)' _TEST = { u"url": u"http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/", - u'file': u'20130811_PhoenixWright.mp4', - u'md5': u'8d42d15381e2dfa81dee86c7956d35ff', + u'file': u'20130811.mp4', + u'md5': u'17f6088f7d0149ff2b46f2714bdb1954', u'info_dict': { - u"title": u"Phoenix Wright: Ace Attorney – Dual Destinies Review", + u"title": u"Phoenix Wright: Ace Attorney \u2013 Dual Destinies Review", u"description": u"Melle en Steven hebben voor de review een week in de rechtbank doorbracht met Phoenix Wright: Ace Attorney - Dual Destinies.", } } @@ -23,17 +23,18 @@ class GamekingsIE(InfoExtractor): mobj = re.match(self._VALID_URL, url) name = mobj.group('name') webpage = self._download_webpage(url, name) - gamekings_url = self._og_search_video_url(webpage) + video_url = self._og_search_video_url(webpage) - video = re.search(r'[0-9]+',gamekings_url) + video = re.search(r'[0-9]+', video_url) video_id = video.group(0) - # Todo: add medium format - gamekings_url = gamekings_url.replace(video_id,'large/' + video_id) + # Todo: add medium format + video_url = video_url.replace(video_id, 'large/' + video_id) - return {'id': video_id, - 'ext': 'mp4', - 'url': gamekings_url, - 'title': self._og_search_title(webpage), - 'description': self._og_search_description(webpage), - } + return { + 'id': video_id, + 'ext': 'mp4', + 'url': video_url, + 'title': self._og_search_title(webpage), + 'description': self._og_search_description(webpage), + } From b5bdc2699a5fead926114f6db6f1178181d75c58 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 13 Nov 2013 10:52:22 +0100 Subject: [PATCH 011/425] Credit @jelly for gamekings extractor (#1759) --- youtube_dl/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 1f1db9f67..254fcd39c 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -32,6 +32,7 @@ __authors__ = ( 'Ismael Mejía', 'Steffan \'Ruirize\' James', 'Andras Elso', + 'Jelle van der Waa', ) __license__ = 'Public Domain' From c3a3028f9f7fa79ee7357b65252ff2c9a062bdc8 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 13 Nov 2013 11:06:53 +0100 Subject: [PATCH 012/425] [tvp] Minor improvements (#1730) --- youtube_dl/extractor/tvp.py | 47 +++++++++++-------------------------- 1 file changed, 14 insertions(+), 33 deletions(-) diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py index 63fb57bbe..32e0f5037 100644 --- a/youtube_dl/extractor/tvp.py +++ b/youtube_dl/extractor/tvp.py @@ -1,23 +1,17 @@ -# encoding: utf-8 -import re import json +import re from .common import InfoExtractor -from ..utils import ( - determine_ext, - ExtractorError, - RegexNotFoundError, -) + class TvpIE(InfoExtractor): IE_NAME = u'tvp.pl' _VALID_URL = r'https?://www\.tvp\.pl/.*?wideo/(?P\d+)/(?P\d+)' - _INFO_URL = 'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s' - _TEST = { u'url': u'http://www.tvp.pl/warszawa/magazyny/campusnews/wideo/31102013/12878238', - u'file': u'31.10.2013-12878238.wmv', + u'md5': u'148408967a6a468953c0a75cbdaf0d7a', + u'file': u'12878238.wmv', u'info_dict': { u'title': u'31.10.2013', u'description': u'31.10.2013', @@ -27,34 +21,21 @@ class TvpIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') - webpage = self._download_webpage(url, video_id, "Downloading video webpage") - json_params = self._download_webpage(self._INFO_URL % video_id, video_id, "Downloading video metadata") - - try: - params = json.loads(json_params) - except: - raise ExtractorError(u'Invalid JSON') + webpage = self._download_webpage(url, video_id) + json_url = 'http://www.tvp.pl/pub/stat/videofileinfo?video_id=%s' % video_id + json_params = self._download_webpage( + json_url, video_id, u"Downloading video metadata") + params = json.loads(json_params) self.report_extraction(video_id) - try: - video_url = params['video_url'] - except KeyError: - raise ExtractorError('Missing JSON parameter: ' + sys.exc_info()[1]) + video_url = params['video_url'] - try: - title = self._og_search_title(webpage) - except RegexNotFoundError: - title = video_id - info = { + title = self._og_search_title(webpage, fatal=True) + return { 'id': video_id, 'title': title, 'ext': 'wmv', 'url': video_url, + 'description': self._og_search_description(webpage), + 'thumbnail': self._og_search_thumbnail(webpage), } - try: - info['description'] = self._og_search_description(webpage) - info['thumbnail'] = self._og_search_thumbnail(webpage) - except RegexNotFoundError: - pass - - return info From d37936386f0217a411efe96483b97e590b7eceb3 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 13 Nov 2013 11:08:07 +0100 Subject: [PATCH 013/425] Credit @saper for tvp IE (#1730) --- youtube_dl/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 254fcd39c..4dee487ab 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -33,6 +33,7 @@ __authors__ = ( 'Steffan \'Ruirize\' James', 'Andras Elso', 'Jelle van der Waa', + 'Marcin Cieślak', ) __license__ = 'Public Domain' From 80b9bbce8687f800b79edb36edf8c193dcf26a78 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 13 Nov 2013 11:09:04 +0100 Subject: [PATCH 014/425] release 2013.11.13 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 338e7ba1f..26b91105f 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.11' +__version__ = '2013.11.13' From dcbb45803f9b70041ec0ef9c3c6547340bd1ef7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Wed, 13 Nov 2013 16:21:24 +0100 Subject: [PATCH 015/425] [youtube:playlist] Don't use the gdata api (closes #1508) Parse the playlist pages instead --- test/test_youtube_lists.py | 14 +++++------ youtube_dl/extractor/youtube.py | 43 ++++++++++----------------------- 2 files changed, 20 insertions(+), 37 deletions(-) diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py index 4b7a7847b..50ad52695 100644 --- a/test/test_youtube_lists.py +++ b/test/test_youtube_lists.py @@ -27,7 +27,7 @@ class TestYoutubeLists(unittest.TestCase): def test_youtube_playlist(self): dl = FakeYDL() ie = YoutubePlaylistIE(dl) - result = ie.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')[0] + result = ie.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re') self.assertIsPlaylist(result) self.assertEqual(result['title'], 'ytdl test PL') ytie_results = [YoutubeIE()._extract_id(url['url']) for url in result['entries']] @@ -44,13 +44,13 @@ class TestYoutubeLists(unittest.TestCase): def test_issue_673(self): dl = FakeYDL() ie = YoutubePlaylistIE(dl) - result = ie.extract('PLBB231211A4F62143')[0] + result = ie.extract('PLBB231211A4F62143') self.assertTrue(len(result['entries']) > 25) def test_youtube_playlist_long(self): dl = FakeYDL() ie = YoutubePlaylistIE(dl) - result = ie.extract('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q')[0] + result = ie.extract('https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q') self.assertIsPlaylist(result) self.assertTrue(len(result['entries']) >= 799) @@ -58,7 +58,7 @@ class TestYoutubeLists(unittest.TestCase): #651 dl = FakeYDL() ie = YoutubePlaylistIE(dl) - result = ie.extract('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC')[0] + result = ie.extract('https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC') ytie_results = [YoutubeIE()._extract_id(url['url']) for url in result['entries']] self.assertFalse('pElCt5oNDuI' in ytie_results) self.assertFalse('KdPEApIVdWM' in ytie_results) @@ -66,7 +66,7 @@ class TestYoutubeLists(unittest.TestCase): def test_youtube_playlist_empty(self): dl = FakeYDL() ie = YoutubePlaylistIE(dl) - result = ie.extract('https://www.youtube.com/playlist?list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx')[0] + result = ie.extract('https://www.youtube.com/playlist?list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx') self.assertIsPlaylist(result) self.assertEqual(len(result['entries']), 0) @@ -74,7 +74,7 @@ class TestYoutubeLists(unittest.TestCase): dl = FakeYDL() ie = YoutubePlaylistIE(dl) # TODO find a > 100 (paginating?) videos course - result = ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')[0] + result = ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8') entries = result['entries'] self.assertEqual(YoutubeIE()._extract_id(entries[0]['url']), 'j9WZyLZCBzs') self.assertEqual(len(entries), 25) @@ -99,7 +99,7 @@ class TestYoutubeLists(unittest.TestCase): def test_youtube_safe_search(self): dl = FakeYDL() ie = YoutubePlaylistIE(dl) - result = ie.extract('PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl')[0] + result = ie.extract('PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl') self.assertEqual(len(result['entries']), 2) def test_youtube_show(self): diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index c992cba97..d97ea8c83 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1506,8 +1506,9 @@ class YoutubePlaylistIE(InfoExtractor): | ((?:PL|EC|UU|FL)[0-9A-Za-z-_]{10,}) )""" - _TEMPLATE_URL = 'https://gdata.youtube.com/feeds/api/playlists/%s?max-results=%i&start-index=%i&v=2&alt=json&safeSearch=none' - _MAX_RESULTS = 50 + _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s&page=%s' + _MORE_PAGES_INDICATOR = r'data-link-type="next"' + _VIDEO_RE = r'href="/watch\?v=([0-9A-Za-z_-]{11})&' IE_NAME = u'youtube:playlist' @classmethod @@ -1532,41 +1533,23 @@ class YoutubePlaylistIE(InfoExtractor): else: self.to_screen(u'Downloading playlist PL%s - add --no-playlist to just download video %s' % (playlist_id, video_id)) - # Download playlist videos from API - videos = [] + # Extract the video ids from the playlist pages + ids = [] for page_num in itertools.count(1): - start_index = self._MAX_RESULTS * (page_num - 1) + 1 - if start_index >= 1000: - self._downloader.report_warning(u'Max number of results reached') - break - url = self._TEMPLATE_URL % (playlist_id, self._MAX_RESULTS, start_index) + url = self._TEMPLATE_URL % (playlist_id, page_num) page = self._download_webpage(url, playlist_id, u'Downloading page #%s' % page_num) + # The ids are duplicated + new_ids = orderedSet(re.findall(self._VIDEO_RE, page)) + ids.extend(new_ids) - try: - response = json.loads(page) - except ValueError as err: - raise ExtractorError(u'Invalid JSON in API response: ' + compat_str(err)) - - if 'feed' not in response: - raise ExtractorError(u'Got a malformed response from YouTube API') - playlist_title = response['feed']['title']['$t'] - if 'entry' not in response['feed']: - # Number of videos is a multiple of self._MAX_RESULTS + if re.search(self._MORE_PAGES_INDICATOR, page) is None: break - for entry in response['feed']['entry']: - index = entry['yt$position']['$t'] - if 'media$group' in entry and 'yt$videoid' in entry['media$group']: - videos.append(( - index, - 'https://www.youtube.com/watch?v=' + entry['media$group']['yt$videoid']['$t'] - )) + playlist_title = self._og_search_title(page) - videos = [v[1] for v in sorted(videos)] - - url_results = [self.url_result(vurl, 'Youtube') for vurl in videos] - return [self.playlist_result(url_results, playlist_id, playlist_title)] + url_results = [self.url_result(vid, 'Youtube') for vid in ids] + return self.playlist_result(url_results, playlist_id, playlist_title) class YoutubeChannelIE(InfoExtractor): From 880e1c529de1d0f7f0a065afc4148320894a25b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Wed, 13 Nov 2013 16:39:11 +0100 Subject: [PATCH 016/425] [youtube:playlist] Login into youtube if requested (fixes #1757) Allows to download private playlists --- youtube_dl/extractor/youtube.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index d97ea8c83..c48c0e24f 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1490,7 +1490,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): }) return results -class YoutubePlaylistIE(InfoExtractor): +class YoutubePlaylistIE(YoutubeBaseInfoExtractor): IE_DESC = u'YouTube.com playlists' _VALID_URL = r"""(?: (?:https?://)? @@ -1516,6 +1516,9 @@ class YoutubePlaylistIE(InfoExtractor): """Receives a URL and returns True if suitable for this IE.""" return re.match(cls._VALID_URL, url, re.VERBOSE) is not None + def _real_initialize(self): + self._login() + def _real_extract(self, url): # Extract playlist id mobj = re.match(self._VALID_URL, url, re.VERBOSE) From ea7a7af1d46ecb51566db0af3e8779ab2a04b516 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Wed, 13 Nov 2013 17:13:06 +0100 Subject: [PATCH 017/425] [gamekings] Fix the test video checksum --- youtube_dl/extractor/gamekings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py index 4b4259447..6aa657ef4 100644 --- a/youtube_dl/extractor/gamekings.py +++ b/youtube_dl/extractor/gamekings.py @@ -11,7 +11,7 @@ class GamekingsIE(InfoExtractor): _TEST = { u"url": u"http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/", u'file': u'20130811.mp4', - u'md5': u'17f6088f7d0149ff2b46f2714bdb1954', + u'md5': u'2f32b1f7b80fdc5cb616efb4f387f8a3', u'info_dict': { u"title": u"Phoenix Wright: Ace Attorney \u2013 Dual Destinies Review", u"description": u"Melle en Steven hebben voor de review een week in de rechtbank doorbracht met Phoenix Wright: Ace Attorney - Dual Destinies.", From ca715127a2f95da30b6700a5e217a2acc904b459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Wed, 13 Nov 2013 17:06:02 +0100 Subject: [PATCH 018/425] Don't assume the 'subtitlesformat' is set in the params dict (fixes #1750) --- youtube_dl/YoutubeDL.py | 2 +- youtube_dl/extractor/youtube.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 5253c39e1..f615911de 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -640,7 +640,7 @@ class YoutubeDL(object): # subtitles download errors are already managed as troubles in relevant IE # that way it will silently go on when used with unsupporting IE subtitles = info_dict['subtitles'] - sub_format = self.params.get('subtitlesformat') + sub_format = self.params.get('subtitlesformat', 'srt') for sub_lang in subtitles.keys(): sub = subtitles[sub_lang] if sub is None: diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index c992cba97..56d30f52a 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1098,7 +1098,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): params = compat_urllib_parse.urlencode({ 'lang': lang, 'v': video_id, - 'fmt': self._downloader.params.get('subtitlesformat'), + 'fmt': self._downloader.params.get('subtitlesformat', 'srt'), 'name': l[0].encode('utf-8'), }) url = u'http://www.youtube.com/api/timedtext?' + params @@ -1111,7 +1111,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): def _get_available_automatic_caption(self, video_id, webpage): """We need the webpage for getting the captions url, pass it as an argument to speed up the process.""" - sub_format = self._downloader.params.get('subtitlesformat') + sub_format = self._downloader.params.get('subtitlesformat', 'srt') self.to_screen(u'%s: Looking for automatic captions' % video_id) mobj = re.search(r';ytplayer.config = ({.*?});', webpage) err_msg = u'Couldn\'t find automatic captions for %s' % video_id From 9771cceb2cfab153d0c2a5b0a7dbc923de90f2c1 Mon Sep 17 00:00:00 2001 From: Janez Troha Date: Wed, 13 Nov 2013 18:34:03 +0100 Subject: [PATCH 019/425] Fix filename extension leaking to json filename Makes writeinfojson behaving exactly as writethumbnail in case where filename contains mediafile extension. Case: video.mp4 converted to music.mp3 would yield music.mp4.info.json instead music.mp3.info.json or music.info.json --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index f615911de..3160d9712 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -655,7 +655,7 @@ class YoutubeDL(object): return if self.params.get('writeinfojson', False): - infofn = filename + u'.info.json' + infofn = os.path.splitext(filename)[0] + u'.info.json' self.report_writeinfojson(infofn) try: json_info_dict = dict((k, v) for k, v in info_dict.items() if not k in ['urlhandle']) From 08bc37cdd02bc861a0b5271abb34623f0ebcd66c Mon Sep 17 00:00:00 2001 From: Janez Troha Date: Wed, 13 Nov 2013 18:55:49 +0100 Subject: [PATCH 020/425] Update test_write_info_json.py --- test/test_write_info_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_write_info_json.py b/test/test_write_info_json.py index a5b6f6972..30c4859fd 100644 --- a/test/test_write_info_json.py +++ b/test/test_write_info_json.py @@ -31,7 +31,7 @@ params = get_params({ TEST_ID = 'BaW_jenozKc' -INFO_JSON_FILE = TEST_ID + '.mp4.info.json' +INFO_JSON_FILE = TEST_ID + '.info.json' DESCRIPTION_FILE = TEST_ID + '.mp4.description' EXPECTED_DESCRIPTION = u'''test chars: "'/\ä↭𝕐 From c66d2baa9cb33327a3318f49cbb89f9ac559c978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Thu, 14 Nov 2013 13:16:32 +0100 Subject: [PATCH 021/425] [livestream] Add an extractor for the original version of livestream (closes #1764) The two versions use different systems. --- youtube_dl/extractor/__init__.py | 2 +- youtube_dl/extractor/livestream.py | 44 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 0594a3666..ee3173468 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -80,7 +80,7 @@ from .keezmovies import KeezMoviesIE from .kickstarter import KickStarterIE from .keek import KeekIE from .liveleak import LiveLeakIE -from .livestream import LivestreamIE +from .livestream import LivestreamIE, LivestreamOriginalIE from .metacafe import MetacafeIE from .metacritic import MetacriticIE from .mit import TechTVMITIE, MITIE diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py index 4531fd6ab..1a3e0ae6b 100644 --- a/youtube_dl/extractor/livestream.py +++ b/youtube_dl/extractor/livestream.py @@ -1,16 +1,19 @@ import re import json +import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( compat_urllib_parse_urlparse, compat_urlparse, get_meta_content, + xpath_with_ns, ExtractorError, ) class LivestreamIE(InfoExtractor): + IE_NAME = u'livestream' _VALID_URL = r'http://new.livestream.com/.*?/(?P.*?)(/videos/(?P\d+))?/?$' _TEST = { u'url': u'http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370', @@ -54,3 +57,44 @@ class LivestreamIE(InfoExtractor): info = json.loads(self._download_webpage(api_url, video_id, u'Downloading video info')) return self._extract_video_info(info) + + +# The original version of Livestream uses a different system +class LivestreamOriginalIE(InfoExtractor): + IE_NAME = u'livestream:original' + _VALID_URL = r'https?://www\.livestream\.com/(?P[^/]+)/video\?.*?clipId=(?P.*?)(&|$)' + _TEST = { + u'url': u'http://www.livestream.com/dealbook/video?clipId=pla_8aa4a3f1-ba15-46a4-893b-902210e138fb', + u'info_dict': { + u'id': u'pla_8aa4a3f1-ba15-46a4-893b-902210e138fb', + u'ext': u'flv', + u'title': u'Spark 1 (BitCoin) with Cameron Winklevoss & Tyler Winklevoss of Winklevoss Capital', + }, + u'params': { + # rtmp + u'skip_download': True, + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + user = mobj.group('user') + api_url = 'http://x{0}x.api.channel.livestream.com/2.0/clipdetails?extendedInfo=true&id={1}'.format(user, video_id) + + api_response = self._download_webpage(api_url, video_id) + info = xml.etree.ElementTree.fromstring(api_response.encode('utf-8')) + item = info.find('channel').find('item') + ns = {'media': 'http://search.yahoo.com/mrss'} + thumbnail_url = item.find(xpath_with_ns('media:thumbnail', ns)).attrib['url'] + # Remove the extension and number from the path (like 1.jpg) + path = self._search_regex(r'(user-files/.+)_.*?\.jpg$', thumbnail_url, u'path') + + return { + 'id': video_id, + 'title': item.find('title').text, + 'url': 'rtmp://extondemand.livestream.com/ondemand', + 'play_path': 'mp4:trans/dv15/mogulus-{0}.mp4'.format(path), + 'ext': 'flv', + 'thumbnail': thumbnail_url, + } From e3b9ab5e187a590143f7f6110e55d43fb78f15bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Thu, 14 Nov 2013 19:45:39 +0100 Subject: [PATCH 022/425] [soundlcoud] Set the correct extension for the tracks (fixes #1766) Some tracks are not in mp3 format, they can be wav files. --- youtube_dl/extractor/soundcloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 4717fbb77..83e1f055f 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -87,7 +87,7 @@ class SoundcloudIE(InfoExtractor): 'uploader': info['user']['username'], 'upload_date': unified_strdate(info['created_at']), 'title': info['title'], - 'ext': u'mp3', + 'ext': info.get('original_format', u'mp3'), 'description': info['description'], 'thumbnail': thumbnail, } From 9f9be844fcc5155ab3e832c8428c8f016bea819b Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 15 Nov 2013 01:45:34 +0100 Subject: [PATCH 023/425] [youtube] Fix protocol-independent URLs (Fixes #1768) --- youtube_dl/extractor/youtube.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index c992cba97..ed82e4fc0 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1019,6 +1019,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): """Turn the encrypted s field into a working signature""" if player_url is not None: + if player_url.startswith(u'//'): + player_url = u'https:' + player_url try: player_id = (player_url, len(s)) if player_id not in self._player_cache: From a25a5cfeecc1c6371ab28d6d458cd066baf7013d Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 15 Nov 2013 01:47:15 +0100 Subject: [PATCH 024/425] release 2013.11.15 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 26b91105f..cd9f0b546 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.13' +__version__ = '2013.11.15' From feee2ecfa9fbc6fd34246c7e167ac9542ae7def2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 15 Nov 2013 11:04:26 +0100 Subject: [PATCH 025/425] Pass the 'download' argument to 'process_video_result' (fixes #1769) --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index f615911de..b5c670dd4 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -385,7 +385,7 @@ class YoutubeDL(object): result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system if result_type == 'video': self.add_extra_info(ie_result, extra_info) - return self.process_video_result(ie_result) + return self.process_video_result(ie_result, download=download) elif result_type == 'url': # We have to add extra_info to the results because it may be # contained in a playlist From b9643eed7c9b081c91e72257b380ccbd92555254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 15 Nov 2013 11:51:45 +0100 Subject: [PATCH 026/425] [youtube:channel] Fix the extraction of autogenerated channels The ajax pages are empty, now it looks directly in the channel's /videos page --- youtube_dl/extractor/youtube.py | 37 +++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 7d7aeb461..8c0e6f252 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1594,20 +1594,31 @@ class YoutubeChannelIE(InfoExtractor): # Download channel page channel_id = mobj.group(1) video_ids = [] + url = 'https://www.youtube.com/channel/%s/videos' % channel_id + channel_page = self._download_webpage(url, channel_id) + if re.search(r'channel-header-autogenerated-label', channel_page) is not None: + autogenerated = True + else: + autogenerated = False - # Download all channel pages using the json-based channel_ajax query - for pagenum in itertools.count(1): - url = self._MORE_PAGES_URL % (pagenum, channel_id) - page = self._download_webpage(url, channel_id, - u'Downloading page #%s' % pagenum) - - page = json.loads(page) - - ids_in_page = self.extract_videos_from_page(page['content_html']) - video_ids.extend(ids_in_page) - - if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']: - break + if autogenerated: + # The videos are contained in a single page + # the ajax pages can't be used, they are empty + video_ids = self.extract_videos_from_page(channel_page) + else: + # Download all channel pages using the json-based channel_ajax query + for pagenum in itertools.count(1): + url = self._MORE_PAGES_URL % (pagenum, channel_id) + page = self._download_webpage(url, channel_id, + u'Downloading page #%s' % pagenum) + + page = json.loads(page) + + ids_in_page = self.extract_videos_from_page(page['content_html']) + video_ids.extend(ids_in_page) + + if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']: + break self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids))) From 85d61685f15bdc62709c699e849af512db78089f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 15 Nov 2013 12:10:22 +0100 Subject: [PATCH 027/425] [tvp] Update the title and the description of the test video --- youtube_dl/extractor/tvp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py index 32e0f5037..76721a986 100644 --- a/youtube_dl/extractor/tvp.py +++ b/youtube_dl/extractor/tvp.py @@ -13,8 +13,8 @@ class TvpIE(InfoExtractor): u'md5': u'148408967a6a468953c0a75cbdaf0d7a', u'file': u'12878238.wmv', u'info_dict': { - u'title': u'31.10.2013', - u'description': u'31.10.2013', + u'title': u'31.10.2013 - Odcinek 2', + u'description': u'31.10.2013 - Odcinek 2', }, } From ab2d524780736249c8988313db021e83642c24d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 15 Nov 2013 12:24:54 +0100 Subject: [PATCH 028/425] Improve the OpenGraph regex * Do not accept '>' between the property and content attributes. * Recognize the properties if the content attribute is before the property attribute using two regexes (fixes the extraction of the description for SlideshareIE). --- youtube_dl/extractor/common.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 9c20d30b4..e02176852 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -315,13 +315,17 @@ class InfoExtractor(object): # Helper functions for extracting OpenGraph info @staticmethod - def _og_regex(prop): - return r']+?property=[\'"]og:%s[\'"][^>]+?content=(?:"(.+?)"|\'(.+?)\')' % esc_prop, + r']+?content=(?:"(.+?)"|\'(.+?)\')[^>]+?property=[\'"]og:%s[\'"]' % esc_prop, + ] def _og_search_property(self, prop, html, name=None, **kargs): if name is None: name = 'OpenGraph %s' % prop - escaped = self._search_regex(self._og_regex(prop), html, name, flags=re.DOTALL, **kargs) + escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, **kargs) if escaped is None: return None return unescapeHTML(escaped) @@ -336,8 +340,8 @@ class InfoExtractor(object): return self._og_search_property('title', html, **kargs) def _og_search_video_url(self, html, name='video url', secure=True, **kargs): - regexes = [self._og_regex('video')] - if secure: regexes.insert(0, self._og_regex('video:secure_url')) + regexes = self._og_regexes('video') + if secure: regexes = self._og_regexes('video:secure_url') + regexes return self._html_search_regex(regexes, html, name, **kargs) def _rta_search(self, html): From 78fb87b2837e15124b5855734a951598dfe025fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 15 Nov 2013 12:54:13 +0100 Subject: [PATCH 029/425] Don't accept '>' inside the content attribute in OpenGraph regexes --- youtube_dl/extractor/common.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index e02176852..45dd01789 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -316,10 +316,12 @@ class InfoExtractor(object): # Helper functions for extracting OpenGraph info @staticmethod def _og_regexes(prop): - esc_prop = re.escape(prop) + content_re = r'content=(?:"([^>]+?)"|\'(.+?)\')' + property_re = r'property=[\'"]og:%s[\'"]' % re.escape(prop) + template = r']+?%s[^>]+?%s' return [ - r']+?property=[\'"]og:%s[\'"][^>]+?content=(?:"(.+?)"|\'(.+?)\')' % esc_prop, - r']+?content=(?:"(.+?)"|\'(.+?)\')[^>]+?property=[\'"]og:%s[\'"]' % esc_prop, + template % (property_re, content_re), + template % (content_re, property_re), ] def _og_search_property(self, prop, html, name=None, **kargs): From d24ffe1cfadca8fdc37a7b2fb5c2f080c785ad1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 15 Nov 2013 12:57:59 +0100 Subject: [PATCH 030/425] [rtlnow] Remove the test for nitro The videos expire. --- youtube_dl/extractor/rtlnow.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py index 9ac7c3be8..2f238de35 100644 --- a/youtube_dl/extractor/rtlnow.py +++ b/youtube_dl/extractor/rtlnow.py @@ -62,18 +62,6 @@ class RTLnowIE(InfoExtractor): u'skip_download': True, }, }, - { - u'url': u'http://www.rtlnitronow.de/recht-ordnung/stadtpolizei-frankfurt-gerichtsvollzieher-leipzig.php?film_id=129679&player=1&season=1', - u'file': u'129679.flv', - u'info_dict': { - u'upload_date': u'20131016', - u'title': u'Recht & Ordnung - Stadtpolizei Frankfurt/ Gerichtsvollzieher...', - u'description': u'Stadtpolizei Frankfurt/ Gerichtsvollzieher Leipzig', - }, - u'params': { - u'skip_download': True, - }, - }, { u'url': u'http://www.n-tvnow.de/top-gear/episode-1-2013-01-01-00-00-00.php?film_id=124903&player=1&season=10', u'file': u'124903.flv', From 463a908705674b9411a01b7a696c84348ec0244e Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 15 Nov 2013 14:06:38 +0100 Subject: [PATCH 031/425] [ted] simplify --- youtube_dl/extractor/ted.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py index 76cfdfb90..8001ca5a3 100644 --- a/youtube_dl/extractor/ted.py +++ b/youtube_dl/extractor/ted.py @@ -85,7 +85,7 @@ class TEDIE(SubtitlesInfoExtractor): 'ext': 'mp4', 'url': stream['file'], 'format': stream['id'] - } for stream in info['htmlStreams']] + } for stream in info['htmlStreams']] video_id = info['id'] @@ -95,7 +95,7 @@ class TEDIE(SubtitlesInfoExtractor): self._list_available_subtitles(video_id, webpage) return - info = { + return { 'id': video_id, 'title': title, 'thumbnail': thumbnail, @@ -104,11 +104,6 @@ class TEDIE(SubtitlesInfoExtractor): 'formats': formats, } - # TODO: Remove when #980 has been merged - info.update(info['formats'][-1]) - - return info - def _get_available_subtitles(self, video_id, webpage): try: options = self._search_regex(r'(?:)', webpage, 'subtitles_language_select', flags=re.DOTALL) From fc2ef392bee9e46564d4106d3b0e5f6b8d71e37b Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 15 Nov 2013 14:33:51 +0100 Subject: [PATCH 032/425] [ted] Fix playlists (Fixes #1770) --- youtube_dl/extractor/ted.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py index 8001ca5a3..2e497c86e 100644 --- a/youtube_dl/extractor/ted.py +++ b/youtube_dl/extractor/ted.py @@ -43,26 +43,25 @@ class TEDIE(SubtitlesInfoExtractor): self.to_screen(u'Getting info of playlist %s: "%s"' % (playlist_id,name)) return [self._playlist_videos_info(url,name,playlist_id)] - def _playlist_videos_info(self,url,name,playlist_id=0): + + def _playlist_videos_info(self, url, name, playlist_id): '''Returns the videos of the playlist''' - video_RE=r''' - (?P.+?)

' - webpage=self._download_webpage(url, playlist_id, 'Downloading playlist webpage') - m_videos=re.finditer(video_RE,webpage,re.VERBOSE) - m_names=re.finditer(video_name_RE,webpage) + + webpage = self._download_webpage( + url, playlist_id, u'Downloading playlist webpage') + matches = re.finditer( + r'/talks/[^"]+\.html)">[^<]*

', + webpage) playlist_title = self._html_search_regex(r'div class="headline">\s*?

\s*?(.*?)', webpage, 'playlist title') - playlist_entries = [] - for m_video, m_name in zip(m_videos,m_names): - talk_url='http://www.ted.com%s' % m_name.group('talk_url') - playlist_entries.append(self.url_result(talk_url, 'TED')) - return self.playlist_result(playlist_entries, playlist_id = playlist_id, playlist_title = playlist_title) + playlist_entries = [ + self.url_result(u'http://www.ted.com' + m.group('talk_url'), 'TED') + for m in matches + ] + return self.playlist_result( + playlist_entries, playlist_id=playlist_id, playlist_title=playlist_title) def _talk_info(self, url, video_id=0): """Return the video for the talk in the url""" From aa13b2dffde73978fe2a169d3f99de7e5e7754cb Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 15 Nov 2013 14:35:00 +0100 Subject: [PATCH 033/425] release 2013.11.15.1 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index cd9f0b546..b04238eb5 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.15' +__version__ = '2013.11.15.1' From 91c7271aabdd74c833ef570db59018e2d9f9d803 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sat, 16 Nov 2013 01:08:43 +0100 Subject: [PATCH 034/425] Add automatic generation of format note based on bitrate and codecs --- youtube_dl/YoutubeDL.py | 18 +++++++++++++++++- youtube_dl/extractor/common.py | 4 ++++ youtube_dl/extractor/vevo.py | 7 ++++--- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index b5c670dd4..9c79af1f2 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -781,12 +781,28 @@ class YoutubeDL(object): return res def list_formats(self, info_dict): + def format_note(fdict): + if fdict.get('format_note') is not None: + return fdict['format_note'] + res = u'' + if fdict.get('vcodec') is not None: + res += fdict['vcodec'] + if fdict.get('vbr') is not None: + res += u'@%4dk' % fdict['vbr'] + if fdict.get('acodec') is not None: + if res: + res += u', ' + res += fdict['acodec'] + if fdict.get('abr') is not None: + res += u'@%3dk' % fdict['abr'] + return res + def line(format): return (u'%-20s%-10s%-12s%s' % ( format['format_id'], format['ext'], self.format_resolution(format), - format.get('format_note', ''), + format_note(format), ) ) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 45dd01789..f787d0a3c 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -71,6 +71,10 @@ class InfoExtractor(object): ("3D" or "DASH video") * width Width of the video, if known * height Height of the video, if known + * abr Average audio bitrate in KBit/s + * acodec Name of the audio codec in use + * vbr Average video bitrate in KBit/s + * vcodec Name of the video codec in use webpage_url: The url to the video webpage, if given to youtube-dl it should allow to get the same result again. (It will be set by YoutubeDL if it's missing) diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index 3f6020f74..4378b1780 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -78,12 +78,13 @@ class VevoIE(InfoExtractor): continue format_url = self._SMIL_BASE_URL + m.group('path') - format_note = ('%(vcodec)s@%(vbr)4sk, %(acodec)s@%(abr)3sk' % - m.groupdict()) formats.append({ 'url': format_url, 'format_id': u'SMIL_' + m.group('cbr'), - 'format_note': format_note, + 'vcodec': m.group('vcodec'), + 'acodec': m.group('acodec'), + 'vbr': int(m.group('vbr')), + 'abr': int(m.group('abr')), 'ext': m.group('ext'), 'width': int(m.group('width')), 'height': int(m.group('height')), From 7150858d49f05a5650a12b5f4694f91dfb9595d3 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sat, 16 Nov 2013 01:33:12 +0100 Subject: [PATCH 035/425] [spiegel] Implement format selection --- youtube_dl/YoutubeDL.py | 10 +++++-- youtube_dl/extractor/spiegel.py | 50 ++++++++++++++++++++++++--------- 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 9c79af1f2..273f7d977 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -786,13 +786,19 @@ class YoutubeDL(object): return fdict['format_note'] res = u'' if fdict.get('vcodec') is not None: - res += fdict['vcodec'] + res += u'%-5s' % fdict['vcodec'] + elif fdict.get('vbr') is not None: + res += u'video' if fdict.get('vbr') is not None: res += u'@%4dk' % fdict['vbr'] if fdict.get('acodec') is not None: if res: res += u', ' - res += fdict['acodec'] + res += u'%-5s' % fdict['acodec'] + elif fdict.get('abr') is not None: + if res: + res += u', ' + res += 'audio' if fdict.get('abr') is not None: res += u'@%3dk' % fdict['abr'] return res diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py index 13c86401c..6dc2eda6d 100644 --- a/youtube_dl/extractor/spiegel.py +++ b/youtube_dl/extractor/spiegel.py @@ -2,18 +2,27 @@ import re import xml.etree.ElementTree from .common import InfoExtractor +from ..utils import determine_ext class SpiegelIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?spiegel\.de/video/[^/]*-(?P[0-9]+)(?:\.html)?(?:#.*)?$' - _TEST = { + _TESTS = [{ u'url': u'http://www.spiegel.de/video/vulkan-tungurahua-in-ecuador-ist-wieder-aktiv-video-1259285.html', u'file': u'1259285.mp4', u'md5': u'2c2754212136f35fb4b19767d242f66e', u'info_dict': { u"title": u"Vulkanausbruch in Ecuador: Der \"Feuerschlund\" ist wieder aktiv" } - } + }, + { + u'url': u'http://www.spiegel.de/video/schach-wm-videoanalyse-des-fuenften-spiels-video-1309159.html', + u'file': u'1309159.mp4', + u'md5': u'f2cdf638d7aa47654e251e1aee360af1', + u'info_dict': { + u'title': u'Schach-WM in der Videoanalyse: Carlsen nutzt die Fehlgriffe des Titelverteidigers' + } + }] def _real_extract(self, url): m = re.match(self._VALID_URL, url) @@ -21,25 +30,38 @@ class SpiegelIE(InfoExtractor): webpage = self._download_webpage(url, video_id) - video_title = self._html_search_regex(r'
(.*?)
', - webpage, u'title') + video_title = self._html_search_regex( + r'
(.*?)
', webpage, u'title') xml_url = u'http://video2.spiegel.de/flash/' + video_id + u'.xml' - xml_code = self._download_webpage(xml_url, video_id, - note=u'Downloading XML', errnote=u'Failed to download XML') + xml_code = self._download_webpage( + xml_url, video_id, + note=u'Downloading XML', errnote=u'Failed to download XML') idoc = xml.etree.ElementTree.fromstring(xml_code) - last_type = idoc[-1] - filename = last_type.findall('./filename')[0].text - duration = float(last_type.findall('./duration')[0].text) - video_url = 'http://video2.spiegel.de/flash/' + filename - video_ext = filename.rpartition('.')[2] + formats = [ + { + 'format_id': n.tag.rpartition('type')[2], + 'url': u'http://video2.spiegel.de/flash/' + n.find('./filename').text, + 'width': int(n.find('./width').text), + 'height': int(n.find('./height').text), + 'abr': int(n.find('./audiobitrate').text), + 'vbr': int(n.find('./videobitrate').text), + 'vcodec': n.find('./codec').text, + 'acodec': 'MP4A', + } + for n in list(idoc) + # Blacklist type 6, it's extremely LQ and not available on the same server + if n.tag.startswith('type') and n.tag != 'type6' + ] + formats.sort(key=lambda f: f['vbr']) + duration = float(idoc[0].findall('./duration')[0].text) + info = { 'id': video_id, - 'url': video_url, - 'ext': video_ext, 'title': video_title, 'duration': duration, + 'formats': formats, } - return [info] + return info From b5349e8721d0580a50519593926726a2ea832c9b Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sat, 16 Nov 2013 01:39:45 +0100 Subject: [PATCH 036/425] Fix indentation of (best) and (worst) in --list-formats --- youtube_dl/YoutubeDL.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 273f7d977..d29e8bec5 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -815,8 +815,8 @@ class YoutubeDL(object): formats = info_dict.get('formats', [info_dict]) formats_s = list(map(line, formats)) if len(formats) > 1: - formats_s[0] += (' ' if formats[0].get('format_note') else '') + '(worst)' - formats_s[-1] += (' ' if formats[-1].get('format_note') else '') + '(best)' + formats_s[0] += (' ' if format_note(formats[0]) else '') + '(worst)' + formats_s[-1] += (' ' if format_note(formats[-1]) else '') + '(best)' header_line = line({ 'format_id': u'format code', 'ext': u'extension', From f058e340114cb599e21001cef25ab3d02cfb194f Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sat, 16 Nov 2013 01:56:23 +0100 Subject: [PATCH 037/425] [dailymotion] Fix playlists --- youtube_dl/extractor/dailymotion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py index e87690f9d..71f5e03ee 100644 --- a/youtube_dl/extractor/dailymotion.py +++ b/youtube_dl/extractor/dailymotion.py @@ -186,7 +186,7 @@ class DailymotionPlaylistIE(DailymotionBaseInfoExtractor): webpage = self._download_webpage(request, id, u'Downloading page %s' % pagenum) - playlist_el = get_element_by_attribute(u'class', u'video_list', webpage) + playlist_el = get_element_by_attribute(u'class', u'row video_list', webpage) video_ids.extend(re.findall(r'data-id="(.+?)"', playlist_el)) if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None: From ce152341a118c94e442f42db8008de95aebec56b Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sat, 16 Nov 2013 01:59:28 +0100 Subject: [PATCH 038/425] [bambuser] Do not test for MD5, seems to be flaky --- youtube_dl/extractor/bambuser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py index f3b36f473..967568c4a 100644 --- a/youtube_dl/extractor/bambuser.py +++ b/youtube_dl/extractor/bambuser.py @@ -15,7 +15,8 @@ class BambuserIE(InfoExtractor): _TEST = { u'url': u'http://bambuser.com/v/4050584', - u'md5': u'fba8f7693e48fd4e8641b3fd5539a641', + # MD5 seems to be flaky, see https://travis-ci.org/rg3/youtube-dl/jobs/14051016#L388 + #u'md5': u'fba8f7693e48fd4e8641b3fd5539a641', u'info_dict': { u'id': u'4050584', u'ext': u'flv', From 52d703d3d1b0d24f6c4c01ef8fcf33bfee78928b Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sat, 16 Nov 2013 02:09:30 +0100 Subject: [PATCH 039/425] [tvp] Skip tests --- youtube_dl/extractor/tvp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/tvp.py b/youtube_dl/extractor/tvp.py index 76721a986..bfed9dd04 100644 --- a/youtube_dl/extractor/tvp.py +++ b/youtube_dl/extractor/tvp.py @@ -16,6 +16,7 @@ class TvpIE(InfoExtractor): u'title': u'31.10.2013 - Odcinek 2', u'description': u'31.10.2013 - Odcinek 2', }, + u'skip': u'Download has to use same server IP as extraction. Therefore, a good (load-balancing) DNS resolver will make the download fail.' } def _real_extract(self, url): From 21ea3e06c9e39ac2cbfa24e7c02ebac936202893 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sat, 16 Nov 2013 02:31:02 +0100 Subject: [PATCH 040/425] [gamekings] remove unnecessary import --- youtube_dl/extractor/gamekings.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py index 6aa657ef4..64715581d 100644 --- a/youtube_dl/extractor/gamekings.py +++ b/youtube_dl/extractor/gamekings.py @@ -1,9 +1,6 @@ import re from .common import InfoExtractor -from ..utils import ( - determine_ext, -) class GamekingsIE(InfoExtractor): From eab2724138670159f1946608359f5e5bd5a7e7af Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sat, 16 Nov 2013 02:32:08 +0100 Subject: [PATCH 041/425] [gamekings] Do not test md5 sum, precise file changes regularly --- youtube_dl/extractor/gamekings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py index 64715581d..c91669b0e 100644 --- a/youtube_dl/extractor/gamekings.py +++ b/youtube_dl/extractor/gamekings.py @@ -8,7 +8,8 @@ class GamekingsIE(InfoExtractor): _TEST = { u"url": u"http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/", u'file': u'20130811.mp4', - u'md5': u'2f32b1f7b80fdc5cb616efb4f387f8a3', + # MD5 is flaky, seems to change regularly + #u'md5': u'2f32b1f7b80fdc5cb616efb4f387f8a3', u'info_dict': { u"title": u"Phoenix Wright: Ace Attorney \u2013 Dual Destinies Review", u"description": u"Melle en Steven hebben voor de review een week in de rechtbank doorbracht met Phoenix Wright: Ace Attorney - Dual Destinies.", From d1c252048bf74add82de0b9812b589956f3c8a69 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sat, 16 Nov 2013 10:30:09 +0100 Subject: [PATCH 042/425] [redtube] Do not test md5, seems to vary --- youtube_dl/extractor/redtube.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py index 994778e16..3bbda128e 100644 --- a/youtube_dl/extractor/redtube.py +++ b/youtube_dl/extractor/redtube.py @@ -8,7 +8,9 @@ class RedTubeIE(InfoExtractor): _TEST = { u'url': u'http://www.redtube.com/66418', u'file': u'66418.mp4', - u'md5': u'7b8c22b5e7098a3e1c09709df1126d2d', + # md5 varies from time to time, as in + # https://travis-ci.org/rg3/youtube-dl/jobs/14052463#L295 + #u'md5': u'7b8c22b5e7098a3e1c09709df1126d2d', u'info_dict': { u"title": u"Sucked on a toilet", u"age_limit": 18, From ba3881dffd241d8719430c31f107156ed8830996 Mon Sep 17 00:00:00 2001 From: Anton Larionov Date: Sat, 16 Nov 2013 18:26:34 +0400 Subject: [PATCH 043/425] Add support for anitube.se (#1417) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/anitube.py | 59 ++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 youtube_dl/extractor/anitube.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index ee3173468..a30de3033 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -1,5 +1,6 @@ from .appletrailers import AppleTrailersIE from .addanime import AddAnimeIE +from .anitube import AnitubeIE from .archiveorg import ArchiveOrgIE from .ard import ARDIE from .arte import ( diff --git a/youtube_dl/extractor/anitube.py b/youtube_dl/extractor/anitube.py new file mode 100644 index 000000000..2954966a6 --- /dev/null +++ b/youtube_dl/extractor/anitube.py @@ -0,0 +1,59 @@ +import re +import xml.etree.ElementTree + +from .common import InfoExtractor + + +class AnitubeIE(InfoExtractor): + IE_NAME = u'anitube.se' + _VALID_URL = r'http?://(?:www\.)?anitube\.se/video/(?P\d+)' + + _TEST = { + u'url': u'http://www.anitube.se/video/36621', + u'md5': u'0c4e4f1051bf50f5982f829f7230f539', + u'info_dict': { + u'id': u'36621', + u'ext': u'mp4', + u'title': u'Recorder to Randoseru 01', + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + + key = self._html_search_regex(r'http://www\.anitube\.se/embed/([A-Za-z0-9_-]*)', + webpage, u'key') + + webpage_config = self._download_webpage('http://www.anitube.se/nuevo/econfig.php?key=%s' % key, + key) + + config_xml = xml.etree.ElementTree.fromstring(webpage_config.encode('utf-8')) + + video_title = config_xml.find('title').text + + + formats = [] + + video_url = config_xml.find('file') + if video_url is not None: + formats.append({ + 'format_id': 'sd', + 'url': video_url.text, + }) + + video_url = config_xml.find('filehd') + if video_url is not None: + formats.append({ + 'format_id': 'hd', + 'url': video_url.text, + }) + + return { + 'id': video_id, + 'title': video_title, + 'ext': 'mp4', + 'formats': formats + } From ddf49c63445d236138846d778b18db6ede067fc8 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 17 Nov 2013 11:05:49 +0100 Subject: [PATCH 044/425] [arte] remove two typos --- youtube_dl/extractor/arte.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py index b35a679e3..44d0b5d70 100644 --- a/youtube_dl/extractor/arte.py +++ b/youtube_dl/extractor/arte.py @@ -69,7 +69,7 @@ class ArteTvIE(InfoExtractor): lang = mobj.group('lang') return self._extract_liveweb(url, name, lang) - if re.search(self._LIVE_URL, video_id) is not None: + if re.search(self._LIVE_URL, url) is not None: raise ExtractorError(u'Arte live streams are not yet supported, sorry') # self.extractLiveStream(url) # return @@ -115,7 +115,7 @@ class ArteTvIE(InfoExtractor): event_doc = config_doc.find('event') url_node = event_doc.find('video').find('urlHd') if url_node is None: - url_node = video_doc.find('urlSd') + url_node = event_doc.find('urlSd') return {'id': video_id, 'title': event_doc.find('name%s' % lang.capitalize()).text, From 1d699755e0978f2ec0932ef8d6562394a4799871 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 17 Nov 2013 11:06:16 +0100 Subject: [PATCH 045/425] [youtube] Add view_count (Fixes #1781) --- youtube_dl/extractor/youtube.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 8c0e6f252..1aa549740 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1301,6 +1301,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): else: raise ExtractorError(u'"token" parameter not in video info for unknown reason') + if 'view_count' in video_info: + view_count = int(video_info['view_count'][0]) + else: + view_count = None + # Check for "rental" videos if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info: raise ExtractorError(u'"rental" videos not supported') @@ -1489,6 +1494,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): 'age_limit': 18 if age_gate else 0, 'annotations': video_annotations, 'webpage_url': 'https://www.youtube.com/watch?v=%s' % video_id, + 'view_count': view_count, }) return results From 1e5b9a95fd2049e024b3ee2f13b4da5c308d2e9c Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 17 Nov 2013 11:39:52 +0100 Subject: [PATCH 046/425] Move console_title to YoutubeDL --- youtube_dl/FileDownloader.py | 17 +++-------------- youtube_dl/YoutubeDL.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 088f59586..e5a542ed5 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -5,9 +5,6 @@ import subprocess import sys import time -if os.name == 'nt': - import ctypes - from .utils import ( compat_urllib_error, compat_urllib_request, @@ -151,16 +148,8 @@ class FileDownloader(object): def to_stderr(self, message): self.ydl.to_screen(message) - def to_cons_title(self, message): - """Set console/terminal window title to message.""" - if not self.params.get('consoletitle', False): - return - if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow(): - # c_wchar_p() might not be necessary if `message` is - # already of type unicode() - ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message)) - elif 'TERM' in os.environ: - self.to_screen('\033]0;%s\007' % message, skip_eol=True) + def to_console_title(self, message): + self.ydl.to_console_title(message) def trouble(self, *args, **kargs): self.ydl.trouble(*args, **kargs) @@ -249,7 +238,7 @@ class FileDownloader(object): else: self.to_screen(u'\r%s[download] %s of %s at %s ETA %s' % (clear_line, percent_str, data_len_str, speed_str, eta_str), skip_eol=True) - self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' % + self.to_console_title(u'youtube-dl - %s of %s at %s ETA %s' % (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip())) def report_resuming_byte(self, resume_len): diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index d29e8bec5..6ea865bd9 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -13,6 +13,9 @@ import sys import time import traceback +if os.name == 'nt': + import ctypes + from .utils import * from .extractor import get_info_extractor, gen_extractors from .FileDownloader import FileDownloader @@ -176,6 +179,16 @@ class YoutubeDL(object): output = output.encode(preferredencoding()) sys.stderr.write(output) + def to_console_title(self, message): + if not self.params.get('consoletitle', False): + return + if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow(): + # c_wchar_p() might not be necessary if `message` is + # already of type unicode() + ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message)) + elif 'TERM' in os.environ: + self.to_screen('\033]0;%s\007' % message, skip_eol=True) + def fixed_template(self): """Checks if the output template is fixed.""" return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None) From ce02ed60f27ea27e66c33af745dc7e716377b46f Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 17 Nov 2013 16:47:52 +0100 Subject: [PATCH 047/425] Remove * imports --- youtube_dl/YoutubeDL.py | 28 ++++++++++++++++++++++++++-- youtube_dl/update.py | 6 +++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 6ea865bd9..6e5ae44d3 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -16,7 +16,31 @@ import traceback if os.name == 'nt': import ctypes -from .utils import * +from .utils import ( + compat_http_client, + compat_print, + compat_str, + compat_urllib_error, + compat_urllib_request, + ContentTooShortError, + date_from_str, + DateRange, + determine_ext, + DownloadError, + encodeFilename, + ExtractorError, + locked_file, + MaxDownloadsReached, + PostProcessingError, + preferredencoding, + SameFileError, + sanitize_filename, + subtitles_filename, + takewhile_inclusive, + UnavailableVideoError, + write_json_file, + write_string, +) from .extractor import get_info_extractor, gen_extractors from .FileDownloader import FileDownloader @@ -267,7 +291,7 @@ class YoutubeDL(object): """Report file has already been fully downloaded.""" try: self.to_screen(u'[download] %s has already been downloaded' % file_name) - except (UnicodeEncodeError) as err: + except UnicodeEncodeError: self.to_screen(u'[download] The file has already been downloaded') def increment_downloads(self): diff --git a/youtube_dl/update.py b/youtube_dl/update.py index 0689a4891..f41b4785a 100644 --- a/youtube_dl/update.py +++ b/youtube_dl/update.py @@ -2,11 +2,15 @@ import io import json import traceback import hashlib +import os import subprocess import sys from zipimport import zipimporter -from .utils import * +from .utils import ( + compat_str, + compat_urllib_request, +) from .version import __version__ def rsa_verify(message, signature, key): From 90b6bbc38c7258358e294e7f1bcb3d46cd56ffd9 Mon Sep 17 00:00:00 2001 From: rzhxeo Date: Sun, 17 Nov 2013 17:42:24 +0100 Subject: [PATCH 048/425] [SouthParkStudiosIE] Also detect urls without http:// or www --- youtube_dl/extractor/southparkstudios.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/southparkstudios.py b/youtube_dl/extractor/southparkstudios.py index b1e96b679..bb0c4b393 100644 --- a/youtube_dl/extractor/southparkstudios.py +++ b/youtube_dl/extractor/southparkstudios.py @@ -5,7 +5,7 @@ from .mtv import MTVIE, _media_xml_tag class SouthParkStudiosIE(MTVIE): IE_NAME = u'southparkstudios.com' - _VALID_URL = r'https?://www\.southparkstudios\.com/(clips|full-episodes)/(?P.+?)(\?|#|$)' + _VALID_URL = r'(https?://)?(www\.)?(?Psouthparkstudios\.com/(clips|full-episodes)/(?P.+?)(\?|#|$))' _FEED_URL = 'http://www.southparkstudios.com/feeds/video-player/mrss' @@ -31,6 +31,7 @@ class SouthParkStudiosIE(MTVIE): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) + url = u'http://www.' + mobj.group(u'url') video_id = mobj.group('id') webpage = self._download_webpage(url, video_id) mgid = self._search_regex(r'swfobject.embedSWF\(".*?(mgid:.*?)"', From 1672647ade97e070fa67eeff68370910ae715573 Mon Sep 17 00:00:00 2001 From: rzhxeo Date: Sun, 17 Nov 2013 17:43:58 +0100 Subject: [PATCH 049/425] [SouthParkStudiosIE] Move from _TEST to _TESTS --- youtube_dl/extractor/southparkstudios.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/southparkstudios.py b/youtube_dl/extractor/southparkstudios.py index bb0c4b393..a75e328a7 100644 --- a/youtube_dl/extractor/southparkstudios.py +++ b/youtube_dl/extractor/southparkstudios.py @@ -9,17 +9,15 @@ class SouthParkStudiosIE(MTVIE): _FEED_URL = 'http://www.southparkstudios.com/feeds/video-player/mrss' - _TEST = { + # Overwrite MTVIE properties we don't want + _TESTS = [{ u'url': u'http://www.southparkstudios.com/clips/104437/bat-daded#tab=featured', u'file': u'a7bff6c2-ed00-11e0-aca6-0026b9414f30.mp4', u'info_dict': { u'title': u'Bat Daded', u'description': u'Randy disqualifies South Park by getting into a fight with Bat Dad.', }, - } - - # Overwrite MTVIE properties we don't want - _TESTS = [] + }] def _get_thumbnail_url(self, uri, itemdoc): search_path = '%s/%s' % (_media_xml_tag('group'), _media_xml_tag('thumbnail')) From 746f491f82cfddd6fafacbf9978205963d7a214d Mon Sep 17 00:00:00 2001 From: rzhxeo Date: Sun, 17 Nov 2013 17:54:47 +0100 Subject: [PATCH 050/425] Add support for southpark.de --- youtube_dl/extractor/__init__.py | 5 ++++- youtube_dl/extractor/southparkstudios.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index ee3173468..2d1e3cdfd 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -116,7 +116,10 @@ from .slashdot import SlashdotIE from .slideshare import SlideshareIE from .sohu import SohuIE from .soundcloud import SoundcloudIE, SoundcloudSetIE, SoundcloudUserIE -from .southparkstudios import SouthParkStudiosIE +from .southparkstudios import ( + SouthParkStudiosIE, + SouthparkDeIE, +) from .space import SpaceIE from .spankwire import SpankwireIE from .spiegel import SpiegelIE diff --git a/youtube_dl/extractor/southparkstudios.py b/youtube_dl/extractor/southparkstudios.py index a75e328a7..a711531e6 100644 --- a/youtube_dl/extractor/southparkstudios.py +++ b/youtube_dl/extractor/southparkstudios.py @@ -35,3 +35,17 @@ class SouthParkStudiosIE(MTVIE): mgid = self._search_regex(r'swfobject.embedSWF\(".*?(mgid:.*?)"', webpage, u'mgid') return self._get_videos_info(mgid) + +class SouthparkDeIE(SouthParkStudiosIE): + IE_NAME = u'southpark.de' + _VALID_URL = r'(https?://)?(www\.)?(?Psouthpark\.de/(clips|alle-episoden)/(?P.+?)(\?|#|$))' + _FEED_URL = 'http://www.southpark.de/feeds/video-player/mrss/' + + _TESTS = [{ + u'url': u'http://www.southpark.de/clips/uygssh/the-government-wont-respect-my-privacy#tab=featured', + u'file': u'85487c96-b3b9-4e39-9127-ad88583d9bf2.mp4', + u'info_dict': { + u'title': u'The Government Won\'t Respect My Privacy', + u'description': u'Cartman explains the benefits of "Shitter" to Stan, Kyle and Craig.', + }, + }] From bdde425cbe01329d8c24e18cf0492465abb21411 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 17 Nov 2013 21:05:14 +0100 Subject: [PATCH 051/425] Save and restore console title (Fixes #1782) --- youtube_dl/YoutubeDL.py | 19 ++++++++ youtube_dl/__init__.py | 97 +++++++++++++++++++++-------------------- 2 files changed, 68 insertions(+), 48 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 6e5ae44d3..4e28f9120 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -213,6 +213,25 @@ class YoutubeDL(object): elif 'TERM' in os.environ: self.to_screen('\033]0;%s\007' % message, skip_eol=True) + def save_console_title(self): + if not self.params.get('consoletitle', False): + return + if 'TERM' in os.environ: + self.to_screen('\033[22t') + + def restore_console_title(self): + if not self.params.get('consoletitle', False): + return + if 'TERM' in os.environ: + self.to_screen('\033[23t') + + def __enter__(self): + self.save_console_title() + return self + + def __exit__(self, *args): + self.restore_console_title() + def fixed_template(self): """Checks if the output template is fixed.""" return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 4dee487ab..af4c9c5c4 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -603,8 +603,7 @@ def _real_main(argv=None): u' file! Use "%%(ext)s" instead of %r' % determine_ext(outtmpl, u'')) - # YoutubeDL - ydl = YoutubeDL({ + ydl_opts = { 'usenetrc': opts.usenetrc, 'username': opts.username, 'password': opts.password, @@ -667,61 +666,63 @@ def _real_main(argv=None): 'youtube_print_sig_code': opts.youtube_print_sig_code, 'age_limit': opts.age_limit, 'download_archive': opts.download_archive, - }) + } - if opts.verbose: - write_string(u'[debug] youtube-dl version ' + __version__ + u'\n') - try: - sp = subprocess.Popen( - ['git', 'rev-parse', '--short', 'HEAD'], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - cwd=os.path.dirname(os.path.abspath(__file__))) - out, err = sp.communicate() - out = out.decode().strip() - if re.match('[0-9a-f]+', out): - write_string(u'[debug] Git HEAD: ' + out + u'\n') - except: + with YoutubeDL(ydl_opts) as ydl: + if opts.verbose: + write_string(u'[debug] youtube-dl version ' + __version__ + u'\n') try: - sys.exc_clear() + sp = subprocess.Popen( + ['git', 'rev-parse', '--short', 'HEAD'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=os.path.dirname(os.path.abspath(__file__))) + out, err = sp.communicate() + out = out.decode().strip() + if re.match('[0-9a-f]+', out): + write_string(u'[debug] Git HEAD: ' + out + u'\n') except: - pass - write_string(u'[debug] Python version %s - %s' %(platform.python_version(), platform_name()) + u'\n') + try: + sys.exc_clear() + except: + pass + write_string(u'[debug] Python version %s - %s' % + (platform.python_version(), platform_name()) + u'\n') - proxy_map = {} - for handler in opener.handlers: - if hasattr(handler, 'proxies'): - proxy_map.update(handler.proxies) - write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n') + proxy_map = {} + for handler in opener.handlers: + if hasattr(handler, 'proxies'): + proxy_map.update(handler.proxies) + write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n') - ydl.add_default_info_extractors() + ydl.add_default_info_extractors() - # PostProcessors - # Add the metadata pp first, the other pps will copy it - if opts.addmetadata: - ydl.add_post_processor(FFmpegMetadataPP()) - if opts.extractaudio: - ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites)) - if opts.recodevideo: - ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo)) - if opts.embedsubtitles: - ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat)) + # PostProcessors + # Add the metadata pp first, the other pps will copy it + if opts.addmetadata: + ydl.add_post_processor(FFmpegMetadataPP()) + if opts.extractaudio: + ydl.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, nopostoverwrites=opts.nopostoverwrites)) + if opts.recodevideo: + ydl.add_post_processor(FFmpegVideoConvertor(preferedformat=opts.recodevideo)) + if opts.embedsubtitles: + ydl.add_post_processor(FFmpegEmbedSubtitlePP(subtitlesformat=opts.subtitlesformat)) - # Update version - if opts.update_self: - update_self(ydl.to_screen, opts.verbose) + # Update version + if opts.update_self: + update_self(ydl.to_screen, opts.verbose) - # Maybe do nothing - if len(all_urls) < 1: - if not opts.update_self: - parser.error(u'you must provide at least one URL') - else: - sys.exit() + # Maybe do nothing + if len(all_urls) < 1: + if not opts.update_self: + parser.error(u'you must provide at least one URL') + else: + sys.exit() - try: - retcode = ydl.download(all_urls) - except MaxDownloadsReached: - ydl.to_screen(u'--max-download limit reached, aborting.') - retcode = 101 + try: + retcode = ydl.download(all_urls) + except MaxDownloadsReached: + ydl.to_screen(u'--max-download limit reached, aborting.') + retcode = 101 # Dump cookie jar if requested if opts.cookiefile is not None: From 749febf4d1513328ca8b9b6c16969d8c70ff4555 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 17 Nov 2013 21:12:50 +0100 Subject: [PATCH 052/425] Allow --console-title when --quiet is given (Fixes #1783) --- youtube_dl/YoutubeDL.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 4e28f9120..20eed96ca 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -211,19 +211,19 @@ class YoutubeDL(object): # already of type unicode() ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message)) elif 'TERM' in os.environ: - self.to_screen('\033]0;%s\007' % message, skip_eol=True) + write_string(u'\033]0;%s\007' % message, self._screen_file) def save_console_title(self): if not self.params.get('consoletitle', False): return if 'TERM' in os.environ: - self.to_screen('\033[22t') + write_string(u'\033[22t', self._screen_file) def restore_console_title(self): if not self.params.get('consoletitle', False): return if 'TERM' in os.environ: - self.to_screen('\033[23t') + write_string(u'\033[23t', self._screen_file) def __enter__(self): self.save_console_title() From 63b7b7224a9cbb33856d3e1f8755ba4bdd3a9d58 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 17 Nov 2013 22:11:39 +0100 Subject: [PATCH 053/425] [MTVIE] Try with RTMP URL if download fails This fixes youtube-dl http://www.southpark.de/clips/155251/cartman-vs-the-dog-whisperer --- youtube_dl/extractor/mtv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py index 24a79ae13..3df7f9b85 100644 --- a/youtube_dl/extractor/mtv.py +++ b/youtube_dl/extractor/mtv.py @@ -48,7 +48,7 @@ class MTVIE(InfoExtractor): def _transform_rtmp_url(rtmp_video_url): m = re.match(r'^rtmpe?://.*?/(?Pgsp\..+?/.*)$', rtmp_video_url) if not m: - raise ExtractorError(u'Cannot transform RTMP url') + return rtmp_video_url base = 'http://mtvnmobile.vo.llnwd.net/kip0/_pxn=1+_pxI0=Ripod-h264+_pxL0=undefined+_pxM0=+_pxK=18639+_pxE=mp4/44620/mtvnorigin/' return base + m.group('finalid') From 73c566695fac926e7e9e6922fe4e6d82c64a1850 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 17 Nov 2013 22:14:13 +0100 Subject: [PATCH 054/425] release 2013.11.17 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index b04238eb5..110058c79 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.15.1' +__version__ = '2013.11.17' From fccd377198e9be2ea95cc761d841704a5f4f5ec5 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 18 Nov 2013 13:05:18 +0100 Subject: [PATCH 055/425] Suppor embed-only videos (Fixes #1746) --- youtube_dl/extractor/youtube.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 1aa549740..514a11f7f 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -139,7 +139,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor): class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): IE_DESC = u'YouTube.com' - _VALID_URL = r"""^ + _VALID_URL = r"""(?xi)^ ( (?:https?://)? # http(s):// (optional) (?:(?:(?:(?:\w+\.)?youtube(?:-nocookie)?\.com/| @@ -363,6 +363,18 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): u"uploader_id": u"justintimberlakeVEVO" } }, + { + u"url": u"https://www.YouTube.com/watch?v=yZIXLfi8CZQ", + u"file": u"yZIXLfi8CZQ.mp4", + u"note": u"Embed-only video (#1746)", + u"info_dict": { + u"upload_date": u"20120608", + u"title": u"Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012", + u"description": u"md5:09b78bd971f1e3e289601dfba15ca4f7", + u"uploader": u"SET India", + u"uploader_id": u"setindia" + } + }, ] @@ -370,7 +382,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): def suitable(cls, url): """Receives a URL and returns True if suitable for this IE.""" if YoutubePlaylistIE.suitable(url): return False - return re.match(cls._VALID_URL, url, re.VERBOSE) is not None + return re.match(cls._VALID_URL, url) is not None def __init__(self, *args, **kwargs): super(YoutubeIE, self).__init__(*args, **kwargs) @@ -1272,7 +1284,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): # We simulate the access to the video from www.youtube.com/v/{video_id} # this can be viewed without login into Youtube data = compat_urllib_parse.urlencode({'video_id': video_id, - 'el': 'embedded', + 'el': 'player_embedded', 'gl': 'US', 'hl': 'en', 'eurl': 'https://youtube.googleapis.com/v/' + video_id, From 96b31b65331710ad6d3582abc7ecbaee0efa605a Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 18 Nov 2013 13:05:58 +0100 Subject: [PATCH 056/425] Add iPhone to UA (#1746) --- youtube_dl/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 1d9785341..659ac859d 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -176,7 +176,7 @@ def compat_ord(c): compiled_regex_type = type(re.compile('')) std_headers = { - 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0 (Chrome)', + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0 (Chrome) (iPhone)', 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Encoding': 'gzip, deflate', From 83aa529330245b9221ae99e6769a40531f4ba216 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 18 Nov 2013 13:18:17 +0100 Subject: [PATCH 057/425] Support protocol-independent URLs (#1787) --- youtube_dl/extractor/youtube.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 514a11f7f..9aa4c6fab 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -141,7 +141,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): IE_DESC = u'YouTube.com' _VALID_URL = r"""(?xi)^ ( - (?:https?://)? # http(s):// (optional) + (?:https?://|//)? # http(s):// or protocol-independent URL (optional) (?:(?:(?:(?:\w+\.)?youtube(?:-nocookie)?\.com/| tube\.majestyc\.net/| youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains @@ -364,7 +364,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): } }, { - u"url": u"https://www.YouTube.com/watch?v=yZIXLfi8CZQ", + u"url": u"//www.YouTube.com/watch?v=yZIXLfi8CZQ", u"file": u"yZIXLfi8CZQ.mp4", u"note": u"Embed-only video (#1746)", u"info_dict": { From 887c6acdf231e6846e0dbb63b644fc7dc3378700 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 18 Nov 2013 13:28:26 +0100 Subject: [PATCH 058/425] Support multiple embedded YouTube URLs (Fixes #1787) --- youtube_dl/extractor/generic.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index c7552fddb..e1d6a2a01 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -162,6 +162,16 @@ class GenericIE(InfoExtractor): raise ExtractorError(u'Failed to download URL: %s' % url) self.report_extraction(video_id) + + # it's tempting to parse this further, but you would + # have to take into account all the variations like + # Video Title - Site Name + # Site Name | Video Title + # Video Title - Tagline | Site Name + # and so on and so forth; it's just not practical + video_title = self._html_search_regex(r'(.*)', + webpage, u'video title', default=u'video', flags=re.DOTALL) + # Look for BrightCove: bc_url = BrightcoveIE._extract_brightcove_url(webpage) if bc_url is not None: @@ -177,11 +187,13 @@ class GenericIE(InfoExtractor): return self.url_result(surl, 'Vimeo') # Look for embedded YouTube player - mobj = re.search( - r']+?src=(["\'])(?Phttps?://(?:www\.)?youtube.com/embed/.+?)\1', webpage) - if mobj: - surl = unescapeHTML(mobj.group(u'url')) - return self.url_result(surl, 'Youtube') + matches = re.findall( + r']+?src=(["\'])(?P(?:https?:)?//(?:www\.)?youtube.com/embed/.+?)\1', webpage) + if matches: + urlrs = [self.url_result(unescapeHTML(tuppl[1]), 'Youtube') + for tuppl in matches] + return self.playlist_result( + urlrs, playlist_id=video_id, playlist_title=video_title) # Look for Bandcamp pages with custom domain mobj = re.search(r']*?content="(.*?bandcamp\.com.*?)"', webpage) @@ -226,15 +238,6 @@ class GenericIE(InfoExtractor): video_extension = os.path.splitext(video_id)[1][1:] video_id = os.path.splitext(video_id)[0] - # it's tempting to parse this further, but you would - # have to take into account all the variations like - # Video Title - Site Name - # Site Name | Video Title - # Video Title - Tagline | Site Name - # and so on and so forth; it's just not practical - video_title = self._html_search_regex(r'(.*)', - webpage, u'video title', default=u'video', flags=re.DOTALL) - # video uploader is domain name video_uploader = self._search_regex(r'(?:https?://)?([^/]*)/.*', url, u'video uploader') From a81b4d5c8fe32611fa79104510d422b208f2a1a2 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 18 Nov 2013 13:30:43 +0100 Subject: [PATCH 059/425] release 2013.11.18 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 110058c79..16e32abe4 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.17' +__version__ = '2013.11.18' From ae8f7871412ae9db40c5060d2d24e5a50f5fe9d0 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 18 Nov 2013 13:52:24 +0100 Subject: [PATCH 060/425] Remove iPhone from user agent. This breaks a lot of extractors In the future, it might be worth investigating whether we get better content when we claime to be an iPhone. --- youtube_dl/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 659ac859d..1d9785341 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -176,7 +176,7 @@ def compat_ord(c): compiled_regex_type = type(re.compile('')) std_headers = { - 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0 (Chrome) (iPhone)', + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0 (Chrome)', 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Encoding': 'gzip, deflate', From 9906d397a05de3c89fb0ba2d60c284c16cb72581 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 18 Nov 2013 13:56:45 +0100 Subject: [PATCH 061/425] [auengine] Simplify --- youtube_dl/extractor/auengine.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/youtube_dl/extractor/auengine.py b/youtube_dl/extractor/auengine.py index 0febbff4f..90dfa9a46 100644 --- a/youtube_dl/extractor/auengine.py +++ b/youtube_dl/extractor/auengine.py @@ -1,10 +1,10 @@ -import os.path import re from .common import InfoExtractor from ..utils import ( compat_urllib_parse, - compat_urllib_parse_urlparse, + determine_ext, + ExtractorError, ) class AUEngineIE(InfoExtractor): @@ -25,22 +25,26 @@ class AUEngineIE(InfoExtractor): title = self._html_search_regex(r'(?P<title>.+?)', webpage, u'title') title = title.strip() - links = re.findall(r'[^A-Za-z0-9]?(?:file|url):\s*["\'](http[^\'"&]*)', webpage) - links = [compat_urllib_parse.unquote(l) for l in links] + links = re.findall(r'\s(?:file|url):\s*["\']([^\'"]+)["\']', webpage) + links = map(compat_urllib_parse.unquote, links) + + thumbnail = None + video_url = None for link in links: - root, pathext = os.path.splitext(compat_urllib_parse_urlparse(link).path) - if pathext == '.png': + if link.endswith('.png'): thumbnail = link - elif pathext == '.mp4': - url = link - ext = pathext + elif '/videos/' in link: + video_url = link + if not video_url: + raise ExtractorError(u'Could not find video URL') + ext = u'.' + determine_ext(video_url) if ext == title[-len(ext):]: title = title[:-len(ext)] - ext = ext[1:] - return [{ + + return { 'id': video_id, - 'url': url, + 'url': video_url, 'ext': ext, 'title': title, 'thumbnail': thumbnail, - }] + } From 9a942a46712330173683f8a0179868ae0c5a9138 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 18 Nov 2013 13:56:53 +0100 Subject: [PATCH 062/425] release 2013.11.18.1 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 16e32abe4..6b3bf39fa 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.18' +__version__ = '2013.11.18.1' From 4113e6ab561e6f2f014fb2548d1c8691352b8304 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 18 Nov 2013 14:36:01 +0100 Subject: [PATCH 063/425] [auengine] Do not return unnecessary ext --- youtube_dl/extractor/auengine.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/extractor/auengine.py b/youtube_dl/extractor/auengine.py index 90dfa9a46..95c038003 100644 --- a/youtube_dl/extractor/auengine.py +++ b/youtube_dl/extractor/auengine.py @@ -44,7 +44,6 @@ class AUEngineIE(InfoExtractor): return { 'id': video_id, 'url': video_url, - 'ext': ext, 'title': title, 'thumbnail': thumbnail, } From efd6c574a2cabc860f54018af726cd291cbec868 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Mon, 18 Nov 2013 16:35:41 +0100 Subject: [PATCH 064/425] Correctly write and restore the console title on the stack (fixes #1782) --- youtube_dl/YoutubeDL.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 20eed96ca..fd5a30a0c 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -217,13 +217,15 @@ class YoutubeDL(object): if not self.params.get('consoletitle', False): return if 'TERM' in os.environ: - write_string(u'\033[22t', self._screen_file) + # Save the title on stack + write_string(u'\033[22;0t', self._screen_file) def restore_console_title(self): if not self.params.get('consoletitle', False): return if 'TERM' in os.environ: - write_string(u'\033[23t', self._screen_file) + # Restore the title from stack + write_string(u'\033[23;0t', self._screen_file) def __enter__(self): self.save_console_title() From cb7dfeeac407ca973b7d838dd3c8f6faa0a65bac Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 18 Nov 2013 16:42:35 +0100 Subject: [PATCH 065/425] [youtube] only allow domain name to be upper-case (#1786) --- youtube_dl/extractor/youtube.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 9aa4c6fab..41838237c 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -139,10 +139,10 @@ class YoutubeBaseInfoExtractor(InfoExtractor): class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): IE_DESC = u'YouTube.com' - _VALID_URL = r"""(?xi)^ + _VALID_URL = r"""(?x)^ ( (?:https?://|//)? # http(s):// or protocol-independent URL (optional) - (?:(?:(?:(?:\w+\.)?youtube(?:-nocookie)?\.com/| + (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/| tube\.majestyc\.net/| youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains (?:.*?\#/)? # handle anchor (#/) redirect urls From 34b3afc7be059cbc882b901b897ec99eeee25e14 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Tue, 19 Nov 2013 12:41:01 +0100 Subject: [PATCH 066/425] release 2013.11.19 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 6b3bf39fa..e9ff3f640 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.18.1' +__version__ = '2013.11.19' From 69545c2affb6b126398fd657a12a560a9857dbee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 19 Nov 2013 20:43:49 +0100 Subject: [PATCH 067/425] [d8] inherit from CanalplusIE it reuses the same extraction process --- youtube_dl/extractor/canalplus.py | 3 ++- youtube_dl/extractor/d8.py | 36 +++---------------------------- 2 files changed, 5 insertions(+), 34 deletions(-) diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py index 1db9b24cf..bfa2a8b40 100644 --- a/youtube_dl/extractor/canalplus.py +++ b/youtube_dl/extractor/canalplus.py @@ -5,6 +5,7 @@ import xml.etree.ElementTree from .common import InfoExtractor from ..utils import unified_strdate + class CanalplusIE(InfoExtractor): _VALID_URL = r'https?://(www\.canalplus\.fr/.*?/(?P.*)|player\.canalplus\.fr/#/(?P\d+))' _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/cplus/%s' @@ -25,7 +26,7 @@ class CanalplusIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('id') + video_id = mobj.groupdict().get('id') if video_id is None: webpage = self._download_webpage(url, mobj.group('path')) video_id = self._search_regex(r'videoId = "(\d+)";', webpage, u'video id') diff --git a/youtube_dl/extractor/d8.py b/youtube_dl/extractor/d8.py index 5ce483b16..a56842b16 100644 --- a/youtube_dl/extractor/d8.py +++ b/youtube_dl/extractor/d8.py @@ -1,11 +1,8 @@ # encoding: utf-8 -import re -import xml.etree.ElementTree +from .canalplus import CanalplusIE -from .common import InfoExtractor -from ..utils import unified_strdate -class D8IE(InfoExtractor): +class D8IE(CanalplusIE): _VALID_URL = r'https?://www\.d8\.tv/.*?/(?P.*)' _VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/d8/%s' IE_NAME = u'd8.tv' @@ -19,34 +16,7 @@ class D8IE(InfoExtractor): u'upload_date': u'20131108', }, u'params': { + # rtmp u'skip_download': True, }, } - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - webpage = self._download_webpage(url, mobj.group('path')) - video_id = self._search_regex(r'videoId = "(\d+)";', webpage, u'video id') - info_url = self._VIDEO_INFO_TEMPLATE % video_id - info_page = self._download_webpage(info_url,video_id, - u'Downloading video info') - - self.report_extraction(video_id) - doc = xml.etree.ElementTree.fromstring(info_page.encode('utf-8')) - video_info = [video for video in doc if video.find('ID').text == video_id][0] - infos = video_info.find('INFOS') - media = video_info.find('MEDIA') - formats = [media.find('VIDEOS/%s' % format) - for format in ['BAS_DEBIT', 'HAUT_DEBIT', 'HD']] - video_url = [format.text for format in formats if format is not None][-1] - - return {'id': video_id, - 'title': u'%s - %s' % (infos.find('TITRAGE/TITRE').text, - infos.find('TITRAGE/SOUS_TITRE').text), - 'url': video_url, - 'ext': 'flv', - 'upload_date': unified_strdate(infos.find('PUBLICATION/DATE').text), - 'thumbnail': media.find('IMAGES/GRAND').text, - 'description': infos.find('DESCRIPTION').text, - 'view_count': int(infos.find('NB_VUES').text), - } From 59040888112e7e5a22ef17bacc27ca88d58a5d92 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 20 Nov 2013 06:13:19 +0100 Subject: [PATCH 068/425] Add support for tou.tv (Fixes #1792) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/common.py | 28 ++++++++++++ youtube_dl/extractor/toutv.py | 75 ++++++++++++++++++++++++++++++++ youtube_dl/utils.py | 2 + 4 files changed, 106 insertions(+) create mode 100644 youtube_dl/extractor/toutv.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 2679d1a8f..b0df1cef7 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -133,6 +133,7 @@ from .techtalks import TechTalksIE from .ted import TEDIE from .tf1 import TF1IE from .thisav import ThisAVIE +from .toutv import TouTvIE from .traileraddict import TrailerAddictIE from .trilulilu import TriluliluIE from .tube8 import Tube8IE diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index f787d0a3c..eb3435c77 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -350,6 +350,17 @@ class InfoExtractor(object): if secure: regexes = self._og_regexes('video:secure_url') + regexes return self._html_search_regex(regexes, html, name, **kargs) + def _html_search_meta(self, name, html, display_name=None): + if display_name is None: + display_name = name + return self._html_search_regex( + r'''(?ix)]+(?:name|property)=["\']%s["\']) + [^>]+content=["\']([^"\']+)["\']''' % re.escape(name), + html, display_name, fatal=False) + + def _dc_search_uploader(self, html): + return self._html_search_meta('dc.creator', html, 'uploader') + def _rta_search(self, html): # See http://www.rtalabel.org/index.php?content=howtofaq#single if re.search(r'(?ix)[a-zA-Z0-9_-]+(?:/(?PS[0-9]+E[0-9]+)))' + + _TEST = { + u'url': u'http://www.tou.tv/30-vies/S04E41', + u'file': u'30-vies_S04E41.mp4', + u'info_dict': { + u'title': u'30 vies Saison 4 / Épisode 41', + u'description': u'md5:da363002db82ccbe4dafeb9cab039b09', + u'age_limit': 8, + u'uploader': u'Groupe des Nouveaux Médias', + u'duration': 1296, + u'upload_date': u'20131118', + u'thumbnail': u'http://static.tou.tv/medias/images/2013-11-18_19_00_00_30VIES_0341_01_L.jpeg', + }, + u'params': { + u'skip_download': True, # Requires rtmpdump + }, + u'xskip': 'Only available in Canada' + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + webpage = self._download_webpage(url, video_id) + + mediaId = self._search_regex( + r'"idMedia":\s*"([^"]+)"', webpage, u'media ID') + + # TODO test from de + streams_url = u'http://release.theplatform.com/content.select?pid=' + mediaId + streams_webpage = self._download_webpage( + streams_url, video_id, note=u'Downloading stream list') + + streams_doc = xml.etree.ElementTree.fromstring( + streams_webpage.encode('utf-8')) + video_url = next(n.text + for n in streams_doc.findall('.//choice/url') + if u'//ad.doubleclick' not in n.text) + if video_url.endswith('/Unavailable.flv'): + raise ExtractorError( + u'Access to this video is blocked from outside of Canada', + expected=True) + + duration_str = self._html_search_meta( + 'video:duration', webpage, u'duration') + duration = int(duration_str) if duration_str else None + upload_date_str = self._html_search_meta( + 'video:release_date', webpage, u'upload date') + upload_date = unified_strdate(upload_date_str) if upload_date_str else None + + return { + 'id': video_id, + 'title': self._og_search_title(webpage), + 'url': video_url, + 'description': self._og_search_description(webpage), + 'uploader': self._dc_search_uploader(webpage), + 'thumbnail': self._og_search_thumbnail(webpage), + 'age_limit': self._media_rating_search(webpage), + 'duration': duration, + 'upload_date': upload_date, + 'ext': 'mp4', + } diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 1d9785341..b50c8166f 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -734,6 +734,8 @@ def unified_strdate(date_str): '%Y/%m/%d %H:%M:%S', '%d.%m.%Y %H:%M', '%Y-%m-%dT%H:%M:%SZ', + '%Y-%m-%dT%H:%M:%S.%fZ', + '%Y-%m-%dT%H:%M:%S.%f0Z', '%Y-%m-%dT%H:%M:%S', ] for expression in format_expressions: From 9d1538182fdc540fa79ec8341385a980cde334e7 Mon Sep 17 00:00:00 2001 From: Mohamedh Fazal Date: Tue, 19 Nov 2013 18:59:22 +0500 Subject: [PATCH 069/425] Add an option to dump json information --- youtube_dl/YoutubeDL.py | 3 +++ youtube_dl/__init__.py | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index fd5a30a0c..d8d55d7d7 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -84,6 +84,7 @@ class YoutubeDL(object): forcethumbnail: Force printing thumbnail URL. forcedescription: Force printing description. forcefilename: Force printing final filename. + forcejson: Force printing json information. simulate: Do not download the video files. format: Video format code. format_limit: Highest quality format to try. @@ -651,6 +652,8 @@ class YoutubeDL(object): compat_print(filename) if self.params.get('forceformat', False): compat_print(info_dict['format']) + if self.params.get('forcejson', False): + compat_print(json.dumps(info_dict)) # Do nothing else if in simulate mode if self.params.get('simulate', False): diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index af4c9c5c4..7ab99bdd1 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -306,6 +306,9 @@ def parseOpts(overrideArguments=None): verbosity.add_option('--get-format', action='store_true', dest='getformat', help='simulate, quiet but print output format', default=False) + verbosity.add_option('-j', '--dump-json', + action='store_true', dest='dumpjson', + help='simulate, quiet but print json information', default=False) verbosity.add_option('--newline', action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False) verbosity.add_option('--no-progress', @@ -608,7 +611,7 @@ def _real_main(argv=None): 'username': opts.username, 'password': opts.password, 'videopassword': opts.videopassword, - 'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat), + 'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.dumpjson), 'forceurl': opts.geturl, 'forcetitle': opts.gettitle, 'forceid': opts.getid, @@ -616,8 +619,9 @@ def _real_main(argv=None): 'forcedescription': opts.getdescription, 'forcefilename': opts.getfilename, 'forceformat': opts.getformat, + 'forcejson': opts.dumpjson, 'simulate': opts.simulate, - 'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat), + 'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.dumpjson), 'format': opts.format, 'format_limit': opts.format_limit, 'listformats': opts.listformats, From 8694c60000d2aed2b37e10da0ffda132249ff10a Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 20 Nov 2013 06:18:24 +0100 Subject: [PATCH 070/425] import json for --dump-json --- youtube_dl/YoutubeDL.py | 3 ++- youtube_dl/__init__.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index d8d55d7d7..bd093add8 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -5,6 +5,7 @@ from __future__ import absolute_import import errno import io +import json import os import re import shutil @@ -84,7 +85,7 @@ class YoutubeDL(object): forcethumbnail: Force printing thumbnail URL. forcedescription: Force printing description. forcefilename: Force printing final filename. - forcejson: Force printing json information. + forcejson: Force printing info_dict as JSON. simulate: Do not download the video files. format: Video format code. format_limit: Highest quality format to try. diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 7ab99bdd1..e27bd4544 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -308,7 +308,7 @@ def parseOpts(overrideArguments=None): help='simulate, quiet but print output format', default=False) verbosity.add_option('-j', '--dump-json', action='store_true', dest='dumpjson', - help='simulate, quiet but print json information', default=False) + help='simulate, quiet but print JSON information', default=False) verbosity.add_option('--newline', action='store_true', dest='progress_with_newline', help='output progress bar as new lines', default=False) verbosity.add_option('--no-progress', From cc13cc0251b287ac88883096fcc36a8072e7ab5d Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 20 Nov 2013 06:25:33 +0100 Subject: [PATCH 071/425] [teamcoco] Correct error --- youtube_dl/extractor/teamcoco.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py index bc48620f0..165d9f88b 100644 --- a/youtube_dl/extractor/teamcoco.py +++ b/youtube_dl/extractor/teamcoco.py @@ -60,7 +60,7 @@ class TeamcocoIE(InfoExtractor): return -1 formats.sort(key=sort_key) if not formats: - raise RegexNotFoundError(u'Unable to extract video URL') + raise ExtractorError(u'Unable to extract video URL') return { 'id': video_id, From f3682997d7a709e7cf74ba83773570d2a99beb12 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 20 Nov 2013 06:27:48 +0100 Subject: [PATCH 072/425] Clean up unused imports and other minor mistakes --- youtube_dl/extractor/collegehumor.py | 4 +--- youtube_dl/extractor/eighttracks.py | 1 - youtube_dl/extractor/facebook.py | 1 - youtube_dl/extractor/fktv.py | 1 - youtube_dl/extractor/gamespot.py | 2 +- youtube_dl/extractor/jeuxvideo.py | 2 +- youtube_dl/extractor/livestream.py | 2 -- youtube_dl/extractor/mtv.py | 1 - youtube_dl/extractor/pornhub.py | 1 - youtube_dl/extractor/soundcloud.py | 1 - youtube_dl/extractor/spankwire.py | 1 - youtube_dl/extractor/spiegel.py | 1 - youtube_dl/extractor/ted.py | 3 +-- youtube_dl/extractor/tube8.py | 2 -- youtube_dl/extractor/xtube.py | 1 - youtube_dl/extractor/zdf.py | 2 +- 16 files changed, 5 insertions(+), 21 deletions(-) diff --git a/youtube_dl/extractor/collegehumor.py b/youtube_dl/extractor/collegehumor.py index 8d4c93d6d..0c29acfb1 100644 --- a/youtube_dl/extractor/collegehumor.py +++ b/youtube_dl/extractor/collegehumor.py @@ -71,10 +71,8 @@ class CollegeHumorIE(InfoExtractor): adoc = xml.etree.ElementTree.fromstring(manifestXml) try: - media_node = adoc.findall('./{http://ns.adobe.com/f4m/1.0}media')[0] - node_id = media_node.attrib['url'] video_id = adoc.findall('./{http://ns.adobe.com/f4m/1.0}id')[0].text - except IndexError as err: + except IndexError: raise ExtractorError(u'Invalid manifest file') url_pr = compat_urllib_parse_urlparse(info['thumbnail']) info['url'] = url_pr.scheme + '://' + url_pr.netloc + video_id[:-2].replace('.csmil','').replace(',','') diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py index 2cfbcd363..f21ef8853 100644 --- a/youtube_dl/extractor/eighttracks.py +++ b/youtube_dl/extractor/eighttracks.py @@ -1,4 +1,3 @@ -import itertools import json import random import re diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py index f8bdfc2d3..3b210710e 100644 --- a/youtube_dl/extractor/facebook.py +++ b/youtube_dl/extractor/facebook.py @@ -1,5 +1,4 @@ import json -import netrc import re import socket diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py index 9c89362ef..dba1a8dc2 100644 --- a/youtube_dl/extractor/fktv.py +++ b/youtube_dl/extractor/fktv.py @@ -39,7 +39,6 @@ class FKTVIE(InfoExtractor): for i, _ in enumerate(files, 1): video_id = '%04d%d' % (episode, i) video_url = 'http://dl%d.fernsehkritik.tv/fernsehkritik%d%s.flv' % (server, episode, '' if i == 1 else '-%d' % i) - video_title = 'Fernsehkritik %d.%d' % (episode, i) videos.append({ 'id': video_id, 'url': video_url, diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py index 098768361..9645b00c3 100644 --- a/youtube_dl/extractor/gamespot.py +++ b/youtube_dl/extractor/gamespot.py @@ -24,7 +24,7 @@ class GameSpotIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) - page_id = video_id = mobj.group('page_id') + page_id = mobj.group('page_id') webpage = self._download_webpage(url, page_id) data_video_json = self._search_regex(r'data-video=\'(.*?)\'', webpage, u'data video') data_video = json.loads(unescapeHTML(data_video_json)) diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py index 6bb54b932..0020c47cf 100644 --- a/youtube_dl/extractor/jeuxvideo.py +++ b/youtube_dl/extractor/jeuxvideo.py @@ -22,7 +22,7 @@ class JeuxVideoIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) - title = re.match(self._VALID_URL, url).group(1) + title = mobj.group(1) webpage = self._download_webpage(url, title) xml_link = self._html_search_regex( r'', diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py index 1a3e0ae6b..5f548437c 100644 --- a/youtube_dl/extractor/livestream.py +++ b/youtube_dl/extractor/livestream.py @@ -6,9 +6,7 @@ from .common import InfoExtractor from ..utils import ( compat_urllib_parse_urlparse, compat_urlparse, - get_meta_content, xpath_with_ns, - ExtractorError, ) diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py index 3df7f9b85..04afd6c4c 100644 --- a/youtube_dl/extractor/mtv.py +++ b/youtube_dl/extractor/mtv.py @@ -59,7 +59,6 @@ class MTVIE(InfoExtractor): if '/error_country_block.swf' in metadataXml: raise ExtractorError(u'This video is not available from your country.', expected=True) mdoc = xml.etree.ElementTree.fromstring(metadataXml.encode('utf-8')) - renditions = mdoc.findall('.//rendition') formats = [] for rendition in mdoc.findall('.//rendition'): diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py index 75cf4bb9f..8b3471919 100644 --- a/youtube_dl/extractor/pornhub.py +++ b/youtube_dl/extractor/pornhub.py @@ -6,7 +6,6 @@ from ..utils import ( compat_urllib_parse_urlparse, compat_urllib_request, compat_urllib_parse, - unescapeHTML, ) from ..aes import ( aes_decrypt_text diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 83e1f055f..687457e10 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -158,7 +158,6 @@ class SoundcloudSetIE(SoundcloudIE): resolv_url = self._resolv_url(url) info_json = self._download_webpage(resolv_url, full_title) - videos = [] info = json.loads(info_json) if 'errors' in info: for err in info['errors']: diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py index 97f9c268a..794550c81 100644 --- a/youtube_dl/extractor/spankwire.py +++ b/youtube_dl/extractor/spankwire.py @@ -6,7 +6,6 @@ from ..utils import ( compat_urllib_parse_urlparse, compat_urllib_request, compat_urllib_parse, - unescapeHTML, ) from ..aes import ( aes_decrypt_text diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py index 6dc2eda6d..19ce585cf 100644 --- a/youtube_dl/extractor/spiegel.py +++ b/youtube_dl/extractor/spiegel.py @@ -2,7 +2,6 @@ import re import xml.etree.ElementTree from .common import InfoExtractor -from ..utils import determine_ext class SpiegelIE(InfoExtractor): diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py index 2e497c86e..4bca62ba0 100644 --- a/youtube_dl/extractor/ted.py +++ b/youtube_dl/extractor/ted.py @@ -4,7 +4,6 @@ import re from .subtitles import SubtitlesInfoExtractor from ..utils import ( - compat_str, RegexNotFoundError, ) @@ -113,6 +112,6 @@ class TEDIE(SubtitlesInfoExtractor): url = 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/srt' % (video_id, l) sub_lang_list[l] = url return sub_lang_list - except RegexNotFoundError as err: + except RegexNotFoundError: self._downloader.report_warning(u'video doesn\'t have subtitles') return {} diff --git a/youtube_dl/extractor/tube8.py b/youtube_dl/extractor/tube8.py index d4b7603c7..4d9d41db3 100644 --- a/youtube_dl/extractor/tube8.py +++ b/youtube_dl/extractor/tube8.py @@ -5,8 +5,6 @@ from .common import InfoExtractor from ..utils import ( compat_urllib_parse_urlparse, compat_urllib_request, - compat_urllib_parse, - unescapeHTML, ) from ..aes import ( aes_decrypt_text diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py index 03ad88bed..e3458d2bd 100644 --- a/youtube_dl/extractor/xtube.py +++ b/youtube_dl/extractor/xtube.py @@ -5,7 +5,6 @@ from .common import InfoExtractor from ..utils import ( compat_urllib_parse_urlparse, compat_urllib_request, - compat_urllib_parse, ) class XTubeIE(InfoExtractor): diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py index faed7ff7f..c6a9d06f2 100644 --- a/youtube_dl/extractor/zdf.py +++ b/youtube_dl/extractor/zdf.py @@ -53,7 +53,7 @@ class ZDFIE(InfoExtractor): video_id, u'Get stream URL') - MMS_STREAM = r'href="(?Pmms://[^"]*)"' + #MMS_STREAM = r'href="(?Pmms://[^"]*)"' RTSP_STREAM = r'(?Prtsp://[^"]*.mp4)' mobj = re.search(self._MEDIA_STREAM, media_link) From 4eb92208a3bf05d0860e1f138380e8b5cae20c14 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 20 Nov 2013 06:34:48 +0100 Subject: [PATCH 073/425] Adapt test to changed .info.json name --- test/test_download.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/test_download.py b/test/test_download.py index 16f200809..fe7f7b8cb 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -103,7 +103,7 @@ def generator(test_case): tc_filename = get_tc_filename(tc) try_rm(tc_filename) try_rm(tc_filename + '.part') - try_rm(tc_filename + '.info.json') + try_rm(os.path.splitext(tc_filename)[0] + '.info.json') try_rm_tcs_files() try: try_num = 1 @@ -130,11 +130,12 @@ def generator(test_case): if not test_case.get('params', {}).get('skip_download', False): self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename) self.assertTrue(tc_filename in finished_hook_called) - self.assertTrue(os.path.exists(tc_filename + '.info.json')) + info_json_fn = os.path.splitext(tc_filename)[0] + '.info.json' + self.assertTrue(os.path.exists(info_json_fn)) if 'md5' in tc: md5_for_file = _file_md5(tc_filename) self.assertEqual(md5_for_file, tc['md5']) - with io.open(tc_filename + '.info.json', encoding='utf-8') as infof: + with io.open(info_json_fn, encoding='utf-8') as infof: info_dict = json.load(infof) for (info_field, expected) in tc.get('info_dict', {}).items(): if isinstance(expected, compat_str) and expected.startswith('md5:'): From 8f053519846d8758de4e3ad67960e4161407d334 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 20 Nov 2013 06:35:02 +0100 Subject: [PATCH 074/425] [anitube] Minor fixes (#1776) --- youtube_dl/extractor/anitube.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/youtube_dl/extractor/anitube.py b/youtube_dl/extractor/anitube.py index 2954966a6..679027f4e 100644 --- a/youtube_dl/extractor/anitube.py +++ b/youtube_dl/extractor/anitube.py @@ -6,11 +6,12 @@ from .common import InfoExtractor class AnitubeIE(InfoExtractor): IE_NAME = u'anitube.se' - _VALID_URL = r'http?://(?:www\.)?anitube\.se/video/(?P\d+)' + _VALID_URL = r'https?://(?:www\.)?anitube\.se/video/(?P\d+)' _TEST = { u'url': u'http://www.anitube.se/video/36621', - u'md5': u'0c4e4f1051bf50f5982f829f7230f539', + u'md5': u'59d0eeae28ea0bc8c05e7af429998d43', + u'file': u'36621.mp4', u'info_dict': { u'id': u'36621', u'ext': u'mp4', @@ -23,27 +24,22 @@ class AnitubeIE(InfoExtractor): video_id = mobj.group('id') webpage = self._download_webpage(url, video_id) - key = self._html_search_regex(r'http://www\.anitube\.se/embed/([A-Za-z0-9_-]*)', webpage, u'key') webpage_config = self._download_webpage('http://www.anitube.se/nuevo/econfig.php?key=%s' % key, key) - config_xml = xml.etree.ElementTree.fromstring(webpage_config.encode('utf-8')) video_title = config_xml.find('title').text - formats = [] - video_url = config_xml.find('file') if video_url is not None: formats.append({ 'format_id': 'sd', 'url': video_url.text, }) - video_url = config_xml.find('filehd') if video_url is not None: formats.append({ @@ -54,6 +50,5 @@ class AnitubeIE(InfoExtractor): return { 'id': video_id, 'title': video_title, - 'ext': 'mp4', 'formats': formats } From 0a120f74b2556f3569d285993c1a0f3dc654f07a Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 20 Nov 2013 06:36:00 +0100 Subject: [PATCH 075/425] Credit @diffycat for anitube --- youtube_dl/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index e27bd4544..64ebf4d48 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -34,6 +34,7 @@ __authors__ = ( 'Andras Elso', 'Jelle van der Waa', 'Marcin Cieślak', + 'Anton Larionov', ) __license__ = 'Public Domain' From 100959a6d9fee68d2876e112f4be260ade756c40 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 20 Nov 2013 06:47:50 +0100 Subject: [PATCH 076/425] [escapist] Add support for HD format (Closes #1755) --- youtube_dl/extractor/escapist.py | 57 +++++++++++++++++++------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py index 3aa2da52c..f9681fd43 100644 --- a/youtube_dl/extractor/escapist.py +++ b/youtube_dl/extractor/escapist.py @@ -11,11 +11,11 @@ from ..utils import ( class EscapistIE(InfoExtractor): - _VALID_URL = r'^(https?://)?(www\.)?escapistmagazine\.com/videos/view/(?P[^/]+)/(?P[^/?]+)[/?]?.*$' + _VALID_URL = r'^https?://?(www\.)?escapistmagazine\.com/videos/view/(?P[^/]+)/(?P[^/?]+)[/?]?.*$' _TEST = { u'url': u'http://www.escapistmagazine.com/videos/view/the-escapist-presents/6618-Breaking-Down-Baldurs-Gate', u'file': u'6618-Breaking-Down-Baldurs-Gate.mp4', - u'md5': u'c6793dbda81388f4264c1ba18684a74d', + u'md5': u'ab3a706c681efca53f0a35f1415cf0d1', u'info_dict': { u"description": u"Baldur's Gate: Original, Modded or Enhanced Edition? I'll break down what you can expect from the new Baldur's Gate: Enhanced Edition.", u"uploader": u"the-escapist-presents", @@ -25,50 +25,61 @@ class EscapistIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) showName = mobj.group('showname') videoId = mobj.group('episode') self.report_extraction(videoId) webpage = self._download_webpage(url, videoId) - videoDesc = self._html_search_regex(' Date: Wed, 20 Nov 2013 06:55:07 +0100 Subject: [PATCH 077/425] [escapist] Fix syntax error --- youtube_dl/extractor/escapist.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py index f9681fd43..03ebb7c48 100644 --- a/youtube_dl/extractor/escapist.py +++ b/youtube_dl/extractor/escapist.py @@ -66,13 +66,12 @@ class EscapistIE(InfoExtractor): }) _add_format(u'normal', configUrl) + hq_url = (configUrl + + ('&hq=1' if '?' in configUrl else configUrl + '?hq=1')) try: - hq_url = (configUrl + - ('&hq=1' if '?' in configUrl else configUrl + '?hq=1')) - try: - _add_format(u'hq', hq_url) - except ExtractorError: - pass # That's fine, we'll just use normal quality + _add_format(u'hq', hq_url) + except ExtractorError: + pass # That's fine, we'll just use normal quality return { 'id': videoId, From c5edcde21f772c6c51bea79edc4ac48f5e5dd8d3 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 20 Nov 2013 06:56:59 +0100 Subject: [PATCH 078/425] [escapist] upper-case URL --- youtube_dl/extractor/escapist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py index 03ebb7c48..2396f163b 100644 --- a/youtube_dl/extractor/escapist.py +++ b/youtube_dl/extractor/escapist.py @@ -35,13 +35,13 @@ class EscapistIE(InfoExtractor): r' Date: Wed, 20 Nov 2013 07:23:23 +0100 Subject: [PATCH 079/425] [escapist] Fix title search --- youtube_dl/extractor/escapist.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py index 2396f163b..b1242f6bc 100644 --- a/youtube_dl/extractor/escapist.py +++ b/youtube_dl/extractor/escapist.py @@ -35,11 +35,11 @@ class EscapistIE(InfoExtractor): r' Date: Wed, 20 Nov 2013 07:25:17 +0100 Subject: [PATCH 080/425] release 2013.11.20 --- README.md | 1 + youtube_dl/version.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6632e5865..580b16004 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,7 @@ which means you can modify it, redistribute it or use it however you like. --get-description simulate, quiet but print video description --get-filename simulate, quiet but print output filename --get-format simulate, quiet but print output format + -j, --dump-json simulate, quiet but print JSON information --newline output progress bar as new lines --no-progress do not print progress bar --console-title display progress in console titlebar diff --git a/youtube_dl/version.py b/youtube_dl/version.py index e9ff3f640..ad3fad818 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.19' +__version__ = '2013.11.20' From f99e0f1ed62e6210a52f1a4bddb860a5b09623ee Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 20 Nov 2013 07:37:07 +0100 Subject: [PATCH 081/425] Adapt age restriction tests to new .info.json filenames --- test/test_age_restriction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_age_restriction.py b/test/test_age_restriction.py index d500c6edc..506572e9e 100644 --- a/test/test_age_restriction.py +++ b/test/test_age_restriction.py @@ -24,7 +24,7 @@ def _download_restricted(url, filename, age): } ydl = YoutubeDL(params) ydl.add_default_info_extractors() - json_filename = filename + '.info.json' + json_filename = os.path.splitext(filename)[0] + '.info.json' try_rm(json_filename) ydl.download([url]) res = os.path.exists(json_filename) From 9a98a466b34206f61543b3efdbcba40d58e51052 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 20 Nov 2013 07:37:14 +0100 Subject: [PATCH 082/425] [toutv] really skip test --- youtube_dl/extractor/toutv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py index 73ea67da9..732083617 100644 --- a/youtube_dl/extractor/toutv.py +++ b/youtube_dl/extractor/toutv.py @@ -28,7 +28,7 @@ class TouTvIE(InfoExtractor): u'params': { u'skip_download': True, # Requires rtmpdump }, - u'xskip': 'Only available in Canada' + u'skip': 'Only available in Canada' } def _real_extract(self, url): From c4864091a1f867c4ce74f459c31ac85d96c417ca Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 20 Nov 2013 07:43:21 +0100 Subject: [PATCH 083/425] [videopremium] Support new crazy redirect scheme --- youtube_dl/extractor/videopremium.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/videopremium.py b/youtube_dl/extractor/videopremium.py index 65f39b982..4800415bd 100644 --- a/youtube_dl/extractor/videopremium.py +++ b/youtube_dl/extractor/videopremium.py @@ -24,12 +24,16 @@ class VideoPremiumIE(InfoExtractor): webpage_url = 'http://videopremium.tv/' + video_id webpage = self._download_webpage(webpage_url, video_id) - self.report_extraction(video_id) + if re.match(r"^]*>window.location\s*=", webpage): + # Download again, we need a cookie + webpage = self._download_webpage( + webpage_url, video_id, + note=u'Downloading webpage again (with cookie)') - video_title = self._html_search_regex(r'\s*(.+?)\s*<', - webpage, u'video title') + video_title = self._html_search_regex( + r'\s*(.+?)\s*<', webpage, u'video title') - return [{ + return { 'id': video_id, 'url': "rtmp://e%d.md.iplay.md/play" % random.randint(1, 16), 'play_path': "mp4:%s.f4v" % video_id, @@ -37,4 +41,4 @@ class VideoPremiumIE(InfoExtractor): 'player_url': "http://videopremium.tv/uplayer/uppod.swf", 'ext': 'f4v', 'title': video_title, - }] + } \ No newline at end of file From 0ad97bbc056c83d49bc771ac202a9eb8354686de Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 20 Nov 2013 07:45:32 +0100 Subject: [PATCH 084/425] [spankwire] fix check for description --- youtube_dl/extractor/spankwire.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py index 794550c81..995b46444 100644 --- a/youtube_dl/extractor/spankwire.py +++ b/youtube_dl/extractor/spankwire.py @@ -35,11 +35,12 @@ class SpankwireIE(InfoExtractor): webpage = self._download_webpage(req, video_id) video_title = self._html_search_regex(r'

([^<]+)', webpage, u'title') - video_uploader = self._html_search_regex(r'by:\s*]*>(.+?)', webpage, u'uploader', fatal=False) - thumbnail = self._html_search_regex(r'flashvars\.image_url = "([^"]+)', webpage, u'thumbnail', fatal=False) - description = self._html_search_regex(r'>\s*Description:\s*<[^>]*>([^<]+)', webpage, u'description', fatal=False) - if len(description) == 0: - description = None + video_uploader = self._html_search_regex( + r'by:\s*]*>(.+?)', webpage, u'uploader', fatal=False) + thumbnail = self._html_search_regex( + r'flashvars\.image_url = "([^"]+)', webpage, u'thumbnail', fatal=False) + description = self._html_search_regex( + r'>\s*Description:\s*<[^>]+>([^<]+)', webpage, u'description', fatal=False) video_urls = list(map(compat_urllib_parse.unquote , re.findall(r'flashvars\.quality_[0-9]{3}p = "([^"]+)', webpage))) if webpage.find('flashvars\.encrypted = "true"') != -1: From f2e87ef4fab5f6d000a7bb152b535a748866bca3 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 20 Nov 2013 07:46:44 +0100 Subject: [PATCH 085/425] [anitube] Skip test (on travis) --- youtube_dl/extractor/anitube.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/anitube.py b/youtube_dl/extractor/anitube.py index 679027f4e..691d5a844 100644 --- a/youtube_dl/extractor/anitube.py +++ b/youtube_dl/extractor/anitube.py @@ -17,6 +17,7 @@ class AnitubeIE(InfoExtractor): u'ext': u'mp4', u'title': u'Recorder to Randoseru 01', }, + u'skip': u'Blocked in the US', } def _real_extract(self, url): From ca872a4c0b5a926be9dccec6dc43ebb5b8a6abb7 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 20 Nov 2013 09:23:53 +0100 Subject: [PATCH 086/425] [spankwire] Fix description search --- youtube_dl/extractor/spankwire.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/spankwire.py b/youtube_dl/extractor/spankwire.py index 995b46444..9e2ad0d99 100644 --- a/youtube_dl/extractor/spankwire.py +++ b/youtube_dl/extractor/spankwire.py @@ -40,7 +40,7 @@ class SpankwireIE(InfoExtractor): thumbnail = self._html_search_regex( r'flashvars\.image_url = "([^"]+)', webpage, u'thumbnail', fatal=False) description = self._html_search_regex( - r'>\s*Description:\s*<[^>]+>([^<]+)', webpage, u'description', fatal=False) + r'([^<]+)<', webpage, u'description', fatal=False) video_urls = list(map(compat_urllib_parse.unquote , re.findall(r'flashvars\.quality_[0-9]{3}p = "([^"]+)', webpage))) if webpage.find('flashvars\.encrypted = "true"') != -1: From 0190eecc00c45b26eb5ef5444181cd996b139f18 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 20 Nov 2013 09:45:22 +0100 Subject: [PATCH 087/425] [nhl] Make NHLVideocenter IE_DESC fit with other descriptions --- youtube_dl/extractor/nhl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py index 224f56ac8..458fe4063 100644 --- a/youtube_dl/extractor/nhl.py +++ b/youtube_dl/extractor/nhl.py @@ -72,7 +72,7 @@ class NHLIE(NHLBaseInfoExtractor): class NHLVideocenterIE(NHLBaseInfoExtractor): IE_NAME = u'nhl.com:videocenter' - IE_DESC = u'Download the first 12 videos from a videocenter category' + IE_DESC = u'NHL videocenter category' _VALID_URL = r'https?://video\.(?P[^.]*)\.nhl\.com/videocenter/(console\?.*?catid=(?P[^&]+))?' @classmethod From 9e4f50a8aee0d3f030be203686c8f0479b32b793 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Wed, 20 Nov 2013 09:59:03 +0100 Subject: [PATCH 088/425] [sztv] skip test, site is undergoing mid-term maintenance --- youtube_dl/extractor/sztvhu.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/sztvhu.py b/youtube_dl/extractor/sztvhu.py index 81fa35c4b..c9359fafb 100644 --- a/youtube_dl/extractor/sztvhu.py +++ b/youtube_dl/extractor/sztvhu.py @@ -15,7 +15,8 @@ class SztvHuIE(InfoExtractor): u'info_dict': { u"title": u"Cserkészek népszerűsítették a környezettudatos életmódot a Savaria téren", u"description": u'A zöld nap játékos ismeretterjesztő programjait a Magyar Cserkész Szövetség szervezte, akik az ország nyolc városában adják át tudásukat az érdeklődőknek. A PET...', - } + }, + u'skip': u'Service temporarily disabled as of 2013-11-20' } def _real_extract(self, url): From 64bb5187f53caea0e6067434c95da60284eef2ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Thu, 21 Nov 2013 13:16:19 +0100 Subject: [PATCH 089/425] [soundcloud] Retrieve the file url using the client_id for the iPhone (fixes #1798) The desktop's client_id always give the rtmp url, but with the iPhone one it returns the http url if it's available. --- youtube_dl/extractor/soundcloud.py | 32 ++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 687457e10..5544325b5 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -59,6 +59,7 @@ class SoundcloudIE(InfoExtractor): ] _CLIENT_ID = 'b45b1aa10f1ac2941910a7f0d10f8e28' + _IPHONE_CLIENT_ID = '376f225bf427445fc4bfb6b99b72e0bf' @classmethod def suitable(cls, url): @@ -83,7 +84,6 @@ class SoundcloudIE(InfoExtractor): thumbnail = thumbnail.replace('-large', '-t500x500') result = { 'id': track_id, - 'url': info['stream_url'] + '?client_id=' + self._CLIENT_ID, 'uploader': info['user']['username'], 'upload_date': unified_strdate(info['created_at']), 'title': info['title'], @@ -92,19 +92,29 @@ class SoundcloudIE(InfoExtractor): 'thumbnail': thumbnail, } if info.get('downloadable', False): + # We can build a direct link to the song result['url'] = 'https://api.soundcloud.com/tracks/{0}/download?client_id={1}'.format(track_id, self._CLIENT_ID) - if not info.get('streamable', False): - # We have to get the rtmp url + else: + # We have to retrieve the url stream_json = self._download_webpage( - 'http://api.soundcloud.com/i1/tracks/{0}/streams?client_id={1}'.format(track_id, self._CLIENT_ID), + 'http://api.soundcloud.com/i1/tracks/{0}/streams?client_id={1}'.format(track_id, self._IPHONE_CLIENT_ID), track_id, u'Downloading track url') - rtmp_url = json.loads(stream_json)['rtmp_mp3_128_url'] - # The url doesn't have an rtmp app, we have to extract the playpath - url, path = rtmp_url.split('mp3:', 1) - result.update({ - 'url': url, - 'play_path': 'mp3:' + path, - }) + # There should be only one entry in the dictionary + key, stream_url = list(json.loads(stream_json).items())[0] + if key.startswith(u'http'): + result['url'] = stream_url + elif key.startswith(u'rtmp'): + # The url doesn't have an rtmp app, we have to extract the playpath + url, path = stream_url.split('mp3:', 1) + result.update({ + 'url': url, + 'play_path': 'mp3:' + path, + }) + else: + # We fallback to the stream_url in the original info, this + # cannot be always used, sometimes it can give an HTTP 404 error + resut['url'] = info['stream_url'] + '?client_id=' + self._CLIENT_ID, + return result def _real_extract(self, url): From 2bb683c2012ad45b0e3008664be5994aa603c196 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Thu, 21 Nov 2013 13:59:33 +0100 Subject: [PATCH 090/425] release 2013.11.21 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index ad3fad818..17ce776a9 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.20' +__version__ = '2013.11.21' From a6a173c2fddf2fa38a69ca750431b7ca6932bcb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Thu, 21 Nov 2013 14:09:28 +0100 Subject: [PATCH 091/425] utils.shell_quote: Convert the args to unicode strings The youtube test video failed with `UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 34: ordinal not in range(128)`, the problem was with the filenames being encoded. --- test/test_utils.py | 6 ++++++ youtube_dl/utils.py | 11 ++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/test/test_utils.py b/test/test_utils.py index f3fbff042..e9e590e74 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -24,6 +24,8 @@ from youtube_dl.utils import ( xpath_with_ns, smuggle_url, unsmuggle_url, + shell_quote, + encodeFilename, ) if sys.version_info < (3, 0): @@ -170,6 +172,10 @@ class TestUtil(unittest.TestCase): self.assertEqual(res_url, url) self.assertEqual(res_data, None) + def test_shell_quote(self): + args = ['ffmpeg', '-i', encodeFilename(u'ñ€ß\'.mp4')] + self.assertEqual(shell_quote(args), u"""ffmpeg -i 'ñ€ß'"'"'.mp4'""") + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index b50c8166f..0720fe9eb 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -951,7 +951,16 @@ class locked_file(object): def shell_quote(args): - return ' '.join(map(pipes.quote, args)) + quoted_args = [] + encoding = sys.getfilesystemencoding() + if encoding is None: + encoding = 'utf-8' + for a in args: + if isinstance(a, bytes): + # We may get a filename encoded with 'encodeFilename' + a = a.decode(encoding) + quoted_args.append(pipes.quote(a)) + return u' '.join(quoted_args) def takewhile_inclusive(pred, seq): From b1c9c669365cea4fb94babd66076665685abc453 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Thu, 21 Nov 2013 23:26:28 +0100 Subject: [PATCH 092/425] Remove unnecessary slash in setup.py (Fixes #1778) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index aa7cfca08..8e24fe679 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ else: 'data_files': [ # Installing system-wide would require sudo... ('etc/bash_completion.d', ['youtube-dl.bash-completion']), ('share/doc/youtube_dl', ['README.txt']), - ('share/man/man1/', ['youtube-dl.1']) + ('share/man/man1', ['youtube-dl.1']) ] } if setuptools_available: From 0980426559741bb9a8b2ea39b581073cf2738f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 22 Nov 2013 16:05:14 +0100 Subject: [PATCH 093/425] [bandcamp] add support for albums (reported in #1270) --- test/test_playlists.py | 9 +++++++++ youtube_dl/extractor/__init__.py | 2 +- youtube_dl/extractor/bandcamp.py | 24 ++++++++++++++++++++++++ youtube_dl/extractor/generic.py | 3 ++- 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/test/test_playlists.py b/test/test_playlists.py index 706b6bdca..d83b3bf51 100644 --- a/test/test_playlists.py +++ b/test/test_playlists.py @@ -22,6 +22,7 @@ from youtube_dl.extractor import ( LivestreamIE, NHLVideocenterIE, BambuserChannelIE, + BandcampAlbumIE ) @@ -103,5 +104,13 @@ class TestPlaylists(unittest.TestCase): self.assertEqual(result['title'], u'pixelversity') self.assertTrue(len(result['entries']) >= 66) + def test_bandcamp_album(self): + dl = FakeYDL() + ie = BandcampAlbumIE(dl) + result = ie.extract('http://mpallante.bandcamp.com/album/nightmare-night-ep') + self.assertIsPlaylist(result) + self.assertEqual(result['title'], u'Nightmare Night EP') + self.assertTrue(len(result['entries']) >= 4) + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index ffb74df9f..802beef21 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -11,7 +11,7 @@ from .arte import ( ) from .auengine import AUEngineIE from .bambuser import BambuserIE, BambuserChannelIE -from .bandcamp import BandcampIE +from .bandcamp import BandcampIE, BandcampAlbumIE from .bliptv import BlipTVIE, BlipTVUserIE from .bloomberg import BloombergIE from .breakcom import BreakIE diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py index 129a20f44..81d5c60e9 100644 --- a/youtube_dl/extractor/bandcamp.py +++ b/youtube_dl/extractor/bandcamp.py @@ -3,11 +3,13 @@ import re from .common import InfoExtractor from ..utils import ( + compat_urlparse, ExtractorError, ) class BandcampIE(InfoExtractor): + IE_NAME = u'Bandcamp' _VALID_URL = r'http://.*?\.bandcamp\.com/track/(?P.*)' _TEST = { u'url': u'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song', @@ -61,3 +63,25 @@ class BandcampIE(InfoExtractor): } return [track_info] + + +class BandcampAlbumIE(InfoExtractor): + IE_NAME = u'Bandcamp:album' + _VALID_URL = r'http://.*?\.bandcamp\.com/album/(?P<title>.*)' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + title = mobj.group('title') + webpage = self._download_webpage(url, title) + tracks_paths = re.findall(r'<a href="(.*?)" itemprop="url">', webpage) + if not tracks_paths: + raise ExtractorError(u'The page doesn\'t contain any track') + entries = [ + self.url_result(compat_urlparse.urljoin(url, t_path), ie=BandcampIE.ie_key()) + for t_path in tracks_paths] + title = self._search_regex(r'album_title : "(.*?)"', webpage, u'title') + return { + '_type': 'playlist', + 'title': title, + 'entries': entries, + } diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index e1d6a2a01..0b5f2b2bb 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -199,7 +199,8 @@ class GenericIE(InfoExtractor): mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage) if mobj is not None: burl = unescapeHTML(mobj.group(1)) - return self.url_result(burl, 'Bandcamp') + # Don't set the extractor because it can be a track url or an album + return self.url_result(burl) # Start with something easy: JW Player in SWFObject mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage) From ab009f59ef4a4e2a4842d37212b2e57466d9f22e Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 22 Nov 2013 17:17:55 +0100 Subject: [PATCH 094/425] [toutv] Fix a typo --- youtube_dl/extractor/soundcloud.py | 2 +- youtube_dl/extractor/toutv.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 5544325b5..67b2dff9c 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -113,7 +113,7 @@ class SoundcloudIE(InfoExtractor): else: # We fallback to the stream_url in the original info, this # cannot be always used, sometimes it can give an HTTP 404 error - resut['url'] = info['stream_url'] + '?client_id=' + self._CLIENT_ID, + result['url'] = info['stream_url'] + '?client_id=' + self._CLIENT_ID, return result diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py index 732083617..2f728d3dc 100644 --- a/youtube_dl/extractor/toutv.py +++ b/youtube_dl/extractor/toutv.py @@ -39,7 +39,6 @@ class TouTvIE(InfoExtractor): mediaId = self._search_regex( r'"idMedia":\s*"([^"]+)"', webpage, u'media ID') - # TODO test from de streams_url = u'http://release.theplatform.com/content.select?pid=' + mediaId streams_webpage = self._download_webpage( streams_url, video_id, note=u'Downloading stream list') From 02e4ebbbad5653b9bbbcf615bdcae6b2c7ea1e30 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 22 Nov 2013 17:19:22 +0100 Subject: [PATCH 095/425] [streamcloud] Add IE (Fixes #1801) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/streamcloud.py | 65 +++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 youtube_dl/extractor/streamcloud.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 802beef21..02f9e2546 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -128,6 +128,7 @@ from .spiegel import SpiegelIE from .stanfordoc import StanfordOpenClassroomIE from .statigram import StatigramIE from .steam import SteamIE +from .streamcloud import StreamcloudIE from .sztvhu import SztvHuIE from .teamcoco import TeamcocoIE from .techtalks import TechTalksIE diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dl/extractor/streamcloud.py new file mode 100644 index 000000000..d476693ec --- /dev/null +++ b/youtube_dl/extractor/streamcloud.py @@ -0,0 +1,65 @@ +# coding: utf-8 +import re +import time + +from .common import InfoExtractor +from ..utils import ( + compat_urllib_parse, + compat_urllib_request, +) + + +class StreamcloudIE(InfoExtractor): + IE_NAME = u'streamcloud.eu' + _VALID_URL = r'https?://streamcloud\.eu/(?P<id>[a-zA-Z0-9_-]+)/(?P<fname>[^#?]*)\.html' + + _TEST = { + u'url': u'http://streamcloud.eu/skp9j99s4bpz/youtube-dl_test_video_____________-BaW_jenozKc.mp4.html', + u'file': u'skp9j99s4bpz.mp4', + u'md5': u'6bea4c7fa5daaacc2a946b7146286686', + u'info_dict': { + u'title': u'youtube-dl test video \'/\\ ä ↭', + u'duration': 9, + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + orig_webpage = self._download_webpage(url, video_id) + + fields = re.findall(r'''(?x)<input\s+ + type="(?:hidden|submit)"\s+ + name="([^"]+)"\s+ + (?:id="[^"]+"\s+)? + value="([^"]*)" + ''', orig_webpage) + post = compat_urllib_parse.urlencode(fields) + + self.to_screen('%s: Waiting for timeout' % video_id) + time.sleep(12) + headers = { + b'Content-Type': b'application/x-www-form-urlencoded', + } + req = compat_urllib_request.Request(url, post, headers) + + webpage = self._download_webpage( + req, video_id, note=u'Downloading video page ...') + title = self._html_search_regex( + r'<h1[^>]*>([^<]+)<', webpage, u'title') + video_url = self._search_regex( + r'file:\s*"([^"]+)"', webpage, u'video URL') + duration_str = self._search_regex( + r'duration:\s*"?([0-9]+)"?', webpage, u'duration', fatal=False) + duration = None if duration_str is None else int(duration_str) + thumbnail = self._search_regex( + r'image:\s*"([^"]+)"', webpage, u'thumbnail URL', fatal=False) + + return { + 'id': video_id, + 'title': title, + 'url': video_url, + 'duration': duration, + 'thumbnail': thumbnail, + } From cffa6aa10757099bfb46c5c4eff5c5d9a3ff97f4 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 22 Nov 2013 17:44:55 +0100 Subject: [PATCH 096/425] [bandcamp] Support trackinfo-style songs (Fixes #1270) --- youtube_dl/extractor/bandcamp.py | 46 ++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py index 81d5c60e9..8b1d56da3 100644 --- a/youtube_dl/extractor/bandcamp.py +++ b/youtube_dl/extractor/bandcamp.py @@ -3,6 +3,7 @@ import re from .common import InfoExtractor from ..utils import ( + compat_str, compat_urlparse, ExtractorError, ) @@ -11,7 +12,7 @@ from ..utils import ( class BandcampIE(InfoExtractor): IE_NAME = u'Bandcamp' _VALID_URL = r'http://.*?\.bandcamp\.com/track/(?P<title>.*)' - _TEST = { + _TESTS = [{ u'url': u'http://youtube-dl.bandcamp.com/track/youtube-dl-test-song', u'file': u'1812978515.mp3', u'md5': u'cdeb30cdae1921719a3cbcab696ef53c', @@ -19,7 +20,28 @@ class BandcampIE(InfoExtractor): u"title": u"youtube-dl test song \"'/\\\u00e4\u21ad" }, u'skip': u'There is a limit of 200 free downloads / month for the test song' - } + }, { + u'url': u'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1', + u'playlist': [ + { + u'file': u'1353101989.mp3', + u'md5': u'39bc1eded3476e927c724321ddf116cf', + u'info_dict': { + u'title': u'Intro', + } + }, + { + u'file': u'38097443.mp3', + u'md5': u'1a2c32e2691474643e912cc6cd4bffaa', + u'info_dict': { + u'title': u'Kero One - Keep It Alive (Blazo remix)', + } + }, + ], + u'params': { + u'playlistend': 2 + } + }] def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) @@ -28,6 +50,26 @@ class BandcampIE(InfoExtractor): # We get the link to the free download page m_download = re.search(r'freeDownloadPage: "(.*?)"', webpage) if m_download is None: + m_trackinfo = re.search(r'trackinfo: (.+),\s*?\n', webpage) + if m_trackinfo: + json_code = m_trackinfo.group(1) + data = json.loads(json_code) + + entries = [] + for d in data: + formats = [{ + 'format_id': 'format_id', + 'url': format_url, + 'ext': format_id.partition('-')[0] + } for format_id, format_url in sorted(d['file'].items())] + entries.append({ + 'id': compat_str(d['id']), + 'title': d['title'], + 'formats': formats, + }) + + return self.playlist_result(entries, title, title) + else: raise ExtractorError(u'No free songs found') download_link = m_download.group(1) From bfe7439a2076452a70946ac146be463fa7277d67 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 22 Nov 2013 17:46:26 +0100 Subject: [PATCH 097/425] release 2013.11.22 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 17ce776a9..c1f581cd6 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.21' +__version__ = '2013.11.22' From 241650c7ff8afa7236598d0f95d20c0898abc02c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Fri, 22 Nov 2013 18:20:31 +0100 Subject: [PATCH 098/425] [vimeo] Fix the extraction of vimeo pro and player.vimeo.com videos --- youtube_dl/extractor/vimeo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index d465bf20b..7d82c2cfa 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -151,7 +151,7 @@ class VimeoIE(InfoExtractor): config = json.loads(config_json) except RegexNotFoundError: # For pro videos or player.vimeo.com urls - config = self._search_regex([r' = {config:({.+?}),assets:', r'c=({.+?);'], + config = self._search_regex([r' = {config:({.+?}),assets:', r'(?:c|b)=({.+?});'], webpage, u'info section', flags=re.DOTALL) config = json.loads(config) except Exception as e: From f143a42fe65ce4932dfab7be1f41c52ffe8a203e Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 22 Nov 2013 19:08:25 +0100 Subject: [PATCH 099/425] [bandcamp] Skip album test --- youtube_dl/extractor/bandcamp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py index 8b1d56da3..359d4174b 100644 --- a/youtube_dl/extractor/bandcamp.py +++ b/youtube_dl/extractor/bandcamp.py @@ -40,7 +40,8 @@ class BandcampIE(InfoExtractor): ], u'params': { u'playlistend': 2 - } + }, + u'skip': u'Bancamp imposes download limits. See test_playlists:test_bandcamp_album for the playlist test' }] def _real_extract(self, url): From 0b63aed8dfd36a5a7f5ae6518b9c385d9a43e735 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 22 Nov 2013 19:15:36 +0100 Subject: [PATCH 100/425] [update] do not assign to unused variables --- youtube_dl/update.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/youtube_dl/update.py b/youtube_dl/update.py index f41b4785a..e5f441707 100644 --- a/youtube_dl/update.py +++ b/youtube_dl/update.py @@ -109,7 +109,7 @@ def update_self(to_screen, verbose): urlh = compat_urllib_request.urlopen(version['exe'][0]) newcontent = urlh.read() urlh.close() - except (IOError, OSError) as err: + except (IOError, OSError): if verbose: to_screen(compat_str(traceback.format_exc())) to_screen(u'ERROR: unable to download latest version') return @@ -122,7 +122,7 @@ def update_self(to_screen, verbose): try: with open(exe + '.new', 'wb') as outf: outf.write(newcontent) - except (IOError, OSError) as err: + except (IOError, OSError): if verbose: to_screen(compat_str(traceback.format_exc())) to_screen(u'ERROR: unable to write the new version') return @@ -141,7 +141,7 @@ start /b "" cmd /c del "%%~f0"&exit /b" subprocess.Popen([bat]) # Continues to run in the background return # Do not show premature success messages - except (IOError, OSError) as err: + except (IOError, OSError): if verbose: to_screen(compat_str(traceback.format_exc())) to_screen(u'ERROR: unable to overwrite current version') return @@ -152,7 +152,7 @@ start /b "" cmd /c del "%%~f0"&exit /b" urlh = compat_urllib_request.urlopen(version['bin'][0]) newcontent = urlh.read() urlh.close() - except (IOError, OSError) as err: + except (IOError, OSError): if verbose: to_screen(compat_str(traceback.format_exc())) to_screen(u'ERROR: unable to download latest version') return @@ -165,7 +165,7 @@ start /b "" cmd /c del "%%~f0"&exit /b" try: with open(filename, 'wb') as outf: outf.write(newcontent) - except (IOError, OSError) as err: + except (IOError, OSError): if verbose: to_screen(compat_str(traceback.format_exc())) to_screen(u'ERROR: unable to overwrite current version') return From dca087205692c934163ec9aca5962056f890cd19 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 22 Nov 2013 19:57:52 +0100 Subject: [PATCH 101/425] Move the opener to the YoutubeDL object. This is the first step towards being able to just import youtube_dl and start using it. Apart from removing global state, this would fix problems like #1805. --- youtube_dl/YoutubeDL.py | 86 ++++++++++++++++++++++++++++- youtube_dl/__init__.py | 98 +++------------------------------- youtube_dl/extractor/common.py | 4 +- youtube_dl/utils.py | 4 +- 4 files changed, 96 insertions(+), 96 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index a2e3df1f9..72ccfa2ae 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -7,8 +7,10 @@ import errno import io import json import os +import platform import re import shutil +import subprocess import socket import sys import time @@ -18,6 +20,7 @@ if os.name == 'nt': import ctypes from .utils import ( + compat_cookiejar, compat_http_client, compat_print, compat_str, @@ -31,8 +34,10 @@ from .utils import ( encodeFilename, ExtractorError, locked_file, + make_HTTPS_handler, MaxDownloadsReached, PostProcessingError, + platform_name, preferredencoding, SameFileError, sanitize_filename, @@ -41,9 +46,11 @@ from .utils import ( UnavailableVideoError, write_json_file, write_string, + YoutubeDLHandler, ) from .extractor import get_info_extractor, gen_extractors from .FileDownloader import FileDownloader +from .version import __version__ class YoutubeDL(object): @@ -120,6 +127,8 @@ class YoutubeDL(object): downloadarchive: File name of a file where all downloads are recorded. Videos already present in the file are not downloaded again. + cookiefile: File name where cookies should be read from and dumped to. + nocheckcertificate Do not verify SSL certificates The following parameters are not used by YoutubeDL itself, they are used by the FileDownloader: @@ -160,6 +169,8 @@ class YoutubeDL(object): if '%(stitle)s' in self.params['outtmpl']: self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.') + self._setup_opener() + def add_info_extractor(self, ie): """Add an InfoExtractor object to the end of the list.""" self._ies.append(ie) @@ -235,6 +246,9 @@ class YoutubeDL(object): def __exit__(self, *args): self.restore_console_title() + + if self.params.get('cookiefile') is not None: + self.cookiejar.save() def fixed_template(self): """Checks if the output template is fixed.""" @@ -774,7 +788,7 @@ class YoutubeDL(object): for url in url_list: try: #It also downloads the videos - videos = self.extract_info(url) + self.extract_info(url) except UnavailableVideoError: self.report_error(u'unable to download video') except MaxDownloadsReached: @@ -885,3 +899,73 @@ class YoutubeDL(object): '_resolution': u'resolution', 'format_note': u'note'}) self.to_screen(u'[info] Available formats for %s:\n%s\n%s' % (info_dict['id'], header_line, u"\n".join(formats_s))) + + def urlopen(self, req): + """ Start an HTTP download """ + return self._opener.open(req) + + def print_debug_header(self): + if not self.params.get('verbose'): + return + write_string(u'[debug] youtube-dl version ' + __version__ + u'\n') + try: + sp = subprocess.Popen( + ['git', 'rev-parse', '--short', 'HEAD'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=os.path.dirname(os.path.abspath(__file__))) + out, err = sp.communicate() + out = out.decode().strip() + if re.match('[0-9a-f]+', out): + write_string(u'[debug] Git HEAD: ' + out + u'\n') + except: + try: + sys.exc_clear() + except: + pass + write_string(u'[debug] Python version %s - %s' % + (platform.python_version(), platform_name()) + u'\n') + + proxy_map = {} + for handler in self._opener.handlers: + if hasattr(handler, 'proxies'): + proxy_map.update(handler.proxies) + write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n') + + def _setup_opener(self, timeout=300): + opts_cookiefile = self.params.get('cookiefile') + opts_proxy = self.params.get('proxy') + + if opts_cookiefile is None: + self.cookiejar = compat_cookiejar.CookieJar() + else: + self.cookiejar = compat_cookiejar.MozillaCookieJar( + opts_cookiefile) + if os.access(opts_cookiefile, os.R_OK): + self.cookiejar.load() + + cookie_processor = compat_urllib_request.HTTPCookieProcessor( + self.cookiejar) + if opts_proxy is not None: + if opts_proxy == '': + proxies = {} + else: + proxies = {'http': opts_proxy, 'https': opts_proxy} + else: + proxies = compat_urllib_request.getproxies() + # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805) + if 'http' in proxies and 'https' not in proxies: + proxies['https'] = proxies['http'] + proxy_handler = compat_urllib_request.ProxyHandler(proxies) + https_handler = make_HTTPS_handler( + self.params.get('nocheckcertificate', False)) + opener = compat_urllib_request.build_opener( + https_handler, proxy_handler, cookie_processor, YoutubeDLHandler()) + # Delete the default user-agent header, which would otherwise apply in + # cases where our custom HTTP handler doesn't come into play + # (See https://github.com/rg3/youtube-dl/issues/1309 for details) + opener.addheaders = [] + self._opener = opener + + # TODO remove this global modification + compat_urllib_request.install_opener(opener) + socket.setdefaulttimeout(timeout) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 64ebf4d48..27886593b 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -40,45 +40,35 @@ __authors__ = ( __license__ = 'Public Domain' import codecs -import collections import getpass import optparse import os import random import re import shlex -import socket import subprocess import sys -import traceback -import platform from .utils import ( - compat_cookiejar, compat_print, - compat_str, - compat_urllib_request, DateRange, decodeOption, determine_ext, DownloadError, get_cachedir, - make_HTTPS_handler, MaxDownloadsReached, - platform_name, preferredencoding, SameFileError, std_headers, write_string, - YoutubeDLHandler, ) from .update import update_self -from .version import __version__ from .FileDownloader import ( FileDownloader, ) from .extractor import gen_extractors +from .version import __version__ from .YoutubeDL import YoutubeDL from .PostProcessor import ( FFmpegMetadataPP, @@ -451,19 +441,6 @@ def _real_main(argv=None): parser, opts, args = parseOpts(argv) - # Open appropriate CookieJar - if opts.cookiefile is None: - jar = compat_cookiejar.CookieJar() - else: - try: - jar = compat_cookiejar.MozillaCookieJar(opts.cookiefile) - if os.access(opts.cookiefile, os.R_OK): - jar.load() - except (IOError, OSError) as err: - if opts.verbose: - traceback.print_exc() - write_string(u'ERROR: unable to open cookie file\n') - sys.exit(101) # Set user agent if opts.user_agent is not None: std_headers['User-Agent'] = opts.user_agent @@ -495,8 +472,6 @@ def _real_main(argv=None): all_urls = batchurls + args all_urls = [url.strip() for url in all_urls] - opener = _setup_opener(jar=jar, opts=opts) - extractors = gen_extractors() if opts.list_extractors: @@ -551,7 +526,7 @@ def _real_main(argv=None): if opts.retries is not None: try: opts.retries = int(opts.retries) - except (TypeError, ValueError) as err: + except (TypeError, ValueError): parser.error(u'invalid retry count specified') if opts.buffersize is not None: numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize) @@ -562,13 +537,13 @@ def _real_main(argv=None): opts.playliststart = int(opts.playliststart) if opts.playliststart <= 0: raise ValueError(u'Playlist start must be positive') - except (TypeError, ValueError) as err: + except (TypeError, ValueError): parser.error(u'invalid playlist start number specified') try: opts.playlistend = int(opts.playlistend) if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart): raise ValueError(u'Playlist end must be greater than playlist start') - except (TypeError, ValueError) as err: + except (TypeError, ValueError): parser.error(u'invalid playlist end number specified') if opts.extractaudio: if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']: @@ -671,34 +646,12 @@ def _real_main(argv=None): 'youtube_print_sig_code': opts.youtube_print_sig_code, 'age_limit': opts.age_limit, 'download_archive': opts.download_archive, + 'cookiefile': opts.cookiefile, + 'nocheckcertificate': opts.no_check_certificate, } with YoutubeDL(ydl_opts) as ydl: - if opts.verbose: - write_string(u'[debug] youtube-dl version ' + __version__ + u'\n') - try: - sp = subprocess.Popen( - ['git', 'rev-parse', '--short', 'HEAD'], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - cwd=os.path.dirname(os.path.abspath(__file__))) - out, err = sp.communicate() - out = out.decode().strip() - if re.match('[0-9a-f]+', out): - write_string(u'[debug] Git HEAD: ' + out + u'\n') - except: - try: - sys.exc_clear() - except: - pass - write_string(u'[debug] Python version %s - %s' % - (platform.python_version(), platform_name()) + u'\n') - - proxy_map = {} - for handler in opener.handlers: - if hasattr(handler, 'proxies'): - proxy_map.update(handler.proxies) - write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n') - + ydl.print_debug_header() ydl.add_default_info_extractors() # PostProcessors @@ -729,46 +682,9 @@ def _real_main(argv=None): ydl.to_screen(u'--max-download limit reached, aborting.') retcode = 101 - # Dump cookie jar if requested - if opts.cookiefile is not None: - try: - jar.save() - except (IOError, OSError): - sys.exit(u'ERROR: unable to save cookie jar') - sys.exit(retcode) -def _setup_opener(jar=None, opts=None, timeout=300): - if opts is None: - FakeOptions = collections.namedtuple( - 'FakeOptions', ['proxy', 'no_check_certificate']) - opts = FakeOptions(proxy=None, no_check_certificate=False) - - cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar) - if opts.proxy is not None: - if opts.proxy == '': - proxies = {} - else: - proxies = {'http': opts.proxy, 'https': opts.proxy} - else: - proxies = compat_urllib_request.getproxies() - # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805) - if 'http' in proxies and 'https' not in proxies: - proxies['https'] = proxies['http'] - proxy_handler = compat_urllib_request.ProxyHandler(proxies) - https_handler = make_HTTPS_handler(opts) - opener = compat_urllib_request.build_opener( - https_handler, proxy_handler, cookie_processor, YoutubeDLHandler()) - # Delete the default user-agent header, which would otherwise apply in - # cases where our custom HTTP handler doesn't come into play - # (See https://github.com/rg3/youtube-dl/issues/1309 for details) - opener.addheaders = [] - compat_urllib_request.install_opener(opener) - socket.setdefaulttimeout(timeout) - return opener - - def main(argv=None): try: _real_main(argv) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index eb3435c77..423e54cea 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -8,7 +8,6 @@ import netrc from ..utils import ( compat_http_client, compat_urllib_error, - compat_urllib_request, compat_str, clean_html, @@ -19,6 +18,7 @@ from ..utils import ( unescapeHTML, ) + class InfoExtractor(object): """Information Extractor class. @@ -156,7 +156,7 @@ class InfoExtractor(object): elif note is not False: self.to_screen(u'%s: %s' % (video_id, note)) try: - return compat_urllib_request.urlopen(url_or_request) + return self._downloader.urlopen(url_or_request) except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: if errnote is None: errnote = u'Unable to download webpage' diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 0720fe9eb..0d2b7bd10 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -535,7 +535,7 @@ def formatSeconds(secs): else: return '%d' % secs -def make_HTTPS_handler(opts): +def make_HTTPS_handler(opts_no_check_certificate): if sys.version_info < (3,2): # Python's 2.x handler is very simplistic return compat_urllib_request.HTTPSHandler() @@ -545,7 +545,7 @@ def make_HTTPS_handler(opts): context.set_default_verify_paths() context.verify_mode = (ssl.CERT_NONE - if opts.no_check_certificate + if opts_no_check_certificate else ssl.CERT_REQUIRED) return compat_urllib_request.HTTPSHandler(context=context) From 00ea0f11eb76e7a67648790524a50f7254b9578f Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 22 Nov 2013 20:00:35 +0100 Subject: [PATCH 102/425] Print full title in --get-title output (#1806) --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index a2e3df1f9..2700051cf 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -639,7 +639,7 @@ class YoutubeDL(object): # Forced printings if self.params.get('forcetitle', False): - compat_print(info_dict['title']) + compat_print(info_dict['fulltitle']) if self.params.get('forceid', False): compat_print(info_dict['id']) if self.params.get('forceurl', False): From 50123be4211e2c16aa5d2fc9ebadbaf72a9becce Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 22 Nov 2013 20:23:55 +0100 Subject: [PATCH 103/425] release 2013.11.22.1 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index c1f581cd6..770b046a5 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.22' +__version__ = '2013.11.22.1' From d35dc6d3b57781e5f1c0a5df308e3c08f66371a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Fri, 22 Nov 2013 21:19:31 +0100 Subject: [PATCH 104/425] [bandcamp] move the album test to the album extractor and return a single track instead of a playlist --- youtube_dl/extractor/bandcamp.py | 52 ++++++++++++++++---------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py index 359d4174b..1aa9dbefd 100644 --- a/youtube_dl/extractor/bandcamp.py +++ b/youtube_dl/extractor/bandcamp.py @@ -20,28 +20,6 @@ class BandcampIE(InfoExtractor): u"title": u"youtube-dl test song \"'/\\\u00e4\u21ad" }, u'skip': u'There is a limit of 200 free downloads / month for the test song' - }, { - u'url': u'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1', - u'playlist': [ - { - u'file': u'1353101989.mp3', - u'md5': u'39bc1eded3476e927c724321ddf116cf', - u'info_dict': { - u'title': u'Intro', - } - }, - { - u'file': u'38097443.mp3', - u'md5': u'1a2c32e2691474643e912cc6cd4bffaa', - u'info_dict': { - u'title': u'Kero One - Keep It Alive (Blazo remix)', - } - }, - ], - u'params': { - u'playlistend': 2 - }, - u'skip': u'Bancamp imposes download limits. See test_playlists:test_bandcamp_album for the playlist test' }] def _real_extract(self, url): @@ -63,13 +41,11 @@ class BandcampIE(InfoExtractor): 'url': format_url, 'ext': format_id.partition('-')[0] } for format_id, format_url in sorted(d['file'].items())] - entries.append({ + return { 'id': compat_str(d['id']), 'title': d['title'], 'formats': formats, - }) - - return self.playlist_result(entries, title, title) + } else: raise ExtractorError(u'No free songs found') @@ -112,6 +88,30 @@ class BandcampAlbumIE(InfoExtractor): IE_NAME = u'Bandcamp:album' _VALID_URL = r'http://.*?\.bandcamp\.com/album/(?P<title>.*)' + _TEST = { + u'url': u'http://blazo.bandcamp.com/album/jazz-format-mixtape-vol-1', + u'playlist': [ + { + u'file': u'1353101989.mp3', + u'md5': u'39bc1eded3476e927c724321ddf116cf', + u'info_dict': { + u'title': u'Intro', + } + }, + { + u'file': u'38097443.mp3', + u'md5': u'1a2c32e2691474643e912cc6cd4bffaa', + u'info_dict': { + u'title': u'Kero One - Keep It Alive (Blazo remix)', + } + }, + ], + u'params': { + u'playlistend': 2 + }, + u'skip': u'Bancamp imposes download limits. See test_playlists:test_bandcamp_album for the playlist test' + } + def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) title = mobj.group('title') From 9f79463803f40a15a6350dc693af75ec215147f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Fri, 22 Nov 2013 21:25:12 +0100 Subject: [PATCH 105/425] [howcast] update test's checksum --- youtube_dl/extractor/howcast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/howcast.py b/youtube_dl/extractor/howcast.py index 46954337f..bafc5826f 100644 --- a/youtube_dl/extractor/howcast.py +++ b/youtube_dl/extractor/howcast.py @@ -8,7 +8,7 @@ class HowcastIE(InfoExtractor): _TEST = { u'url': u'http://www.howcast.com/videos/390161-How-to-Tie-a-Square-Knot-Properly', u'file': u'390161.mp4', - u'md5': u'1d7ba54e2c9d7dc6935ef39e00529138', + u'md5': u'8b743df908c42f60cf6496586c7f12c3', u'info_dict': { u"description": u"The square knot, also known as the reef knot, is one of the oldest, most basic knots to tie, and can be used in many different ways. Here's the proper way to tie a square knot.", u"title": u"How to Tie a Square Knot Properly" From d3b30148edb6795aadc96b3a464c492b239a2242 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Fri, 22 Nov 2013 21:26:31 +0100 Subject: [PATCH 106/425] [bambuser:channel] Update test --- test/test_playlists.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_playlists.py b/test/test_playlists.py index d83b3bf51..7c67239a4 100644 --- a/test/test_playlists.py +++ b/test/test_playlists.py @@ -102,7 +102,7 @@ class TestPlaylists(unittest.TestCase): result = ie.extract('http://bambuser.com/channel/pixelversity') self.assertIsPlaylist(result) self.assertEqual(result['title'], u'pixelversity') - self.assertTrue(len(result['entries']) >= 66) + self.assertTrue(len(result['entries']) >= 60) def test_bandcamp_album(self): dl = FakeYDL() From 7012b23c947fc1ed146e314a30d3c70a5fde70e7 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 22 Nov 2013 22:46:46 +0100 Subject: [PATCH 107/425] Match --download-archive during playlist processing (Fixes #1745) --- test/test_youtube_lists.py | 6 ++--- youtube_dl/YoutubeDL.py | 43 +++++++++++++++++++++++---------- youtube_dl/extractor/common.py | 4 ++- youtube_dl/extractor/youtube.py | 26 ++++++++++++-------- 4 files changed, 52 insertions(+), 27 deletions(-) diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py index 50ad52695..938517a2d 100644 --- a/test/test_youtube_lists.py +++ b/test/test_youtube_lists.py @@ -84,16 +84,16 @@ class TestYoutubeLists(unittest.TestCase): dl = FakeYDL() ie = YoutubeChannelIE(dl) #test paginated channel - result = ie.extract('https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w')[0] + result = ie.extract('https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w') self.assertTrue(len(result['entries']) > 90) #test autogenerated channel - result = ie.extract('https://www.youtube.com/channel/HCtnHdj3df7iM/videos')[0] + result = ie.extract('https://www.youtube.com/channel/HCtnHdj3df7iM/videos') self.assertTrue(len(result['entries']) >= 18) def test_youtube_user(self): dl = FakeYDL() ie = YoutubeUserIE(dl) - result = ie.extract('https://www.youtube.com/user/TheLinuxFoundation')[0] + result = ie.extract('https://www.youtube.com/user/TheLinuxFoundation') self.assertTrue(len(result['entries']) >= 320) def test_youtube_safe_search(self): diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 2700051cf..beb7d0cd1 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -355,15 +355,17 @@ class YoutubeDL(object): def _match_entry(self, info_dict): """ Returns None iff the file should be downloaded """ - title = info_dict['title'] - matchtitle = self.params.get('matchtitle', False) - if matchtitle: - if not re.search(matchtitle, title, re.IGNORECASE): - return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"' - rejecttitle = self.params.get('rejecttitle', False) - if rejecttitle: - if re.search(rejecttitle, title, re.IGNORECASE): - return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"' + if 'title' in info_dict: + # This can happen when we're just evaluating the playlist + title = info_dict['title'] + matchtitle = self.params.get('matchtitle', False) + if matchtitle: + if not re.search(matchtitle, title, re.IGNORECASE): + return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"' + rejecttitle = self.params.get('rejecttitle', False) + if rejecttitle: + if re.search(rejecttitle, title, re.IGNORECASE): + return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"' date = info_dict.get('upload_date', None) if date is not None: dateRange = self.params.get('daterange', DateRange()) @@ -374,8 +376,8 @@ class YoutubeDL(object): if age_limit < info_dict.get('age_limit', 0): return u'Skipping "' + title + '" because it is age restricted' if self.in_download_archive(info_dict): - return (u'%(title)s has already been recorded in archive' - % info_dict) + return (u'%s has already been recorded in archive' + % info_dict.get('title', info_dict.get('id', u'video'))) return None @staticmethod @@ -454,7 +456,7 @@ class YoutubeDL(object): ie_key=ie_result.get('ie_key'), extra_info=extra_info) elif result_type == 'playlist': - self.add_extra_info(ie_result, extra_info) + # We process each entry in the playlist playlist = ie_result.get('title', None) or ie_result.get('id', None) self.to_screen(u'[download] Downloading playlist: %s' % playlist) @@ -484,6 +486,12 @@ class YoutubeDL(object): 'webpage_url': ie_result['webpage_url'], 'extractor_key': ie_result['extractor_key'], } + + reason = self._match_entry(entry) + if reason is not None: + self.to_screen(u'[download] ' + reason) + continue + entry_result = self.process_ie_result(entry, download=download, extra_info=extra) @@ -810,7 +818,16 @@ class YoutubeDL(object): fn = self.params.get('download_archive') if fn is None: return False - vid_id = info_dict['extractor'] + u' ' + info_dict['id'] + extractor = info_dict.get('extractor_id') + if extractor is None: + if 'id' in info_dict: + extractor = info_dict.get('ie_key') # key in a playlist + if extractor is None: + return False # Incomplete video information + # Future-proof against any change in case + # and backwards compatibility with prior versions + extractor = extractor.lower() + vid_id = extractor + u' ' + info_dict['id'] try: with locked_file(fn, 'r', encoding='utf-8') as archive_file: for line in archive_file: diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index eb3435c77..3cebeaf29 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -229,12 +229,14 @@ class InfoExtractor(object): self.to_screen(u'Logging in') #Methods for following #608 - def url_result(self, url, ie=None): + def url_result(self, url, ie=None, video_id=None): """Returns a url that points to a page that should be processed""" #TODO: ie should be the class used for getting the info video_info = {'_type': 'url', 'url': url, 'ie_key': ie} + if video_id is not None: + video_info['id'] = video_id return video_info def playlist_result(self, entries, playlist_id=None, playlist_title=None): """Returns a playlist""" diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 9b09793eb..126688652 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1552,7 +1552,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor): video_id = query_dict['v'][0] if self._downloader.params.get('noplaylist'): self.to_screen(u'Downloading just video %s because of --no-playlist' % video_id) - return self.url_result('https://www.youtube.com/watch?v=' + video_id, 'Youtube') + return self.url_result(video_id, 'Youtube', video_id=video_id) else: self.to_screen(u'Downloading playlist PL%s - add --no-playlist to just download video %s' % (playlist_id, video_id)) @@ -1571,7 +1571,8 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor): playlist_title = self._og_search_title(page) - url_results = [self.url_result(vid, 'Youtube') for vid in ids] + url_results = [self.url_result(video_id, 'Youtube', video_id=video_id) + for video_id in ids] return self.playlist_result(url_results, playlist_id, playlist_title) @@ -1626,9 +1627,9 @@ class YoutubeChannelIE(InfoExtractor): self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids))) - urls = ['http://www.youtube.com/watch?v=%s' % id for id in video_ids] - url_entries = [self.url_result(eurl, 'Youtube') for eurl in urls] - return [self.playlist_result(url_entries, channel_id)] + url_entries = [self.url_result(video_id, 'Youtube', video_id=video_id) + for video_id in video_ids] + return self.playlist_result(url_entries, channel_id) class YoutubeUserIE(InfoExtractor): @@ -1692,9 +1693,11 @@ class YoutubeUserIE(InfoExtractor): if len(ids_in_page) < self._GDATA_PAGE_SIZE: break - urls = ['http://www.youtube.com/watch?v=%s' % video_id for video_id in video_ids] - url_results = [self.url_result(rurl, 'Youtube') for rurl in urls] - return [self.playlist_result(url_results, playlist_title = username)] + url_results = [ + self.url_result(video_id, 'Youtube', video_id=video_id) + for video_id in video_ids] + return self.playlist_result(url_results, playlist_title=username) + class YoutubeSearchIE(SearchInfoExtractor): IE_DESC = u'YouTube.com searches' @@ -1735,7 +1738,8 @@ class YoutubeSearchIE(SearchInfoExtractor): if len(video_ids) > n: video_ids = video_ids[:n] - videos = [self.url_result('http://www.youtube.com/watch?v=%s' % id, 'Youtube') for id in video_ids] + videos = [self.url_result(video_id, 'Youtube', video_id=video_id) + for video_id in video_ids] return self.playlist_result(videos, query) class YoutubeSearchDateIE(YoutubeSearchIE): @@ -1795,7 +1799,9 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor): feed_html = info['feed_html'] m_ids = re.finditer(r'"/watch\?v=(.*?)["&]', feed_html) ids = orderedSet(m.group(1) for m in m_ids) - feed_entries.extend(self.url_result(id, 'Youtube') for id in ids) + feed_entries.extend( + self.url_result(video_id, 'Youtube', video_id=video_id) + for video_id in ids) if info['paging'] is None: break return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE) From d7386f6276b7b01ff4254136524d29c8f243721e Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 22 Nov 2013 23:05:56 +0100 Subject: [PATCH 108/425] [update] Check if version from repository is newer before updating Closes #1704 --- youtube_dl/update.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/youtube_dl/update.py b/youtube_dl/update.py index e5f441707..be7800e8b 100644 --- a/youtube_dl/update.py +++ b/youtube_dl/update.py @@ -41,6 +41,7 @@ def rsa_verify(message, signature, key): if signature != sha256(message).digest(): return False return True + def update_self(to_screen, verbose): """Update the program file with the latest version from the repository""" @@ -82,6 +83,13 @@ def update_self(to_screen, verbose): return version_id = versions_info['latest'] + + def version_tuple(version_str): + return tuple(map(int, version_str.split('.'))) + if version_tuple(__version__) >= version_tuple(version_str): + to_screen(u'youtube-dl is up to date (%s)' % __version__) + return + to_screen(u'Updating to version ' + version_id + '...') version = versions_info['versions'][version_id] From a87b0615aa311083923e607c3d1a5cdceab818f7 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 22 Nov 2013 23:08:15 +0100 Subject: [PATCH 109/425] release 2013.11.22.2 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 770b046a5..f6d18f945 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.22.1' +__version__ = '2013.11.22.2' From 43afe2858870d140b2a133fda2a0cbbd642a3bfc Mon Sep 17 00:00:00 2001 From: Itay Brandes <Brandes.Itay@gmail.com> Date: Sat, 23 Nov 2013 10:22:18 +0200 Subject: [PATCH 110/425] Log to an external logger (fixes #1810) Sadly applications using youtube-dl's python sources can't directly access it's log stream. It's pretty much limited to stdout and stderr only. It should log to logging.Logger instance passed to YoutubeDL's params dictionary. --- youtube_dl/YoutubeDL.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index beb7d0cd1..6729d53ad 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -97,6 +97,7 @@ class YoutubeDL(object): playlistend: Playlist item to end at. matchtitle: Download only matching titles. rejecttitle: Reject downloads for matching titles. + logger: Log messages to a logging.Logger instance. logtostderr: Log messages to stderr instead of stdout. writedescription: Write the video description to a .description file writeinfojson: Write the video description to a .info.json file @@ -192,7 +193,9 @@ class YoutubeDL(object): def to_screen(self, message, skip_eol=False): """Print message to stdout if not in quiet mode.""" - if not self.params.get('quiet', False): + if self.params.get('logger', False): + self.params['logger'].debug(message) + elif not self.params.get('quiet', False): terminator = [u'\n', u''][skip_eol] output = message + terminator write_string(output, self._screen_file) @@ -200,10 +203,13 @@ class YoutubeDL(object): def to_stderr(self, message): """Print message to stderr.""" assert type(message) == type(u'') - output = message + u'\n' - if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr - output = output.encode(preferredencoding()) - sys.stderr.write(output) + if self.params.get('logger', False): + self.params['logger'].error(message) + else: + output = message + u'\n' + if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr + output = output.encode(preferredencoding()) + sys.stderr.write(output) def to_console_title(self, message): if not self.params.get('consoletitle', False): From 52ad14aeb0178a187a861e7ce2259b7046702281 Mon Sep 17 00:00:00 2001 From: Takuya Tsuchida <takuya0301@gmail.com> Date: Sat, 23 Nov 2013 18:19:44 +0900 Subject: [PATCH 111/425] Add support for niconico --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/niconico.py | 190 +++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 youtube_dl/extractor/niconico.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 02f9e2546..f443f11f6 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -98,6 +98,7 @@ from .nba import NBAIE from .nbc import NBCNewsIE from .newgrounds import NewgroundsIE from .nhl import NHLIE, NHLVideocenterIE +from .niconico import NiconicoIE from .nowvideo import NowVideoIE from .ooyala import OoyalaIE from .orf import ORFIE diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py new file mode 100644 index 000000000..8638a8ee8 --- /dev/null +++ b/youtube_dl/extractor/niconico.py @@ -0,0 +1,190 @@ +# encoding: utf-8 + +import re +import socket +import xml.etree.ElementTree + +from .common import InfoExtractor +from ..utils import ( + compat_http_client, + compat_urllib_error, + compat_urllib_parse, + compat_urllib_request, + compat_urlparse, + compat_str, + + ExtractorError, + unified_strdate, +) + +class NiconicoIE(InfoExtractor): + IE_NAME = u'niconico' + IE_DESC = u'ニコニコ動画' + + _TEST = { + u'url': u'http://www.nicovideo.jp/watch/sm22312215', + u'file': u'sm22312215.mp4', + u'md5': u'd1a75c0823e2f629128c43e1212760f9', + u'info_dict': { + u'title': u'Big Buck Bunny', + u'uploader': u'takuya0301', + u'uploader_id': u'2698420', + u'upload_date': u'20131123', + u'description': u'(c) copyright 2008, Blender Foundation / www.bigbuckbunny.org', + }, + u'params': { + u'username': u'ydl.niconico@gmail.com', + u'password': u'youtube-dl', + }, + } + + _VALID_URL = r'^(?:https?://)?(?:www\.)?nicovideo\.jp/watch/([a-z][a-z][0-9]+)(?:.*)$' + _LOGIN_URL = 'https://secure.nicovideo.jp/secure/login' + _NETRC_MACHINE = 'niconico' + # If True it will raise an error if no login info is provided + _LOGIN_REQUIRED = True + + def _real_initialize(self): + self._login() + + def _login(self): + (username, password) = self._get_login_info() + # No authentication to be performed + if username is None: + if self._LOGIN_REQUIRED: + raise ExtractorError(u'No login info available, needed for using %s.' % self.IE_NAME, expected=True) + return False + + # Log in + login_form_strs = { + u'mail': username, + u'password': password, + } + # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode + # chokes on unicode + login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in login_form_strs.items()) + login_data = compat_urllib_parse.urlencode(login_form).encode('ascii') + request = compat_urllib_request.Request(self._LOGIN_URL, login_data) + try: + self.report_login() + login_results = compat_urllib_request.urlopen(request).read().decode('utf-8') + if re.search(r'(?i)<h1 class="mb8p4">Log in error</h1>', login_results) is not None: + self._downloader.report_warning(u'unable to log in: bad username or password') + return False + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + self._downloader.report_warning(u'unable to log in: %s' % compat_str(err)) + return False + return True + + def _real_extract(self, url): + video_id = self._extract_id(url) + + # Get video webpage + self.report_video_webpage_download(video_id) + url = 'http://www.nicovideo.jp/watch/' + video_id + request = compat_urllib_request.Request(url) + try: + video_webpage = compat_urllib_request.urlopen(request).read() + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + raise ExtractorError(u'Unable to download video webpage: %s' % compat_str(err)) + + # Get video info + self.report_video_info_webpage_download(video_id) + url = 'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id + request = compat_urllib_request.Request(url) + try: + video_info_webpage = compat_urllib_request.urlopen(request).read() + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + raise ExtractorError(u'Unable to download video info webpage: %s' % compat_str(err)) + + # Get flv info + self.report_flv_info_webpage_download(video_id) + url = 'http://flapi.nicovideo.jp/api/getflv?v=' + video_id + request = compat_urllib_request.Request(url) + try: + flv_info_webpage = compat_urllib_request.urlopen(request).read() + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + raise ExtractorError(u'Unable to download flv info webpage: %s' % compat_str(err)) + + # Start extracting information + self.report_information_extraction(video_id) + video_info = xml.etree.ElementTree.fromstring(video_info_webpage) + + # url + video_real_url = compat_urlparse.parse_qs(flv_info_webpage.decode('utf-8'))['url'][0] + + # title + video_title = video_info.find('.//title').text + + # ext + video_extension = video_info.find('.//movie_type').text + + # format + video_format = video_extension.upper() + + # thumbnail + video_thumbnail = video_info.find('.//thumbnail_url').text + + # description + video_description = video_info.find('.//description').text + + # uploader_id + video_uploader_id = video_info.find('.//user_id').text + + # uploader + url = 'http://seiga.nicovideo.jp/api/user/info?id=' + video_uploader_id + request = compat_urllib_request.Request(url) + try: + user_info_webpage = compat_urllib_request.urlopen(request).read() + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + self._downloader.report_warning(u'Unable to download user info webpage: %s' % compat_str(err)) + + user_info = xml.etree.ElementTree.fromstring(user_info_webpage) + video_uploader = user_info.find('.//nickname').text + + # uploder_date + video_upload_date = unified_strdate(video_info.find('.//first_retrieve').text.split('+')[0]) + + # view_count + video_view_count = video_info.find('.//view_counter').text + + # webpage_url + video_webpage_url = video_info.find('.//watch_url').text + + return { + 'id': video_id, + 'url': video_real_url, + 'title': video_title, + 'ext': video_extension, + 'format': video_format, + 'thumbnail': video_thumbnail, + 'description': video_description, + 'uploader': video_uploader, + 'upload_date': video_upload_date, + 'uploader_id': video_uploader_id, + 'view_count': video_view_count, + 'webpage_url': video_webpage_url, + } + + def _extract_id(self, url): + mobj = re.match(self._VALID_URL, url) + if mobj is None: + raise ExtractorError(u'Invalid URL: %s' % url) + video_id = mobj.group(1) + return video_id + + def report_video_webpage_download(self, video_id): + """Report attempt to download video webpage.""" + self.to_screen(u'%s: Downloading video webpage' % video_id) + + def report_video_info_webpage_download(self, video_id): + """Report attempt to download video info webpage.""" + self.to_screen(u'%s: Downloading video info webpage' % video_id) + + def report_flv_info_webpage_download(self, video_id): + """Report attempt to download flv info webpage.""" + self.to_screen(u'%s: Downloading flv info webpage' % video_id) + + def report_information_extraction(self, video_id): + """Report attempt to extract video information.""" + self.to_screen(u'%s: Extracting video information' % video_id) From e5c146d58682dbdda2b46fc71a16ddc5b1fcc9fc Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sat, 23 Nov 2013 15:57:42 +0100 Subject: [PATCH 112/425] [streamcloud] skip test on travis --- youtube_dl/extractor/streamcloud.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dl/extractor/streamcloud.py index d476693ec..9faf3a5e3 100644 --- a/youtube_dl/extractor/streamcloud.py +++ b/youtube_dl/extractor/streamcloud.py @@ -21,6 +21,7 @@ class StreamcloudIE(InfoExtractor): u'title': u'youtube-dl test video \'/\\ ä ↭', u'duration': 9, }, + u'skip': u'Only available from the EU' } def _real_extract(self, url): From 36de0a0e1a49f6324616f9b508920ff7d06136c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Sat, 23 Nov 2013 23:26:06 +0100 Subject: [PATCH 113/425] [brightcove] Set the 'videoPlayer' value to the 'videoId' if it's missing in the parameters (fixes #1815) --- youtube_dl/extractor/brightcove.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py index d8c35465a..74a7d13e3 100644 --- a/youtube_dl/extractor/brightcove.py +++ b/youtube_dl/extractor/brightcove.py @@ -75,14 +75,17 @@ class BrightcoveIE(InfoExtractor): params = {'flashID': object_doc.attrib['id'], 'playerID': find_xpath_attr(object_doc, './param', 'name', 'playerID').attrib['value'], } - playerKey = find_xpath_attr(object_doc, './param', 'name', 'playerKey') + def find_param(name): + return find_xpath_attr(object_doc, './param', 'name', name) + playerKey = find_param('playerKey') # Not all pages define this value if playerKey is not None: params['playerKey'] = playerKey.attrib['value'] - videoPlayer = find_xpath_attr(object_doc, './param', 'name', '@videoPlayer') + # The three fields hold the id of the video + videoPlayer = find_param('@videoPlayer') or find_param('videoId') or find_param('videoID') if videoPlayer is not None: params['@videoPlayer'] = videoPlayer.attrib['value'] - linkBase = find_xpath_attr(object_doc, './param', 'name', 'linkBaseURL') + linkBase = find_param('linkBaseURL') if linkBase is not None: params['linkBaseURL'] = linkBase.attrib['value'] data = compat_urllib_parse.urlencode(params) From 8bf9319e9c0c02f5516b00509476abff89eb3d41 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 24 Nov 2013 06:08:11 +0100 Subject: [PATCH 114/425] Simplify logger code(#1811) --- youtube_dl/YoutubeDL.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 6729d53ad..d7e2417ac 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -97,7 +97,7 @@ class YoutubeDL(object): playlistend: Playlist item to end at. matchtitle: Download only matching titles. rejecttitle: Reject downloads for matching titles. - logger: Log messages to a logging.Logger instance. + logger: Log messages to a logging.Logger instance. logtostderr: Log messages to stderr instead of stdout. writedescription: Write the video description to a .description file writeinfojson: Write the video description to a .info.json file @@ -193,7 +193,7 @@ class YoutubeDL(object): def to_screen(self, message, skip_eol=False): """Print message to stdout if not in quiet mode.""" - if self.params.get('logger', False): + if self.params.get('logger'): self.params['logger'].debug(message) elif not self.params.get('quiet', False): terminator = [u'\n', u''][skip_eol] @@ -203,7 +203,7 @@ class YoutubeDL(object): def to_stderr(self, message): """Print message to stderr.""" assert type(message) == type(u'') - if self.params.get('logger', False): + if self.params.get('logger'): self.params['logger'].error(message) else: output = message + u'\n' From 13ebea791fb4293acf939730ad5a9c07e553005f Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 24 Nov 2013 06:37:14 +0100 Subject: [PATCH 115/425] [niconico] Simplify and make work with old Python versions The website requires SSLv3, otherwise it just times out during SSL negotiation. --- youtube_dl/extractor/niconico.py | 121 ++++++++----------------------- youtube_dl/utils.py | 29 ++++++-- 2 files changed, 55 insertions(+), 95 deletions(-) diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py index 8638a8ee8..22898b5a1 100644 --- a/youtube_dl/extractor/niconico.py +++ b/youtube_dl/extractor/niconico.py @@ -17,6 +17,7 @@ from ..utils import ( unified_strdate, ) + class NiconicoIE(InfoExtractor): IE_NAME = u'niconico' IE_DESC = u'ニコニコ動画' @@ -38,8 +39,7 @@ class NiconicoIE(InfoExtractor): }, } - _VALID_URL = r'^(?:https?://)?(?:www\.)?nicovideo\.jp/watch/([a-z][a-z][0-9]+)(?:.*)$' - _LOGIN_URL = 'https://secure.nicovideo.jp/secure/login' + _VALID_URL = r'^https?://(?:www\.|secure\.)?nicovideo\.jp/watch/([a-z][a-z][0-9]+)(?:.*)$' _NETRC_MACHINE = 'niconico' # If True it will raise an error if no login info is provided _LOGIN_REQUIRED = True @@ -57,99 +57,63 @@ class NiconicoIE(InfoExtractor): # Log in login_form_strs = { - u'mail': username, - u'password': password, + u'mail': username, + u'password': password, } # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode # chokes on unicode login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in login_form_strs.items()) - login_data = compat_urllib_parse.urlencode(login_form).encode('ascii') - request = compat_urllib_request.Request(self._LOGIN_URL, login_data) - try: - self.report_login() - login_results = compat_urllib_request.urlopen(request).read().decode('utf-8') - if re.search(r'(?i)<h1 class="mb8p4">Log in error</h1>', login_results) is not None: - self._downloader.report_warning(u'unable to log in: bad username or password') - return False - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.report_warning(u'unable to log in: %s' % compat_str(err)) + login_data = compat_urllib_parse.urlencode(login_form).encode('utf-8') + request = compat_urllib_request.Request( + u'https://secure.nicovideo.jp/secure/login', login_data) + login_results = self._download_webpage( + request, u'', note=u'Logging in', errnote=u'Unable to log in') + if re.search(r'(?i)<h1 class="mb8p4">Log in error</h1>', login_results) is not None: + self._downloader.report_warning(u'unable to log in: bad username or password') return False return True def _real_extract(self, url): - video_id = self._extract_id(url) + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group(1) # Get video webpage - self.report_video_webpage_download(video_id) - url = 'http://www.nicovideo.jp/watch/' + video_id - request = compat_urllib_request.Request(url) - try: - video_webpage = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to download video webpage: %s' % compat_str(err)) + video_webpage = self._download_webpage( + 'http://www.nicovideo.jp/watch/' + video_id, video_id) - # Get video info - self.report_video_info_webpage_download(video_id) - url = 'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id - request = compat_urllib_request.Request(url) - try: - video_info_webpage = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to download video info webpage: %s' % compat_str(err)) + video_info_webpage = self._download_webpage( + 'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id, video_id, + note=u'Downloading video info page') # Get flv info - self.report_flv_info_webpage_download(video_id) - url = 'http://flapi.nicovideo.jp/api/getflv?v=' + video_id - request = compat_urllib_request.Request(url) - try: - flv_info_webpage = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to download flv info webpage: %s' % compat_str(err)) + flv_info_webpage = self._download_webpage( + u'http://flapi.nicovideo.jp/api/getflv?v=' + video_id, + video_id, u'Downloading flv info') + video_real_url = compat_urlparse.parse_qs(flv_info_webpage)['url'][0] # Start extracting information - self.report_information_extraction(video_id) video_info = xml.etree.ElementTree.fromstring(video_info_webpage) - - # url - video_real_url = compat_urlparse.parse_qs(flv_info_webpage.decode('utf-8'))['url'][0] - - # title video_title = video_info.find('.//title').text - - # ext video_extension = video_info.find('.//movie_type').text - - # format video_format = video_extension.upper() - - # thumbnail video_thumbnail = video_info.find('.//thumbnail_url').text - - # description video_description = video_info.find('.//description').text - - # uploader_id video_uploader_id = video_info.find('.//user_id').text + video_upload_date = unified_strdate(video_info.find('.//first_retrieve').text.split('+')[0]) + video_view_count = video_info.find('.//view_counter').text + video_webpage_url = video_info.find('.//watch_url').text # uploader + video_uploader = video_uploader_id url = 'http://seiga.nicovideo.jp/api/user/info?id=' + video_uploader_id - request = compat_urllib_request.Request(url) try: - user_info_webpage = compat_urllib_request.urlopen(request).read() + user_info_webpage = self._download_webpage( + url, video_id, note=u'Downloading user information') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.report_warning(u'Unable to download user info webpage: %s' % compat_str(err)) - - user_info = xml.etree.ElementTree.fromstring(user_info_webpage) - video_uploader = user_info.find('.//nickname').text - - # uploder_date - video_upload_date = unified_strdate(video_info.find('.//first_retrieve').text.split('+')[0]) - - # view_count - video_view_count = video_info.find('.//view_counter').text - - # webpage_url - video_webpage_url = video_info.find('.//watch_url').text + else: + user_info = xml.etree.ElementTree.fromstring(user_info_webpage) + video_uploader = user_info.find('.//nickname').text return { 'id': video_id, @@ -165,26 +129,3 @@ class NiconicoIE(InfoExtractor): 'view_count': video_view_count, 'webpage_url': video_webpage_url, } - - def _extract_id(self, url): - mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - video_id = mobj.group(1) - return video_id - - def report_video_webpage_download(self, video_id): - """Report attempt to download video webpage.""" - self.to_screen(u'%s: Downloading video webpage' % video_id) - - def report_video_info_webpage_download(self, video_id): - """Report attempt to download video info webpage.""" - self.to_screen(u'%s: Downloading video info webpage' % video_id) - - def report_flv_info_webpage_download(self, video_id): - """Report attempt to download flv info webpage.""" - self.to_screen(u'%s: Downloading flv info webpage' % video_id) - - def report_information_extraction(self, video_id): - """Report attempt to extract video information.""" - self.to_screen(u'%s: Extracting video information' % video_id) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 0720fe9eb..34b3d19e0 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -12,6 +12,7 @@ import os import pipes import platform import re +import ssl import socket import sys import traceback @@ -535,13 +536,31 @@ def formatSeconds(secs): else: return '%d' % secs + def make_HTTPS_handler(opts): - if sys.version_info < (3,2): - # Python's 2.x handler is very simplistic - return compat_urllib_request.HTTPSHandler() + if sys.version_info < (3, 2): + import httplib + + class HTTPSConnectionV3(httplib.HTTPSConnection): + def __init__(self, *args, **kwargs): + httplib.HTTPSConnection.__init__(self, *args, **kwargs) + + def connect(self): + sock = socket.create_connection((self.host, self.port), self.timeout) + if self._tunnel_host: + self.sock = sock + self._tunnel() + try: + self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv3) + except ssl.SSLError as e: + self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23) + + class HTTPSHandlerV3(compat_urllib_request.HTTPSHandler): + def https_open(self, req): + return self.do_open(HTTPSConnectionV3, req) + return HTTPSHandlerV3() else: - import ssl - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context = ssl.SSLContext(ssl.PROTOCOL_SSLv3) context.set_default_verify_paths() context.verify_mode = (ssl.CERT_NONE From 38b2db6a666e094896927217aa293750a732e81d Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 24 Nov 2013 06:39:49 +0100 Subject: [PATCH 116/425] Credit @takuya0301 for niconico --- youtube_dl/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 64ebf4d48..19904dbfd 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -35,6 +35,7 @@ __authors__ = ( 'Jelle van der Waa', 'Marcin Cieślak', 'Anton Larionov', + 'Takuya Tsuchida', ) __license__ = 'Public Domain' From 2e767313e49b43400b3baae247e0f4c9e9e24992 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 24 Nov 2013 06:52:21 +0100 Subject: [PATCH 117/425] [update] fix error --- youtube_dl/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/update.py b/youtube_dl/update.py index be7800e8b..cd9670166 100644 --- a/youtube_dl/update.py +++ b/youtube_dl/update.py @@ -86,7 +86,7 @@ def update_self(to_screen, verbose): def version_tuple(version_str): return tuple(map(int, version_str.split('.'))) - if version_tuple(__version__) >= version_tuple(version_str): + if version_tuple(__version__) >= version_tuple(version_id): to_screen(u'youtube-dl is up to date (%s)' % __version__) return From 23e6d50d73188eab26944e41f164a5a1ab7f547a Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 24 Nov 2013 06:52:53 +0100 Subject: [PATCH 118/425] [bandcamp] Remove unused variable --- youtube_dl/extractor/bandcamp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py index 1aa9dbefd..3a32c14c5 100644 --- a/youtube_dl/extractor/bandcamp.py +++ b/youtube_dl/extractor/bandcamp.py @@ -34,7 +34,6 @@ class BandcampIE(InfoExtractor): json_code = m_trackinfo.group(1) data = json.loads(json_code) - entries = [] for d in data: formats = [{ 'format_id': 'format_id', From bd49928f7a0254eeb8d5f918c5649ce4eb78ef36 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 24 Nov 2013 06:53:50 +0100 Subject: [PATCH 119/425] [niconico] Clarify download --- youtube_dl/extractor/niconico.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py index 22898b5a1..729607ea3 100644 --- a/youtube_dl/extractor/niconico.py +++ b/youtube_dl/extractor/niconico.py @@ -77,9 +77,9 @@ class NiconicoIE(InfoExtractor): mobj = re.match(self._VALID_URL, url) video_id = mobj.group(1) - # Get video webpage - video_webpage = self._download_webpage( - 'http://www.nicovideo.jp/watch/' + video_id, video_id) + # Get video webpage. We are not actually interested in it, but need + # the cookies in order to be able to download the info webpage + self._download_webpage('http://www.nicovideo.jp/watch/' + video_id, video_id) video_info_webpage = self._download_webpage( 'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id, video_id, From 66ec0192406bbf1bffcb6c4e72fe1529f1e21195 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 24 Nov 2013 06:54:26 +0100 Subject: [PATCH 120/425] [youtube] do not use variable name twice --- youtube_dl/extractor/youtube.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 126688652..07a457f4d 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1571,8 +1571,8 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor): playlist_title = self._og_search_title(page) - url_results = [self.url_result(video_id, 'Youtube', video_id=video_id) - for video_id in ids] + url_results = [self.url_result(vid_id, 'Youtube', video_id=vid_id) + for vid_id in ids] return self.playlist_result(url_results, playlist_id, playlist_title) From 382ed50e0ecfb2fa692049030c858b99159c791b Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 24 Nov 2013 07:30:05 +0100 Subject: [PATCH 121/425] [viki] Add extractor (fixes #1813) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/viki.py | 91 ++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 youtube_dl/extractor/viki.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index f443f11f6..867734fa2 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -157,6 +157,7 @@ from .videofyme import VideofyMeIE from .videopremium import VideoPremiumIE from .vimeo import VimeoIE, VimeoChannelIE from .vine import VineIE +from .viki import VikiIE from .vk import VKIE from .wat import WatIE from .websurg import WeBSurgIE diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py new file mode 100644 index 000000000..78d03c079 --- /dev/null +++ b/youtube_dl/extractor/viki.py @@ -0,0 +1,91 @@ +import re + +from ..utils import ( + unified_strdate, +) +from .subtitles import SubtitlesInfoExtractor + + +class VikiIE(SubtitlesInfoExtractor): + IE_NAME = u'viki' + + _VALID_URL = r'^https?://(?:www\.)?viki\.com/videos/(?P<id>[0-9]+v)' + _TEST = { + u'url': u'http://www.viki.com/videos/1023585v-heirs-episode-14', + u'file': u'1023585v.mp4', + u'md5': u'a21454021c2646f5433514177e2caa5f', + u'info_dict': { + u'title': u'Heirs Episode 14', + u'uploader': u'SBS', + u'description': u'md5:c4b17b9626dd4b143dcc4d855ba3474e', + u'upload_date': u'20131121', + u'age_limit': 13, + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group(1) + + webpage = self._download_webpage(url, video_id) + title = self._og_search_title(webpage) + description = self._og_search_description(webpage) + thumbnail = self._og_search_thumbnail(webpage) + + uploader = self._html_search_regex( + r'<strong>Broadcast Network: </strong>\s*([^<]*)<', webpage, + u'uploader') + if uploader is not None: + uploader = uploader.strip() + + rating_str = self._html_search_regex( + r'<strong>Rating: </strong>\s*([^<]*)<', webpage, + u'rating information', default='').strip() + RATINGS = { + 'G': 0, + 'PG': 10, + 'PG-13': 13, + 'R': 16, + 'NC': 18, + } + age_limit = RATINGS.get(rating_str) + + info_url = 'http://www.viki.com/player5_fragment/%s?action=show&controller=videos' % video_id + info_webpage = self._download_webpage(info_url, video_id) + video_url = self._html_search_regex( + r'<source[^>]+src="([^"]+)"', info_webpage, u'video URL') + + upload_date_str = self._html_search_regex( + r'"created_at":"([^"]+)"', info_webpage, u'upload date') + upload_date = ( + unified_strdate(upload_date_str) + if upload_date_str is not None + else None + ) + + # subtitles + video_subtitles = self.extract_subtitles(video_id, info_webpage) + if self._downloader.params.get('listsubtitles', False): + self._list_available_subtitles(video_id, info_webpage) + return + + return { + 'id': video_id, + 'title': title, + 'url': video_url, + 'description': description, + 'thumbnail': thumbnail, + 'age_limit': age_limit, + 'uploader': uploader, + 'subtitles': video_subtitles, + 'upload_date': upload_date, + } + + def _get_available_subtitles(self, video_id, info_webpage): + res = {} + for sturl in re.findall(r'<track src="([^"]+)"/>'): + m = re.search(r'/(?P<lang>[a-z]+)\.vtt', sturl) + if not m: + continue + res[m.group('lang')] = sturl + return res From eaaafc59c2f8ffaee4df06092a57f65eec1b6eaa Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 24 Nov 2013 07:30:34 +0100 Subject: [PATCH 122/425] release 2013.11.24 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index f6d18f945..68ef46a30 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.22.2' +__version__ = '2013.11.24' From 0c7c19d6bc55a624532f2426d080aea51962cfe0 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 24 Nov 2013 07:51:44 +0100 Subject: [PATCH 123/425] [clipfish] Add extractor (Fixes #1760) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/clipfish.py | 53 ++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 youtube_dl/extractor/clipfish.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 867734fa2..4c280fa5e 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -20,6 +20,7 @@ from .c56 import C56IE from .canalplus import CanalplusIE from .canalc2 import Canalc2IE from .cinemassacre import CinemassacreIE +from .clipfish import ClipfishIE from .cnn import CNNIE from .collegehumor import CollegeHumorIE from .comedycentral import ComedyCentralIE diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py new file mode 100644 index 000000000..95449da3c --- /dev/null +++ b/youtube_dl/extractor/clipfish.py @@ -0,0 +1,53 @@ +import re +import time +import xml.etree.ElementTree + +from .common import InfoExtractor + + +class ClipfishIE(InfoExtractor): + IE_NAME = u'clipfish' + + _VALID_URL = r'^https?://(?:www\.)?clipfish\.de/.*?/video/(?P<id>[0-9]+)/' + _TEST = { + u'url': u'http://www.clipfish.de/special/supertalent/video/4028320/supertalent-2013-ivana-opacak-singt-nobodys-perfect/', + u'file': u'4028320.f4v', + u'md5': u'5e38bda8c329fbfb42be0386a3f5a382', + u'info_dict': { + u'title': u'Supertalent 2013: Ivana Opacak singt Nobody\'s Perfect', + u'duration': 399, + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group(1) + + info_url = ('http://www.clipfish.de/devxml/videoinfo/%s?ts=%d' % + (video_id, int(time.time()))) + info_xml = self._download_webpage( + info_url, video_id, note=u'Downloading info page') + doc = xml.etree.ElementTree.fromstring(info_xml) + title = doc.find('title').text + video_url = doc.find('filename').text + thumbnail = doc.find('imageurl').text + duration_str = doc.find('duration').text + m = re.match( + r'^(?P<hours>[0-9]+):(?P<minutes>[0-9]{2}):(?P<seconds>[0-9]{2}):(?P<ms>[0-9]*)$', + duration_str) + if m: + duration = ( + (int(m.group('hours')) * 60 * 60) + + (int(m.group('minutes')) * 60) + + (int(m.group('seconds'))) + ) + else: + duration = None + + return { + 'id': video_id, + 'title': title, + 'url': video_url, + 'thumbnail': thumbnail, + 'duration': duration, + } From 138df537ffaeda182789440c4086f009a739dde3 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 24 Nov 2013 07:51:56 +0100 Subject: [PATCH 124/425] release 2013.11.24.1 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 68ef46a30..de92411bb 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.24' +__version__ = '2013.11.24.1' From d214fdb8fe796e92485e28038ee72d28caa3ad10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Sun, 24 Nov 2013 11:02:34 +0100 Subject: [PATCH 125/425] [brightcove] Don't use 'or' with the xml nodes, use the 'value' attribute instead --- youtube_dl/extractor/brightcove.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py index 74a7d13e3..66fe0ac9a 100644 --- a/youtube_dl/extractor/brightcove.py +++ b/youtube_dl/extractor/brightcove.py @@ -76,18 +76,21 @@ class BrightcoveIE(InfoExtractor): 'playerID': find_xpath_attr(object_doc, './param', 'name', 'playerID').attrib['value'], } def find_param(name): - return find_xpath_attr(object_doc, './param', 'name', name) + node = find_xpath_attr(object_doc, './param', 'name', name) + if node is not None: + return node.attrib['value'] + return None playerKey = find_param('playerKey') # Not all pages define this value if playerKey is not None: - params['playerKey'] = playerKey.attrib['value'] + params['playerKey'] = playerKey # The three fields hold the id of the video videoPlayer = find_param('@videoPlayer') or find_param('videoId') or find_param('videoID') if videoPlayer is not None: - params['@videoPlayer'] = videoPlayer.attrib['value'] + params['@videoPlayer'] = videoPlayer linkBase = find_param('linkBaseURL') if linkBase is not None: - params['linkBaseURL'] = linkBase.attrib['value'] + params['linkBaseURL'] = linkBase data = compat_urllib_parse.urlencode(params) return cls._FEDERATED_URL_TEMPLATE % data From dc65dcbb6d709ef6e38f336fe77c14767d6c8f9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Sun, 24 Nov 2013 11:28:44 +0100 Subject: [PATCH 126/425] [mixcloud] The description field may be missing (fixes #1819) --- youtube_dl/extractor/mixcloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py index a200dcd74..e2baf44d7 100644 --- a/youtube_dl/extractor/mixcloud.py +++ b/youtube_dl/extractor/mixcloud.py @@ -60,7 +60,7 @@ class MixcloudIE(InfoExtractor): 'title': info['name'], 'url': final_song_url, 'ext': 'mp3', - 'description': info['description'], + 'description': info.get('description'), 'thumbnail': info['pictures'].get('extra_large'), 'uploader': info['user']['name'], 'uploader_id': info['user']['username'], From f459d17018812dc896324f8208cdfe2ada04ea50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Sun, 24 Nov 2013 14:33:50 +0100 Subject: [PATCH 127/425] [youtube] Add an extractor for downloading the watch history (closes #1821) --- test/test_all_urls.py | 1 + youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/youtube.py | 14 ++++++++++++++ 3 files changed, 16 insertions(+) diff --git a/test/test_all_urls.py b/test/test_all_urls.py index 56e5f80e1..42813da1a 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -100,6 +100,7 @@ class TestAllURLsMatching(unittest.TestCase): def test_keywords(self): self.assertMatch(':ytsubs', ['youtube:subscriptions']) self.assertMatch(':ytsubscriptions', ['youtube:subscriptions']) + self.assertMatch(':ythistory', ['youtube:history']) self.assertMatch(':thedailyshow', ['ComedyCentral']) self.assertMatch(':tds', ['ComedyCentral']) self.assertMatch(':colbertreport', ['ComedyCentral']) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 4c280fa5e..1fbd10bc5 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -186,6 +186,7 @@ from .youtube import ( YoutubeTruncatedURLIE, YoutubeWatchLaterIE, YoutubeFavouritesIE, + YoutubeHistoryIE, ) from .zdf import ZDFIE diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 07a457f4d..64d4c2445 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1826,6 +1826,20 @@ class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor): _PAGING_STEP = 100 _PERSONAL_FEED = True +class YoutubeHistoryIE(YoutubeFeedsInfoExtractor): + IE_DESC = u'Youtube watch history, "ythistory" keyword (requires authentication)' + _VALID_URL = u'https?://www\.youtube\.com/feed/history|:ythistory' + _FEED_NAME = 'history' + _PERSONAL_FEED = True + _PLAYLIST_TITLE = u'Youtube Watch History' + + def _real_extract(self, url): + webpage = self._download_webpage('https://www.youtube.com/feed/history', u'History') + data_paging = self._search_regex(r'data-paging="(\d+)"', webpage, u'data-paging') + # The step is actually a ridiculously big number (like 1374343569725646) + self._PAGING_STEP = int(data_paging) + return super(YoutubeHistoryIE, self)._real_extract(url) + class YoutubeFavouritesIE(YoutubeBaseInfoExtractor): IE_NAME = u'youtube:favorites' IE_DESC = u'YouTube.com favourite videos, "ytfav" keyword (requires authentication)' From 267ed0c5d3547c68f1d34203c2ae4b0d826a29d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Sun, 24 Nov 2013 14:59:19 +0100 Subject: [PATCH 128/425] [collegehumor] Encode the xml before calling xml.etree.ElementTree.fromstring (fixes #1822) Uses a new helper method in InfoExtractor: _download_xml --- youtube_dl/extractor/collegehumor.py | 7 ++----- youtube_dl/extractor/common.py | 6 ++++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/collegehumor.py b/youtube_dl/extractor/collegehumor.py index 0c29acfb1..b27c1dfc5 100644 --- a/youtube_dl/extractor/collegehumor.py +++ b/youtube_dl/extractor/collegehumor.py @@ -1,5 +1,4 @@ import re -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( @@ -46,11 +45,10 @@ class CollegeHumorIE(InfoExtractor): self.report_extraction(video_id) xmlUrl = 'http://www.collegehumor.com/moogaloop/video/' + video_id - metaXml = self._download_webpage(xmlUrl, video_id, + mdoc = self._download_xml(xmlUrl, video_id, u'Downloading info XML', u'Unable to download video info XML') - mdoc = xml.etree.ElementTree.fromstring(metaXml) try: videoNode = mdoc.findall('./video')[0] youtubeIdNode = videoNode.find('./youtubeID') @@ -65,11 +63,10 @@ class CollegeHumorIE(InfoExtractor): if next_url.endswith(u'manifest.f4m'): manifest_url = next_url + '?hdcore=2.10.3' - manifestXml = self._download_webpage(manifest_url, video_id, + adoc = self._download_xml(manifest_url, video_id, u'Downloading XML manifest', u'Unable to download video info XML') - adoc = xml.etree.ElementTree.fromstring(manifestXml) try: video_id = adoc.findall('./{http://ns.adobe.com/f4m/1.0}id')[0].text except IndexError: diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 3cebeaf29..482a231ec 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -4,6 +4,7 @@ import re import socket import sys import netrc +import xml.etree.ElementTree from ..utils import ( compat_http_client, @@ -208,6 +209,11 @@ class InfoExtractor(object): """ Returns the data of the page as a string """ return self._download_webpage_handle(url_or_request, video_id, note, errnote)[0] + def _download_xml(self, url_or_request, video_id, note=u'Downloading XML', errnote=u'Unable to downloand XML'): + """Return the xml as an xml.etree.ElementTree.Element""" + xml_string = self._download_webpage(url_or_request, video_id, note, errnote) + return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8')) + def to_screen(self, msg): """Print msg to screen, prefixing it with '[ie_name]'""" self._downloader.to_screen(u'[%s] %s' % (self.IE_NAME, msg)) From a1ee09e815cb413d67cee17ad686224b26182dfb Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 24 Nov 2013 15:03:25 +0100 Subject: [PATCH 129/425] Document proxy --- youtube_dl/YoutubeDL.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 72ccfa2ae..0a845a344 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -128,7 +128,8 @@ class YoutubeDL(object): Videos already present in the file are not downloaded again. cookiefile: File name where cookies should be read from and dumped to. - nocheckcertificate Do not verify SSL certificates + nocheckcertificate:Do not verify SSL certificates + proxy: URL of the proxy server to use The following parameters are not used by YoutubeDL itself, they are used by the FileDownloader: From b7553b25543175c27c885b0c6ab77d91b270a520 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 24 Nov 2013 15:20:16 +0100 Subject: [PATCH 130/425] [vik] Clarify output --- youtube_dl/extractor/viki.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py index 78d03c079..8088dcf0b 100644 --- a/youtube_dl/extractor/viki.py +++ b/youtube_dl/extractor/viki.py @@ -51,7 +51,8 @@ class VikiIE(SubtitlesInfoExtractor): age_limit = RATINGS.get(rating_str) info_url = 'http://www.viki.com/player5_fragment/%s?action=show&controller=videos' % video_id - info_webpage = self._download_webpage(info_url, video_id) + info_webpage = self._download_webpage( + info_url, video_id, note=u'Downloading info page') video_url = self._html_search_regex( r'<source[^>]+src="([^"]+)"', info_webpage, u'video URL') From 6d88bc37a32d5d624c09d68cd19e64e6095fa5de Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 24 Nov 2013 15:28:33 +0100 Subject: [PATCH 131/425] [viki] Skip travis test Also provide a better error message for geoblocked videos. --- youtube_dl/extractor/viki.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py index 8088dcf0b..7b3a58de8 100644 --- a/youtube_dl/extractor/viki.py +++ b/youtube_dl/extractor/viki.py @@ -1,6 +1,7 @@ import re from ..utils import ( + ExtractorError, unified_strdate, ) from .subtitles import SubtitlesInfoExtractor @@ -20,7 +21,8 @@ class VikiIE(SubtitlesInfoExtractor): u'description': u'md5:c4b17b9626dd4b143dcc4d855ba3474e', u'upload_date': u'20131121', u'age_limit': 13, - } + }, + u'skip': u'Blocked in the US', } def _real_extract(self, url): @@ -53,6 +55,10 @@ class VikiIE(SubtitlesInfoExtractor): info_url = 'http://www.viki.com/player5_fragment/%s?action=show&controller=videos' % video_id info_webpage = self._download_webpage( info_url, video_id, note=u'Downloading info page') + if re.match(r'\s*<div\s+class="video-error', info_webpage): + raise ExtractorError( + u'Video %s is blocked from your location.' % video_id, + expected=True) video_url = self._html_search_regex( r'<source[^>]+src="([^"]+)"', info_webpage, u'video URL') From 66cfab4226296c1596fbf37c27758bbdb6846d53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Sun, 24 Nov 2013 21:18:35 +0100 Subject: [PATCH 132/425] [comedycentral] Add support for comedycentral.com videos (closes #1824) It's a subclass of MTVIE The extractor for colbertnation.com and thedailyshow.com is called now ComedyCentralShowsIE --- youtube_dl/extractor/__init__.py | 2 +- youtube_dl/extractor/comedycentral.py | 33 ++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 1fbd10bc5..0b4d086b7 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -23,7 +23,7 @@ from .cinemassacre import CinemassacreIE from .clipfish import ClipfishIE from .cnn import CNNIE from .collegehumor import CollegeHumorIE -from .comedycentral import ComedyCentralIE +from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE from .condenast import CondeNastIE from .criterion import CriterionIE from .cspan import CSpanIE diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py index 69b2beece..725849d2e 100644 --- a/youtube_dl/extractor/comedycentral.py +++ b/youtube_dl/extractor/comedycentral.py @@ -2,6 +2,7 @@ import re import xml.etree.ElementTree from .common import InfoExtractor +from .mtv import MTVIE, _media_xml_tag from ..utils import ( compat_str, compat_urllib_parse, @@ -11,7 +12,37 @@ from ..utils import ( ) -class ComedyCentralIE(InfoExtractor): +class ComedyCentralIE(MTVIE): + _VALID_URL = r'http://www.comedycentral.com/(video-clips|episodes|cc-studios)/(?P<title>.*)' + _FEED_URL = u'http://comedycentral.com/feeds/mrss/' + + _TEST = { + u'url': u'http://www.comedycentral.com/video-clips/kllhuv/stand-up-greg-fitzsimmons--uncensored---too-good-of-a-mother', + u'md5': u'4167875aae411f903b751a21f357f1ee', + u'info_dict': { + u'id': u'cef0cbb3-e776-4bc9-b62e-8016deccb354', + u'ext': u'mp4', + u'title': u'Uncensored - Greg Fitzsimmons - Too Good of a Mother', + u'description': u'After a certain point, breastfeeding becomes c**kblocking.', + }, + } + # Overwrite MTVIE properties we don't want + _TESTS = [] + + def _get_thumbnail_url(self, uri, itemdoc): + search_path = '%s/%s' % (_media_xml_tag('group'), _media_xml_tag('thumbnail')) + return itemdoc.find(search_path).attrib['url'] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + title = mobj.group('title') + webpage = self._download_webpage(url, title) + mgid = self._search_regex(r'data-mgid="(?P<mgid>mgid:.*?)"', + webpage, u'mgid') + return self._get_videos_info(mgid) + + +class ComedyCentralShowsIE(InfoExtractor): IE_DESC = u'The Daily Show / Colbert Report' # urls can be abbreviations like :thedailyshow or :colbert # urls for episodes like: From 16e055849ebfa5a942aef4411728b36bf53ebaa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Sun, 24 Nov 2013 22:13:20 +0100 Subject: [PATCH 133/425] Update the keywords tests for the rename of the old ComedyCentralIE --- test/test_all_urls.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_all_urls.py b/test/test_all_urls.py index 42813da1a..1f1adb6b4 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -101,10 +101,10 @@ class TestAllURLsMatching(unittest.TestCase): self.assertMatch(':ytsubs', ['youtube:subscriptions']) self.assertMatch(':ytsubscriptions', ['youtube:subscriptions']) self.assertMatch(':ythistory', ['youtube:history']) - self.assertMatch(':thedailyshow', ['ComedyCentral']) - self.assertMatch(':tds', ['ComedyCentral']) - self.assertMatch(':colbertreport', ['ComedyCentral']) - self.assertMatch(':cr', ['ComedyCentral']) + self.assertMatch(':thedailyshow', ['ComedyCentralShows']) + self.assertMatch(':tds', ['ComedyCentralShows']) + self.assertMatch(':colbertreport', ['ComedyCentralShows']) + self.assertMatch(':cr', ['ComedyCentralShows']) if __name__ == '__main__': From 1fb2bcbbf748e07d05f98110cc27d440506a9b77 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 02:02:34 +0100 Subject: [PATCH 134/425] [viki] Make uploader field optional (#1813) --- youtube_dl/extractor/viki.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py index 7b3a58de8..cd986a749 100644 --- a/youtube_dl/extractor/viki.py +++ b/youtube_dl/extractor/viki.py @@ -34,11 +34,12 @@ class VikiIE(SubtitlesInfoExtractor): description = self._og_search_description(webpage) thumbnail = self._og_search_thumbnail(webpage) - uploader = self._html_search_regex( - r'<strong>Broadcast Network: </strong>\s*([^<]*)<', webpage, - u'uploader') - if uploader is not None: - uploader = uploader.strip() + uploader_m = re.search( + r'<strong>Broadcast Network: </strong>\s*([^<]*)<', webpage) + if uploader_m is None: + uploader = None + else: + uploader = uploader.group(1).strip() rating_str = self._html_search_regex( r'<strong>Rating: </strong>\s*([^<]*)<', webpage, From 02dbf93f0e98a56ed04b4a9e6a6d62efd6d801f9 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 03:12:26 +0100 Subject: [PATCH 135/425] [zdf/common] Use API in ZDF extractor. This also comes with a lot of extra format fields Fixes #1518 --- youtube_dl/FileDownloader.py | 20 +----- youtube_dl/YoutubeDL.py | 24 ++++--- youtube_dl/extractor/common.py | 2 + youtube_dl/extractor/zdf.py | 115 ++++++++++++++++++++------------- youtube_dl/utils.py | 21 ++++++ 5 files changed, 112 insertions(+), 70 deletions(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index e5a542ed5..2b4fb0b31 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -1,4 +1,3 @@ -import math import os import re import subprocess @@ -11,6 +10,7 @@ from .utils import ( ContentTooShortError, determine_ext, encodeFilename, + format_bytes, sanitize_open, timeconvert, ) @@ -53,20 +53,6 @@ class FileDownloader(object): self._progress_hooks = [] self.params = params - @staticmethod - def format_bytes(bytes): - if bytes is None: - return 'N/A' - if type(bytes) is str: - bytes = float(bytes) - if bytes == 0.0: - exponent = 0 - else: - exponent = int(math.log(bytes, 1024.0)) - suffix = ['B','KiB','MiB','GiB','TiB','PiB','EiB','ZiB','YiB'][exponent] - converted = float(bytes) / float(1024 ** exponent) - return '%.2f%s' % (converted, suffix) - @staticmethod def format_seconds(seconds): (mins, secs) = divmod(seconds, 60) @@ -117,7 +103,7 @@ class FileDownloader(object): def format_speed(speed): if speed is None: return '%10s' % '---b/s' - return '%10s' % ('%s/s' % FileDownloader.format_bytes(speed)) + return '%10s' % ('%s/s' % format_bytes(speed)) @staticmethod def best_block_size(elapsed_time, bytes): @@ -525,7 +511,7 @@ class FileDownloader(object): self.to_screen(u'\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len)) return False - data_len_str = self.format_bytes(data_len) + data_len_str = format_bytes(data_len) byte_counter = 0 + resume_len block_size = self.params.get('buffersize', 1024) start = time.time() diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index d7e2417ac..0578fe6c1 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -30,6 +30,7 @@ from .utils import ( DownloadError, encodeFilename, ExtractorError, + format_bytes, locked_file, MaxDownloadsReached, PostProcessingError, @@ -867,9 +868,11 @@ class YoutubeDL(object): def list_formats(self, info_dict): def format_note(fdict): - if fdict.get('format_note') is not None: - return fdict['format_note'] res = u'' + if fdict.get('format_note') is not None: + res += fdict['format_note'] + u' ' + if fdict.get('quality_name') is not None: + res += u'%s ' % fdict['quality_name'] if fdict.get('vcodec') is not None: res += u'%-5s' % fdict['vcodec'] elif fdict.get('vbr') is not None: @@ -886,25 +889,30 @@ class YoutubeDL(object): res += 'audio' if fdict.get('abr') is not None: res += u'@%3dk' % fdict['abr'] + if fdict.get('filesize') is not None: + if res: + res += u', ' + res += format_bytes(fdict['filesize']) return res - def line(format): - return (u'%-20s%-10s%-12s%s' % ( + def line(format, idlen=20): + return ((u'%-' + compat_str(idlen + 1) + u's%-10s%-12s%s') % ( format['format_id'], format['ext'], self.format_resolution(format), format_note(format), - ) - ) + )) formats = info_dict.get('formats', [info_dict]) - formats_s = list(map(line, formats)) + idlen = max(len(u'format code'), + max(len(f['format_id']) for f in formats)) + formats_s = [line(f, idlen) for f in formats] if len(formats) > 1: formats_s[0] += (' ' if format_note(formats[0]) else '') + '(worst)' formats_s[-1] += (' ' if format_note(formats[-1]) else '') + '(best)' header_line = line({ 'format_id': u'format code', 'ext': u'extension', - '_resolution': u'resolution', 'format_note': u'note'}) + '_resolution': u'resolution', 'format_note': u'note'}, idlen=idlen) self.to_screen(u'[info] Available formats for %s:\n%s\n%s' % (info_dict['id'], header_line, u"\n".join(formats_s))) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 482a231ec..3c4781121 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -76,6 +76,8 @@ class InfoExtractor(object): * acodec Name of the audio codec in use * vbr Average video bitrate in KBit/s * vcodec Name of the video codec in use + * quality_name Human-readable name of the video quality. + * filesize The number of bytes, if known in advance webpage_url: The url to the video webpage, if given to youtube-dl it should allow to get the same result again. (It will be set by YoutubeDL if it's missing) diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py index c6a9d06f2..a8d899883 100644 --- a/youtube_dl/extractor/zdf.py +++ b/youtube_dl/extractor/zdf.py @@ -1,75 +1,100 @@ +import operator import re from .common import InfoExtractor from ..utils import ( - determine_ext, - ExtractorError, + parse_xml_doc, + unified_strdate, ) class ZDFIE(InfoExtractor): _VALID_URL = r'^http://www\.zdf\.de\/ZDFmediathek(?P<hash>#)?\/(.*beitrag\/video\/)(?P<video_id>[^/\?]+)(?:\?.*)?' - _MEDIA_STREAM = r'<a href="(?P<video_url>.+(?P<media_type>.streaming).+/zdf/(?P<quality>[^\/]+)/[^"]*)".+class="play".+>' def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) video_id = mobj.group('video_id') - if mobj.group('hash'): - url = url.replace(u'#', u'', 1) + xml_url = u'http://www.zdf.de/ZDFmediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id + info_xml = self._download_webpage( + xml_url, video_id, note=u'Downloading video info') + doc = parse_xml_doc(info_xml) - html = self._download_webpage(url, video_id) - streams = [m.groupdict() for m in re.finditer(self._MEDIA_STREAM, html)] - if streams is None: - raise ExtractorError(u'No media url found.') + title = doc.find('.//information/title').text + description = doc.find('.//information/detail').text + uploader_node = doc.find('.//details/originChannelTitle') + uploader = None if uploader_node is None else uploader_node.text + duration_str = doc.find('.//details/length').text + duration_m = re.match(r'''(?x)^ + (?P<hours>[0-9]{2}) + :(?P<minutes>[0-9]{2}) + :(?P<seconds>[0-9]{2}) + (?:\.(?P<ms>[0-9]+)?) + ''', duration_str) + duration = ( + ( + (int(duration_m.group('hours')) * 60 * 60) + + (int(duration_m.group('minutes')) * 60) + + int(duration_m.group('seconds')) + ) + if duration_m + else None + ) + upload_date = unified_strdate(doc.find('.//details/airtime').text) - # s['media_type'] == 'wstreaming' -> use 'Windows Media Player' and mms url - # s['media_type'] == 'hstreaming' -> use 'Quicktime' and rtsp url - # choose first/default media type and highest quality for now - def stream_pref(s): - TYPE_ORDER = ['ostreaming', 'hstreaming', 'wstreaming'] + def xml_to_format(fnode): + video_url = fnode.find('url').text + is_available = u'http://www.metafilegenerator' not in video_url + + format_id = fnode.attrib['basetype'] + format_m = re.match(r'''(?x) + (?P<vcodec>[^_]+)_(?P<acodec>[^_]+)_(?P<container>[^_]+)_ + (?P<proto>[^_]+)_(?P<index>[^_]+)_(?P<indexproto>[^_]+) + ''', format_id) + + PROTO_ORDER = ['http', 'rtmp', 'rtsp'] try: - type_pref = TYPE_ORDER.index(s['media_type']) + proto_pref = -PROTO_ORDER.index(format_m.group('proto')) except ValueError: - type_pref = 999 + proto_pref = 999 - QUALITY_ORDER = ['veryhigh', '300'] + quality = fnode.find('./quality').text + QUALITY_ORDER = ['veryhigh', '300', 'high', 'med', 'low'] try: - quality_pref = QUALITY_ORDER.index(s['quality']) + quality_pref = -QUALITY_ORDER.index(quality) except ValueError: quality_pref = 999 - return (type_pref, quality_pref) + abr = int(fnode.find('./audioBitrate').text) // 1000 + vbr = int(fnode.find('./videoBitrate').text) // 1000 + pref = (is_available, proto_pref, quality_pref, vbr, abr) - sorted_streams = sorted(streams, key=stream_pref) - if not sorted_streams: - raise ExtractorError(u'No stream found.') - stream = sorted_streams[0] + return { + 'format_id': format_id, + 'url': video_url, + 'ext': format_m.group('container'), + 'acodec': format_m.group('acodec'), + 'vcodec': format_m.group('vcodec'), + 'abr': abr, + 'vbr': vbr, + 'width': int(fnode.find('./width').text), + 'height': int(fnode.find('./height').text), + 'quality_name': quality, + 'filesize': int(fnode.find('./filesize').text), + 'format_note': None if is_available else u'(unavailable)', + '_pref': pref, + } - media_link = self._download_webpage( - stream['video_url'], - video_id, - u'Get stream URL') - - #MMS_STREAM = r'href="(?P<video_url>mms://[^"]*)"' - RTSP_STREAM = r'(?P<video_url>rtsp://[^"]*.mp4)' - - mobj = re.search(self._MEDIA_STREAM, media_link) - if mobj is None: - mobj = re.search(RTSP_STREAM, media_link) - if mobj is None: - raise ExtractorError(u'Cannot extract mms:// or rtsp:// URL') - video_url = mobj.group('video_url') - - title = self._html_search_regex( - r'<h1(?: class="beitragHeadline")?>(.*?)</h1>', - html, u'title') + format_nodes = doc.findall('.//formitaeten/formitaet') + formats = sorted(map(xml_to_format, format_nodes), + key=operator.itemgetter('_pref')) return { 'id': video_id, - 'url': video_url, 'title': title, - 'ext': determine_ext(video_url) + 'formats': formats, + 'description': description, + 'uploader': uploader, + 'duration': duration, + 'upload_date': upload_date, } diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 34b3d19e0..ad0a06287 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -8,6 +8,7 @@ import gzip import io import json import locale +import math import os import pipes import platform @@ -16,6 +17,7 @@ import ssl import socket import sys import traceback +import xml.etree.ElementTree import zlib try: @@ -1006,3 +1008,22 @@ def unsmuggle_url(smug_url): jsond = compat_parse_qs(sdata)[u'__youtubedl_smuggle'][0] data = json.loads(jsond) return url, data + + +def parse_xml_doc(s): + assert isinstance(s, type(u'')) + return xml.etree.ElementTree.fromstring(s.encode('utf-8')) + + +def format_bytes(bytes): + if bytes is None: + return u'N/A' + if type(bytes) is str: + bytes = float(bytes) + if bytes == 0.0: + exponent = 0 + else: + exponent = int(math.log(bytes, 1024.0)) + suffix = [u'B', u'KiB', u'MiB', u'GiB', u'TiB', u'PiB', u'EiB', u'ZiB', u'YiB'][exponent] + converted = float(bytes) / float(1024 ** exponent) + return u'%.2f%s' % (converted, suffix) From c059bdd432911cff8c7426380a876c9679855ab5 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 03:28:55 +0100 Subject: [PATCH 136/425] Remove quality_name field and improve zdf extractor --- youtube_dl/YoutubeDL.py | 2 -- youtube_dl/extractor/common.py | 1 - youtube_dl/extractor/zdf.py | 23 +++++++++++++++++------ 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 0578fe6c1..a896d9e63 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -871,8 +871,6 @@ class YoutubeDL(object): res = u'' if fdict.get('format_note') is not None: res += fdict['format_note'] + u' ' - if fdict.get('quality_name') is not None: - res += u'%s ' % fdict['quality_name'] if fdict.get('vcodec') is not None: res += u'%-5s' % fdict['vcodec'] elif fdict.get('vbr') is not None: diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 3c4781121..3d8ac8ba2 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -76,7 +76,6 @@ class InfoExtractor(object): * acodec Name of the audio codec in use * vbr Average video bitrate in KBit/s * vcodec Name of the video codec in use - * quality_name Human-readable name of the video quality. * filesize The number of bytes, if known in advance webpage_url: The url to the video webpage, if given to youtube-dl it should allow to get the same result again. (It will be set diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py index a8d899883..07f830e80 100644 --- a/youtube_dl/extractor/zdf.py +++ b/youtube_dl/extractor/zdf.py @@ -52,6 +52,9 @@ class ZDFIE(InfoExtractor): (?P<proto>[^_]+)_(?P<index>[^_]+)_(?P<indexproto>[^_]+) ''', format_id) + ext = format_m.group('container') + is_supported = ext != 'f4f' + PROTO_ORDER = ['http', 'rtmp', 'rtsp'] try: proto_pref = -PROTO_ORDER.index(format_m.group('proto')) @@ -67,26 +70,34 @@ class ZDFIE(InfoExtractor): abr = int(fnode.find('./audioBitrate').text) // 1000 vbr = int(fnode.find('./videoBitrate').text) // 1000 - pref = (is_available, proto_pref, quality_pref, vbr, abr) + pref = (is_available, is_supported, + proto_pref, quality_pref, vbr, abr) + + format_note = u'' + if not is_supported: + format_note += u'(unsupported)' + if not format_note: + format_note = None return { - 'format_id': format_id, + 'format_id': format_id + u'-' + quality, 'url': video_url, - 'ext': format_m.group('container'), + 'ext': ext, 'acodec': format_m.group('acodec'), 'vcodec': format_m.group('vcodec'), 'abr': abr, 'vbr': vbr, 'width': int(fnode.find('./width').text), 'height': int(fnode.find('./height').text), - 'quality_name': quality, 'filesize': int(fnode.find('./filesize').text), - 'format_note': None if is_available else u'(unavailable)', + 'format_note': format_note, '_pref': pref, + '_available': is_available, } format_nodes = doc.findall('.//formitaeten/formitaet') - formats = sorted(map(xml_to_format, format_nodes), + formats = sorted(filter(lambda f: f['_available'], + map(xml_to_format, format_nodes)), key=operator.itemgetter('_pref')) return { From 113577e155b10d6775f38e00b897f8e1d743a17e Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 03:35:52 +0100 Subject: [PATCH 137/425] [generic] Improve detection Allow download of http://goo.gl/7X5tOk Fixes #1818 --- youtube_dl/extractor/generic.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 0b5f2b2bb..37671430a 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -209,7 +209,7 @@ class GenericIE(InfoExtractor): mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage) if mobj is None: # Broaden the search a little bit: JWPlayer JS loader - mobj = re.search(r'[^A-Za-z0-9]?file["\']?:\s*["\'](http[^\'"&]*)', webpage) + mobj = re.search(r'[^A-Za-z0-9]?file["\']?:\s*["\'](http[^\'"]*)', webpage) if mobj is None: # Try to find twitter cards info mobj = re.search(r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage) @@ -236,18 +236,16 @@ class GenericIE(InfoExtractor): video_id = compat_urllib_parse.unquote(os.path.basename(video_url)) # here's a fun little line of code for you: - video_extension = os.path.splitext(video_id)[1][1:] video_id = os.path.splitext(video_id)[0] # video uploader is domain name video_uploader = self._search_regex(r'(?:https?://)?([^/]*)/.*', url, u'video uploader') - return [{ + return { 'id': video_id, 'url': video_url, 'uploader': video_uploader, 'upload_date': None, 'title': video_title, - 'ext': video_extension, - }] + } From ac05067d3dbc68cd50e8e07d51700b5a8a698a29 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 03:37:49 +0100 Subject: [PATCH 138/425] release 2013.11.25 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index de92411bb..91b36e55c 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.24.1' +__version__ = '2013.11.25' From d0efb9ec9a85662fa43f026339821513ac2f039c Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 03:47:32 +0100 Subject: [PATCH 139/425] [tests] Remove global_setup function --- test/helper.py | 4 ---- test/test_age_restriction.py | 3 +-- test/test_download.py | 2 -- test/test_playlists.py | 3 +-- test/test_subtitles.py | 3 +-- test/test_write_annotations.py | 3 +-- test/test_write_info_json.py | 3 +-- test/test_youtube_lists.py | 3 +-- test/test_youtube_signature.py | 3 --- 9 files changed, 6 insertions(+), 21 deletions(-) diff --git a/test/helper.py b/test/helper.py index d7bf7a828..b1f421ac5 100644 --- a/test/helper.py +++ b/test/helper.py @@ -12,10 +12,6 @@ from youtube_dl import YoutubeDL from youtube_dl.utils import preferredencoding -def global_setup(): - youtube_dl._setup_opener(timeout=10) - - def get_params(override=None): PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json") diff --git a/test/test_age_restriction.py b/test/test_age_restriction.py index 506572e9e..c9cdb96cb 100644 --- a/test/test_age_restriction.py +++ b/test/test_age_restriction.py @@ -6,8 +6,7 @@ import sys import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import global_setup, try_rm -global_setup() +from test.helper import try_rm from youtube_dl import YoutubeDL diff --git a/test/test_download.py b/test/test_download.py index fe7f7b8cb..dd5818dba 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -9,12 +9,10 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import ( get_params, get_testcases, - global_setup, try_rm, md5, report_warning ) -global_setup() import hashlib diff --git a/test/test_playlists.py b/test/test_playlists.py index 7c67239a4..167801ae2 100644 --- a/test/test_playlists.py +++ b/test/test_playlists.py @@ -8,8 +8,7 @@ import sys import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import FakeYDL, global_setup -global_setup() +from test.helper import FakeYDL from youtube_dl.extractor import ( diff --git a/test/test_subtitles.py b/test/test_subtitles.py index 06a304879..94a1f771d 100644 --- a/test/test_subtitles.py +++ b/test/test_subtitles.py @@ -6,8 +6,7 @@ import sys import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import FakeYDL, global_setup, md5 -global_setup() +from test.helper import FakeYDL, md5 from youtube_dl.extractor import ( diff --git a/test/test_write_annotations.py b/test/test_write_annotations.py index 35defb895..eac53b285 100644 --- a/test/test_write_annotations.py +++ b/test/test_write_annotations.py @@ -7,8 +7,7 @@ import sys import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import get_params, global_setup, try_rm -global_setup() +from test.helper import get_params, try_rm import io diff --git a/test/test_write_info_json.py b/test/test_write_info_json.py index 30c4859fd..d7177611b 100644 --- a/test/test_write_info_json.py +++ b/test/test_write_info_json.py @@ -7,8 +7,7 @@ import sys import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import get_params, global_setup -global_setup() +from test.helper import get_params import io diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py index 938517a2d..8fd073f31 100644 --- a/test/test_youtube_lists.py +++ b/test/test_youtube_lists.py @@ -6,8 +6,7 @@ import sys import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import FakeYDL, global_setup -global_setup() +from test.helper import FakeYDL from youtube_dl.extractor import ( diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py index 5e1ff5eb0..056700614 100644 --- a/test/test_youtube_signature.py +++ b/test/test_youtube_signature.py @@ -6,9 +6,6 @@ import sys import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import global_setup -global_setup() - import io import re From 07e40358799158e51453e2d2c493d265a495b9e0 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 05:57:55 +0100 Subject: [PATCH 140/425] [viki] Fix uploader extraction --- youtube_dl/extractor/viki.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py index cd986a749..20e8bbf7e 100644 --- a/youtube_dl/extractor/viki.py +++ b/youtube_dl/extractor/viki.py @@ -39,7 +39,7 @@ class VikiIE(SubtitlesInfoExtractor): if uploader_m is None: uploader = None else: - uploader = uploader.group(1).strip() + uploader = uploader_m.group(1).strip() rating_str = self._html_search_regex( r'<strong>Rating: </strong>\s*([^<]*)<', webpage, From 94ccb6fa2e3ec014bb995d05bfe634cf986d6198 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 05:58:04 +0100 Subject: [PATCH 141/425] [viki] Fix subtitles extraction --- youtube_dl/extractor/viki.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py index 20e8bbf7e..ac199d410 100644 --- a/youtube_dl/extractor/viki.py +++ b/youtube_dl/extractor/viki.py @@ -91,7 +91,7 @@ class VikiIE(SubtitlesInfoExtractor): def _get_available_subtitles(self, video_id, info_webpage): res = {} - for sturl in re.findall(r'<track src="([^"]+)"/>'): + for sturl in re.findall(r'<track src="([^"]+)"/>', info_webpage): m = re.search(r'/(?P<lang>[a-z]+)\.vtt', sturl) if not m: continue From de79c46c8fa86dd3cb2383fd46cdd19a48e2f81f Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 06:06:18 +0100 Subject: [PATCH 142/425] [viki] Fix subtitle extraction --- youtube_dl/extractor/viki.py | 4 +++- youtube_dl/utils.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py index ac199d410..2206a06d5 100644 --- a/youtube_dl/extractor/viki.py +++ b/youtube_dl/extractor/viki.py @@ -2,6 +2,7 @@ import re from ..utils import ( ExtractorError, + unescapeHTML, unified_strdate, ) from .subtitles import SubtitlesInfoExtractor @@ -91,7 +92,8 @@ class VikiIE(SubtitlesInfoExtractor): def _get_available_subtitles(self, video_id, info_webpage): res = {} - for sturl in re.findall(r'<track src="([^"]+)"/>', info_webpage): + for sturl_html in re.findall(r'<track src="([^"]+)"/>', info_webpage): + sturl = unescapeHTML(sturl_html) m = re.search(r'/(?P<lang>[a-z]+)\.vtt', sturl) if not m: continue diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index caec00e37..946e90e93 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -553,7 +553,7 @@ def make_HTTPS_handler(opts_no_check_certificate): self._tunnel() try: self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv3) - except ssl.SSLError as e: + except ssl.SSLError: self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23) class HTTPSHandlerV3(compat_urllib_request.HTTPSHandler): From daa0dd2973212fc1b2837c9572b1502f91f6acbc Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 06:06:39 +0100 Subject: [PATCH 143/425] release 2013.11.25.1 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 91b36e55c..2af23040f 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.25' +__version__ = '2013.11.25.1' From d0d2b49ab728e70b8b325298e7825760fa7b3775 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 06:17:41 +0100 Subject: [PATCH 144/425] [FileDownloader] use moved format_bytes method --- youtube_dl/FileDownloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index c6276d194..27684d0f6 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -289,7 +289,7 @@ class FileDownloader(object): data_len = None if percent > 0: data_len = int(downloaded_data_len * 100 / percent) - data_len_str = u'~'+self.format_bytes(data_len) + data_len_str = u'~' + format_bytes(data_len) self.report_progress(percent, data_len_str, speed, eta) cursor_in_new_line = False self._hook_progress({ From 5db07df634713fe73e15e98de62f70ffe3a47871 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 15:46:54 +0100 Subject: [PATCH 145/425] Fix --download-archive (Fixes #1826) --- youtube_dl/YoutubeDL.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 30ba94666..a1ef3a94a 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -836,20 +836,26 @@ class YoutubeDL(object): except (IOError, OSError): self.report_warning(u'Unable to remove downloaded video file') - def in_download_archive(self, info_dict): - fn = self.params.get('download_archive') - if fn is None: - return False - extractor = info_dict.get('extractor_id') + def _make_archive_id(self, info_dict): + # Future-proof against any change in case + # and backwards compatibility with prior versions + extractor = info_dict.get('extractor') if extractor is None: if 'id' in info_dict: extractor = info_dict.get('ie_key') # key in a playlist if extractor is None: + return None # Incomplete video information + return extractor.lower() + u' ' + info_dict['id'] + + def in_download_archive(self, info_dict): + fn = self.params.get('download_archive') + if fn is None: + return False + + vid_id = self._make_archive_id(info_dict) + if vid_id is None: return False # Incomplete video information - # Future-proof against any change in case - # and backwards compatibility with prior versions - extractor = extractor.lower() - vid_id = extractor + u' ' + info_dict['id'] + try: with locked_file(fn, 'r', encoding='utf-8') as archive_file: for line in archive_file: @@ -864,7 +870,8 @@ class YoutubeDL(object): fn = self.params.get('download_archive') if fn is None: return - vid_id = info_dict['extractor'] + u' ' + info_dict['id'] + vid_id = self._make_archive_id(info_dict) + assert vid_id with locked_file(fn, 'a', encoding='utf-8') as archive_file: archive_file.write(vid_id + u'\n') From bb2bebdbe1ef06adc3c1cb2d078e061f44cf7d29 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 15:47:14 +0100 Subject: [PATCH 146/425] release 2013.11.25.2 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 2af23040f..aed0c4e75 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.25.1' +__version__ = '2013.11.25.2' From d46cc192d763f66655247ee122e397626481caca Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 19:11:01 +0100 Subject: [PATCH 147/425] Reduce socket timeout --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index a1ef3a94a..0cef1daf3 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -968,7 +968,7 @@ class YoutubeDL(object): proxy_map.update(handler.proxies) write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n') - def _setup_opener(self, timeout=300): + def _setup_opener(self, timeout=20): opts_cookiefile = self.params.get('cookiefile') opts_proxy = self.params.get('proxy') From 2a15e7063bceed326bcbc0a01ba77324f0373a0c Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 20:30:41 +0100 Subject: [PATCH 148/425] [soundcloud] Prefer HTTP over RTMP (#1798) --- youtube_dl/extractor/soundcloud.py | 66 ++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 18 deletions(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 67b2dff9c..ee8da227e 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -76,44 +76,74 @@ class SoundcloudIE(InfoExtractor): def _extract_info_dict(self, info, full_title=None, quiet=False): track_id = compat_str(info['id']) name = full_title or track_id - if quiet == False: + if quiet: self.report_extraction(name) thumbnail = info['artwork_url'] if thumbnail is not None: thumbnail = thumbnail.replace('-large', '-t500x500') + ext = info.get('original_format', u'mp3') result = { - 'id': track_id, + 'id': track_id, 'uploader': info['user']['username'], 'upload_date': unified_strdate(info['created_at']), - 'title': info['title'], - 'ext': info.get('original_format', u'mp3'), + 'title': info['title'], 'description': info['description'], 'thumbnail': thumbnail, } if info.get('downloadable', False): # We can build a direct link to the song - result['url'] = 'https://api.soundcloud.com/tracks/{0}/download?client_id={1}'.format(track_id, self._CLIENT_ID) + format_url = ( + u'https://api.soundcloud.com/tracks/{0}/download?client_id={1}'.format( + track_id, self._CLIENT_ID)) + result['formats'] = [{ + 'format_id': 'download', + 'ext': ext, + 'url': format_url, + }] else: # We have to retrieve the url stream_json = self._download_webpage( 'http://api.soundcloud.com/i1/tracks/{0}/streams?client_id={1}'.format(track_id, self._IPHONE_CLIENT_ID), track_id, u'Downloading track url') - # There should be only one entry in the dictionary - key, stream_url = list(json.loads(stream_json).items())[0] - if key.startswith(u'http'): - result['url'] = stream_url - elif key.startswith(u'rtmp'): - # The url doesn't have an rtmp app, we have to extract the playpath - url, path = stream_url.split('mp3:', 1) - result.update({ - 'url': url, - 'play_path': 'mp3:' + path, - }) - else: + + formats = [] + format_dict = json.loads(stream_json) + for key, stream_url in format_dict.items(): + if key.startswith(u'http'): + formats.append({ + 'format_id': key, + 'ext': ext, + 'url': stream_url, + }) + elif key.startswith(u'rtmp'): + # The url doesn't have an rtmp app, we have to extract the playpath + url, path = stream_url.split('mp3:', 1) + formats.append({ + 'format_id': key, + 'url': url, + 'play_path': 'mp3:' + path, + 'ext': ext, + }) + + if not formats: # We fallback to the stream_url in the original info, this # cannot be always used, sometimes it can give an HTTP 404 error - result['url'] = info['stream_url'] + '?client_id=' + self._CLIENT_ID, + formats.append({ + 'format_id': u'fallback', + 'url': info['stream_url'] + '?client_id=' + self._CLIENT_ID, + 'ext': ext, + }) + + def format_pref(f): + if f['format_id'].startswith('http'): + return 2 + if f['format_id'].startswith('rtmp'): + return 1 + return 0 + + formats.sort(key=format_pref) + result['formats'] = formats return result From 1a62c18f6521803ab41483f5da56fc72957d7655 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Mon, 25 Nov 2013 22:03:20 +0100 Subject: [PATCH 149/425] [bambuser] Skip the download in the test It doesn't respect the 'Range' header. --- youtube_dl/extractor/bambuser.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py index 967568c4a..b80508efe 100644 --- a/youtube_dl/extractor/bambuser.py +++ b/youtube_dl/extractor/bambuser.py @@ -25,6 +25,11 @@ class BambuserIE(InfoExtractor): u'uploader': u'pixelversity', u'uploader_id': u'344706', }, + u'params': { + # It doesn't respect the 'Range' header, it would download the whole video + # caused the travis builds to fail: https://travis-ci.org/rg3/youtube-dl/jobs/14493845#L59 + u'skip_download': True, + }, } def _real_extract(self, url): From a3927cf7eefd2318cdfb44cdb213b3810ea7627b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Mon, 25 Nov 2013 21:55:20 +0100 Subject: [PATCH 150/425] Allow to initialize a YoutubeDL object without parameters Having to pass the 'outtmpl' parameter feels really strange when you just want to extract the info of a video. --- youtube_dl/YoutubeDL.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index a1ef3a94a..46635bce1 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -146,7 +146,7 @@ class YoutubeDL(object): _num_downloads = None _screen_file = None - def __init__(self, params): + def __init__(self, params={}): """Create a FileDownloader object with the given options.""" self._ies = [] self._ies_instances = {} @@ -169,7 +169,7 @@ class YoutubeDL(object): self.params = params self.fd = FileDownloader(self, self.params) - if '%(stitle)s' in self.params['outtmpl']: + if '%(stitle)s' in self.params.get('outtmpl', ''): self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.') self._setup_opener() From 0c75c3fa7a24c05a74891ec49e5a18de4f2792f1 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 22:15:20 +0100 Subject: [PATCH 151/425] Do not warn about fixed output template if --max-downloads is 1 Fixes #1828 --- youtube_dl/YoutubeDL.py | 8 +++----- youtube_dl/__init__.py | 4 +++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 0cef1daf3..50f750593 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -258,10 +258,6 @@ class YoutubeDL(object): if self.params.get('cookiefile') is not None: self.cookiejar.save() - def fixed_template(self): - """Checks if the output template is fixed.""" - return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None) - def trouble(self, message=None, tb=None): """Determine action to take when a download problem appears. @@ -798,7 +794,9 @@ class YoutubeDL(object): def download(self, url_list): """Download a given list of URLs.""" - if len(url_list) > 1 and self.fixed_template(): + if (len(url_list) > 1 and + '%' not in self.params['outtmpl'] + and self.params.get('max_downloads') != 1): raise SameFileError(self.params['outtmpl']) for url in url_list: diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 1f15c7eaa..102508cf9 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -206,7 +206,9 @@ def parseOpts(overrideArguments=None): dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1) selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)') selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)') - selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None) + selection.add_option('--max-downloads', metavar='NUMBER', + dest='max_downloads', type=int, default=None, + help='Abort after downloading NUMBER files') selection.add_option('--min-filesize', metavar='SIZE', dest='min_filesize', help="Do not download any videos smaller than SIZE (e.g. 50k or 44.6m)", default=None) selection.add_option('--max-filesize', metavar='SIZE', dest='max_filesize', help="Do not download any videos larger than SIZE (e.g. 50k or 44.6m)", default=None) selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None) From d9b011f201ef61c10ce63b6078cd1e21b6da4d4a Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 22:31:27 +0100 Subject: [PATCH 152/425] Fix rtmpdump with non-ASCII filenames on Windows on 2.x Reported in #1798 --- youtube_dl/FileDownloader.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 27684d0f6..3ff9716b3 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -339,13 +339,29 @@ class FileDownloader(object): if live: basic_args += ['--live'] args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)] + + if sys.platform == 'win32' and sys.version_info < (3, 0): + # Windows subprocess module does not actually support Unicode + # on Python 2.x + # See http://stackoverflow.com/a/9951851/35070 + subprocess_encoding = sys.getfilesystemencoding() + args = [a.encode(subprocess_encoding, 'ignore') for a in args] + else: + subprocess_encoding = None + if self.params.get('verbose', False): + if subprocess_encoding: + str_args = [ + a.decode(subprocess_encoding) if isinstance(a, bytes) else a + for a in args] + else: + str_args = args try: import pipes - shell_quote = lambda args: ' '.join(map(pipes.quote, args)) + shell_quote = lambda args: ' '.join(map(pipes.quote, str_args)) except ImportError: shell_quote = repr - self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args)) + self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(str_args)) retval = run_rtmpdump(args) From fb04e40396509fd2bd41250eec3b07adf1aa1125 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 22:34:56 +0100 Subject: [PATCH 153/425] [soundcloud] Support for listing of audio-only files --- youtube_dl/YoutubeDL.py | 5 ++++- youtube_dl/extractor/soundcloud.py | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index d0aab1bbd..87635e173 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -875,6 +875,8 @@ class YoutubeDL(object): @staticmethod def format_resolution(format, default='unknown'): + if format.get('vcodec') == 'none': + return 'audio only' if format.get('_resolution') is not None: return format['_resolution'] if format.get('height') is not None: @@ -891,7 +893,8 @@ class YoutubeDL(object): res = u'' if fdict.get('format_note') is not None: res += fdict['format_note'] + u' ' - if fdict.get('vcodec') is not None: + if (fdict.get('vcodec') is not None and + fdict.get('vcodec') != 'none'): res += u'%-5s' % fdict['vcodec'] elif fdict.get('vbr') is not None: res += u'video' diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index ee8da227e..3a19ab172 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -100,6 +100,7 @@ class SoundcloudIE(InfoExtractor): 'format_id': 'download', 'ext': ext, 'url': format_url, + 'vcodec': 'none', }] else: # We have to retrieve the url @@ -115,6 +116,7 @@ class SoundcloudIE(InfoExtractor): 'format_id': key, 'ext': ext, 'url': stream_url, + 'vcodec': 'none', }) elif key.startswith(u'rtmp'): # The url doesn't have an rtmp app, we have to extract the playpath @@ -124,6 +126,7 @@ class SoundcloudIE(InfoExtractor): 'url': url, 'play_path': 'mp3:' + path, 'ext': ext, + 'vcodec': 'none', }) if not formats: @@ -133,6 +136,7 @@ class SoundcloudIE(InfoExtractor): 'format_id': u'fallback', 'url': info['stream_url'] + '?client_id=' + self._CLIENT_ID, 'ext': ext, + 'vcodec': 'none', }) def format_pref(f): From 781a7d054657d813527fa0f98f831679675f8ea7 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 25 Nov 2013 22:36:18 +0100 Subject: [PATCH 154/425] release 2013.11.25.3 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index aed0c4e75..fc0881201 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.25.2' +__version__ = '2013.11.25.3' From 529a2e2cc35df8c77418f9d02b0f5b4730b95b06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Mon, 25 Nov 2013 22:52:09 +0100 Subject: [PATCH 155/425] Fix typo in the documentation of the 'download_archive' param --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 87635e173..e23042c48 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -126,7 +126,7 @@ class YoutubeDL(object): noplaylist: Download single video instead of a playlist if in doubt. age_limit: An integer representing the user's age in years. Unsuitable videos for the given age are skipped. - downloadarchive: File name of a file where all downloads are recorded. + download_archive: File name of a file where all downloads are recorded. Videos already present in the file are not downloaded again. cookiefile: File name where cookies should be read from and dumped to. From d31209a1449d0bd9315e063be4cf7f5d45726563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Mon, 25 Nov 2013 22:57:15 +0100 Subject: [PATCH 156/425] Use the 'extractor_key' field for the download archive file It has the same value as the ie_key. --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index e23042c48..e86e8a090 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -837,7 +837,7 @@ class YoutubeDL(object): def _make_archive_id(self, info_dict): # Future-proof against any change in case # and backwards compatibility with prior versions - extractor = info_dict.get('extractor') + extractor = info_dict.get('extractor_key') if extractor is None: if 'id' in info_dict: extractor = info_dict.get('ie_key') # key in a playlist From c2e52508cca307113ff0c3aedcc0519d92c48f00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Tue, 26 Nov 2013 08:03:11 +0100 Subject: [PATCH 157/425] Include the proxy in the parameters for YoutubeDL (fixes #1831) --- youtube_dl/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 102508cf9..0704515df 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -651,6 +651,7 @@ def _real_main(argv=None): 'download_archive': opts.download_archive, 'cookiefile': opts.cookiefile, 'nocheckcertificate': opts.no_check_certificate, + 'proxy': opts.proxy, } with YoutubeDL(ydl_opts) as ydl: From c5ed4e8f7efaa258c74dd3179a7c691208874e41 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 26 Nov 2013 10:41:35 +0100 Subject: [PATCH 158/425] release 2013.11.26 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index fc0881201..99a5e0505 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.25.3' +__version__ = '2013.11.26' From 4a98cdbf3b19b07c7a885d348e79ddf79318f133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Tue, 26 Nov 2013 18:53:36 +0100 Subject: [PATCH 159/425] YoutubeDL: set the 'params' property before any message/warning/error is sent (fixes #1840) If it sets the 'restrictfilenames' param, it will first report a warning. It will try to get the logger from the 'params' property, which would be set at that moment to None, raising the error 'AttributeError: 'NoneType' object has no attribute 'get'' --- youtube_dl/YoutubeDL.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index e86e8a090..711b5d79e 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -155,6 +155,7 @@ class YoutubeDL(object): self._download_retcode = 0 self._num_downloads = 0 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] + self.params = params if (sys.version_info >= (3,) and sys.platform != 'win32' and sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] @@ -164,9 +165,8 @@ class YoutubeDL(object): u'Assuming --restrict-filenames since file system encoding ' u'cannot encode all charactes. ' u'Set the LC_ALL environment variable to fix this.') - params['restrictfilenames'] = True + self.params['restrictfilenames'] = True - self.params = params self.fd = FileDownloader(self, self.params) if '%(stitle)s' in self.params.get('outtmpl', ''): From 6e47b51eef26dbaa3634b73914e4ee7213ad38f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Tue, 26 Nov 2013 19:09:14 +0100 Subject: [PATCH 160/425] [youtube:playlist] Remove the link with index 0 It's not the first video of the playlist, it appears in the 'Play all' button (see the test course for an example) --- youtube_dl/extractor/youtube.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 1bf9cb7d4..4c43d5739 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1528,7 +1528,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor): )""" _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s&page=%s' _MORE_PAGES_INDICATOR = r'data-link-type="next"' - _VIDEO_RE = r'href="/watch\?v=([0-9A-Za-z_-]{11})&' + _VIDEO_RE = r'href="/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&[^"]*?index=(?P<index>\d+)' IE_NAME = u'youtube:playlist' @classmethod @@ -1562,8 +1562,10 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor): for page_num in itertools.count(1): url = self._TEMPLATE_URL % (playlist_id, page_num) page = self._download_webpage(url, playlist_id, u'Downloading page #%s' % page_num) - # The ids are duplicated - new_ids = orderedSet(re.findall(self._VIDEO_RE, page)) + matches = re.finditer(self._VIDEO_RE, page) + # We remove the duplicates and the link with index 0 + # (it's not the first video of the playlist) + new_ids = orderedSet(m.group('id') for m in matches if m.group('index') != '0') ids.extend(new_ids) if re.search(self._MORE_PAGES_INDICATOR, page) is None: From e26f8712289c727a43d74a4669aee4924b9f75f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Tue, 26 Nov 2013 18:48:52 +0100 Subject: [PATCH 161/425] Use the new '_download_xml' helper in more extractors --- youtube_dl/extractor/anitube.py | 4 +--- youtube_dl/extractor/arte.py | 7 ++----- youtube_dl/extractor/canalplus.py | 4 +--- youtube_dl/extractor/clipfish.py | 4 +--- youtube_dl/extractor/cnn.py | 4 +--- youtube_dl/extractor/comedycentral.py | 7 ++----- youtube_dl/extractor/daum.py | 10 +++------- youtube_dl/extractor/dreisat.py | 4 +--- youtube_dl/extractor/ebaumsworld.py | 4 +--- youtube_dl/extractor/faz.py | 4 +--- youtube_dl/extractor/francetv.py | 4 +--- youtube_dl/extractor/internetvideoarchive.py | 7 ++----- youtube_dl/extractor/jeuxvideo.py | 8 ++------ youtube_dl/extractor/justintv.py | 4 +--- youtube_dl/extractor/livestream.py | 4 +--- youtube_dl/extractor/mtv.py | 3 +-- youtube_dl/extractor/myspass.py | 4 +--- youtube_dl/extractor/naver.py | 7 ++----- youtube_dl/extractor/nbc.py | 5 ++--- youtube_dl/extractor/nhl.py | 4 +--- youtube_dl/extractor/niconico.py | 10 +++------- youtube_dl/extractor/sina.py | 4 +--- youtube_dl/extractor/spiegel.py | 5 +---- youtube_dl/extractor/teamcoco.py | 4 +--- youtube_dl/extractor/toutv.py | 5 +---- youtube_dl/extractor/trilulilu.py | 5 +---- youtube_dl/extractor/videofyme.py | 4 +--- youtube_dl/extractor/youtube.py | 4 +--- 28 files changed, 38 insertions(+), 105 deletions(-) diff --git a/youtube_dl/extractor/anitube.py b/youtube_dl/extractor/anitube.py index 691d5a844..2b019daa9 100644 --- a/youtube_dl/extractor/anitube.py +++ b/youtube_dl/extractor/anitube.py @@ -1,5 +1,4 @@ import re -import xml.etree.ElementTree from .common import InfoExtractor @@ -28,9 +27,8 @@ class AnitubeIE(InfoExtractor): key = self._html_search_regex(r'http://www\.anitube\.se/embed/([A-Za-z0-9_-]*)', webpage, u'key') - webpage_config = self._download_webpage('http://www.anitube.se/nuevo/econfig.php?key=%s' % key, + config_xml = self._download_xml('http://www.anitube.se/nuevo/econfig.php?key=%s' % key, key) - config_xml = xml.etree.ElementTree.fromstring(webpage_config.encode('utf-8')) video_title = config_xml.find('title').text diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py index 44d0b5d70..8b62ee774 100644 --- a/youtube_dl/extractor/arte.py +++ b/youtube_dl/extractor/arte.py @@ -1,7 +1,6 @@ # encoding: utf-8 import re import json -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( @@ -78,8 +77,7 @@ class ArteTvIE(InfoExtractor): """Extract from videos.arte.tv""" ref_xml_url = url.replace('/videos/', '/do_delegate/videos/') ref_xml_url = ref_xml_url.replace('.html', ',view,asPlayerXml.xml') - ref_xml = self._download_webpage(ref_xml_url, video_id, note=u'Downloading metadata') - ref_xml_doc = xml.etree.ElementTree.fromstring(ref_xml) + ref_xml_doc = self._download_xml(ref_xml_url, video_id, note=u'Downloading metadata') config_node = find_xpath_attr(ref_xml_doc, './/video', 'lang', lang) config_xml_url = config_node.attrib['ref'] config_xml = self._download_webpage(config_xml_url, video_id, note=u'Downloading configuration') @@ -109,9 +107,8 @@ class ArteTvIE(InfoExtractor): """Extract form http://liveweb.arte.tv/""" webpage = self._download_webpage(url, name) video_id = self._search_regex(r'eventId=(\d+?)("|&)', webpage, u'event id') - config_xml = self._download_webpage('http://download.liveweb.arte.tv/o21/liveweb/events/event-%s.xml' % video_id, + config_doc = self._download_xml('http://download.liveweb.arte.tv/o21/liveweb/events/event-%s.xml' % video_id, video_id, u'Downloading information') - config_doc = xml.etree.ElementTree.fromstring(config_xml.encode('utf-8')) event_doc = config_doc.find('event') url_node = event_doc.find('video').find('urlHd') if url_node is None: diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py index bfa2a8b40..7cdcd8399 100644 --- a/youtube_dl/extractor/canalplus.py +++ b/youtube_dl/extractor/canalplus.py @@ -1,6 +1,5 @@ # encoding: utf-8 import re -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import unified_strdate @@ -31,11 +30,10 @@ class CanalplusIE(InfoExtractor): webpage = self._download_webpage(url, mobj.group('path')) video_id = self._search_regex(r'videoId = "(\d+)";', webpage, u'video id') info_url = self._VIDEO_INFO_TEMPLATE % video_id - info_page = self._download_webpage(info_url,video_id, + doc = self._download_xml(info_url,video_id, u'Downloading video info') self.report_extraction(video_id) - doc = xml.etree.ElementTree.fromstring(info_page.encode('utf-8')) video_info = [video for video in doc if video.find('ID').text == video_id][0] infos = video_info.find('INFOS') media = video_info.find('MEDIA') diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py index 95449da3c..5f0b5602f 100644 --- a/youtube_dl/extractor/clipfish.py +++ b/youtube_dl/extractor/clipfish.py @@ -1,6 +1,5 @@ import re import time -import xml.etree.ElementTree from .common import InfoExtractor @@ -25,9 +24,8 @@ class ClipfishIE(InfoExtractor): info_url = ('http://www.clipfish.de/devxml/videoinfo/%s?ts=%d' % (video_id, int(time.time()))) - info_xml = self._download_webpage( + doc = self._download_xml( info_url, video_id, note=u'Downloading info page') - doc = xml.etree.ElementTree.fromstring(info_xml) title = doc.find('title').text video_url = doc.find('filename').text thumbnail = doc.find('imageurl').text diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py index 34adf6dda..a034bb2fb 100644 --- a/youtube_dl/extractor/cnn.py +++ b/youtube_dl/extractor/cnn.py @@ -1,5 +1,4 @@ import re -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import determine_ext @@ -33,8 +32,7 @@ class CNNIE(InfoExtractor): path = mobj.group('path') page_title = mobj.group('title') info_url = u'http://cnn.com/video/data/3.0/%s/index.xml' % path - info_xml = self._download_webpage(info_url, page_title) - info = xml.etree.ElementTree.fromstring(info_xml.encode('utf-8')) + info = self._download_xml(info_url, page_title) formats = [] for f in info.findall('files/file'): diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py index 725849d2e..23647f99e 100644 --- a/youtube_dl/extractor/comedycentral.py +++ b/youtube_dl/extractor/comedycentral.py @@ -1,5 +1,4 @@ import re -import xml.etree.ElementTree from .common import InfoExtractor from .mtv import MTVIE, _media_xml_tag @@ -158,13 +157,12 @@ class ComedyCentralShowsIE(InfoExtractor): uri = mMovieParams[0][1] indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + compat_urllib_parse.urlencode({'uri': uri}) - indexXml = self._download_webpage(indexUrl, epTitle, + idoc = self._download_xml(indexUrl, epTitle, u'Downloading show index', u'unable to download episode index') results = [] - idoc = xml.etree.ElementTree.fromstring(indexXml) itemEls = idoc.findall('.//item') for partNum,itemEl in enumerate(itemEls): mediaId = itemEl.findall('./guid')[0].text @@ -175,10 +173,9 @@ class ComedyCentralShowsIE(InfoExtractor): configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' + compat_urllib_parse.urlencode({'uri': mediaId})) - configXml = self._download_webpage(configUrl, epTitle, + cdoc = self._download_xml(configUrl, epTitle, u'Downloading configuration for %s' % shortMediaId) - cdoc = xml.etree.ElementTree.fromstring(configXml) turls = [] for rendition in cdoc.findall('.//rendition'): finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text) diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py index a804e83bd..3d1dcb793 100644 --- a/youtube_dl/extractor/daum.py +++ b/youtube_dl/extractor/daum.py @@ -1,6 +1,5 @@ # encoding: utf-8 import re -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( @@ -32,14 +31,12 @@ class DaumIE(InfoExtractor): full_id = self._search_regex(r'<link rel="video_src" href=".+?vid=(.+?)"', webpage, u'full id') query = compat_urllib_parse.urlencode({'vid': full_id}) - info_xml = self._download_webpage( + info = self._download_xml( 'http://tvpot.daum.net/clip/ClipInfoXml.do?' + query, video_id, u'Downloading video info') - urls_xml = self._download_webpage( + urls = self._download_xml( 'http://videofarm.daum.net/controller/api/open/v1_2/MovieData.apixml?' + query, video_id, u'Downloading video formats info') - info = xml.etree.ElementTree.fromstring(info_xml.encode('utf-8')) - urls = xml.etree.ElementTree.fromstring(urls_xml.encode('utf-8')) self.to_screen(u'%s: Getting video urls' % video_id) formats = [] @@ -49,10 +46,9 @@ class DaumIE(InfoExtractor): 'vid': full_id, 'profile': profile, }) - url_xml = self._download_webpage( + url_doc = self._download_xml( 'http://videofarm.daum.net/controller/api/open/v1_2/MovieLocation.apixml?' + format_query, video_id, note=False) - url_doc = xml.etree.ElementTree.fromstring(url_xml.encode('utf-8')) format_url = url_doc.find('result/url').text formats.append({ 'url': format_url, diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py index 765cb1f37..3cb382e12 100644 --- a/youtube_dl/extractor/dreisat.py +++ b/youtube_dl/extractor/dreisat.py @@ -1,7 +1,6 @@ # coding: utf-8 import re -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( @@ -30,8 +29,7 @@ class DreiSatIE(InfoExtractor): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') details_url = 'http://www.3sat.de/mediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id - details_xml = self._download_webpage(details_url, video_id, note=u'Downloading video details') - details_doc = xml.etree.ElementTree.fromstring(details_xml.encode('utf-8')) + details_doc = self._download_xml(details_url, video_id, note=u'Downloading video details') thumbnail_els = details_doc.findall('.//teaserimage') thumbnails = [{ diff --git a/youtube_dl/extractor/ebaumsworld.py b/youtube_dl/extractor/ebaumsworld.py index f02c6998b..877113d63 100644 --- a/youtube_dl/extractor/ebaumsworld.py +++ b/youtube_dl/extractor/ebaumsworld.py @@ -1,5 +1,4 @@ import re -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import determine_ext @@ -21,9 +20,8 @@ class EbaumsWorldIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') - config_xml = self._download_webpage( + config = self._download_xml( 'http://www.ebaumsworld.com/video/player/%s' % video_id, video_id) - config = xml.etree.ElementTree.fromstring(config_xml.encode('utf-8')) video_url = config.find('file').text return { diff --git a/youtube_dl/extractor/faz.py b/youtube_dl/extractor/faz.py index 89ed08db4..c0169de04 100644 --- a/youtube_dl/extractor/faz.py +++ b/youtube_dl/extractor/faz.py @@ -1,6 +1,5 @@ # encoding: utf-8 import re -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( @@ -28,9 +27,8 @@ class FazIE(InfoExtractor): webpage = self._download_webpage(url, video_id) config_xml_url = self._search_regex(r'writeFLV\(\'(.+?)\',', webpage, u'config xml url') - config_xml = self._download_webpage(config_xml_url, video_id, + config = self._download_xml(config_xml_url, video_id, u'Downloading config xml') - config = xml.etree.ElementTree.fromstring(config_xml.encode('utf-8')) encodings = config.find('ENCODINGS') formats = [] diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py index 086cafca0..6e1971043 100644 --- a/youtube_dl/extractor/francetv.py +++ b/youtube_dl/extractor/francetv.py @@ -1,6 +1,5 @@ # encoding: utf-8 import re -import xml.etree.ElementTree import json from .common import InfoExtractor @@ -11,11 +10,10 @@ from ..utils import ( class FranceTVBaseInfoExtractor(InfoExtractor): def _extract_video(self, video_id): - xml_desc = self._download_webpage( + info = self._download_xml( 'http://www.francetvinfo.fr/appftv/webservices/video/' 'getInfosOeuvre.php?id-diffusion=' + video_id, video_id, 'Downloading XML config') - info = xml.etree.ElementTree.fromstring(xml_desc.encode('utf-8')) manifest_url = info.find('videos/video/url').text video_url = manifest_url.replace('manifest.f4m', 'index_2_av.m3u8') diff --git a/youtube_dl/extractor/internetvideoarchive.py b/youtube_dl/extractor/internetvideoarchive.py index be8e05f53..16a6f73c8 100644 --- a/youtube_dl/extractor/internetvideoarchive.py +++ b/youtube_dl/extractor/internetvideoarchive.py @@ -1,5 +1,4 @@ import re -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( @@ -43,9 +42,8 @@ class InternetVideoArchiveIE(InfoExtractor): video_id = query_dic['publishedid'][0] url = self._build_url(query) - flashconfiguration_xml = self._download_webpage(url, video_id, + flashconfiguration = self._download_xml(url, video_id, u'Downloading flash configuration') - flashconfiguration = xml.etree.ElementTree.fromstring(flashconfiguration_xml.encode('utf-8')) file_url = flashconfiguration.find('file').text file_url = file_url.replace('/playlist.aspx', '/mrssplaylist.aspx') # Replace some of the parameters in the query to get the best quality @@ -53,9 +51,8 @@ class InternetVideoArchiveIE(InfoExtractor): file_url = re.sub(r'(?<=\?)(.+)$', lambda m: self._clean_query(m.group()), file_url) - info_xml = self._download_webpage(file_url, video_id, + info = self._download_xml(file_url, video_id, u'Downloading video info') - info = xml.etree.ElementTree.fromstring(info_xml.encode('utf-8')) item = info.find('channel/item') def _bp(p): diff --git a/youtube_dl/extractor/jeuxvideo.py b/youtube_dl/extractor/jeuxvideo.py index 0020c47cf..caf9d8c85 100644 --- a/youtube_dl/extractor/jeuxvideo.py +++ b/youtube_dl/extractor/jeuxvideo.py @@ -2,7 +2,6 @@ import json import re -import xml.etree.ElementTree from .common import InfoExtractor @@ -32,12 +31,9 @@ class JeuxVideoIE(InfoExtractor): r'http://www\.jeuxvideo\.com/config/\w+/\d+/(.*?)/\d+_player\.xml', xml_link, u'video ID') - xml_config = self._download_webpage( + config = self._download_xml( xml_link, title, u'Downloading XML config') - config = xml.etree.ElementTree.fromstring(xml_config.encode('utf-8')) - info_json = self._search_regex( - r'(?sm)<format\.json>(.*?)</format\.json>', - xml_config, u'JSON information') + info_json = config.find('format.json').text info = json.loads(info_json)['versions'][0] video_url = 'http://video720.jeuxvideo.com/' + info['file'] diff --git a/youtube_dl/extractor/justintv.py b/youtube_dl/extractor/justintv.py index f60017992..e9bde0c18 100644 --- a/youtube_dl/extractor/justintv.py +++ b/youtube_dl/extractor/justintv.py @@ -1,7 +1,6 @@ import json import os import re -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( @@ -94,10 +93,9 @@ class JustinTVIE(InfoExtractor): archive_id = m.group(1) api = api_base + '/broadcast/by_chapter/%s.xml' % chapter_id - chapter_info_xml = self._download_webpage(api, chapter_id, + doc = self._download_xml(api, chapter_id, note=u'Downloading chapter information', errnote=u'Chapter information download failed') - doc = xml.etree.ElementTree.fromstring(chapter_info_xml) for a in doc.findall('.//archive'): if archive_id == a.find('./id').text: break diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py index 5f548437c..9bc35b115 100644 --- a/youtube_dl/extractor/livestream.py +++ b/youtube_dl/extractor/livestream.py @@ -1,6 +1,5 @@ import re import json -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( @@ -80,8 +79,7 @@ class LivestreamOriginalIE(InfoExtractor): user = mobj.group('user') api_url = 'http://x{0}x.api.channel.livestream.com/2.0/clipdetails?extendedInfo=true&id={1}'.format(user, video_id) - api_response = self._download_webpage(api_url, video_id) - info = xml.etree.ElementTree.fromstring(api_response.encode('utf-8')) + info = self._download_xml(api_url, video_id) item = info.find('channel').find('item') ns = {'media': 'http://search.yahoo.com/mrss'} thumbnail_url = item.find(xpath_with_ns('media:thumbnail', ns)).attrib['url'] diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py index 04afd6c4c..42aee58be 100644 --- a/youtube_dl/extractor/mtv.py +++ b/youtube_dl/extractor/mtv.py @@ -109,9 +109,8 @@ class MTVIE(InfoExtractor): def _get_videos_info(self, uri): video_id = self._id_from_uri(uri) data = compat_urllib_parse.urlencode({'uri': uri}) - infoXml = self._download_webpage(self._FEED_URL +'?' + data, video_id, + idoc = self._download_xml(self._FEED_URL +'?' + data, video_id, u'Downloading info') - idoc = xml.etree.ElementTree.fromstring(infoXml.encode('utf-8')) return [self._get_video_info(item) for item in idoc.findall('.//item')] def _real_extract(self, url): diff --git a/youtube_dl/extractor/myspass.py b/youtube_dl/extractor/myspass.py index 107665d15..0067bf134 100644 --- a/youtube_dl/extractor/myspass.py +++ b/youtube_dl/extractor/myspass.py @@ -1,5 +1,4 @@ import os.path -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( @@ -33,8 +32,7 @@ class MySpassIE(InfoExtractor): # get metadata metadata_url = META_DATA_URL_TEMPLATE % video_id - metadata_text = self._download_webpage(metadata_url, video_id) - metadata = xml.etree.ElementTree.fromstring(metadata_text.encode('utf-8')) + metadata = self._download_xml(metadata_url, video_id) # extract values from metadata url_flv_el = metadata.find('url_flv') diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py index 9df236d69..d290397c7 100644 --- a/youtube_dl/extractor/naver.py +++ b/youtube_dl/extractor/naver.py @@ -1,6 +1,5 @@ # encoding: utf-8 import re -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( @@ -38,14 +37,12 @@ class NaverIE(InfoExtractor): 'protocol': 'p2p', 'inKey': key, }) - info_xml = self._download_webpage( + info = self._download_xml( 'http://serviceapi.rmcnmv.naver.com/flash/videoInfo.nhn?' + query, video_id, u'Downloading video info') - urls_xml = self._download_webpage( + urls = self._download_xml( 'http://serviceapi.rmcnmv.naver.com/flash/playableEncodingOption.nhn?' + query_urls, video_id, u'Downloading video formats info') - info = xml.etree.ElementTree.fromstring(info_xml.encode('utf-8')) - urls = xml.etree.ElementTree.fromstring(urls_xml.encode('utf-8')) formats = [] for format_el in urls.findall('EncodingOptions/EncodingOption'): diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py index 3bc9dae6d..e8bbfff7b 100644 --- a/youtube_dl/extractor/nbc.py +++ b/youtube_dl/extractor/nbc.py @@ -1,5 +1,4 @@ import re -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import find_xpath_attr, compat_str @@ -21,8 +20,8 @@ class NBCNewsIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') - info_xml = self._download_webpage('http://www.nbcnews.com/id/%s/displaymode/1219' % video_id, video_id) - info = xml.etree.ElementTree.fromstring(info_xml.encode('utf-8')).find('video') + all_info = self._download_xml('http://www.nbcnews.com/id/%s/displaymode/1219' % video_id, video_id) + info = all_info.find('video') return {'id': video_id, 'title': info.find('headline').text, diff --git a/youtube_dl/extractor/nhl.py b/youtube_dl/extractor/nhl.py index 458fe4063..2edd806a3 100644 --- a/youtube_dl/extractor/nhl.py +++ b/youtube_dl/extractor/nhl.py @@ -1,6 +1,5 @@ import re import json -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( @@ -26,9 +25,8 @@ class NHLBaseInfoExtractor(InfoExtractor): 'path': initial_video_url.replace('.mp4', '_sd.mp4'), }) path_url = 'http://video.nhl.com/videocenter/servlets/encryptvideopath?' + data - path_response = self._download_webpage(path_url, video_id, + path_doc = self._download_xml(path_url, video_id, u'Downloading final video url') - path_doc = xml.etree.ElementTree.fromstring(path_response) video_url = path_doc.find('path').text join = compat_urlparse.urljoin diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py index 729607ea3..46774317c 100644 --- a/youtube_dl/extractor/niconico.py +++ b/youtube_dl/extractor/niconico.py @@ -2,7 +2,6 @@ import re import socket -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( @@ -81,7 +80,7 @@ class NiconicoIE(InfoExtractor): # the cookies in order to be able to download the info webpage self._download_webpage('http://www.nicovideo.jp/watch/' + video_id, video_id) - video_info_webpage = self._download_webpage( + video_info = self._download_xml( 'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id, video_id, note=u'Downloading video info page') @@ -92,7 +91,6 @@ class NiconicoIE(InfoExtractor): video_real_url = compat_urlparse.parse_qs(flv_info_webpage)['url'][0] # Start extracting information - video_info = xml.etree.ElementTree.fromstring(video_info_webpage) video_title = video_info.find('.//title').text video_extension = video_info.find('.//movie_type').text video_format = video_extension.upper() @@ -107,13 +105,11 @@ class NiconicoIE(InfoExtractor): video_uploader = video_uploader_id url = 'http://seiga.nicovideo.jp/api/user/info?id=' + video_uploader_id try: - user_info_webpage = self._download_webpage( + user_info = self._download_xml( url, video_id, note=u'Downloading user information') + video_uploader = user_info.find('.//nickname').text except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.report_warning(u'Unable to download user info webpage: %s' % compat_str(err)) - else: - user_info = xml.etree.ElementTree.fromstring(user_info_webpage) - video_uploader = user_info.find('.//nickname').text return { 'id': video_id, diff --git a/youtube_dl/extractor/sina.py b/youtube_dl/extractor/sina.py index 14b1c656c..74a87fe56 100644 --- a/youtube_dl/extractor/sina.py +++ b/youtube_dl/extractor/sina.py @@ -1,7 +1,6 @@ # coding: utf-8 import re -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( @@ -35,12 +34,11 @@ class SinaIE(InfoExtractor): def _extract_video(self, video_id): data = compat_urllib_parse.urlencode({'vid': video_id}) - url_page = self._download_webpage('http://v.iask.com/v_play.php?%s' % data, + url_doc = self._download_xml('http://v.iask.com/v_play.php?%s' % data, video_id, u'Downloading video url') image_page = self._download_webpage( 'http://interface.video.sina.com.cn/interface/common/getVideoImage.php?%s' % data, video_id, u'Downloading thumbnail info') - url_doc = xml.etree.ElementTree.fromstring(url_page.encode('utf-8')) return {'id': video_id, 'url': url_doc.find('./durl/url').text, diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py index 19ce585cf..695520524 100644 --- a/youtube_dl/extractor/spiegel.py +++ b/youtube_dl/extractor/spiegel.py @@ -1,5 +1,4 @@ import re -import xml.etree.ElementTree from .common import InfoExtractor @@ -33,12 +32,10 @@ class SpiegelIE(InfoExtractor): r'<div class="module-title">(.*?)</div>', webpage, u'title') xml_url = u'http://video2.spiegel.de/flash/' + video_id + u'.xml' - xml_code = self._download_webpage( + idoc = self._download_xml( xml_url, video_id, note=u'Downloading XML', errnote=u'Failed to download XML') - idoc = xml.etree.ElementTree.fromstring(xml_code) - formats = [ { 'format_id': n.tag.rpartition('type')[2], diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py index 165d9f88b..2bf26d056 100644 --- a/youtube_dl/extractor/teamcoco.py +++ b/youtube_dl/extractor/teamcoco.py @@ -1,5 +1,4 @@ import re -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( @@ -32,8 +31,7 @@ class TeamcocoIE(InfoExtractor): self.report_extraction(video_id) data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id - data_xml = self._download_webpage(data_url, video_id, 'Downloading data webpage') - data = xml.etree.ElementTree.fromstring(data_xml.encode('utf-8')) + data = self._download_xml(data_url, video_id, 'Downloading data webpage') qualities = ['500k', '480p', '1000k', '720p', '1080p'] diff --git a/youtube_dl/extractor/toutv.py b/youtube_dl/extractor/toutv.py index 2f728d3dc..1e9598ef6 100644 --- a/youtube_dl/extractor/toutv.py +++ b/youtube_dl/extractor/toutv.py @@ -1,6 +1,5 @@ # coding: utf-8 import re -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( @@ -40,11 +39,9 @@ class TouTvIE(InfoExtractor): r'"idMedia":\s*"([^"]+)"', webpage, u'media ID') streams_url = u'http://release.theplatform.com/content.select?pid=' + mediaId - streams_webpage = self._download_webpage( + streams_doc = self._download_xml( streams_url, video_id, note=u'Downloading stream list') - streams_doc = xml.etree.ElementTree.fromstring( - streams_webpage.encode('utf-8')) video_url = next(n.text for n in streams_doc.findall('.//choice/url') if u'//ad.doubleclick' not in n.text) diff --git a/youtube_dl/extractor/trilulilu.py b/youtube_dl/extractor/trilulilu.py index 0bf028f61..1c49e580d 100644 --- a/youtube_dl/extractor/trilulilu.py +++ b/youtube_dl/extractor/trilulilu.py @@ -1,6 +1,5 @@ import json import re -import xml.etree.ElementTree from .common import InfoExtractor @@ -36,12 +35,10 @@ class TriluliluIE(InfoExtractor): format_url = (u'http://fs%(server)s.trilulilu.ro/%(hash)s/' u'video-formats2' % log) - format_str = self._download_webpage( + format_doc = self._download_xml( format_url, video_id, note=u'Downloading formats', errnote=u'Error while downloading formats') - - format_doc = xml.etree.ElementTree.fromstring(format_str) video_url_template = ( u'http://fs%(server)s.trilulilu.ro/stream.php?type=video' diff --git a/youtube_dl/extractor/videofyme.py b/youtube_dl/extractor/videofyme.py index 94f64ffa5..912802d9a 100644 --- a/youtube_dl/extractor/videofyme.py +++ b/youtube_dl/extractor/videofyme.py @@ -1,5 +1,4 @@ import re -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( @@ -27,9 +26,8 @@ class VideofyMeIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') - config_xml = self._download_webpage('http://sunshine.videofy.me/?videoId=%s' % video_id, + config = self._download_xml('http://sunshine.videofy.me/?videoId=%s' % video_id, video_id) - config = xml.etree.ElementTree.fromstring(config_xml.encode('utf-8')) video = config.find('video') sources = video.find('sources') url_node = next(node for node in [find_xpath_attr(sources, 'source', 'id', 'HQ %s' % key) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 4c43d5739..a76a9071a 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -11,7 +11,6 @@ import socket import string import struct import traceback -import xml.etree.ElementTree import zlib from .common import InfoExtractor, SearchInfoExtractor @@ -1144,8 +1143,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): 'asrs': 1, }) list_url = caption_url + '&' + list_params - list_page = self._download_webpage(list_url, video_id) - caption_list = xml.etree.ElementTree.fromstring(list_page.encode('utf-8')) + caption_list = self._download_xml(list_url, video_id) original_lang_node = caption_list.find('track') if original_lang_node is None or original_lang_node.attrib.get('kind') != 'asr' : self._downloader.report_warning(u'Video doesn\'t have automatic captions') From 652cdaa269725dfbf9effdc18a8fd0b369100399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Tue, 26 Nov 2013 21:35:03 +0100 Subject: [PATCH 162/425] [youtube:playlist] Add support for YouTube mixes (fixes #1839) --- test/test_youtube_lists.py | 9 +++++++++ youtube_dl/extractor/youtube.py | 24 ++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py index 8fd073f31..95f07d129 100644 --- a/test/test_youtube_lists.py +++ b/test/test_youtube_lists.py @@ -107,5 +107,14 @@ class TestYoutubeLists(unittest.TestCase): result = ie.extract('http://www.youtube.com/show/airdisasters') self.assertTrue(len(result) >= 3) + def test_youtube_mix(self): + dl = FakeYDL() + ie = YoutubePlaylistIE(dl) + result = ie.extract('http://www.youtube.com/watch?v=lLJf9qJHR3E&list=RDrjFaenf1T-Y') + entries = result['entries'] + self.assertTrue(len(entries) >= 20) + original_video = entries[0] + self.assertEqual(original_video['id'], 'rjFaenf1T-Y') + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index a76a9071a..9ef5fecce 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -28,6 +28,7 @@ from ..utils import ( clean_html, get_cachedir, get_element_by_id, + get_element_by_attribute, ExtractorError, unescapeHTML, unified_strdate, @@ -1537,6 +1538,22 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor): def _real_initialize(self): self._login() + def _ids_to_results(self, ids): + return [self.url_result(vid_id, 'Youtube', video_id=vid_id) + for vid_id in ids] + + def _extract_mix(self, playlist_id): + # The mixes are generated from a a single video + # the id of the playlist is just 'RD' + video_id + url = 'https://youtube.com/watch?v=%s&list=%s' % (playlist_id[2:], playlist_id) + webpage = self._download_webpage(url, playlist_id, u'Downloading Youtube mix') + title = clean_html(get_element_by_attribute('class', 'title long-title', webpage)) + video_re = r'data-index="\d+".*?href="/watch\?v=([0-9A-Za-z_-]{11})&[^"]*?list=%s' % re.escape(playlist_id) + ids = orderedSet(re.findall(video_re, webpage)) + url_results = self._ids_to_results(ids) + + return self.playlist_result(url_results, playlist_id, title) + def _real_extract(self, url): # Extract playlist id mobj = re.match(self._VALID_URL, url, re.VERBOSE) @@ -1554,6 +1571,10 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor): else: self.to_screen(u'Downloading playlist PL%s - add --no-playlist to just download video %s' % (playlist_id, video_id)) + if len(playlist_id) == 13: # 'RD' + 11 characters for the video id + # Mixes require a custom extraction process + return self._extract_mix(playlist_id) + # Extract the video ids from the playlist pages ids = [] @@ -1571,8 +1592,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor): playlist_title = self._og_search_title(page) - url_results = [self.url_result(vid_id, 'Youtube', video_id=vid_id) - for vid_id in ids] + url_results = self._ids_to_results(ids) return self.playlist_result(url_results, playlist_id, playlist_title) From c1f9c59d11c0a96be7caa0b4c6e90d900e3161c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Wed, 27 Nov 2013 00:41:30 +0100 Subject: [PATCH 163/425] [bash-completion] Complete filenames or directories if the previous option requires it --- devscripts/bash-completion.in | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/devscripts/bash-completion.in b/devscripts/bash-completion.in index ce893fcbe..cc469366d 100644 --- a/devscripts/bash-completion.in +++ b/devscripts/bash-completion.in @@ -1,10 +1,21 @@ __youtube_dl() { - local cur prev opts + local cur prev opts fileopts diropts COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" opts="{{flags}}" keywords=":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater" + fileopts="-a|--batch-file|--download-archive|--cookies" + diropts="--cache-dir" + + if [[ ${prev} =~ ${fileopts} ]]; then + COMPREPLY=( $(compgen -f -- ${cur}) ) + return 0 + elif [[ ${prev} =~ ${diropts} ]]; then + COMPREPLY=( $(compgen -d -- ${cur}) ) + return 0 + fi if [[ ${cur} =~ : ]]; then COMPREPLY=( $(compgen -W "${keywords}" -- ${cur}) ) From 5f09bbff4df9e11d803e3af066c08167f7f7dcb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Wed, 27 Nov 2013 00:42:59 +0100 Subject: [PATCH 164/425] [bash-completion] Complete the ':ythistory' keyword --- devscripts/bash-completion.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/devscripts/bash-completion.in b/devscripts/bash-completion.in index cc469366d..3af87a378 100644 --- a/devscripts/bash-completion.in +++ b/devscripts/bash-completion.in @@ -1,11 +1,11 @@ __youtube_dl() { - local cur prev opts fileopts diropts + local cur prev opts fileopts diropts keywords COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" opts="{{flags}}" - keywords=":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater" + keywords=":ytfavorites :ytrecommended :ytsubscriptions :ytwatchlater :ythistory" fileopts="-a|--batch-file|--download-archive|--cookies" diropts="--cache-dir" From 4b19e3895492a472c5b63d9da5777bc29d44e25c Mon Sep 17 00:00:00 2001 From: Filippo Valsorda <filosottile.wiki@gmail.com> Date: Wed, 27 Nov 2013 02:54:51 +0100 Subject: [PATCH 165/425] [videopremium] support new .me domain --- youtube_dl/extractor/videopremium.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/videopremium.py b/youtube_dl/extractor/videopremium.py index 4800415bd..acae81448 100644 --- a/youtube_dl/extractor/videopremium.py +++ b/youtube_dl/extractor/videopremium.py @@ -5,7 +5,7 @@ from .common import InfoExtractor class VideoPremiumIE(InfoExtractor): - _VALID_URL = r'(?:https?://)?(?:www\.)?videopremium\.tv/(?P<id>\w+)(?:/.*)?' + _VALID_URL = r'(?:https?://)?(?:www\.)?videopremium\.(?:tv|me)/(?P<id>\w+)(?:/.*)?' _TEST = { u'url': u'http://videopremium.tv/4w7oadjsf156', u'file': u'4w7oadjsf156.f4v', @@ -41,4 +41,4 @@ class VideoPremiumIE(InfoExtractor): 'player_url': "http://videopremium.tv/uplayer/uppod.swf", 'ext': 'f4v', 'title': video_title, - } \ No newline at end of file + } From dcca796ce431da0d8b6927609c08938f22ba44cf Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Wed, 27 Nov 2013 18:33:51 +0100 Subject: [PATCH 166/425] [clipfish] Effect a better error message (#1842) --- youtube_dl/extractor/clipfish.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py index 5f0b5602f..05afce338 100644 --- a/youtube_dl/extractor/clipfish.py +++ b/youtube_dl/extractor/clipfish.py @@ -1,5 +1,6 @@ import re import time +import xml.etree.ElementTree from .common import InfoExtractor @@ -28,6 +29,10 @@ class ClipfishIE(InfoExtractor): info_url, video_id, note=u'Downloading info page') title = doc.find('title').text video_url = doc.find('filename').text + if video_url is None: + xml_bytes = xml.etree.ElementTree.tostring(doc) + raise ExtractorError(u'Cannot find video URL in document %r' % + xml_bytes) thumbnail = doc.find('imageurl').text duration_str = doc.find('duration').text m = re.match( From 76d1700b283ee482288eec12a6903a345742eead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Wed, 27 Nov 2013 20:01:51 +0100 Subject: [PATCH 167/425] [youtube:playlist] Fix the extraction of the title for some mixes (#1844) Like https://www.youtube.com/watch?v=g8jDB5xOiuE&list=RDIh2gxLqR7HM --- youtube_dl/extractor/youtube.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 9ef5fecce..fb61f47e8 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1547,7 +1547,9 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor): # the id of the playlist is just 'RD' + video_id url = 'https://youtube.com/watch?v=%s&list=%s' % (playlist_id[2:], playlist_id) webpage = self._download_webpage(url, playlist_id, u'Downloading Youtube mix') - title = clean_html(get_element_by_attribute('class', 'title long-title', webpage)) + title_span = (get_element_by_attribute('class', 'title long-title', webpage) or + get_element_by_attribute('class', 'title ', webpage)) + title = clean_html(title_span) video_re = r'data-index="\d+".*?href="/watch\?v=([0-9A-Za-z_-]{11})&[^"]*?list=%s' % re.escape(playlist_id) ids = orderedSet(re.findall(video_re, webpage)) url_results = self._ids_to_results(ids) From 35907e23ec4d7e754ff239693500e05886b80ee7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Wed, 27 Nov 2013 21:24:55 +0100 Subject: [PATCH 168/425] [yahoo] Fix video extraction and use the new format system exclusively --- youtube_dl/extractor/yahoo.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py index 34e6afb20..617e3bb06 100644 --- a/youtube_dl/extractor/yahoo.py +++ b/youtube_dl/extractor/yahoo.py @@ -46,7 +46,7 @@ class YahooIE(InfoExtractor): video_id = mobj.group('id') webpage = self._download_webpage(url, video_id) - items_json = self._search_regex(r'YVIDEO_INIT_ITEMS = ({.*?});$', + items_json = self._search_regex(r'mediaItems: ({.*?})$', webpage, u'items', flags=re.MULTILINE) items = json.loads(items_json) info = items['mediaItems']['query']['results']['mediaObj'][0] @@ -91,17 +91,13 @@ class YahooIE(InfoExtractor): formats.append(format_info) formats = sorted(formats, key=lambda f:(f['height'], f['width'])) - info = { + return { 'id': video_id, 'title': meta['title'], 'formats': formats, 'description': clean_html(meta['description']), 'thumbnail': meta['thumbnail'], } - # TODO: Remove when #980 has been merged - info.update(formats[-1]) - - return info class YahooSearchIE(SearchInfoExtractor): From 0e44d8381a439c84dd23477d32f7da4bb0a06293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Thu, 28 Nov 2013 00:33:27 +0100 Subject: [PATCH 169/425] [youtube:feeds] Use the 'paging' value from the downloaded json information (fixes #1845) --- youtube_dl/extractor/youtube.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index fb61f47e8..765b4a9bf 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1791,7 +1791,6 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor): Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties. """ _LOGIN_REQUIRED = True - _PAGING_STEP = 30 # use action_load_personal_feed instead of action_load_system_feed _PERSONAL_FEED = False @@ -1811,9 +1810,8 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor): def _real_extract(self, url): feed_entries = [] - # The step argument is available only in 2.7 or higher - for i in itertools.count(0): - paging = i*self._PAGING_STEP + paging = 0 + for i in itertools.count(1): info = self._download_webpage(self._FEED_TEMPLATE % paging, u'%s feed' % self._FEED_NAME, u'Downloading page %s' % i) @@ -1826,6 +1824,7 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor): for video_id in ids) if info['paging'] is None: break + paging = info['paging'] return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE) class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor): @@ -1845,7 +1844,6 @@ class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor): _VALID_URL = r'https?://www\.youtube\.com/feed/watch_later|:ytwatchlater' _FEED_NAME = 'watch_later' _PLAYLIST_TITLE = u'Youtube Watch Later' - _PAGING_STEP = 100 _PERSONAL_FEED = True class YoutubeHistoryIE(YoutubeFeedsInfoExtractor): @@ -1855,13 +1853,6 @@ class YoutubeHistoryIE(YoutubeFeedsInfoExtractor): _PERSONAL_FEED = True _PLAYLIST_TITLE = u'Youtube Watch History' - def _real_extract(self, url): - webpage = self._download_webpage('https://www.youtube.com/feed/history', u'History') - data_paging = self._search_regex(r'data-paging="(\d+)"', webpage, u'data-paging') - # The step is actually a ridiculously big number (like 1374343569725646) - self._PAGING_STEP = int(data_paging) - return super(YoutubeHistoryIE, self)._real_extract(url) - class YoutubeFavouritesIE(YoutubeBaseInfoExtractor): IE_NAME = u'youtube:favorites' IE_DESC = u'YouTube.com favourite videos, "ytfav" keyword (requires authentication)' From a2e6db365c11d8c9eaaaeb8de53d59add648f978 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Thu, 28 Nov 2013 05:47:20 +0100 Subject: [PATCH 170/425] [zdf] add a pseudo-testcase and fix URL matching --- youtube_dl/extractor/zdf.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py index 07f830e80..3c01cc041 100644 --- a/youtube_dl/extractor/zdf.py +++ b/youtube_dl/extractor/zdf.py @@ -1,3 +1,5 @@ +# coding: utf-8 + import operator import re @@ -9,7 +11,19 @@ from ..utils import ( class ZDFIE(InfoExtractor): - _VALID_URL = r'^http://www\.zdf\.de\/ZDFmediathek(?P<hash>#)?\/(.*beitrag\/video\/)(?P<video_id>[^/\?]+)(?:\?.*)?' + _VALID_URL = r'^https?://www\.zdf\.de/ZDFmediathek(?P<hash>#)?/(.*beitrag/(?:video/)?)(?P<video_id>[0-9]+)(?:/[^/?]+)?(?:\?.*)?' + + _TEST = { + u"url": u"http://www.zdf.de/ZDFmediathek/beitrag/video/2037704/ZDFspezial---Ende-des-Machtpokers--?bc=sts;stt", + u"file": u"2037704.webm", + u"info_dict": { + u"upload_date": u"20131127", + u"description": u"Union und SPD haben sich auf einen Koalitionsvertrag geeinigt. Aber was bedeutet das für die Bürger? Sehen Sie hierzu das ZDFspezial \"Ende des Machtpokers - Große Koalition für Deutschland\".", + u"uploader": u"spezial", + u"title": u"ZDFspezial - Ende des Machtpokers" + }, + u"skip": u"Videos on ZDF.de are depublicised in short order", + } def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) From 2a275ab007d6d336b44a6a0cd4fac6783ba63cb8 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Thu, 28 Nov 2013 05:47:50 +0100 Subject: [PATCH 171/425] [zdf] Use _download_xml --- youtube_dl/extractor/common.py | 3 ++- youtube_dl/extractor/zdf.py | 8 ++++---- youtube_dl/utils.py | 5 ----- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 5656445a3..4f1b50880 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -210,7 +210,8 @@ class InfoExtractor(object): """ Returns the data of the page as a string """ return self._download_webpage_handle(url_or_request, video_id, note, errnote)[0] - def _download_xml(self, url_or_request, video_id, note=u'Downloading XML', errnote=u'Unable to downloand XML'): + def _download_xml(self, url_or_request, video_id, + note=u'Downloading XML', errnote=u'Unable to download XML'): """Return the xml as an xml.etree.ElementTree.Element""" xml_string = self._download_webpage(url_or_request, video_id, note, errnote) return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8')) diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py index 3c01cc041..689f19735 100644 --- a/youtube_dl/extractor/zdf.py +++ b/youtube_dl/extractor/zdf.py @@ -5,7 +5,6 @@ import re from .common import InfoExtractor from ..utils import ( - parse_xml_doc, unified_strdate, ) @@ -30,9 +29,10 @@ class ZDFIE(InfoExtractor): video_id = mobj.group('video_id') xml_url = u'http://www.zdf.de/ZDFmediathek/xmlservice/web/beitragsDetails?ak=web&id=%s' % video_id - info_xml = self._download_webpage( - xml_url, video_id, note=u'Downloading video info') - doc = parse_xml_doc(info_xml) + doc = self._download_xml( + xml_url, video_id, + note=u'Downloading video info', + errnote=u'Failed to download video info') title = doc.find('.//information/title').text description = doc.find('.//information/detail').text diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 946e90e93..c486ef8ec 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1009,11 +1009,6 @@ def unsmuggle_url(smug_url): return url, data -def parse_xml_doc(s): - assert isinstance(s, type(u'')) - return xml.etree.ElementTree.fromstring(s.encode('utf-8')) - - def format_bytes(bytes): if bytes is None: return u'N/A' From ea07dbb8b108d7c77b6b822fba98817063a8457a Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Thu, 28 Nov 2013 05:48:32 +0100 Subject: [PATCH 172/425] release 2013.11.28 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 99a5e0505..03cb283bd 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.26' +__version__ = '2013.11.28' From f8f60d27931421f969c7ec0a2a45caa743549994 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Thu, 28 Nov 2013 05:54:46 +0100 Subject: [PATCH 173/425] [clipfish] Fix imports (#1842) --- youtube_dl/extractor/clipfish.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py index 05afce338..ba5623572 100644 --- a/youtube_dl/extractor/clipfish.py +++ b/youtube_dl/extractor/clipfish.py @@ -3,6 +3,7 @@ import time import xml.etree.ElementTree from .common import InfoExtractor +from ..utils import ExtractorError class ClipfishIE(InfoExtractor): From fc9e1cc69706ef079fca0ee32529503ecedae578 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Thu, 28 Nov 2013 06:10:37 +0100 Subject: [PATCH 174/425] [clipfish] Use FIFA trailer as testcase (#1842) --- youtube_dl/extractor/clipfish.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py index ba5623572..0d18e9a7a 100644 --- a/youtube_dl/extractor/clipfish.py +++ b/youtube_dl/extractor/clipfish.py @@ -11,12 +11,12 @@ class ClipfishIE(InfoExtractor): _VALID_URL = r'^https?://(?:www\.)?clipfish\.de/.*?/video/(?P<id>[0-9]+)/' _TEST = { - u'url': u'http://www.clipfish.de/special/supertalent/video/4028320/supertalent-2013-ivana-opacak-singt-nobodys-perfect/', - u'file': u'4028320.f4v', - u'md5': u'5e38bda8c329fbfb42be0386a3f5a382', + u'url': u'http://www.clipfish.de/special/game-trailer/video/3966754/fifa-14-e3-2013-trailer/', + u'file': u'3966754.mp4', + u'md5': u'2521cd644e862936cf2e698206e47385', u'info_dict': { - u'title': u'Supertalent 2013: Ivana Opacak singt Nobody\'s Perfect', - u'duration': 399, + u'title': u'FIFA 14 - E3 2013 Trailer', + u'duration': 82, } } From 4e0084d92e589d385f28ac98bfb847240d61dc93 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Thu, 28 Nov 2013 06:14:17 +0100 Subject: [PATCH 175/425] [youtube/subtitles] Change MD5 of vtt subtitle in test --- test/test_subtitles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_subtitles.py b/test/test_subtitles.py index 94a1f771d..23a653124 100644 --- a/test/test_subtitles.py +++ b/test/test_subtitles.py @@ -72,7 +72,7 @@ class TestYoutubeSubtitles(BaseTestSubtitles): self.DL.params['writesubtitles'] = True self.DL.params['subtitlesformat'] = 'vtt' subtitles = self.getSubtitles() - self.assertEqual(md5(subtitles['en']), '356cdc577fde0c6783b9b822e7206ff7') + self.assertEqual(md5(subtitles['en']), '3cb210999d3e021bd6c7f0ea751eab06') def test_youtube_list_subtitles(self): self.DL.expect_warning(u'Video doesn\'t have automatic captions') From 2be54167d085c5b4c956c66ad0367fdcfb68b891 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Thu, 28 Nov 2013 06:17:56 +0100 Subject: [PATCH 176/425] release 2013.11.28.1 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 03cb283bd..9cae97ee2 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.28' +__version__ = '2013.11.28.1' From d8d6148628b972b6998a8c2a5465f031a44f4004 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Thu, 28 Nov 2013 13:32:49 +0100 Subject: [PATCH 177/425] Add an extractor for Internet Movie Database trailers (closes #1832) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/imdb.py | 59 ++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 youtube_dl/extractor/imdb.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 0b4d086b7..30e4a9105 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -71,6 +71,7 @@ from .hotnewhiphop import HotNewHipHopIE from .howcast import HowcastIE from .hypem import HypemIE from .ign import IGNIE, OneUPIE +from .imdb import ImdbIE from .ina import InaIE from .infoq import InfoQIE from .instagram import InstagramIE diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py new file mode 100644 index 000000000..07e4f7d29 --- /dev/null +++ b/youtube_dl/extractor/imdb.py @@ -0,0 +1,59 @@ +import re +import json + +from .common import InfoExtractor +from ..utils import ( + compat_urlparse, + get_element_by_attribute, +) + + +class ImdbIE(InfoExtractor): + IE_NAME = u'imdb' + IE_DESC = u'Internet Movie Database trailers' + _VALID_URL = r'http://www\.imdb\.com/video/imdb/vi(?P<id>\d+)' + + _TEST = { + u'url': u'http://www.imdb.com/video/imdb/vi2524815897', + u'md5': u'9f34fa777ade3a6e57a054fdbcb3a068', + u'info_dict': { + u'id': u'2524815897', + u'ext': u'mp4', + u'title': u'Ice Age: Continental Drift Trailer (No. 2) - IMDb', + u'description': u'md5:9061c2219254e5d14e03c25c98e96a81', + u'duration': 151, + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + webpage = self._download_webpage(url,video_id) + descr = get_element_by_attribute('itemprop', 'description', webpage) + available_formats = re.findall( + r'case \'(?P<f_id>.*?)\' :$\s+url = \'(?P<path>.*?)\'', webpage, + flags=re.MULTILINE) + formats = [] + for f_id, f_path in available_formats: + format_page = self._download_webpage( + compat_urlparse.urljoin(url, f_path), + u'Downloading info for %s format' % f_id) + json_data = get_element_by_attribute('class', 'imdb-player-data', + format_page) + info = json.loads(json_data) + format_info = info['videoPlayerObject']['video'] + formats.append({ + 'format_id': f_id, + 'url': format_info['url'], + 'height': format_info['height'], + 'width': format_info['width'], + }) + + return { + 'id': video_id, + 'title': self._og_search_title(webpage), + 'formats': formats, + 'description': descr, + 'thumbnail': format_info['slate'], + 'duration': int(info['titleObject']['title']['duration_seconds']), + } From b03d0d064c0e198aa281faacb2b5a74af7628b74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Thu, 28 Nov 2013 13:49:00 +0100 Subject: [PATCH 178/425] [imdb] Fix extraction in python 2.6 Using a regular expression because the html cannot be parsed. --- youtube_dl/extractor/imdb.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py index 07e4f7d29..520edc7d0 100644 --- a/youtube_dl/extractor/imdb.py +++ b/youtube_dl/extractor/imdb.py @@ -38,8 +38,9 @@ class ImdbIE(InfoExtractor): format_page = self._download_webpage( compat_urlparse.urljoin(url, f_path), u'Downloading info for %s format' % f_id) - json_data = get_element_by_attribute('class', 'imdb-player-data', - format_page) + json_data = self._search_regex( + r'<script[^>]+class="imdb-player-data"[^>]*?>(.*?)</script>', + format_page, u'json data', flags=re.DOTALL) info = json.loads(json_data) format_info = info['videoPlayerObject']['video'] formats.append({ From 3862402ff3a991e7fb58470ac38fba82ba9b18d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Thu, 28 Nov 2013 14:38:10 +0100 Subject: [PATCH 179/425] Add an extractor for Clipsyndicate (closes #1744) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/clipsyndicate.py | 52 +++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 youtube_dl/extractor/clipsyndicate.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 30e4a9105..1e4f36aa3 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -21,6 +21,7 @@ from .canalplus import CanalplusIE from .canalc2 import Canalc2IE from .cinemassacre import CinemassacreIE from .clipfish import ClipfishIE +from .clipsyndicate import ClipsyndicateIE from .cnn import CNNIE from .collegehumor import CollegeHumorIE from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE diff --git a/youtube_dl/extractor/clipsyndicate.py b/youtube_dl/extractor/clipsyndicate.py new file mode 100644 index 000000000..d4fc86973 --- /dev/null +++ b/youtube_dl/extractor/clipsyndicate.py @@ -0,0 +1,52 @@ +import re +import xml.etree.ElementTree + +from .common import InfoExtractor +from ..utils import ( + find_xpath_attr, +) + + +class ClipsyndicateIE(InfoExtractor): + _VALID_URL = r'http://www\.clipsyndicate\.com/video/play(list/\d+)?/(?P<id>\d+)' + + _TEST = { + u'url': u'http://www.clipsyndicate.com/video/play/4629301/brick_briscoe', + u'md5': u'4d7d549451bad625e0ff3d7bd56d776c', + u'info_dict': { + u'id': u'4629301', + u'ext': u'mp4', + u'title': u'Brick Briscoe', + u'duration': 612, + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + js_player = self._download_webpage( + 'http://eplayer.clipsyndicate.com/embed/player.js?va_id=%s' % video_id, + video_id, u'Downlaoding player') + # it includes a required token + flvars = self._search_regex(r'flvars: "(.*?)"', js_player, u'flvars') + + playlist_page = self._download_webpage( + 'http://eplayer.clipsyndicate.com/osmf/playlist?%s' % flvars, + video_id, u'Downloading video info') + # Fix broken xml + playlist_page = re.sub('&', '&', playlist_page) + pdoc = xml.etree.ElementTree.fromstring(playlist_page.encode('utf-8')) + + track_doc = pdoc.find('trackList/track') + def find_param(name): + node = find_xpath_attr(track_doc, './/param', 'name', name) + if node is not None: + return node.attrib['value'] + + return { + 'id': video_id, + 'title': find_param('title'), + 'url': track_doc.find('location').text, + 'thumbnail': find_param('thumbnail'), + 'duration': int(find_param('duration')), + } From 677c18092d8fd5ca6e08b25985c8533b6a0738d5 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 29 Nov 2013 03:33:25 +0100 Subject: [PATCH 180/425] [podomatic] Add extractor --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/podomatic.py | 49 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 youtube_dl/extractor/podomatic.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 1e4f36aa3..fd890e251 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -107,6 +107,7 @@ from .ooyala import OoyalaIE from .orf import ORFIE from .pbs import PBSIE from .photobucket import PhotobucketIE +from .podomatic import PodomaticIE from .pornhub import PornHubIE from .pornotube import PornotubeIE from .rbmaradio import RBMARadioIE diff --git a/youtube_dl/extractor/podomatic.py b/youtube_dl/extractor/podomatic.py new file mode 100644 index 000000000..58200971b --- /dev/null +++ b/youtube_dl/extractor/podomatic.py @@ -0,0 +1,49 @@ +import json +import re + +from .common import InfoExtractor + + +class PodomaticIE(InfoExtractor): + IE_NAME = 'podomatic' + _VALID_URL = r'^(?P<proto>https?)://(?P<channel>[^.]+)\.podomatic\.com/entry/(?P<id>[^?]+)' + + _TEST = { + u"url": u"http://scienceteachingtips.podomatic.com/entry/2009-01-02T16_03_35-08_00", + u"file": u"2009-01-02T16_03_35-08_00.mp3", + u"md5": u"84bb855fcf3429e6bf72460e1eed782d", + u"info_dict": { + u"uploader": u"Science Teaching Tips", + u"uploader_id": u"scienceteachingtips", + u"title": u"64. When the Moon Hits Your Eye", + u"duration": 446, + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + channel = mobj.group('channel') + + json_url = (('%s://%s.podomatic.com/entry/embed_params/%s' + + '?permalink=true&rtmp=0') % + (mobj.group('proto'), channel, video_id)) + data_json = self._download_webpage( + json_url, video_id, note=u'Downloading video info') + data = json.loads(data_json) + + video_url = data['downloadLink'] + uploader = data['podcast'] + title = data['title'] + thumbnail = data['imageLocation'] + duration = int(data['length'] / 1000.0) + + return { + 'id': video_id, + 'url': video_url, + 'title': title, + 'uploader': uploader, + 'uploader_id': channel, + 'thumbnail': thumbnail, + 'duration': duration, + } From 17769d5a6c24eb8f5d609aa99f84debc3fe4adec Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 29 Nov 2013 03:34:26 +0100 Subject: [PATCH 181/425] release 2013.11.29 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 9cae97ee2..a73d7fb5c 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.28.1' +__version__ = '2013.11.29' From acf37ca151d67ee28034775662318d9a0a1eb6f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Fri, 29 Nov 2013 07:56:14 +0100 Subject: [PATCH 182/425] [imdb] Fix the resolution values (fixes #1847) We were using the size of the player, it was the same for all the formats --- youtube_dl/extractor/imdb.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py index 520edc7d0..d8e9712a7 100644 --- a/youtube_dl/extractor/imdb.py +++ b/youtube_dl/extractor/imdb.py @@ -46,8 +46,7 @@ class ImdbIE(InfoExtractor): formats.append({ 'format_id': f_id, 'url': format_info['url'], - 'height': format_info['height'], - 'width': format_info['width'], + 'height': int(info['titleObject']['encoding']['selected'][:-1]), }) return { From e1f900d6a4c449b2a7c7ed74dbe8eca74cbccf13 Mon Sep 17 00:00:00 2001 From: Nicolas Kaiser <nikai@nikai.net> Date: Fri, 29 Nov 2013 09:44:05 +0100 Subject: [PATCH 183/425] fix typo in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 580b16004..af4d969d6 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ which means you can modify it, redistribute it or use it however you like. directory ## Video Format Options: - -f, --format FORMAT video format code, specifiy the order of + -f, --format FORMAT video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported --all-formats download all available video formats From 9986238ba9a486ca76334c50562760a312ab20fa Mon Sep 17 00:00:00 2001 From: Nicolas Kaiser <nikai@nikai.net> Date: Fri, 29 Nov 2013 09:48:38 +0100 Subject: [PATCH 184/425] fix typo in help --- youtube_dl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 0704515df..8f8422cc7 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -235,7 +235,7 @@ def parseOpts(overrideArguments=None): video_format.add_option('-f', '--format', action='store', dest='format', metavar='FORMAT', default='best', - help='video format code, specifiy the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported') + help='video format code, specify the order of preference using slashes: "-f 22/17/18". "-f mp4" and "-f flv" are also supported') video_format.add_option('--all-formats', action='store_const', dest='format', help='download all available video formats', const='all') video_format.add_option('--prefer-free-formats', From a3fb4675fb67b061e2a71cec78a5dbd8695b8ef0 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 29 Nov 2013 15:25:09 +0100 Subject: [PATCH 185/425] Do not mutate default arguments In this case, it looks rather harmless (since the conditions for --restrict-filenames should not change while a process is running), but just to be sure. This also simplifies the interface for callers, who can just pass in the idiomatic None for "I don't care, whatever is the default". --- youtube_dl/YoutubeDL.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 711b5d79e..b822930cb 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -146,7 +146,7 @@ class YoutubeDL(object): _num_downloads = None _screen_file = None - def __init__(self, params={}): + def __init__(self, params=None): """Create a FileDownloader object with the given options.""" self._ies = [] self._ies_instances = {} @@ -155,7 +155,7 @@ class YoutubeDL(object): self._download_retcode = 0 self._num_downloads = 0 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] - self.params = params + self.params = {} if params is None else params if (sys.version_info >= (3,) and sys.platform != 'win32' and sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] From befd88b786dc41ff075693fd17bafbc7fa4c100e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Fri, 29 Nov 2013 15:25:43 +0100 Subject: [PATCH 186/425] [yahoo] Add an extractor for yahoo news (closes #1849) --- youtube_dl/extractor/__init__.py | 6 +++++- youtube_dl/extractor/yahoo.py | 34 +++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index fd890e251..664639b53 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -172,7 +172,11 @@ from .xhamster import XHamsterIE from .xnxx import XNXXIE from .xvideos import XVideosIE from .xtube import XTubeIE -from .yahoo import YahooIE, YahooSearchIE +from .yahoo import ( + YahooIE, + YahooNewsIE, + YahooSearchIE, +) from .youjizz import YouJizzIE from .youku import YoukuIE from .youporn import YouPornIE diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py index 617e3bb06..2d87e81b2 100644 --- a/youtube_dl/extractor/yahoo.py +++ b/youtube_dl/extractor/yahoo.py @@ -53,8 +53,11 @@ class YahooIE(InfoExtractor): # The 'meta' field is not always in the video webpage, we request it # from another page long_id = info['id'] + return self._get_info(info['id'], video_id) + + def _get_info(self, long_id, video_id): query = ('SELECT * FROM yahoo.media.video.streams WHERE id="%s"' - ' AND plrs="86Gj0vCaSzV_Iuf6hNylf2"' % long_id) + ' AND plrs="86Gj0vCaSzV_Iuf6hNylf2" AND region="US"' % long_id) data = compat_urllib_parse.urlencode({ 'q': query, 'env': 'prod', @@ -100,6 +103,35 @@ class YahooIE(InfoExtractor): } +class YahooNewsIE(YahooIE): + IE_NAME = 'yahoo:news' + _VALID_URL = r'http://news\.yahoo\.com/video/.*?-(?P<id>\d*?)\.html' + + _TEST = { + u'url': u'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html', + u'info_dict': { + u'id': u'104538833', + u'ext': u'flv', + u'title': u'China Moses Is Crazy About the Blues', + u'description': u'md5:9900ab8cd5808175c7b3fe55b979bed0', + }, + u'params': { + # Requires rtmpdump + u'skip_download': True, + }, + } + + # Overwrite YahooIE properties we don't want + _TESTS = [] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + webpage = self._download_webpage(url, video_id) + long_id = self._search_regex(r'contentId: \'(.+?)\',', webpage, u'long id') + return self._get_info(long_id, video_id) + + class YahooSearchIE(SearchInfoExtractor): IE_DESC = u'Yahoo screen search' _MAX_RESULTS = 1000 From 323ec6ae566af9744edce97a23e623d99eea8a1f Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 29 Nov 2013 15:57:43 +0100 Subject: [PATCH 187/425] Clarify --download-archive help --- youtube_dl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 0704515df..c63d62986 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -220,7 +220,7 @@ def parseOpts(overrideArguments=None): default=None, type=int) selection.add_option('--download-archive', metavar='FILE', dest='download_archive', - help='Download only videos not present in the archive file. Record all downloaded videos in it.') + help='Download only videos not present in the archive file. Record the IDs of all downloaded videos in it.') authentication.add_option('-u', '--username', From c5171c454b4392f7276b7f9e94c25d7f1ad60375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Fri, 29 Nov 2013 22:06:17 +0100 Subject: [PATCH 188/425] [yahoo] Force use of the http protocol for downloading the videos. --- youtube_dl/extractor/yahoo.py | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py index 2d87e81b2..e457c4707 100644 --- a/youtube_dl/extractor/yahoo.py +++ b/youtube_dl/extractor/yahoo.py @@ -17,27 +17,21 @@ class YahooIE(InfoExtractor): _TESTS = [ { u'url': u'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html', - u'file': u'214727115.flv', + u'file': u'214727115.mp4', + u'md5': u'4962b075c08be8690a922ee026d05e69', u'info_dict': { u'title': u'Julian Smith & Travis Legg Watch Julian Smith', u'description': u'Julian and Travis watch Julian Smith', }, - u'params': { - # Requires rtmpdump - u'skip_download': True, - }, }, { u'url': u'http://screen.yahoo.com/wired/codefellas-s1-ep12-cougar-lies-103000935.html', - u'file': u'103000935.flv', + u'file': u'103000935.mp4', + u'md5': u'd6e6fc6e1313c608f316ddad7b82b306', u'info_dict': { u'title': u'Codefellas - The Cougar Lies with Spanish Moss', u'description': u'Agent Topple\'s mustache does its dirty work, and Nicole brokers a deal for peace. But why is the NSA collecting millions of Instagram brunch photos? And if your waffles have nothing to hide, what are they so worried about?', }, - u'params': { - # Requires rtmpdump - u'skip_download': True, - }, }, ] @@ -57,7 +51,8 @@ class YahooIE(InfoExtractor): def _get_info(self, long_id, video_id): query = ('SELECT * FROM yahoo.media.video.streams WHERE id="%s"' - ' AND plrs="86Gj0vCaSzV_Iuf6hNylf2" AND region="US"' % long_id) + ' AND plrs="86Gj0vCaSzV_Iuf6hNylf2" AND region="US"' + ' AND protocol="http"' % long_id) data = compat_urllib_parse.urlencode({ 'q': query, 'env': 'prod', @@ -109,16 +104,13 @@ class YahooNewsIE(YahooIE): _TEST = { u'url': u'http://news.yahoo.com/video/china-moses-crazy-blues-104538833.html', + u'md5': u'67010fdf3a08d290e060a4dd96baa07b', u'info_dict': { u'id': u'104538833', - u'ext': u'flv', + u'ext': u'mp4', u'title': u'China Moses Is Crazy About the Blues', u'description': u'md5:9900ab8cd5808175c7b3fe55b979bed0', }, - u'params': { - # Requires rtmpdump - u'skip_download': True, - }, } # Overwrite YahooIE properties we don't want From 06dcbb71d8e19947eb6e71390a6a0640abe3dad0 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sat, 30 Nov 2013 00:42:43 +0100 Subject: [PATCH 189/425] Clarify help of --write-pages (#1853) --- youtube_dl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index c63d62986..42ab572f2 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -317,7 +317,7 @@ def parseOpts(overrideArguments=None): help='print downloaded pages to debug problems(very verbose)') verbosity.add_option('--write-pages', action='store_true', dest='write_pages', default=False, - help='Write downloaded pages to files in the current directory') + help='Write downloaded intermediary pages to files in the current directory to debug problems') verbosity.add_option('--youtube-print-sig-code', action='store_true', dest='youtube_print_sig_code', default=False, help=optparse.SUPPRESS_HELP) From 0a688bc0b28c970e9af965b3fa0c7927507eeb97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Sat, 30 Nov 2013 14:56:51 +0100 Subject: [PATCH 190/425] [youtube] Add support for downloading top lists (fixes #1868) It needs to know the channel and the title of the list, because the ids change every time you browse the channels and are attached to a 'VISITOR_INFO1_LIVE' cookie. --- test/test_youtube_lists.py | 8 ++++++++ youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/youtube.py | 35 ++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py index 95f07d129..33db09f43 100644 --- a/test/test_youtube_lists.py +++ b/test/test_youtube_lists.py @@ -15,6 +15,7 @@ from youtube_dl.extractor import ( YoutubeIE, YoutubeChannelIE, YoutubeShowIE, + YoutubeTopListIE, ) @@ -116,5 +117,12 @@ class TestYoutubeLists(unittest.TestCase): original_video = entries[0] self.assertEqual(original_video['id'], 'rjFaenf1T-Y') + def test_youtube_toplist(self): + dl = FakeYDL() + ie = YoutubeTopListIE(dl) + result = ie.extract('yttoplist:music:Top Tracks') + entries = result['entries'] + self.assertTrue(len(entries) >= 9) + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 664639b53..0abf86e44 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -194,6 +194,7 @@ from .youtube import ( YoutubeWatchLaterIE, YoutubeFavouritesIE, YoutubeHistoryIE, + YoutubeTopListIE, ) from .zdf import ZDFIE diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 765b4a9bf..a1a4d896d 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1576,6 +1576,9 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor): if len(playlist_id) == 13: # 'RD' + 11 characters for the video id # Mixes require a custom extraction process return self._extract_mix(playlist_id) + if playlist_id.startswith('TL'): + raise ExtractorError(u'For downloading YouTube.com top lists, use ' + u'the "yttoplist" keyword, for example "youtube-dl \'yttoplist:music:Top Tracks\'"', expected=True) # Extract the video ids from the playlist pages ids = [] @@ -1598,6 +1601,38 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor): return self.playlist_result(url_results, playlist_id, playlist_title) +class YoutubeTopListIE(YoutubePlaylistIE): + IE_NAME = u'youtube:toplist' + IE_DESC = (u'YouTube.com top lists, "yttoplist:{channel}:{list title}"' + u' (Example: "yttoplist:music:Top Tracks")') + _VALID_URL = r'yttoplist:(?P<chann>.*?):(?P<title>.*?)$' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + channel = mobj.group('chann') + title = mobj.group('title') + query = compat_urllib_parse.urlencode({'title': title}) + playlist_re = 'href="([^"]+?%s[^"]+?)"' % re.escape(query) + channel_page = self._download_webpage('https://www.youtube.com/%s' % channel, title) + link = self._html_search_regex(playlist_re, channel_page, u'list') + url = compat_urlparse.urljoin('https://www.youtube.com/', link) + + video_re = r'data-index="\d+".*?data-video-id="([0-9A-Za-z_-]{11})"' + ids = [] + # sometimes the webpage doesn't contain the videos + # retry until we get them + for i in itertools.count(0): + msg = u'Downloading Youtube mix' + if i > 0: + msg += ', retry #%d' % i + webpage = self._download_webpage(url, title, msg) + ids = orderedSet(re.findall(video_re, webpage)) + if ids: + break + url_results = self._ids_to_results(ids) + return self.playlist_result(url_results, playlist_title=title) + + class YoutubeChannelIE(InfoExtractor): IE_DESC = u'YouTube.com channels' _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)" From 5e09d6abbd09de92869cbb8ed204d18f9cd04931 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 1 Dec 2013 01:16:20 +0100 Subject: [PATCH 191/425] [clipfish] Skip test on travis --- youtube_dl/extractor/clipfish.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/clipfish.py b/youtube_dl/extractor/clipfish.py index 0d18e9a7a..43efb08bf 100644 --- a/youtube_dl/extractor/clipfish.py +++ b/youtube_dl/extractor/clipfish.py @@ -17,7 +17,8 @@ class ClipfishIE(InfoExtractor): u'info_dict': { u'title': u'FIFA 14 - E3 2013 Trailer', u'duration': 82, - } + }, + u'skip': 'Blocked in the US' } def _real_extract(self, url): From 355e4fd07e7f9c0632d9d78415675f8b5cc3c2ce Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 1 Dec 2013 01:21:33 +0100 Subject: [PATCH 192/425] [generic] Find embedded dailymotion videos (Fixes #1848) --- youtube_dl/extractor/generic.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 37671430a..10ae06263 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -195,6 +195,15 @@ class GenericIE(InfoExtractor): return self.playlist_result( urlrs, playlist_id=video_id, playlist_title=video_title) + # Look for embedded Dailymotion player + matches = re.findall( + r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion.com/embed/video/.+?)\1', webpage) + if matches: + urlrs = [self.url_result(unescapeHTML(tuppl[1]), 'Dailymotion') + for tuppl in matches] + return self.playlist_result( + urlrs, playlist_id=video_id, playlist_title=video_title) + # Look for Bandcamp pages with custom domain mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage) if mobj is not None: From e344693b65a42436eb40efe85095c01f767a502d Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 1 Dec 2013 11:42:02 +0100 Subject: [PATCH 193/425] Make socket timeout configurable, and bump default to 10 minutes (#1862) --- test/parameters.json | 3 ++- youtube_dl/YoutubeDL.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/test/parameters.json b/test/parameters.json index f042880ed..487a46d56 100644 --- a/test/parameters.json +++ b/test/parameters.json @@ -39,5 +39,6 @@ "writeinfojson": true, "writesubtitles": false, "allsubtitles": false, - "listssubtitles": false + "listssubtitles": false, + "socket_timeout": 20 } diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index b822930cb..b7393fd79 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -132,6 +132,7 @@ class YoutubeDL(object): cookiefile: File name where cookies should be read from and dumped to. nocheckcertificate:Do not verify SSL certificates proxy: URL of the proxy server to use + socket_timeout: Time to wait for unresponsive hosts, in seconds The following parameters are not used by YoutubeDL itself, they are used by the FileDownloader: @@ -969,7 +970,8 @@ class YoutubeDL(object): proxy_map.update(handler.proxies) write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n') - def _setup_opener(self, timeout=20): + def _setup_opener(self): + timeout = float(self.params.get('socket_timeout', 600)) opts_cookiefile = self.params.get('cookiefile') opts_proxy = self.params.get('proxy') From 55a10eab48776197245d3d87b86195f182d8d82a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Sun, 1 Dec 2013 22:36:18 +0100 Subject: [PATCH 194/425] [vimeo] Add an extractor for users (closes #1871) --- test/test_all_urls.py | 4 ++++ test/test_playlists.py | 9 +++++++ youtube_dl/extractor/__init__.py | 6 ++++- youtube_dl/extractor/vimeo.py | 41 ++++++++++++++++++++++++-------- 4 files changed, 49 insertions(+), 11 deletions(-) diff --git a/test/test_all_urls.py b/test/test_all_urls.py index 1f1adb6b4..6b9764c67 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -106,6 +106,10 @@ class TestAllURLsMatching(unittest.TestCase): self.assertMatch(':colbertreport', ['ComedyCentralShows']) self.assertMatch(':cr', ['ComedyCentralShows']) + def test_vimeo_matching(self): + self.assertMatch('http://vimeo.com/channels/tributes', ['vimeo:channel']) + self.assertMatch('http://vimeo.com/user7108434', ['vimeo:user']) + if __name__ == '__main__': unittest.main() diff --git a/test/test_playlists.py b/test/test_playlists.py index 167801ae2..13a6f4b2f 100644 --- a/test/test_playlists.py +++ b/test/test_playlists.py @@ -15,6 +15,7 @@ from youtube_dl.extractor import ( DailymotionPlaylistIE, DailymotionUserIE, VimeoChannelIE, + VimeoUserIE, UstreamChannelIE, SoundcloudSetIE, SoundcloudUserIE, @@ -54,6 +55,14 @@ class TestPlaylists(unittest.TestCase): self.assertEqual(result['title'], u'Vimeo Tributes') self.assertTrue(len(result['entries']) > 24) + def test_vimeo_user(self): + dl = FakeYDL() + ie = VimeoUserIE(dl) + result = ie.extract('http://vimeo.com/nkistudio/videos') + self.assertIsPlaylist(result) + self.assertEqual(result['title'], u'Nki') + self.assertTrue(len(result['entries']) > 65) + def test_ustream_channel(self): dl = FakeYDL() ie = UstreamChannelIE(dl) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 664639b53..cc93e619c 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -159,7 +159,11 @@ from .viddler import ViddlerIE from .videodetective import VideoDetectiveIE from .videofyme import VideofyMeIE from .videopremium import VideoPremiumIE -from .vimeo import VimeoIE, VimeoChannelIE +from .vimeo import ( + VimeoIE, + VimeoChannelIE, + VimeoUserIE, +) from .vine import VineIE from .viki import VikiIE from .vk import VKIE diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index 7d82c2cfa..f27763ae2 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -249,25 +249,46 @@ class VimeoChannelIE(InfoExtractor): IE_NAME = u'vimeo:channel' _VALID_URL = r'(?:https?://)?vimeo.\com/channels/(?P<id>[^/]+)' _MORE_PAGES_INDICATOR = r'<a.+?rel="next"' + _TITLE_RE = r'<link rel="alternate"[^>]+?title="(.*?)"' - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - channel_id = mobj.group('id') + def _extract_videos(self, list_id, base_url): video_ids = [] - for pagenum in itertools.count(1): - webpage = self._download_webpage('http://vimeo.com/channels/%s/videos/page:%d' % (channel_id, pagenum), - channel_id, u'Downloading page %s' % pagenum) + webpage = self._download_webpage( + '%s/videos/page:%d/' % (base_url, pagenum),list_id, + u'Downloading page %s' % pagenum) video_ids.extend(re.findall(r'id="clip_(\d+?)"', webpage)) if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None: break entries = [self.url_result('http://vimeo.com/%s' % video_id, 'Vimeo') for video_id in video_ids] - channel_title = self._html_search_regex(r'<a href="/channels/%s">(.*?)</a>' % channel_id, - webpage, u'channel title') + list_title = self._html_search_regex(self._TITLE_RE, webpage, + u'list title') return {'_type': 'playlist', - 'id': channel_id, - 'title': channel_title, + 'id': list_id, + 'title': list_title, 'entries': entries, } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + channel_id = mobj.group('id') + return self._extract_videos(channel_id, 'http://vimeo.com/channels/%s' % channel_id) + + +class VimeoUserIE(VimeoChannelIE): + IE_NAME = u'vimeo:user' + _VALID_URL = r'(?:https?://)?vimeo.\com/(?P<name>[^/]+)' + _TITLE_RE = r'<a[^>]+?class="user">([^<>]+?)</a>' + + @classmethod + def suitable(cls, url): + if VimeoChannelIE.suitable(url) or VimeoIE.suitable(url): + return False + return super(VimeoUserIE, cls).suitable(url) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + name = mobj.group('name') + return self._extract_videos(name, 'http://vimeo.com/%s' % name) From 6ad14cab599b05a658756fef47d3837281429da7 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 2 Dec 2013 13:37:05 +0100 Subject: [PATCH 195/425] Add --socket-timeout option --- youtube_dl/YoutubeDL.py | 4 +++- youtube_dl/__init__.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index b7393fd79..b68b110a4 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -971,7 +971,9 @@ class YoutubeDL(object): write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n') def _setup_opener(self): - timeout = float(self.params.get('socket_timeout', 600)) + timeout_val = self.params.get('socket_timeout') + timeout = 600 if timeout_val is None else float(timeout_val) + opts_cookiefile = self.params.get('cookiefile') opts_proxy = self.params.get('proxy') diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 92e583744..799eca566 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -198,6 +198,9 @@ def parseOpts(overrideArguments=None): general.add_option( '--no-cache-dir', action='store_const', const=None, dest='cachedir', help='Disable filesystem caching') + general.add_option( + '--socket-timeout', dest='socket_timeout', + type=float, default=None, help=optparse.SUPPRESS_HELP) selection.add_option('--playlist-start', @@ -652,6 +655,7 @@ def _real_main(argv=None): 'cookiefile': opts.cookiefile, 'nocheckcertificate': opts.no_check_certificate, 'proxy': opts.proxy, + 'socket_timeout': opts.socket_timeout, } with YoutubeDL(ydl_opts) as ydl: From 0037e02921e7f70409ce113fb060765a6f24a27e Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 2 Dec 2013 13:37:26 +0100 Subject: [PATCH 196/425] release 2013.12.02 --- README.md | 7 ++++--- youtube_dl/version.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index af4d969d6..031e436b6 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,8 @@ which means you can modify it, redistribute it or use it however you like. --no-playlist download only the currently playing video --age-limit YEARS download only videos suitable for the given age --download-archive FILE Download only videos not present in the archive - file. Record all downloaded videos in it. + file. Record the IDs of all downloaded videos in + it. ## Download Options: -r, --rate-limit LIMIT maximum download rate in bytes per second (e.g. @@ -130,8 +131,8 @@ which means you can modify it, redistribute it or use it however you like. -v, --verbose print various debugging information --dump-intermediate-pages print downloaded pages to debug problems(very verbose) - --write-pages Write downloaded pages to files in the current - directory + --write-pages Write downloaded intermediary pages to files in + the current directory to debug problems ## Video Format Options: -f, --format FORMAT video format code, specify the order of diff --git a/youtube_dl/version.py b/youtube_dl/version.py index a73d7fb5c..d8f341ab9 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.11.29' +__version__ = '2013.12.02' From 5270d8cb1389a9b26fa698137bf4861d4bab6a25 Mon Sep 17 00:00:00 2001 From: dst <dstftw@gmail.com> Date: Mon, 2 Dec 2013 20:10:19 +0700 Subject: [PATCH 197/425] Added extractors for smotri.com --- test/test_playlists.py | 22 ++- youtube_dl/extractor/__init__.py | 5 + youtube_dl/extractor/smotri.py | 239 +++++++++++++++++++++++++++++++ 3 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 youtube_dl/extractor/smotri.py diff --git a/test/test_playlists.py b/test/test_playlists.py index 13a6f4b2f..00c950109 100644 --- a/test/test_playlists.py +++ b/test/test_playlists.py @@ -22,7 +22,9 @@ from youtube_dl.extractor import ( LivestreamIE, NHLVideocenterIE, BambuserChannelIE, - BandcampAlbumIE + BandcampAlbumIE, + SmotriCommunityIE, + SmotriUserIE ) @@ -119,6 +121,24 @@ class TestPlaylists(unittest.TestCase): self.assertIsPlaylist(result) self.assertEqual(result['title'], u'Nightmare Night EP') self.assertTrue(len(result['entries']) >= 4) + + def test_smotri_community(self): + dl = FakeYDL() + ie = SmotriCommunityIE(dl) + result = ie.extract('http://smotri.com/community/video/kommuna') + self.assertIsPlaylist(result) + self.assertEqual(result['id'], u'kommuna') + self.assertEqual(result['title'], u'КПРФ') + self.assertTrue(len(result['entries']) >= 4) + + def test_smotri_user(self): + dl = FakeYDL() + ie = SmotriUserIE(dl) + result = ie.extract('http://smotri.com/user/inspector') + self.assertIsPlaylist(result) + self.assertEqual(result['id'], u'inspector') + self.assertEqual(result['title'], u'Inspector') + self.assertTrue(len(result['entries']) >= 9) if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index cc93e619c..bd996483b 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -121,6 +121,11 @@ from .rutube import RutubeIE from .sina import SinaIE from .slashdot import SlashdotIE from .slideshare import SlideshareIE +from .smotri import ( + SmotriIE, + SmotriCommunityIE, + SmotriUserIE, +) from .sohu import SohuIE from .soundcloud import SoundcloudIE, SoundcloudSetIE, SoundcloudUserIE from .southparkstudios import ( diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py new file mode 100644 index 000000000..ea42d5320 --- /dev/null +++ b/youtube_dl/extractor/smotri.py @@ -0,0 +1,239 @@ +# encoding: utf-8 + +import re +import json +import hashlib + +from .common import InfoExtractor +from ..utils import ( + determine_ext, + ExtractorError +) + + +class SmotriIE(InfoExtractor): + IE_DESC = u'Smotri.com' + IE_NAME = u'smotri' + _VALID_URL = r'^(?:http://)?(?:www\.)?(?P<url>smotri\.com/video/view/\?id=(?P<videoid>v(?P<realvideoid>[0-9]+)[a-z0-9]{4}))' + + _TESTS = [ + # real video id 2610366 + { + u'url': u'http://smotri.com/video/view/?id=v261036632ab', + u'file': u'v261036632ab.mp4', + u'md5': u'46a72e83a6ad8862b64fa6953fa93f8a', + u'info_dict': { + u'title': u'катастрофа с камер видеонаблюдения', + u'uploader': u'rbc2008', + u'uploader_id': u'rbc08', + u'upload_date': u'20131118', + u'thumbnail': u'http://frame6.loadup.ru/8b/a9/2610366.3.3.jpg' + }, + }, + # real video id 57591 + { + u'url': u'http://smotri.com/video/view/?id=v57591cb20', + u'file': u'v57591cb20.flv', + u'md5': u'9eae59f6dda7087bf39a140e2fff5757', + u'info_dict': { + u'title': u'test', + u'uploader': u'Support Photofile@photofile', + u'uploader_id': u'support-photofile', + u'upload_date': u'20070704', + u'thumbnail': u'http://frame4.loadup.ru/03/ed/57591.2.3.jpg' + }, + }, + # video-password + { + u'url': u'http://smotri.com/video/view/?id=v1390466a13c', + u'file': u'v1390466a13c.mp4', + u'md5': u'fe4dd9357558d5ee3c8fc0ef0d39de66', + u'info_dict': { + u'title': u'TOCCA_A_NOI_-_LE_COSE_NON_VANNO_CAMBIAMOLE_ORA-1', + u'uploader': u'timoxa40', + u'uploader_id': u'timoxa40', + u'upload_date': u'20100404', + u'thumbnail': u'http://frame7.loadup.ru/af/3f/1390466.3.3.jpg' + }, + u'params': { + u'videopassword': u'qwerty', + }, + }, + # age limit + video-password + { + u'url': u'http://smotri.com/video/view/?id=v15408898bcf', + u'file': u'v15408898bcf.flv', + u'md5': u'c66a5d61379ac6fde06f07eebe436316', + u'info_dict': { + u'title': u'этот ролик не покажут по ТВ', + u'uploader': u'zzxxx', + u'uploader_id': u'ueggb', + u'upload_date': u'20101001', + u'thumbnail': u'http://frame3.loadup.ru/75/75/1540889.1.3.jpg', + u'age_limit': 18 + }, + u'params': { + u'videopassword': u'333' + } + } + ] + + _SUCCESS = 0 + _PASSWORD_NOT_VERIFIED = 1 + _PASSWORD_DETECTED = 2 + _VIDEO_NOT_FOUND = 3 + + def _search_meta(self, name, html, display_name=None): + if display_name is None: + display_name = name + return self._html_search_regex( + r'<meta itemprop="%s" content="([^"]+)" />' % re.escape(name), + html, display_name, fatal=False) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('videoid') + real_video_id = mobj.group('realvideoid') + + # Download video JSON data + video_json_url = 'http://smotri.com/vt.php?id=%s' % real_video_id + video_json_page = self._download_webpage(video_json_url, video_id, u'Downloading video JSON') + video_json = json.loads(video_json_page) + + status = video_json['status'] + if status == self._VIDEO_NOT_FOUND: + raise ExtractorError(u'Video %s does not exist' % video_id, expected=True) + elif status == self._PASSWORD_DETECTED: # The video is protected by a password, retry with + # video-password set + video_password = self._downloader.params.get('videopassword', None) + if not video_password: + raise ExtractorError(u'This video is protected by a password, use the --video-password option', expected=True) + video_json_url += '&md5pass=%s' % hashlib.md5(video_password).hexdigest() + video_json_page = self._download_webpage(video_json_url, video_id, u'Downloading video JSON (video-password set)') + video_json = json.loads(video_json_page) + status = video_json['status'] + if status == self._PASSWORD_NOT_VERIFIED: + raise ExtractorError(u'Video password is invalid', expected=True) + + if status != self._SUCCESS: + raise ExtractorError(u'Unexpected status value %s' % status) + + # Extract the URL of the video + video_url = video_json['file_data'] + video_ext = determine_ext(video_url) + + # Video JSON does not provide enough meta data + # We will extract some from the video web page instead + video_page_url = 'http://' + mobj.group('url') + video_page = self._download_webpage(video_page_url, video_id, u'Downloading video page') + + # Adult content + if re.search(u'EroConfirmText">', video_page) is not None: + self.report_age_confirmation() + confirm_string = self._html_search_regex( + ur'<a href="/video/view/\?id=%s&confirm=([^"]+)" title="[^"]+">' % video_id, + video_page, u'confirm string') + confirm_url = video_page_url + '&confirm=%s' % confirm_string + video_page = self._download_webpage(confirm_url, video_id, u'Downloading video page (age confirmed)') + adult_content = True + else: + adult_content = False + + # Extract the rest of meta data + video_title = self._search_meta(u'name', video_page, u'title') + if not video_title: + video_title = video_url.rsplit('/', 1)[-1] + + video_description = self._search_meta(u'description', video_page) + video_thumbnail = self._search_meta(u'thumbnail', video_page) + + upload_date_str = self._search_meta(u'uploadDate', video_page, u'upload date') + upload_date_m = re.search(r'(?P<year>\d{4})\.(?P<month>\d{2})\.(?P<day>\d{2})T', upload_date_str) + video_upload_date = ( + ( + upload_date_m.group('year') + + upload_date_m.group('month') + + upload_date_m.group('day') + ) + if upload_date_m else None + ) + + duration_str = self._search_meta(u'duration', video_page) + duration_m = re.search(r'T(?P<hours>[0-9]{2})H(?P<minutes>[0-9]{2})M(?P<seconds>[0-9]{2})S', duration_str) + video_duration = ( + ( + (int(duration_m.group('hours')) * 60 * 60) + + (int(duration_m.group('minutes')) * 60) + + int(duration_m.group('seconds')) + ) + if duration_m else None + ) + + video_uploader = self._html_search_regex( + ur'<div class="DescrUser"><div>Автор.*?onmouseover="popup_user_info[^"]+">(.*?)</a>', + video_page, u'uploader', fatal=False, flags=re.MULTILINE|re.DOTALL) + + video_uploader_id = self._html_search_regex( + ur'<div class="DescrUser"><div>Автор.*?onmouseover="popup_user_info\(.*?\'([^\']+)\'\);">', + video_page, u'uploader id', fatal=False, flags=re.MULTILINE|re.DOTALL) + + video_view_count = self._html_search_regex( + ur'Общее количество просмотров.*?<span class="Number">(\d+)</span>', + video_page, u'view count', fatal=False, flags=re.MULTILINE|re.DOTALL) + + return { + 'id': video_id, + 'url': video_url, + 'title': video_title, + 'ext': video_ext, + 'thumbnail': video_thumbnail, + 'description': video_description, + 'uploader': video_uploader, + 'upload_date': video_upload_date, + 'uploader_id': video_uploader_id, + 'video_duration': video_duration, + 'view_count': video_view_count, + 'age_limit': 18 if adult_content else 0, + 'video_page_url': video_page_url + } + +class SmotriCommunityIE(InfoExtractor): + IE_DESC = u'Smotri.com community videos' + IE_NAME = u'smotri:community' + _VALID_URL = r'^(?:http://)?(?:www\.)?smotri\.com/community/video/(?P<communityid>[0-9A-Za-z_\'-]+)' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + community_id = mobj.group('communityid') + + url = 'http://smotri.com/export/rss/video/by/community/-/%s/video.xml' % community_id + rss = self._download_xml(url, community_id, u'Downloading community RSS') + + entries = [self.url_result(video_url.text, 'Smotri') + for video_url in rss.findall('./channel/item/link')] + + community_title = self._html_search_regex( + ur'^Видео сообщества "([^"]+)"$', rss.find('./channel/description').text, u'community title') + + return self.playlist_result(entries, community_id, community_title) + +class SmotriUserIE(InfoExtractor): + IE_DESC = u'Smotri.com user videos' + IE_NAME = u'smotri:user' + _VALID_URL = r'^(?:http://)?(?:www\.)?smotri\.com/user/(?P<userid>[0-9A-Za-z_\'-]+)' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url); + user_id = mobj.group('userid') + + url = 'http://smotri.com/export/rss/user/video/-/%s/video.xml' % user_id + rss = self._download_xml(url, user_id, u'Downloading user RSS') + + entries = [self.url_result(video_url.text, 'Smotri') + for video_url in rss.findall('./channel/item/link')] + + user_nickname = self._html_search_regex( + ur'^Видео режиссера (.*)$', rss.find('./channel/description').text, u'user nickname') + + return self.playlist_result(entries, user_id, user_nickname) + \ No newline at end of file From aaebed13a8447961e23cca9c75b097732c246476 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 2 Dec 2013 17:08:17 +0100 Subject: [PATCH 198/425] [smotri] Simplify --- youtube_dl/extractor/common.py | 3 +- youtube_dl/extractor/smotri.py | 91 +++++++++++++++++++--------------- 2 files changed, 54 insertions(+), 40 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 4f1b50880..1b049082d 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -364,7 +364,8 @@ class InfoExtractor(object): if display_name is None: display_name = name return self._html_search_regex( - r'''(?ix)<meta(?=[^>]+(?:name|property)=["\']%s["\']) + r'''(?ix)<meta + (?=[^>]+(?:itemprop|name|property)=["\']%s["\']) [^>]+content=["\']([^"\']+)["\']''' % re.escape(name), html, display_name, fatal=False) diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py index ea42d5320..f035a3214 100644 --- a/youtube_dl/extractor/smotri.py +++ b/youtube_dl/extractor/smotri.py @@ -14,46 +14,49 @@ from ..utils import ( class SmotriIE(InfoExtractor): IE_DESC = u'Smotri.com' IE_NAME = u'smotri' - _VALID_URL = r'^(?:http://)?(?:www\.)?(?P<url>smotri\.com/video/view/\?id=(?P<videoid>v(?P<realvideoid>[0-9]+)[a-z0-9]{4}))' - + _VALID_URL = r'^https?://(?:www\.)?(?P<url>smotri\.com/video/view/\?id=(?P<videoid>v(?P<realvideoid>[0-9]+)[a-z0-9]{4}))' + _TESTS = [ # real video id 2610366 { u'url': u'http://smotri.com/video/view/?id=v261036632ab', u'file': u'v261036632ab.mp4', - u'md5': u'46a72e83a6ad8862b64fa6953fa93f8a', + u'md5': u'2a7b08249e6f5636557579c368040eb9', u'info_dict': { u'title': u'катастрофа с камер видеонаблюдения', u'uploader': u'rbc2008', u'uploader_id': u'rbc08', u'upload_date': u'20131118', - u'thumbnail': u'http://frame6.loadup.ru/8b/a9/2610366.3.3.jpg' + u'description': u'катастрофа с камер видеонаблюдения, видео катастрофа с камер видеонаблюдения', + u'thumbnail': u'http://frame6.loadup.ru/8b/a9/2610366.3.3.jpg', }, }, # real video id 57591 { u'url': u'http://smotri.com/video/view/?id=v57591cb20', u'file': u'v57591cb20.flv', - u'md5': u'9eae59f6dda7087bf39a140e2fff5757', + u'md5': u'830266dfc21f077eac5afd1883091bcd', u'info_dict': { u'title': u'test', u'uploader': u'Support Photofile@photofile', u'uploader_id': u'support-photofile', u'upload_date': u'20070704', - u'thumbnail': u'http://frame4.loadup.ru/03/ed/57591.2.3.jpg' - }, + u'description': u'test, видео test', + u'thumbnail': u'http://frame4.loadup.ru/03/ed/57591.2.3.jpg', + }, }, # video-password { u'url': u'http://smotri.com/video/view/?id=v1390466a13c', u'file': u'v1390466a13c.mp4', - u'md5': u'fe4dd9357558d5ee3c8fc0ef0d39de66', + u'md5': u'f6331cef33cad65a0815ee482a54440b', u'info_dict': { u'title': u'TOCCA_A_NOI_-_LE_COSE_NON_VANNO_CAMBIAMOLE_ORA-1', u'uploader': u'timoxa40', u'uploader_id': u'timoxa40', u'upload_date': u'20100404', - u'thumbnail': u'http://frame7.loadup.ru/af/3f/1390466.3.3.jpg' + u'thumbnail': u'http://frame7.loadup.ru/af/3f/1390466.3.3.jpg', + u'description': u'TOCCA_A_NOI_-_LE_COSE_NON_VANNO_CAMBIAMOLE_ORA-1, видео TOCCA_A_NOI_-_LE_COSE_NON_VANNO_CAMBIAMOLE_ORA-1', }, u'params': { u'videopassword': u'qwerty', @@ -63,15 +66,16 @@ class SmotriIE(InfoExtractor): { u'url': u'http://smotri.com/video/view/?id=v15408898bcf', u'file': u'v15408898bcf.flv', - u'md5': u'c66a5d61379ac6fde06f07eebe436316', + u'md5': u'91e909c9f0521adf5ee86fbe073aad70', u'info_dict': { u'title': u'этот ролик не покажут по ТВ', u'uploader': u'zzxxx', u'uploader_id': u'ueggb', u'upload_date': u'20101001', u'thumbnail': u'http://frame3.loadup.ru/75/75/1540889.1.3.jpg', - u'age_limit': 18 - }, + u'age_limit': 18, + u'description': u'этот ролик не покажут по ТВ, видео этот ролик не покажут по ТВ', + }, u'params': { u'videopassword': u'333' } @@ -82,14 +86,15 @@ class SmotriIE(InfoExtractor): _PASSWORD_NOT_VERIFIED = 1 _PASSWORD_DETECTED = 2 _VIDEO_NOT_FOUND = 3 - + def _search_meta(self, name, html, display_name=None): if display_name is None: display_name = name return self._html_search_regex( r'<meta itemprop="%s" content="([^"]+)" />' % re.escape(name), html, display_name, fatal=False) - + return self._html_search_meta(name, html, display_name) + def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('videoid') @@ -103,12 +108,12 @@ class SmotriIE(InfoExtractor): status = video_json['status'] if status == self._VIDEO_NOT_FOUND: raise ExtractorError(u'Video %s does not exist' % video_id, expected=True) - elif status == self._PASSWORD_DETECTED: # The video is protected by a password, retry with + elif status == self._PASSWORD_DETECTED: # The video is protected by a password, retry with # video-password set video_password = self._downloader.params.get('videopassword', None) if not video_password: raise ExtractorError(u'This video is protected by a password, use the --video-password option', expected=True) - video_json_url += '&md5pass=%s' % hashlib.md5(video_password).hexdigest() + video_json_url += '&md5pass=%s' % hashlib.md5(video_password.encode('utf-8')).hexdigest() video_json_page = self._download_webpage(video_json_url, video_id, u'Downloading video JSON (video-password set)') video_json = json.loads(video_json_page) status = video_json['status'] @@ -120,7 +125,6 @@ class SmotriIE(InfoExtractor): # Extract the URL of the video video_url = video_json['file_data'] - video_ext = determine_ext(video_url) # Video JSON does not provide enough meta data # We will extract some from the video web page instead @@ -131,7 +135,7 @@ class SmotriIE(InfoExtractor): if re.search(u'EroConfirmText">', video_page) is not None: self.report_age_confirmation() confirm_string = self._html_search_regex( - ur'<a href="/video/view/\?id=%s&confirm=([^"]+)" title="[^"]+">' % video_id, + r'<a href="/video/view/\?id=%s&confirm=([^"]+)" title="[^"]+">' % video_id, video_page, u'confirm string') confirm_url = video_page_url + '&confirm=%s' % confirm_string video_page = self._download_webpage(confirm_url, video_id, u'Downloading video page (age confirmed)') @@ -143,11 +147,17 @@ class SmotriIE(InfoExtractor): video_title = self._search_meta(u'name', video_page, u'title') if not video_title: video_title = video_url.rsplit('/', 1)[-1] - + video_description = self._search_meta(u'description', video_page) + END_TEXT = u' на сайте Smotri.com' + if video_description.endswith(END_TEXT): + video_description = video_description[:-len(END_TEXT)] + START_TEXT = u'Смотреть онлайн ролик ' + if video_description.startswith(START_TEXT): + video_description = video_description[len(START_TEXT):] video_thumbnail = self._search_meta(u'thumbnail', video_page) - - upload_date_str = self._search_meta(u'uploadDate', video_page, u'upload date') + + upload_date_str = self._search_meta(u'uploadDate', video_page, u'upload date') upload_date_m = re.search(r'(?P<year>\d{4})\.(?P<month>\d{2})\.(?P<day>\d{2})T', upload_date_str) video_upload_date = ( ( @@ -170,22 +180,21 @@ class SmotriIE(InfoExtractor): ) video_uploader = self._html_search_regex( - ur'<div class="DescrUser"><div>Автор.*?onmouseover="popup_user_info[^"]+">(.*?)</a>', + u'<div class="DescrUser"><div>Автор.*?onmouseover="popup_user_info[^"]+">(.*?)</a>', video_page, u'uploader', fatal=False, flags=re.MULTILINE|re.DOTALL) video_uploader_id = self._html_search_regex( - ur'<div class="DescrUser"><div>Автор.*?onmouseover="popup_user_info\(.*?\'([^\']+)\'\);">', + u'<div class="DescrUser"><div>Автор.*?onmouseover="popup_user_info\\(.*?\'([^\']+)\'\\);">', video_page, u'uploader id', fatal=False, flags=re.MULTILINE|re.DOTALL) video_view_count = self._html_search_regex( - ur'Общее количество просмотров.*?<span class="Number">(\d+)</span>', + u'Общее количество просмотров.*?<span class="Number">(\\d+)</span>', video_page, u'view count', fatal=False, flags=re.MULTILINE|re.DOTALL) return { 'id': video_id, 'url': video_url, 'title': video_title, - 'ext': video_ext, 'thumbnail': video_thumbnail, 'description': video_description, 'uploader': video_uploader, @@ -197,43 +206,47 @@ class SmotriIE(InfoExtractor): 'video_page_url': video_page_url } + class SmotriCommunityIE(InfoExtractor): IE_DESC = u'Smotri.com community videos' IE_NAME = u'smotri:community' - _VALID_URL = r'^(?:http://)?(?:www\.)?smotri\.com/community/video/(?P<communityid>[0-9A-Za-z_\'-]+)' + _VALID_URL = r'^https?://(?:www\.)?smotri\.com/community/video/(?P<communityid>[0-9A-Za-z_\'-]+)' def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) community_id = mobj.group('communityid') - + url = 'http://smotri.com/export/rss/video/by/community/-/%s/video.xml' % community_id rss = self._download_xml(url, community_id, u'Downloading community RSS') - + entries = [self.url_result(video_url.text, 'Smotri') for video_url in rss.findall('./channel/item/link')] - + + description_text = rss.find('./channel/description').text community_title = self._html_search_regex( - ur'^Видео сообщества "([^"]+)"$', rss.find('./channel/description').text, u'community title') + u'^Видео сообщества "([^"]+)"$', description_text, u'community title') return self.playlist_result(entries, community_id, community_title) - + + class SmotriUserIE(InfoExtractor): IE_DESC = u'Smotri.com user videos' IE_NAME = u'smotri:user' - _VALID_URL = r'^(?:http://)?(?:www\.)?smotri\.com/user/(?P<userid>[0-9A-Za-z_\'-]+)' - + _VALID_URL = r'^https?://(?:www\.)?smotri\.com/user/(?P<userid>[0-9A-Za-z_\'-]+)' + def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url); + mobj = re.match(self._VALID_URL, url) user_id = mobj.group('userid') - + url = 'http://smotri.com/export/rss/user/video/-/%s/video.xml' % user_id rss = self._download_xml(url, user_id, u'Downloading user RSS') - + entries = [self.url_result(video_url.text, 'Smotri') for video_url in rss.findall('./channel/item/link')] - + + description_text = rss.find('./channel/description').text user_nickname = self._html_search_regex( - ur'^Видео режиссера (.*)$', rss.find('./channel/description').text, u'user nickname') + u'^Видео режиссера (.*)$', description_text, + u'user nickname') return self.playlist_result(entries, user_id, user_nickname) - \ No newline at end of file From 87968574293ef87b98f51cf0d7c0958b9f496a7a Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 2 Dec 2013 17:43:22 +0100 Subject: [PATCH 199/425] Credit @dstftw for smotri IE --- youtube_dl/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 799eca566..2eeef2ae9 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -36,6 +36,7 @@ __authors__ = ( 'Marcin Cieślak', 'Anton Larionov', 'Takuya Tsuchida', + 'Sergey M.', ) __license__ = 'Public Domain' From 36a826a50dc5e53af8355f1233cc4f3ceba2e61b Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 3 Dec 2013 11:54:52 +0100 Subject: [PATCH 200/425] Clarify --download-archive help (#1757) --- youtube_dl/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 2eeef2ae9..48137ebe5 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -224,7 +224,7 @@ def parseOpts(overrideArguments=None): default=None, type=int) selection.add_option('--download-archive', metavar='FILE', dest='download_archive', - help='Download only videos not present in the archive file. Record the IDs of all downloaded videos in it.') + help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.') authentication.add_option('-u', '--username', From 1b753cb3344837fb69e9bfde89d03161d33ba3ff Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 3 Dec 2013 13:04:02 +0100 Subject: [PATCH 201/425] Add Windows configuration file locations (#1881) --- README.md | 2 +- youtube_dl/__init__.py | 29 ++++++++++++++++++++++------- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 031e436b6..0ff6ff8b9 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,7 @@ which means you can modify it, redistribute it or use it however you like. # CONFIGURATION -You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl.conf`. +You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl.conf`. On Windows, the configuration file locations are `%APPDATA%\youtube-dl\config` and `C:\Users\<Yourname>\youtube-dl.conf`. # OUTPUT TEMPLATE diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 48137ebe5..32490b24e 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -81,15 +81,13 @@ from .PostProcessor import ( def parseOpts(overrideArguments=None): - def _readOptions(filename_bytes): + def _readOptions(filename_bytes, def=[]): try: optionf = open(filename_bytes) except IOError: - return [] # silently skip if file is not present + return def # silently skip if file is not present try: - res = [] - for l in optionf: - res += shlex.split(l, comments=True) + res = [shlex.split(l, comments=True) for l in optionf] finally: optionf.close() return res @@ -419,6 +417,8 @@ def parseOpts(overrideArguments=None): if opts.verbose: write_string(u'[debug] Override config: ' + repr(overrideArguments) + '\n') else: + systemConf = _readOptions('/etc/youtube-dl.conf') + xdg_config_home = os.environ.get('XDG_CONFIG_HOME') if xdg_config_home: userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config') @@ -428,8 +428,23 @@ def parseOpts(overrideArguments=None): userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config') if not os.path.isfile(userConfFile): userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf') - systemConf = _readOptions('/etc/youtube-dl.conf') - userConf = _readOptions(userConfFile) + userConf = _readOptions(userConfFile, None) + + if userConf is None: + appdata_dir = os.environ.get('appdata') + if appdata_dir: + userConf = _readOptions( + os.path.join(appdata_dir, 'youtube-dl', 'config'), + def=None) + + if userConf is None: + userConfFile = _readOptions( + os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'), + def=None) + + if userConf is None: + userConf = [] + commandLineConf = sys.argv[1:] argv = systemConf + userConf + commandLineConf opts, args = parser.parse_args(argv) From fb27c2295e0e9d6f2f6ac45ed5906987b4710d0a Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 3 Dec 2013 13:09:48 +0100 Subject: [PATCH 202/425] Correct configuration file locations --- README.md | 2 +- youtube_dl/__init__.py | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0ff6ff8b9..85af7cf7e 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,7 @@ which means you can modify it, redistribute it or use it however you like. # CONFIGURATION -You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl.conf`. On Windows, the configuration file locations are `%APPDATA%\youtube-dl\config` and `C:\Users\<Yourname>\youtube-dl.conf`. +You can configure youtube-dl by placing default arguments (such as `--extract-audio --no-mtime` to always extract the audio and not copy the mtime) into `/etc/youtube-dl.conf` and/or `~/.config/youtube-dl.conf`. On Windows, the configuration file locations are `%APPDATA%\youtube-dl\config.txt` and `C:\Users\<Yourname>\youtube-dl.conf`. # OUTPUT TEMPLATE diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 32490b24e..9c8a694f0 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -81,11 +81,11 @@ from .PostProcessor import ( def parseOpts(overrideArguments=None): - def _readOptions(filename_bytes, def=[]): + def _readOptions(filename_bytes, default=[]): try: optionf = open(filename_bytes) except IOError: - return def # silently skip if file is not present + return default # silently skip if file is not present try: res = [shlex.split(l, comments=True) for l in optionf] finally: @@ -435,12 +435,20 @@ def parseOpts(overrideArguments=None): if appdata_dir: userConf = _readOptions( os.path.join(appdata_dir, 'youtube-dl', 'config'), - def=None) + default=None) + if userConf is None: + userConf = _readOptions( + os.path.join(appdata_dir, 'youtube-dl', 'config.txt'), + default=None) if userConf is None: - userConfFile = _readOptions( + userConf = _readOptions( os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'), - def=None) + default=None) + if userConf is None: + userConf = _readOptions( + os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'), + default=None) if userConf is None: userConf = [] From a0eaa341e1ce6254179c1a00a11704da1887e124 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 3 Dec 2013 13:11:20 +0100 Subject: [PATCH 203/425] [configuration] Undo code breakage --- youtube_dl/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 9c8a694f0..fff295e8c 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -87,7 +87,9 @@ def parseOpts(overrideArguments=None): except IOError: return default # silently skip if file is not present try: - res = [shlex.split(l, comments=True) for l in optionf] + res = [] + for l in optionf: + res += shlex.split(l, comments=True) finally: optionf.close() return res From 731e3dde299844fc3b0f369d5a161fa4df0eb718 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 3 Dec 2013 13:13:09 +0100 Subject: [PATCH 204/425] release 2013.12.03 --- README.md | 2 +- youtube_dl/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 85af7cf7e..00975ab5e 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ which means you can modify it, redistribute it or use it however you like. --dateafter DATE download only videos uploaded after this date --no-playlist download only the currently playing video --age-limit YEARS download only videos suitable for the given age - --download-archive FILE Download only videos not present in the archive + --download-archive FILE Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it. diff --git a/youtube_dl/version.py b/youtube_dl/version.py index d8f341ab9..f9a339c02 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.02' +__version__ = '2013.12.03' From cf6758d2040816033ec47afe9c1d497e4c2abd4d Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 3 Dec 2013 13:33:07 +0100 Subject: [PATCH 205/425] Document disabling proxy (#1882) --- youtube_dl/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index fff295e8c..d2446b670 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -191,7 +191,9 @@ def parseOpts(overrideArguments=None): general.add_option('--extractor-descriptions', action='store_true', dest='list_extractor_descriptions', help='Output descriptions of all supported extractors', default=False) - general.add_option('--proxy', dest='proxy', default=None, help='Use the specified HTTP/HTTPS proxy', metavar='URL') + general.add_option( + '--proxy', dest='proxy', default=None, metavar='URL', + help='Use the specified HTTP/HTTPS proxy. Pass in an empty string (--proxy "") for direct connection') general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.') general.add_option( '--cache-dir', dest='cachedir', default=get_cachedir(), metavar='DIR', From cb7fb54600a96bcced33020b925f2cfc9428bd4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Tue, 3 Dec 2013 13:55:25 +0100 Subject: [PATCH 206/425] Change the ie_name of YoutubeSearchDateIE It produced a duplicate entry when listing the extractors with '--list-extractors' and generates noise in the commit log when generating the supported sites webpage (like in 09f355f73bf1657ecacfd05eda21d2c4bf1cc4a8) --- youtube_dl/extractor/youtube.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 765b4a9bf..66f5af000 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1765,6 +1765,7 @@ class YoutubeSearchIE(SearchInfoExtractor): return self.playlist_result(videos, query) class YoutubeSearchDateIE(YoutubeSearchIE): + IE_NAME = YoutubeSearchIE.IE_NAME + ':date' _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc&orderby=published' _SEARCH_KEY = 'ytsearchdate' IE_DESC = u'YouTube.com searches, newest videos first' From e9d8e302aafdb6fcf72c44d582c1f6d4447cd5fc Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 3 Dec 2013 14:06:08 +0100 Subject: [PATCH 207/425] [xhamster] Change test checksum --- youtube_dl/extractor/xhamster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index 7444d3393..279f75e7a 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -26,7 +26,7 @@ class XHamsterIE(InfoExtractor): { u'url': u'http://xhamster.com/movies/2221348/britney_spears_sexy_booty.html?hd', u'file': u'2221348.flv', - u'md5': u'e767b9475de189320f691f49c679c4c7', + u'md5': u'970a94178ca4118c5aa3aaea21211b81', u'info_dict': { u"upload_date": u"20130914", u"uploader_id": u"jojo747400", From 938384c587c33696bcdb9c28b982e2b744695b3d Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 3 Dec 2013 14:08:16 +0100 Subject: [PATCH 208/425] [redtube] Fix search for title --- youtube_dl/extractor/redtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py index 3bbda128e..c2254ae8a 100644 --- a/youtube_dl/extractor/redtube.py +++ b/youtube_dl/extractor/redtube.py @@ -30,7 +30,7 @@ class RedTubeIE(InfoExtractor): r'<source src="(.+?)" type="video/mp4">', webpage, u'video URL') video_title = self._html_search_regex( - r'<h1 class="videoTitle slidePanelMovable">(.+?)</h1>', + r'<h1 class="videoTitle[^"]*">(.+?)</h1>', webpage, u'title') # No self-labeling, but they describe themselves as From ce93879a9b3b1661db3e65ec43649c5b6a08778c Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 3 Dec 2013 14:16:58 +0100 Subject: [PATCH 209/425] [daum] Fix real video ID extraction --- youtube_dl/extractor/daum.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py index 3d1dcb793..fe7cfb064 100644 --- a/youtube_dl/extractor/daum.py +++ b/youtube_dl/extractor/daum.py @@ -28,7 +28,8 @@ class DaumIE(InfoExtractor): video_id = mobj.group(1) canonical_url = 'http://tvpot.daum.net/v/%s' % video_id webpage = self._download_webpage(canonical_url, video_id) - full_id = self._search_regex(r'<link rel="video_src" href=".+?vid=(.+?)"', + full_id = self._search_regex( + r'<iframe src="http://videofarm.daum.net/controller/video/viewer/Video.html\?.*?vid=(.+?)[&"]', webpage, u'full id') query = compat_urllib_parse.urlencode({'vid': full_id}) info = self._download_xml( From fb7abb31af93a2a1d84ba17beb0f389dd09eafdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Tue, 3 Dec 2013 14:21:06 +0100 Subject: [PATCH 210/425] Remove the compatibility code used before the new format system was implemented --- youtube_dl/extractor/appletrailers.py | 9 ++------- youtube_dl/extractor/archiveorg.py | 11 ++--------- youtube_dl/extractor/comedycentral.py | 9 ++------- youtube_dl/extractor/daum.py | 5 +---- youtube_dl/extractor/dreisat.py | 7 +------ youtube_dl/extractor/faz.py | 5 +---- youtube_dl/extractor/gamespot.py | 5 +---- youtube_dl/extractor/metacritic.py | 5 +---- youtube_dl/extractor/mtv.py | 7 +------ youtube_dl/extractor/naver.py | 5 +---- youtube_dl/extractor/trilulilu.py | 6 +----- youtube_dl/extractor/viddler.py | 8 +------- 12 files changed, 15 insertions(+), 67 deletions(-) diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py index 6d6237f8a..4befff394 100644 --- a/youtube_dl/extractor/appletrailers.py +++ b/youtube_dl/extractor/appletrailers.py @@ -113,7 +113,7 @@ class AppleTrailersIE(InfoExtractor): }) formats = sorted(formats, key=lambda f: (f['height'], f['width'])) - info = { + playlist.append({ '_type': 'video', 'id': video_id, 'title': title, @@ -124,12 +124,7 @@ class AppleTrailersIE(InfoExtractor): 'upload_date': upload_date, 'uploader_id': uploader_id, 'user_agent': 'QuickTime compatible (youtube-dl)', - } - # TODO: Remove when #980 has been merged - info['url'] = formats[-1]['url'] - info['ext'] = formats[-1]['ext'] - - playlist.append(info) + }) return { '_type': 'playlist', diff --git a/youtube_dl/extractor/archiveorg.py b/youtube_dl/extractor/archiveorg.py index 61ce4469a..3ae0aebb1 100644 --- a/youtube_dl/extractor/archiveorg.py +++ b/youtube_dl/extractor/archiveorg.py @@ -49,7 +49,7 @@ class ArchiveOrgIE(InfoExtractor): for f in formats: f['ext'] = determine_ext(f['url']) - info = { + return { '_type': 'video', 'id': video_id, 'title': title, @@ -57,12 +57,5 @@ class ArchiveOrgIE(InfoExtractor): 'description': description, 'uploader': uploader, 'upload_date': upload_date, + 'thumbnail': data.get('misc', {}).get('image'), } - thumbnail = data.get('misc', {}).get('image') - if thumbnail: - info['thumbnail'] = thumbnail - - # TODO: Remove when #980 has been merged - info.update(formats[-1]) - - return info diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py index 23647f99e..41ef9ad47 100644 --- a/youtube_dl/extractor/comedycentral.py +++ b/youtube_dl/extractor/comedycentral.py @@ -197,7 +197,7 @@ class ComedyCentralShowsIE(InfoExtractor): }) effTitle = showId + u'-' + epTitle + u' part ' + compat_str(partNum+1) - info = { + results.append({ 'id': shortMediaId, 'formats': formats, 'uploader': showId, @@ -205,11 +205,6 @@ class ComedyCentralShowsIE(InfoExtractor): 'title': effTitle, 'thumbnail': None, 'description': compat_str(officialTitle), - } - - # TODO: Remove when #980 has been merged - info.update(info['formats'][-1]) - - results.append(info) + }) return results diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py index fe7cfb064..d418ce4a8 100644 --- a/youtube_dl/extractor/daum.py +++ b/youtube_dl/extractor/daum.py @@ -57,7 +57,7 @@ class DaumIE(InfoExtractor): 'format_id': profile, }) - info = { + return { 'id': video_id, 'title': info.find('TITLE').text, 'formats': formats, @@ -66,6 +66,3 @@ class DaumIE(InfoExtractor): 'duration': int(info.find('DURATION').text), 'upload_date': info.find('REGDTTM').text[:8], } - # TODO: Remove when #980 has been merged - info.update(formats[-1]) - return info diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py index 3cb382e12..24ce79425 100644 --- a/youtube_dl/extractor/dreisat.py +++ b/youtube_dl/extractor/dreisat.py @@ -65,7 +65,7 @@ class DreiSatIE(InfoExtractor): return (qidx, prefer_http, format['video_bitrate']) formats.sort(key=_sortkey) - info = { + return { '_type': 'video', 'id': video_id, 'title': video_title, @@ -76,8 +76,3 @@ class DreiSatIE(InfoExtractor): 'uploader': video_uploader, 'upload_date': upload_date, } - - # TODO: Remove when #980 has been merged - info.update(formats[-1]) - - return info diff --git a/youtube_dl/extractor/faz.py b/youtube_dl/extractor/faz.py index c0169de04..d0dfde694 100644 --- a/youtube_dl/extractor/faz.py +++ b/youtube_dl/extractor/faz.py @@ -44,13 +44,10 @@ class FazIE(InfoExtractor): }) descr = self._html_search_regex(r'<p class="Content Copy">(.*?)</p>', webpage, u'description') - info = { + return { 'id': video_id, 'title': self._og_search_title(webpage), 'formats': formats, 'description': descr, 'thumbnail': config.find('STILL/STILL_BIG').text, } - # TODO: Remove when #980 has been merged - info.update(formats[-1]) - return info diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py index 9645b00c3..26b7d2ae5 100644 --- a/youtube_dl/extractor/gamespot.py +++ b/youtube_dl/extractor/gamespot.py @@ -47,13 +47,10 @@ class GameSpotIE(InfoExtractor): 'format_id': q, }) - info = { + return { 'id': data_video['guid'], 'title': compat_urllib_parse.unquote(data_video['title']), 'formats': formats, 'description': get_meta_content('description', webpage), 'thumbnail': self._og_search_thumbnail(webpage), } - # TODO: Remove when #980 has been merged - info.update(formats[-1]) - return info diff --git a/youtube_dl/extractor/metacritic.py b/youtube_dl/extractor/metacritic.py index 449138b56..6b95b4998 100644 --- a/youtube_dl/extractor/metacritic.py +++ b/youtube_dl/extractor/metacritic.py @@ -43,13 +43,10 @@ class MetacriticIE(InfoExtractor): description = self._html_search_regex(r'<b>Description:</b>(.*?)</p>', webpage, u'description', flags=re.DOTALL) - info = { + return { 'id': video_id, 'title': clip.find('title').text, 'formats': formats, 'description': description, 'duration': int(clip.find('duration').text), } - # TODO: Remove when #980 has been merged - info.update(formats[-1]) - return info diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py index 42aee58be..972336782 100644 --- a/youtube_dl/extractor/mtv.py +++ b/youtube_dl/extractor/mtv.py @@ -93,7 +93,7 @@ class MTVIE(InfoExtractor): else: description = None - info = { + return { 'title': itemdoc.find('title').text, 'formats': self._extract_video_formats(mediagen_page), 'id': video_id, @@ -101,11 +101,6 @@ class MTVIE(InfoExtractor): 'description': description, } - # TODO: Remove when #980 has been merged - info.update(info['formats'][-1]) - - return info - def _get_videos_info(self, uri): video_id = self._id_from_uri(uri) data = compat_urllib_parse.urlencode({'uri': uri}) diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py index d290397c7..c012ec0cf 100644 --- a/youtube_dl/extractor/naver.py +++ b/youtube_dl/extractor/naver.py @@ -56,7 +56,7 @@ class NaverIE(InfoExtractor): 'height': int(format_el.find('height').text), }) - info = { + return { 'id': video_id, 'title': info.find('Subject').text, 'formats': formats, @@ -65,6 +65,3 @@ class NaverIE(InfoExtractor): 'upload_date': info.find('WriteDate').text.replace('.', ''), 'view_count': int(info.find('PlayCount').text), } - # TODO: Remove when #980 has been merged - info.update(formats[-1]) - return info diff --git a/youtube_dl/extractor/trilulilu.py b/youtube_dl/extractor/trilulilu.py index 1c49e580d..d64aaa41f 100644 --- a/youtube_dl/extractor/trilulilu.py +++ b/youtube_dl/extractor/trilulilu.py @@ -55,7 +55,7 @@ class TriluliluIE(InfoExtractor): for fnode in format_doc.findall('./formats/format') ] - info = { + return { '_type': 'video', 'id': video_id, 'formats': formats, @@ -64,7 +64,3 @@ class TriluliluIE(InfoExtractor): 'thumbnail': thumbnail, } - # TODO: Remove when #980 has been merged - info.update(formats[-1]) - - return info diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py index 826804af3..75335dfb8 100644 --- a/youtube_dl/extractor/viddler.py +++ b/youtube_dl/extractor/viddler.py @@ -47,7 +47,7 @@ class ViddlerIE(InfoExtractor): r"thumbnail\s*:\s*'([^']*)'", webpage, u'thumbnail', fatal=False) - info = { + return { '_type': 'video', 'id': video_id, 'title': title, @@ -56,9 +56,3 @@ class ViddlerIE(InfoExtractor): 'duration': duration, 'formats': formats, } - - # TODO: Remove when #980 has been merged - info['formats'][-1]['ext'] = determine_ext(info['formats'][-1]['url']) - info.update(info['formats'][-1]) - - return info From 84db81815af6787d91188ca065cc9ced4d83a4ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Tue, 3 Dec 2013 14:58:24 +0100 Subject: [PATCH 211/425] Move common code for extractors based in MTV services to a new base class Removes the duplication of the thumbnail extraction code (only MTVIE needs to override it) --- youtube_dl/extractor/comedycentral.py | 10 +--- youtube_dl/extractor/gametrailers.py | 16 ++---- youtube_dl/extractor/mtv.py | 69 ++++++++++++++---------- youtube_dl/extractor/southparkstudios.py | 13 +---- 4 files changed, 48 insertions(+), 60 deletions(-) diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py index 41ef9ad47..53579aa27 100644 --- a/youtube_dl/extractor/comedycentral.py +++ b/youtube_dl/extractor/comedycentral.py @@ -1,7 +1,7 @@ import re from .common import InfoExtractor -from .mtv import MTVIE, _media_xml_tag +from .mtv import MTVServicesInfoExtractor from ..utils import ( compat_str, compat_urllib_parse, @@ -11,7 +11,7 @@ from ..utils import ( ) -class ComedyCentralIE(MTVIE): +class ComedyCentralIE(MTVServicesInfoExtractor): _VALID_URL = r'http://www.comedycentral.com/(video-clips|episodes|cc-studios)/(?P<title>.*)' _FEED_URL = u'http://comedycentral.com/feeds/mrss/' @@ -25,12 +25,6 @@ class ComedyCentralIE(MTVIE): u'description': u'After a certain point, breastfeeding becomes c**kblocking.', }, } - # Overwrite MTVIE properties we don't want - _TESTS = [] - - def _get_thumbnail_url(self, uri, itemdoc): - search_path = '%s/%s' % (_media_xml_tag('group'), _media_xml_tag('thumbnail')) - return itemdoc.find(search_path).attrib['url'] def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) diff --git a/youtube_dl/extractor/gametrailers.py b/youtube_dl/extractor/gametrailers.py index 3cc02d97e..3a8bef250 100644 --- a/youtube_dl/extractor/gametrailers.py +++ b/youtube_dl/extractor/gametrailers.py @@ -1,13 +1,11 @@ import re -from .mtv import MTVIE, _media_xml_tag +from .mtv import MTVServicesInfoExtractor -class GametrailersIE(MTVIE): - """ - Gametrailers use the same videos system as MTVIE, it just changes the feed - url, where the uri is and the method to get the thumbnails. - """ + +class GametrailersIE(MTVServicesInfoExtractor): _VALID_URL = r'http://www.gametrailers.com/(?P<type>videos|reviews|full-episodes)/(?P<id>.*?)/(?P<title>.*)' + _TEST = { u'url': u'http://www.gametrailers.com/videos/zbvr8i/mirror-s-edge-2-e3-2013--debut-trailer', u'file': u'70e9a5d7-cf25-4a10-9104-6f3e7342ae0d.mp4', @@ -17,15 +15,9 @@ class GametrailersIE(MTVIE): u'description': u'Faith is back! Check out the World Premiere trailer for Mirror\'s Edge 2 straight from the EA Press Conference at E3 2013!', }, } - # Overwrite MTVIE properties we don't want - _TESTS = [] _FEED_URL = 'http://www.gametrailers.com/feeds/mrss' - def _get_thumbnail_url(self, uri, itemdoc): - search_path = '%s/%s' % (_media_xml_tag('group'), _media_xml_tag('thumbnail')) - return itemdoc.find(search_path).attrib['url'] - def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py index 972336782..6b3feb560 100644 --- a/youtube_dl/extractor/mtv.py +++ b/youtube_dl/extractor/mtv.py @@ -10,35 +10,8 @@ from ..utils import ( def _media_xml_tag(tag): return '{http://search.yahoo.com/mrss/}%s' % tag -class MTVIE(InfoExtractor): - _VALID_URL = r'^https?://(?:www\.)?mtv\.com/videos/.+?/(?P<videoid>[0-9]+)/[^/]+$' - - _FEED_URL = 'http://www.mtv.com/player/embed/AS3/rss/' - - _TESTS = [ - { - u'url': u'http://www.mtv.com/videos/misc/853555/ours-vh1-storytellers.jhtml', - u'file': u'853555.mp4', - u'md5': u'850f3f143316b1e71fa56a4edfd6e0f8', - u'info_dict': { - u'title': u'Taylor Swift - "Ours (VH1 Storytellers)"', - u'description': u'Album: Taylor Swift performs "Ours" for VH1 Storytellers at Harvey Mudd College.', - }, - }, - { - u'add_ie': ['Vevo'], - u'url': u'http://www.mtv.com/videos/taylor-swift/916187/everything-has-changed-ft-ed-sheeran.jhtml', - u'file': u'USCJY1331283.mp4', - u'md5': u'73b4e7fcadd88929292fe52c3ced8caf', - u'info_dict': { - u'title': u'Everything Has Changed', - u'upload_date': u'20130606', - u'uploader': u'Taylor Swift', - }, - u'skip': u'VEVO is only available in some countries', - }, - ] +class MTVServicesInfoExtractor(InfoExtractor): @staticmethod def _id_from_uri(uri): return uri.split(':')[-1] @@ -53,7 +26,12 @@ class MTVIE(InfoExtractor): return base + m.group('finalid') def _get_thumbnail_url(self, uri, itemdoc): - return 'http://mtv.mtvnimages.com/uri/' + uri + search_path = '%s/%s' % (_media_xml_tag('group'), _media_xml_tag('thumbnail')) + thumb_node = itemdoc.find(search_path) + if thumb_node is None: + return None + else: + return thumb_node.attrib['url'] def _extract_video_formats(self, metadataXml): if '/error_country_block.swf' in metadataXml: @@ -108,6 +86,39 @@ class MTVIE(InfoExtractor): u'Downloading info') return [self._get_video_info(item) for item in idoc.findall('.//item')] + +class MTVIE(MTVServicesInfoExtractor): + _VALID_URL = r'^https?://(?:www\.)?mtv\.com/videos/.+?/(?P<videoid>[0-9]+)/[^/]+$' + + _FEED_URL = 'http://www.mtv.com/player/embed/AS3/rss/' + + _TESTS = [ + { + u'url': u'http://www.mtv.com/videos/misc/853555/ours-vh1-storytellers.jhtml', + u'file': u'853555.mp4', + u'md5': u'850f3f143316b1e71fa56a4edfd6e0f8', + u'info_dict': { + u'title': u'Taylor Swift - "Ours (VH1 Storytellers)"', + u'description': u'Album: Taylor Swift performs "Ours" for VH1 Storytellers at Harvey Mudd College.', + }, + }, + { + u'add_ie': ['Vevo'], + u'url': u'http://www.mtv.com/videos/taylor-swift/916187/everything-has-changed-ft-ed-sheeran.jhtml', + u'file': u'USCJY1331283.mp4', + u'md5': u'73b4e7fcadd88929292fe52c3ced8caf', + u'info_dict': { + u'title': u'Everything Has Changed', + u'upload_date': u'20130606', + u'uploader': u'Taylor Swift', + }, + u'skip': u'VEVO is only available in some countries', + }, + ] + + def _get_thumbnail_url(self, uri, itemdoc): + return 'http://mtv.mtvnimages.com/uri/' + uri + def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('videoid') diff --git a/youtube_dl/extractor/southparkstudios.py b/youtube_dl/extractor/southparkstudios.py index a711531e6..fd90cc5dd 100644 --- a/youtube_dl/extractor/southparkstudios.py +++ b/youtube_dl/extractor/southparkstudios.py @@ -1,15 +1,14 @@ import re -from .mtv import MTVIE, _media_xml_tag +from .mtv import MTVServicesInfoExtractor -class SouthParkStudiosIE(MTVIE): +class SouthParkStudiosIE(MTVServicesInfoExtractor): IE_NAME = u'southparkstudios.com' _VALID_URL = r'(https?://)?(www\.)?(?P<url>southparkstudios\.com/(clips|full-episodes)/(?P<id>.+?)(\?|#|$))' _FEED_URL = 'http://www.southparkstudios.com/feeds/video-player/mrss' - # Overwrite MTVIE properties we don't want _TESTS = [{ u'url': u'http://www.southparkstudios.com/clips/104437/bat-daded#tab=featured', u'file': u'a7bff6c2-ed00-11e0-aca6-0026b9414f30.mp4', @@ -19,14 +18,6 @@ class SouthParkStudiosIE(MTVIE): }, }] - def _get_thumbnail_url(self, uri, itemdoc): - search_path = '%s/%s' % (_media_xml_tag('group'), _media_xml_tag('thumbnail')) - thumb_node = itemdoc.find(search_path) - if thumb_node is None: - return None - else: - return thumb_node.attrib['url'] - def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) url = u'http://www.' + mobj.group(u'url') From 1dcc4c0cad886457c0fa5f874c38f95f0510ea4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Fri, 22 Nov 2013 14:57:53 +0100 Subject: [PATCH 212/425] Add --load-info option (#972) It just calls the 'YoutubeDL.process_ie_result' with the dictionary from the json file --- youtube_dl/YoutubeDL.py | 6 ++++++ youtube_dl/__init__.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index b68b110a4..80c056dc8 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -812,6 +812,12 @@ class YoutubeDL(object): return self._download_retcode + def download_with_info_file(self, info_filename): + with open(info_filename, 'r') as f: + # TODO: Check for errors + info = json.load(f) + self.process_ie_result(info, download=True) + def post_process(self, filename, ie_info): """Run all the postprocessors on the given file.""" info = dict(ie_info) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index d2446b670..b0d9a6763 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -360,6 +360,9 @@ def parseOpts(overrideArguments=None): help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False) filesystem.add_option('-a', '--batch-file', dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)') + filesystem.add_option('--load-info', + dest='load_info_filename', metavar='FILE', + help='json file containing the video information (created with the "--write-json" option') filesystem.add_option('-w', '--no-overwrites', action='store_true', dest='nooverwrites', help='do not overwrite files', default=False) filesystem.add_option('-c', '--continue', @@ -706,14 +709,17 @@ def _real_main(argv=None): update_self(ydl.to_screen, opts.verbose) # Maybe do nothing - if len(all_urls) < 1: + if (len(all_urls) < 1) and (opts.load_info_filename is None): if not opts.update_self: parser.error(u'you must provide at least one URL') else: sys.exit() try: - retcode = ydl.download(all_urls) + if opts.load_info_filename is not None: + retcode = ydl.download_with_info_file(opts.load_info_filename) + else: + retcode = ydl.download(all_urls) except MaxDownloadsReached: ydl.to_screen(u'--max-download limit reached, aborting.') retcode = 101 From d494389821de832874dc78abc2fe16365b5fe815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Tue, 3 Dec 2013 20:16:52 +0100 Subject: [PATCH 213/425] Option '--load-info': if the download fails, try extracting the info with the 'webpage_url' field of the info dict The video url may have expired. --- youtube_dl/YoutubeDL.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 80c056dc8..77339dddf 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -816,7 +816,16 @@ class YoutubeDL(object): with open(info_filename, 'r') as f: # TODO: Check for errors info = json.load(f) - self.process_ie_result(info, download=True) + try: + self.process_ie_result(info, download=True) + except DownloadError: + webpage_url = info.get('webpage_url') + if webpage_url is not None: + self.report_warning(u'The info failed to download, trying with "%s"' % webpage_url) + return self.download([webpage_url]) + else: + raise + return self._download_retcode def post_process(self, filename, ie_info): """Run all the postprocessors on the given file.""" From 55f6597c67dd04729dbc1b83d81bfbd63d7e9c0a Mon Sep 17 00:00:00 2001 From: dst <dstftw@gmail.com> Date: Wed, 4 Dec 2013 08:41:09 +0700 Subject: [PATCH 214/425] [smotri] Add an extractor for live rtmp broadcasts --- youtube_dl/FileDownloader.py | 49 ++++++++++++-- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/smotri.py | 106 ++++++++++++++++++++++++++++++- 3 files changed, 148 insertions(+), 8 deletions(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 3ff9716b3..de1dc66bb 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -226,6 +226,22 @@ class FileDownloader(object): (clear_line, percent_str, data_len_str, speed_str, eta_str), skip_eol=True) self.to_console_title(u'youtube-dl - %s of %s at %s ETA %s' % (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip())) + + def report_progress_live_stream(self, downloaded_data_len, speed, elapsed): + if self.params.get('noprogress', False): + return + clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'') + downloaded_str = format_bytes(downloaded_data_len) + speed_str = self.format_speed(speed) + elapsed_str = FileDownloader.format_seconds(elapsed) + if self.params.get('progress_with_newline', False): + self.to_screen(u'[download] %s at %s' % + (downloaded_str, speed_str)) + else: + self.to_screen(u'\r%s[download] %s at %s ET %s' % + (clear_line, downloaded_str, speed_str, elapsed_str), skip_eol=True) + self.to_console_title(u'youtube-dl - %s at %s ET %s' % + (downloaded_str.strip(), speed_str.strip(), elapsed_str.strip())) def report_resuming_byte(self, resume_len): """Report attempt to resume at given byte.""" @@ -255,7 +271,7 @@ class FileDownloader(object): self.to_screen(u'\r%s[download] 100%% of %s in %s' % (clear_line, data_len_str, self.format_seconds(tot_time))) - def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url, live): + def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url, live, conn): def run_rtmpdump(args): start = time.time() resume_percent = None @@ -301,11 +317,27 @@ class FileDownloader(object): 'eta': eta, 'speed': speed, }) - elif self.params.get('verbose', False): - if not cursor_in_new_line: - self.to_screen(u'') - cursor_in_new_line = True - self.to_screen(u'[rtmpdump] '+line) + else: + # no percent for live streams + mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec', line) + if mobj: + downloaded_data_len = int(float(mobj.group(1))*1024) + time_now = time.time() + speed = self.calc_speed(start, time_now, downloaded_data_len) + self.report_progress_live_stream(downloaded_data_len, speed, time_now - start) + cursor_in_new_line = False + self._hook_progress({ + 'downloaded_bytes': downloaded_data_len, + 'tmpfilename': tmpfilename, + 'filename': filename, + 'status': 'downloading', + 'speed': speed, + }) + elif self.params.get('verbose', False): + if not cursor_in_new_line: + self.to_screen(u'') + cursor_in_new_line = True + self.to_screen(u'[rtmpdump] '+line) proc.wait() if not cursor_in_new_line: self.to_screen(u'') @@ -338,6 +370,8 @@ class FileDownloader(object): basic_args += ['--stop', '1'] if live: basic_args += ['--live'] + if conn: + basic_args += ['--conn', conn] args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)] if sys.platform == 'win32' and sys.version_info < (3, 0): @@ -479,7 +513,8 @@ class FileDownloader(object): info_dict.get('page_url', None), info_dict.get('play_path', None), info_dict.get('tc_url', None), - info_dict.get('rtmp_live', False)) + info_dict.get('rtmp_live', False), + info_dict.get('rtmp_conn', None)) # Attempt to download using mplayer if url.startswith('mms') or url.startswith('rtsp'): diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index bd996483b..60e2d6ebd 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -125,6 +125,7 @@ from .smotri import ( SmotriIE, SmotriCommunityIE, SmotriUserIE, + SmotriBroadcastIE, ) from .sohu import SohuIE from .soundcloud import SoundcloudIE, SoundcloudSetIE, SoundcloudUserIE diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py index f035a3214..f86ee8388 100644 --- a/youtube_dl/extractor/smotri.py +++ b/youtube_dl/extractor/smotri.py @@ -3,10 +3,12 @@ import re import json import hashlib +import uuid from .common import InfoExtractor from ..utils import ( - determine_ext, + compat_urllib_parse, + compat_urllib_request, ExtractorError ) @@ -250,3 +252,105 @@ class SmotriUserIE(InfoExtractor): u'user nickname') return self.playlist_result(entries, user_id, user_nickname) + + +class SmotriBroadcastIE(InfoExtractor): + IE_DESC = u'Smotri.com broadcasts' + IE_NAME = u'smotri:broadcast' + _VALID_URL = r'^https?://(?:www\.)?(?P<url>smotri\.com/live/(?P<broadcastid>[^/]+))/?.*' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + broadcast_id = mobj.group('broadcastid') + + broadcast_url = 'http://' + mobj.group('url') + broadcast_page = self._download_webpage(broadcast_url, broadcast_id, u'Downloading broadcast page') + + if re.search(u'>Режиссер с логином <br/>"%s"<br/> <span>не существует<' % broadcast_id, broadcast_page) is not None: + raise ExtractorError(u'Broadcast %s does not exist' % broadcast_id, expected=True) + + # Adult content + if re.search(u'EroConfirmText">', broadcast_page) is not None: + + (username, password) = self._get_login_info() + if username is None: + raise ExtractorError(u'Erotic broadcasts allowed only for registered users, ' + u'use --username and --password options to provide account credentials.', expected=True) + + # Log in + login_form_strs = { + u'login-hint53': '1', + u'confirm_erotic': '1', + u'login': username, + u'password': password, + } + # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode + # chokes on unicode + login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in login_form_strs.items()) + login_data = compat_urllib_parse.urlencode(login_form).encode('utf-8') + login_url = broadcast_url + '/?no_redirect=1' + request = compat_urllib_request.Request(login_url, login_data) + request.add_header('Content-Type', 'application/x-www-form-urlencoded') + broadcast_page = self._download_webpage( + request, broadcast_id, note=u'Logging in and confirming age') + + if re.search(u'>Неверный логин или пароль<', broadcast_page) is not None: + raise ExtractorError(u'Unable to log in: bad username or password', expected=True) + + adult_content = True + else: + adult_content = False + + ticket = self._html_search_regex( + u'window.broadcast_control.addFlashVar\\(\'file\', \'([^\']+)\'\\);', + broadcast_page, u'broadcast ticket') + + url = 'http://smotri.com/broadcast/view/url/?ticket=%s' % ticket + + broadcast_password = self._downloader.params.get('videopassword', None) + if broadcast_password: + url += '&pass=%s' % hashlib.md5(broadcast_password.encode('utf-8')).hexdigest() + + broadcast_json_page = self._download_webpage(url, broadcast_id, u'Downloading broadcast JSON') + + try: + broadcast_json = json.loads(broadcast_json_page) + + protected_broadcast = broadcast_json['_pass_protected'] == 1 + if protected_broadcast and not broadcast_password: + raise ExtractorError(u'This broadcast is protected by a password, use the --video-password option', expected=True) + + broadcast_offline = broadcast_json['is_play'] == 0 + if broadcast_offline: + raise ExtractorError(u'Broadcast %s is offline' % broadcast_id, expected=True) + + rtmp_url = broadcast_json['_server'] + if not rtmp_url.startswith('rtmp://'): + raise ExtractorError(u'Unexpected broadcast rtmp URL') + + broadcast_playpath = broadcast_json['_streamName'] + broadcast_thumbnail = broadcast_json['_imgURL'] + broadcast_title = broadcast_json['title'] + broadcast_description = broadcast_json['description'] + broadcaster_nick = broadcast_json['nick'] + broadcaster_login = broadcast_json['login'] + rtmp_conn = 'S:%s' % uuid.uuid4().hex + except KeyError: + if protected_broadcast: + raise ExtractorError(u'Bad broadcast password', expected=True) + raise ExtractorError(u'Unexpected broadcast JSON') + + return { + 'id': broadcast_id, + 'url': rtmp_url, + 'title': broadcast_title, + 'thumbnail': broadcast_thumbnail, + 'description': broadcast_description, + 'uploader': broadcaster_nick, + 'uploader_id': broadcaster_login, + 'age_limit': 18 if adult_content else 0, + 'ext': 'flv', + 'play_path': broadcast_playpath, + 'rtmp_live': True, + 'rtmp_conn': rtmp_conn + } \ No newline at end of file From 8aff7b9bc47795288c65399d6fcac7a8c48004e9 Mon Sep 17 00:00:00 2001 From: dst <dstftw@gmail.com> Date: Wed, 4 Dec 2013 12:36:12 +0700 Subject: [PATCH 215/425] [smotri] Fix broadcast ticket regex --- youtube_dl/extractor/smotri.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py index f86ee8388..ff539ea0c 100644 --- a/youtube_dl/extractor/smotri.py +++ b/youtube_dl/extractor/smotri.py @@ -302,7 +302,7 @@ class SmotriBroadcastIE(InfoExtractor): adult_content = False ticket = self._html_search_regex( - u'window.broadcast_control.addFlashVar\\(\'file\', \'([^\']+)\'\\);', + u'window\.broadcast_control\.addFlashVar\\(\'file\', \'([^\']+)\'\\);', broadcast_page, u'broadcast ticket') url = 'http://smotri.com/broadcast/view/url/?ticket=%s' % ticket From 27dcce19045670fc348ff1119c0d2283aaed3ae2 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Wed, 4 Dec 2013 14:16:52 +0100 Subject: [PATCH 216/425] [youtube] Resolve URLs in comments --- youtube_dl/extractor/youtube.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 66f5af000..7fff761bd 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -336,7 +336,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): u"uploader": u"Philipp Hagemeister", u"uploader_id": u"phihag", u"upload_date": u"20121002", - u"description": u"test chars: \"'/\\ä↭𝕐\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de ." + u"description": u"test chars: \"'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de ." } }, { @@ -1366,6 +1366,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): # description video_description = get_element_by_id("eow-description", video_webpage) if video_description: + video_description = re.sub(r'''(?x) + <a\s+ + (?:[a-zA-Z-]+="[^"]+"\s+)*? + title="([^"]+)"\s+ + (?:[a-zA-Z-]+="[^"]+"\s+)*? + class="yt-uix-redirect-link"\s*> + [^<]+ + </a> + ''', r'\1', video_description) video_description = clean_html(video_description) else: fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage) From 671c0f151d5a7bb5c32a59f483a8e330f1f9a15b Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Wed, 4 Dec 2013 14:19:07 +0100 Subject: [PATCH 217/425] release 2013.12.04 --- README.md | 3 ++- youtube_dl/version.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 00975ab5e..029c418d1 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,8 @@ which means you can modify it, redistribute it or use it however you like. --list-extractors List all supported extractors and the URLs they would handle --extractor-descriptions Output descriptions of all supported extractors - --proxy URL Use the specified HTTP/HTTPS proxy + --proxy URL Use the specified HTTP/HTTPS proxy. Pass in an + empty string (--proxy "") for direct connection --no-check-certificate Suppress HTTPS certificate validation. --cache-dir DIR Location in the filesystem where youtube-dl can store downloaded information permanently. By diff --git a/youtube_dl/version.py b/youtube_dl/version.py index f9a339c02..68b30bfd4 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.03' +__version__ = '2013.12.04' From c0ade33e167d1668c4aa8a6684e7083e6c71dd6e Mon Sep 17 00:00:00 2001 From: dst <dstftw@gmail.com> Date: Wed, 4 Dec 2013 20:34:47 +0700 Subject: [PATCH 218/425] Correct some extractor _VALID_URL regexes --- youtube_dl/extractor/addanime.py | 2 +- youtube_dl/extractor/appletrailers.py | 2 +- youtube_dl/extractor/archiveorg.py | 2 +- youtube_dl/extractor/arte.py | 4 ++-- youtube_dl/extractor/auengine.py | 2 +- youtube_dl/extractor/bambuser.py | 2 +- youtube_dl/extractor/bloomberg.py | 2 +- youtube_dl/extractor/comedycentral.py | 2 +- youtube_dl/extractor/cspan.py | 2 +- youtube_dl/extractor/dreisat.py | 2 +- youtube_dl/extractor/eighttracks.py | 2 +- youtube_dl/extractor/exfm.py | 2 +- youtube_dl/extractor/faz.py | 2 +- youtube_dl/extractor/fktv.py | 4 ++-- youtube_dl/extractor/francetv.py | 2 +- youtube_dl/extractor/gamekings.py | 2 +- youtube_dl/extractor/gametrailers.py | 2 +- youtube_dl/extractor/ign.py | 2 +- youtube_dl/extractor/instagram.py | 2 +- youtube_dl/extractor/jukebox.py | 2 +- youtube_dl/extractor/liveleak.py | 2 +- youtube_dl/extractor/livestream.py | 2 +- youtube_dl/extractor/muzu.py | 2 +- youtube_dl/extractor/myspass.py | 2 +- youtube_dl/extractor/orf.py | 2 +- youtube_dl/extractor/pbs.py | 2 +- youtube_dl/extractor/rutube.py | 2 +- youtube_dl/extractor/slashdot.py | 2 +- youtube_dl/extractor/soundcloud.py | 4 ++-- youtube_dl/extractor/space.py | 2 +- youtube_dl/extractor/stanfordoc.py | 2 +- youtube_dl/extractor/tf1.py | 2 +- youtube_dl/extractor/unistra.py | 2 +- youtube_dl/extractor/veehd.py | 2 +- youtube_dl/extractor/vevo.py | 2 +- youtube_dl/extractor/vice.py | 2 +- youtube_dl/extractor/viddler.py | 2 +- youtube_dl/extractor/videofyme.py | 2 +- youtube_dl/extractor/wat.py | 2 +- youtube_dl/extractor/youjizz.py | 2 +- 40 files changed, 43 insertions(+), 43 deletions(-) diff --git a/youtube_dl/extractor/addanime.py b/youtube_dl/extractor/addanime.py index b99d4b966..a3a1b999d 100644 --- a/youtube_dl/extractor/addanime.py +++ b/youtube_dl/extractor/addanime.py @@ -13,7 +13,7 @@ from ..utils import ( class AddAnimeIE(InfoExtractor): - _VALID_URL = r'^http://(?:\w+\.)?add-anime\.net/watch_video.php\?(?:.*?)v=(?P<video_id>[\w_]+)(?:.*)' + _VALID_URL = r'^http://(?:\w+\.)?add-anime\.net/watch_video\.php\?(?:.*?)v=(?P<video_id>[\w_]+)(?:.*)' IE_NAME = u'AddAnime' _TEST = { u'url': u'http://www.add-anime.net/watch_video.php?v=24MR3YO5SAS9', diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py index 6d6237f8a..5b522552a 100644 --- a/youtube_dl/extractor/appletrailers.py +++ b/youtube_dl/extractor/appletrailers.py @@ -10,7 +10,7 @@ from ..utils import ( class AppleTrailersIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?trailers.apple.com/trailers/(?P<company>[^/]+)/(?P<movie>[^/]+)' + _VALID_URL = r'https?://(?:www\.)?trailers\.apple\.com/trailers/(?P<company>[^/]+)/(?P<movie>[^/]+)' _TEST = { u"url": u"http://trailers.apple.com/trailers/wb/manofsteel/", u"playlist": [ diff --git a/youtube_dl/extractor/archiveorg.py b/youtube_dl/extractor/archiveorg.py index 61ce4469a..a8394bfb0 100644 --- a/youtube_dl/extractor/archiveorg.py +++ b/youtube_dl/extractor/archiveorg.py @@ -11,7 +11,7 @@ from ..utils import ( class ArchiveOrgIE(InfoExtractor): IE_NAME = 'archive.org' IE_DESC = 'archive.org videos' - _VALID_URL = r'(?:https?://)?(?:www\.)?archive.org/details/(?P<id>[^?/]+)(?:[?].*)?$' + _VALID_URL = r'(?:https?://)?(?:www\.)?archive\.org/details/(?P<id>[^?/]+)(?:[?].*)?$' _TEST = { u"url": u"http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect", u'file': u'XD300-23_68HighlightsAResearchCntAugHumanIntellect.ogv', diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py index 8b62ee774..56a5d009f 100644 --- a/youtube_dl/extractor/arte.py +++ b/youtube_dl/extractor/arte.py @@ -17,8 +17,8 @@ from ..utils import ( # add tests. class ArteTvIE(InfoExtractor): - _VIDEOS_URL = r'(?:http://)?videos.arte.tv/(?P<lang>fr|de)/.*-(?P<id>.*?).html' - _LIVEWEB_URL = r'(?:http://)?liveweb.arte.tv/(?P<lang>fr|de)/(?P<subpage>.+?)/(?P<name>.+)' + _VIDEOS_URL = r'(?:http://)?videos\.arte\.tv/(?P<lang>fr|de)/.*-(?P<id>.*?)\.html' + _LIVEWEB_URL = r'(?:http://)?liveweb\.arte\.tv/(?P<lang>fr|de)/(?P<subpage>.+?)/(?P<name>.+)' _LIVE_URL = r'index-[0-9]+\.html$' IE_NAME = u'arte.tv' diff --git a/youtube_dl/extractor/auengine.py b/youtube_dl/extractor/auengine.py index 95c038003..bcccc0b7a 100644 --- a/youtube_dl/extractor/auengine.py +++ b/youtube_dl/extractor/auengine.py @@ -16,7 +16,7 @@ class AUEngineIE(InfoExtractor): u"title": u"[Commie]The Legend of the Legendary Heroes - 03 - Replication Eye (Alpha Stigma)[F9410F5A]" } } - _VALID_URL = r'(?:http://)?(?:www\.)?auengine\.com/embed.php\?.*?file=([^&]+).*?' + _VALID_URL = r'(?:http://)?(?:www\.)?auengine\.com/embed\.php\?.*?file=([^&]+).*?' def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) diff --git a/youtube_dl/extractor/bambuser.py b/youtube_dl/extractor/bambuser.py index b80508efe..d48c0c38d 100644 --- a/youtube_dl/extractor/bambuser.py +++ b/youtube_dl/extractor/bambuser.py @@ -54,7 +54,7 @@ class BambuserIE(InfoExtractor): class BambuserChannelIE(InfoExtractor): IE_NAME = u'bambuser:channel' - _VALID_URL = r'http://bambuser.com/channel/(?P<user>.*?)(?:/|#|\?|$)' + _VALID_URL = r'https?://bambuser\.com/channel/(?P<user>.*?)(?:/|#|\?|$)' # The maximum number we can get with each request _STEP = 50 diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py index 3666a780b..755d9c9ef 100644 --- a/youtube_dl/extractor/bloomberg.py +++ b/youtube_dl/extractor/bloomberg.py @@ -4,7 +4,7 @@ from .common import InfoExtractor class BloombergIE(InfoExtractor): - _VALID_URL = r'https?://www\.bloomberg\.com/video/(?P<name>.+?).html' + _VALID_URL = r'https?://www\.bloomberg\.com/video/(?P<name>.+?)\.html' _TEST = { u'url': u'http://www.bloomberg.com/video/shah-s-presentation-on-foreign-exchange-strategies-qurhIVlJSB6hzkVi229d8g.html', diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py index 23647f99e..caea446ea 100644 --- a/youtube_dl/extractor/comedycentral.py +++ b/youtube_dl/extractor/comedycentral.py @@ -12,7 +12,7 @@ from ..utils import ( class ComedyCentralIE(MTVIE): - _VALID_URL = r'http://www.comedycentral.com/(video-clips|episodes|cc-studios)/(?P<title>.*)' + _VALID_URL = r'https?://(?:www\.)?comedycentral\.com/(video-clips|episodes|cc-studios)/(?P<title>.*)' _FEED_URL = u'http://comedycentral.com/feeds/mrss/' _TEST = { diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py index 7bf03c584..d5730684d 100644 --- a/youtube_dl/extractor/cspan.py +++ b/youtube_dl/extractor/cspan.py @@ -6,7 +6,7 @@ from ..utils import ( ) class CSpanIE(InfoExtractor): - _VALID_URL = r'http://www.c-spanvideo.org/program/(.*)' + _VALID_URL = r'http://www\.c-spanvideo\.org/program/(.*)' _TEST = { u'url': u'http://www.c-spanvideo.org/program/HolderonV', u'file': u'315139.flv', diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py index 3cb382e12..008c99699 100644 --- a/youtube_dl/extractor/dreisat.py +++ b/youtube_dl/extractor/dreisat.py @@ -11,7 +11,7 @@ from ..utils import ( class DreiSatIE(InfoExtractor): IE_NAME = '3sat' - _VALID_URL = r'(?:http://)?(?:www\.)?3sat.de/mediathek/index.php\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$' + _VALID_URL = r'(?:http://)?(?:www\.)?3sat\.de/mediathek/index\.php\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$' _TEST = { u"url": u"http://www.3sat.de/mediathek/index.php?obj=36983", u'file': u'36983.webm', diff --git a/youtube_dl/extractor/eighttracks.py b/youtube_dl/extractor/eighttracks.py index f21ef8853..88f5526b8 100644 --- a/youtube_dl/extractor/eighttracks.py +++ b/youtube_dl/extractor/eighttracks.py @@ -10,7 +10,7 @@ from ..utils import ( class EightTracksIE(InfoExtractor): IE_NAME = '8tracks' - _VALID_URL = r'https?://8tracks.com/(?P<user>[^/]+)/(?P<id>[^/#]+)(?:#.*)?$' + _VALID_URL = r'https?://8tracks\.com/(?P<user>[^/]+)/(?P<id>[^/#]+)(?:#.*)?$' _TEST = { u"name": u"EightTracks", u"url": u"http://8tracks.com/ytdl/youtube-dl-test-tracks-a", diff --git a/youtube_dl/extractor/exfm.py b/youtube_dl/extractor/exfm.py index a51d79b08..682901d16 100644 --- a/youtube_dl/extractor/exfm.py +++ b/youtube_dl/extractor/exfm.py @@ -8,7 +8,7 @@ class ExfmIE(InfoExtractor): IE_NAME = u'exfm' IE_DESC = u'ex.fm' _VALID_URL = r'(?:http://)?(?:www\.)?ex\.fm/song/([^/]+)' - _SOUNDCLOUD_URL = r'(?:http://)?(?:www\.)?api\.soundcloud.com/tracks/([^/]+)/stream' + _SOUNDCLOUD_URL = r'(?:http://)?(?:www\.)?api\.soundcloud\.com/tracks/([^/]+)/stream' _TESTS = [ { u'url': u'http://ex.fm/song/eh359', diff --git a/youtube_dl/extractor/faz.py b/youtube_dl/extractor/faz.py index c0169de04..615674baf 100644 --- a/youtube_dl/extractor/faz.py +++ b/youtube_dl/extractor/faz.py @@ -9,7 +9,7 @@ from ..utils import ( class FazIE(InfoExtractor): IE_NAME = u'faz.net' - _VALID_URL = r'https?://www\.faz\.net/multimedia/videos/.*?-(?P<id>\d+).html' + _VALID_URL = r'https?://www\.faz\.net/multimedia/videos/.*?-(?P<id>\d+)\.html' _TEST = { u'url': u'http://www.faz.net/multimedia/videos/stockholm-chemie-nobelpreis-fuer-drei-amerikanische-forscher-12610585.html', diff --git a/youtube_dl/extractor/fktv.py b/youtube_dl/extractor/fktv.py index dba1a8dc2..d7048c8c1 100644 --- a/youtube_dl/extractor/fktv.py +++ b/youtube_dl/extractor/fktv.py @@ -12,7 +12,7 @@ from ..utils import ( class FKTVIE(InfoExtractor): IE_NAME = u'fernsehkritik.tv' - _VALID_URL = r'(?:http://)?(?:www\.)?fernsehkritik.tv/folge-(?P<ep>[0-9]+)(?:/.*)?' + _VALID_URL = r'(?:http://)?(?:www\.)?fernsehkritik\.tv/folge-(?P<ep>[0-9]+)(?:/.*)?' _TEST = { u'url': u'http://fernsehkritik.tv/folge-1', @@ -52,7 +52,7 @@ class FKTVIE(InfoExtractor): class FKTVPosteckeIE(InfoExtractor): IE_NAME = u'fernsehkritik.tv:postecke' - _VALID_URL = r'(?:http://)?(?:www\.)?fernsehkritik.tv/inline-video/postecke.php\?(.*&)?ep=(?P<ep>[0-9]+)(&|$)' + _VALID_URL = r'(?:http://)?(?:www\.)?fernsehkritik\.tv/inline-video/postecke\.php\?(.*&)?ep=(?P<ep>[0-9]+)(&|$)' _TEST = { u'url': u'http://fernsehkritik.tv/inline-video/postecke.php?iframe=true&width=625&height=440&ep=120', u'file': u'0120.flv', diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py index 6e1971043..66aa3aa0d 100644 --- a/youtube_dl/extractor/francetv.py +++ b/youtube_dl/extractor/francetv.py @@ -45,7 +45,7 @@ class PluzzIE(FranceTVBaseInfoExtractor): class FranceTvInfoIE(FranceTVBaseInfoExtractor): IE_NAME = u'francetvinfo.fr' - _VALID_URL = r'https?://www\.francetvinfo\.fr/replay.*/(?P<title>.+).html' + _VALID_URL = r'https?://www\.francetvinfo\.fr/replay.*/(?P<title>.+)\.html' _TEST = { u'url': u'http://www.francetvinfo.fr/replay-jt/france-3/soir-3/jt-grand-soir-3-lundi-26-aout-2013_393427.html', diff --git a/youtube_dl/extractor/gamekings.py b/youtube_dl/extractor/gamekings.py index c91669b0e..a3a5251fe 100644 --- a/youtube_dl/extractor/gamekings.py +++ b/youtube_dl/extractor/gamekings.py @@ -4,7 +4,7 @@ from .common import InfoExtractor class GamekingsIE(InfoExtractor): - _VALID_URL = r'http?://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-]+)' + _VALID_URL = r'http://www\.gamekings\.tv/videos/(?P<name>[0-9a-z\-]+)' _TEST = { u"url": u"http://www.gamekings.tv/videos/phoenix-wright-ace-attorney-dual-destinies-review/", u'file': u'20130811.mp4', diff --git a/youtube_dl/extractor/gametrailers.py b/youtube_dl/extractor/gametrailers.py index 3cc02d97e..88f656031 100644 --- a/youtube_dl/extractor/gametrailers.py +++ b/youtube_dl/extractor/gametrailers.py @@ -7,7 +7,7 @@ class GametrailersIE(MTVIE): Gametrailers use the same videos system as MTVIE, it just changes the feed url, where the uri is and the method to get the thumbnails. """ - _VALID_URL = r'http://www.gametrailers.com/(?P<type>videos|reviews|full-episodes)/(?P<id>.*?)/(?P<title>.*)' + _VALID_URL = r'http://www\.gametrailers\.com/(?P<type>videos|reviews|full-episodes)/(?P<id>.*?)/(?P<title>.*)' _TEST = { u'url': u'http://www.gametrailers.com/videos/zbvr8i/mirror-s-edge-2-e3-2013--debut-trailer', u'file': u'70e9a5d7-cf25-4a10-9104-6f3e7342ae0d.mp4', diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py index c52146f7d..57b79a336 100644 --- a/youtube_dl/extractor/ign.py +++ b/youtube_dl/extractor/ign.py @@ -103,7 +103,7 @@ class IGNIE(InfoExtractor): class OneUPIE(IGNIE): """Extractor for 1up.com, it uses the ign videos system.""" - _VALID_URL = r'https?://gamevideos.1up.com/(?P<type>video)/id/(?P<name_or_id>.+)' + _VALID_URL = r'https?://gamevideos\.1up\.com/(?P<type>video)/id/(?P<name_or_id>.+)' IE_NAME = '1up.com' _DESCRIPTION_RE = r'<div id="vid_summary">(.+?)</div>' diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py index 213aac428..660573d02 100644 --- a/youtube_dl/extractor/instagram.py +++ b/youtube_dl/extractor/instagram.py @@ -3,7 +3,7 @@ import re from .common import InfoExtractor class InstagramIE(InfoExtractor): - _VALID_URL = r'(?:http://)?instagram.com/p/(.*?)/' + _VALID_URL = r'(?:http://)?instagram\.com/p/(.*?)/' _TEST = { u'url': u'http://instagram.com/p/aye83DjauH/?foo=bar#abc', u'file': u'aye83DjauH.mp4', diff --git a/youtube_dl/extractor/jukebox.py b/youtube_dl/extractor/jukebox.py index c7bb234fe..592c64e1d 100644 --- a/youtube_dl/extractor/jukebox.py +++ b/youtube_dl/extractor/jukebox.py @@ -8,7 +8,7 @@ from ..utils import ( ) class JukeboxIE(InfoExtractor): - _VALID_URL = r'^http://www\.jukebox?\..+?\/.+[,](?P<video_id>[a-z0-9\-]+).html' + _VALID_URL = r'^http://www\.jukebox?\..+?\/.+[,](?P<video_id>[a-z0-9\-]+)\.html' _IFRAME = r'<iframe .*src="(?P<iframe>[^"]*)".*>' _VIDEO_URL = r'"config":{"file":"(?P<video_url>http:[^"]+[.](?P<video_ext>[^.?]+)[?]mdtk=[0-9]+)"' _TITLE = r'<h1 class="inline">(?P<title>[^<]+)</h1>.*<span id="infos_article_artist">(?P<artist>[^<]+)</span>' diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py index dd062a14e..5ae57a77c 100644 --- a/youtube_dl/extractor/liveleak.py +++ b/youtube_dl/extractor/liveleak.py @@ -8,7 +8,7 @@ from ..utils import ( class LiveLeakIE(InfoExtractor): - _VALID_URL = r'^(?:http?://)?(?:\w+\.)?liveleak\.com/view\?(?:.*?)i=(?P<video_id>[\w_]+)(?:.*)' + _VALID_URL = r'^(?:http://)?(?:\w+\.)?liveleak\.com/view\?(?:.*?)i=(?P<video_id>[\w_]+)(?:.*)' IE_NAME = u'liveleak' _TEST = { u'url': u'http://www.liveleak.com/view?i=757_1364311680', diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py index 9bc35b115..1dcd1fb2d 100644 --- a/youtube_dl/extractor/livestream.py +++ b/youtube_dl/extractor/livestream.py @@ -11,7 +11,7 @@ from ..utils import ( class LivestreamIE(InfoExtractor): IE_NAME = u'livestream' - _VALID_URL = r'http://new.livestream.com/.*?/(?P<event_name>.*?)(/videos/(?P<id>\d+))?/?$' + _VALID_URL = r'http://new\.livestream\.com/.*?/(?P<event_name>.*?)(/videos/(?P<id>\d+))?/?$' _TEST = { u'url': u'http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370', u'file': u'4719370.mp4', diff --git a/youtube_dl/extractor/muzu.py b/youtube_dl/extractor/muzu.py index 03e31ea1c..1772b7f9a 100644 --- a/youtube_dl/extractor/muzu.py +++ b/youtube_dl/extractor/muzu.py @@ -9,7 +9,7 @@ from ..utils import ( class MuzuTVIE(InfoExtractor): - _VALID_URL = r'https?://www.muzu.tv/(.+?)/(.+?)/(?P<id>\d+)' + _VALID_URL = r'https?://www\.muzu\.tv/(.+?)/(.+?)/(?P<id>\d+)' IE_NAME = u'muzu.tv' _TEST = { diff --git a/youtube_dl/extractor/myspass.py b/youtube_dl/extractor/myspass.py index 0067bf134..4becddee6 100644 --- a/youtube_dl/extractor/myspass.py +++ b/youtube_dl/extractor/myspass.py @@ -9,7 +9,7 @@ from ..utils import ( class MySpassIE(InfoExtractor): - _VALID_URL = r'http://www.myspass.de/.*' + _VALID_URL = r'http://www\.myspass\.de/.*' _TEST = { u'url': u'http://www.myspass.de/myspass/shows/tvshows/absolute-mehrheit/Absolute-Mehrheit-vom-17022013-Die-Highlights-Teil-2--/11741/', u'file': u'11741.mp4', diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py index cfca2a063..b42eae89a 100644 --- a/youtube_dl/extractor/orf.py +++ b/youtube_dl/extractor/orf.py @@ -12,7 +12,7 @@ from ..utils import ( ) class ORFIE(InfoExtractor): - _VALID_URL = r'https?://tvthek.orf.at/(programs/.+?/episodes|topics/.+?)/(?P<id>\d+)' + _VALID_URL = r'https?://tvthek\.orf\.at/(programs/.+?/episodes|topics/.+?)/(?P<id>\d+)' def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py index 65462d867..25f019231 100644 --- a/youtube_dl/extractor/pbs.py +++ b/youtube_dl/extractor/pbs.py @@ -5,7 +5,7 @@ from .common import InfoExtractor class PBSIE(InfoExtractor): - _VALID_URL = r'https?://video.pbs.org/video/(?P<id>\d+)/?' + _VALID_URL = r'https?://video\.pbs\.org/video/(?P<id>\d+)/?' _TEST = { u'url': u'http://video.pbs.org/video/2365006249/', diff --git a/youtube_dl/extractor/rutube.py b/youtube_dl/extractor/rutube.py index a18034fe2..e3e9bc07f 100644 --- a/youtube_dl/extractor/rutube.py +++ b/youtube_dl/extractor/rutube.py @@ -11,7 +11,7 @@ from ..utils import ( class RutubeIE(InfoExtractor): - _VALID_URL = r'https?://rutube.ru/video/(?P<long_id>\w+)' + _VALID_URL = r'https?://rutube\.ru/video/(?P<long_id>\w+)' _TEST = { u'url': u'http://rutube.ru/video/3eac3b4561676c17df9132a9a1e62e3e/', diff --git a/youtube_dl/extractor/slashdot.py b/youtube_dl/extractor/slashdot.py index f5003c7f9..d68646d24 100644 --- a/youtube_dl/extractor/slashdot.py +++ b/youtube_dl/extractor/slashdot.py @@ -4,7 +4,7 @@ from .common import InfoExtractor class SlashdotIE(InfoExtractor): - _VALID_URL = r'https?://tv.slashdot.org/video/\?embed=(?P<id>.*?)(&|$)' + _VALID_URL = r'https?://tv\.slashdot\.org/video/\?embed=(?P<id>.*?)(&|$)' _TEST = { u'add_ie': ['Ooyala'], diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 3a19ab172..cb6dedab7 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -25,7 +25,7 @@ class SoundcloudIE(InfoExtractor): _VALID_URL = r'''^(?:https?://)? (?:(?:(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)/?(?:[?].*)?$) |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)) - |(?P<widget>w.soundcloud.com/player/?.*?url=.*) + |(?P<widget>w\.soundcloud\.com/player/?.*?url=.*) ) ''' IE_NAME = u'soundcloud' @@ -217,7 +217,7 @@ class SoundcloudSetIE(SoundcloudIE): class SoundcloudUserIE(SoundcloudIE): - _VALID_URL = r'https?://(www\.)?soundcloud.com/(?P<user>[^/]+)(/?(tracks/)?)?(\?.*)?$' + _VALID_URL = r'https?://(www\.)?soundcloud\.com/(?P<user>[^/]+)(/?(tracks/)?)?(\?.*)?$' IE_NAME = u'soundcloud:user' # it's in tests/test_playlists.py diff --git a/youtube_dl/extractor/space.py b/youtube_dl/extractor/space.py index 0d32a0688..11455e0fa 100644 --- a/youtube_dl/extractor/space.py +++ b/youtube_dl/extractor/space.py @@ -6,7 +6,7 @@ from ..utils import RegexNotFoundError, ExtractorError class SpaceIE(InfoExtractor): - _VALID_URL = r'https?://www\.space\.com/\d+-(?P<title>[^/\.\?]*?)-video.html' + _VALID_URL = r'https?://www\.space\.com/\d+-(?P<title>[^/\.\?]*?)-video\.html' _TEST = { u'add_ie': ['Brightcove'], u'url': u'http://www.space.com/23373-huge-martian-landforms-detail-revealed-by-european-probe-video.html', diff --git a/youtube_dl/extractor/stanfordoc.py b/youtube_dl/extractor/stanfordoc.py index b27838bf9..d54e01a12 100644 --- a/youtube_dl/extractor/stanfordoc.py +++ b/youtube_dl/extractor/stanfordoc.py @@ -18,7 +18,7 @@ from ..utils import ( class StanfordOpenClassroomIE(InfoExtractor): IE_NAME = u'stanfordoc' IE_DESC = u'Stanford Open ClassRoom' - _VALID_URL = r'^(?:https?://)?openclassroom.stanford.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$' + _VALID_URL = r'^(?:https?://)?openclassroom\.stanford\.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$' _TEST = { u'url': u'http://openclassroom.stanford.edu/MainFolder/VideoPage.php?course=PracticalUnix&video=intro-environment&speed=100', u'file': u'PracticalUnix_intro-environment.mp4', diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py index 772134a12..2c5c88be8 100644 --- a/youtube_dl/extractor/tf1.py +++ b/youtube_dl/extractor/tf1.py @@ -7,7 +7,7 @@ from .common import InfoExtractor class TF1IE(InfoExtractor): """TF1 uses the wat.tv player.""" - _VALID_URL = r'http://videos.tf1.fr/.*-(.*?).html' + _VALID_URL = r'http://videos\.tf1\.fr/.*-(.*?)\.html' _TEST = { u'url': u'http://videos.tf1.fr/auto-moto/citroen-grand-c4-picasso-2013-presentation-officielle-8062060.html', u'file': u'10635995.mp4', diff --git a/youtube_dl/extractor/unistra.py b/youtube_dl/extractor/unistra.py index 516e18914..474610eec 100644 --- a/youtube_dl/extractor/unistra.py +++ b/youtube_dl/extractor/unistra.py @@ -3,7 +3,7 @@ import re from .common import InfoExtractor class UnistraIE(InfoExtractor): - _VALID_URL = r'http://utv.unistra.fr/(?:index|video).php\?id_video\=(\d+)' + _VALID_URL = r'http://utv\.unistra\.fr/(?:index|video)\.php\?id_video\=(\d+)' _TEST = { u'url': u'http://utv.unistra.fr/video.php?id_video=154', diff --git a/youtube_dl/extractor/veehd.py b/youtube_dl/extractor/veehd.py index 3a99a29c6..3cf8c853d 100644 --- a/youtube_dl/extractor/veehd.py +++ b/youtube_dl/extractor/veehd.py @@ -9,7 +9,7 @@ from ..utils import ( ) class VeeHDIE(InfoExtractor): - _VALID_URL = r'https?://veehd.com/video/(?P<id>\d+)' + _VALID_URL = r'https?://veehd\.com/video/(?P<id>\d+)' _TEST = { u'url': u'http://veehd.com/video/4686958', diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index 4378b1780..d8bfcd155 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -15,7 +15,7 @@ class VevoIE(InfoExtractor): Accepts urls from vevo.com or in the format 'vevo:{id}' (currently used by MTVIE) """ - _VALID_URL = r'((http://www.vevo.com/watch/.*?/.*?/)|(vevo:))(?P<id>.*?)(\?|$)' + _VALID_URL = r'((http://www\.vevo\.com/watch/.*?/.*?/)|(vevo:))(?P<id>.*?)(\?|$)' _TESTS = [{ u'url': u'http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280', u'file': u'GB1101300280.mp4', diff --git a/youtube_dl/extractor/vice.py b/youtube_dl/extractor/vice.py index 6b93afa50..87812d6af 100644 --- a/youtube_dl/extractor/vice.py +++ b/youtube_dl/extractor/vice.py @@ -6,7 +6,7 @@ from ..utils import ExtractorError class ViceIE(InfoExtractor): - _VALID_URL = r'http://www.vice.com/.*?/(?P<name>.+)' + _VALID_URL = r'http://www\.vice\.com/.*?/(?P<name>.+)' _TEST = { u'url': u'http://www.vice.com/Fringes/cowboy-capitalists-part-1', diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py index 826804af3..36d1bde08 100644 --- a/youtube_dl/extractor/viddler.py +++ b/youtube_dl/extractor/viddler.py @@ -8,7 +8,7 @@ from ..utils import ( class ViddlerIE(InfoExtractor): - _VALID_URL = r'(?P<domain>https?://(?:www\.)?viddler.com)/(?:v|embed|player)/(?P<id>[a-z0-9]+)' + _VALID_URL = r'(?P<domain>https?://(?:www\.)?viddler\.com)/(?:v|embed|player)/(?P<id>[a-z0-9]+)' _TEST = { u"url": u"http://www.viddler.com/v/43903784", u'file': u'43903784.mp4', diff --git a/youtube_dl/extractor/videofyme.py b/youtube_dl/extractor/videofyme.py index 912802d9a..f75169041 100644 --- a/youtube_dl/extractor/videofyme.py +++ b/youtube_dl/extractor/videofyme.py @@ -7,7 +7,7 @@ from ..utils import ( ) class VideofyMeIE(InfoExtractor): - _VALID_URL = r'https?://(www.videofy.me/.+?|p.videofy.me/v)/(?P<id>\d+)(&|#|$)' + _VALID_URL = r'https?://(www\.videofy\.me/.+?|p\.videofy\.me/v)/(?P<id>\d+)(&|#|$)' IE_NAME = u'videofy.me' _TEST = { diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py index 29c25f0e3..4fab6c6e8 100644 --- a/youtube_dl/extractor/wat.py +++ b/youtube_dl/extractor/wat.py @@ -11,7 +11,7 @@ from ..utils import ( class WatIE(InfoExtractor): - _VALID_URL=r'http://www.wat.tv/.*-(?P<shortID>.*?)_.*?.html' + _VALID_URL=r'http://www\.wat\.tv/.*-(?P<shortID>.*?)_.*?\.html' IE_NAME = 'wat.tv' _TEST = { u'url': u'http://www.wat.tv/video/world-war-philadelphia-vost-6bv55_2fjr7_.html', diff --git a/youtube_dl/extractor/youjizz.py b/youtube_dl/extractor/youjizz.py index 1fcc518ac..e971b5b4b 100644 --- a/youtube_dl/extractor/youjizz.py +++ b/youtube_dl/extractor/youjizz.py @@ -7,7 +7,7 @@ from ..utils import ( class YouJizzIE(InfoExtractor): - _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youjizz\.com/videos/(?P<videoid>[^.]+).html$' + _VALID_URL = r'^(?:https?://)?(?:\w+\.)?youjizz\.com/videos/(?P<videoid>[^.]+)\.html$' _TEST = { u'url': u'http://www.youjizz.com/videos/zeichentrick-1-2189178.html', u'file': u'2189178.flv', From 6a656a843a629ceef6979976a353d177c97b9527 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Wed, 4 Dec 2013 20:35:00 +0100 Subject: [PATCH 219/425] Update description value for the write_info_json test (required after 27dcce19045670fc348ff1119c0d2283aaed3ae2) --- test/test_write_info_json.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_write_info_json.py b/test/test_write_info_json.py index d7177611b..90426a559 100644 --- a/test/test_write_info_json.py +++ b/test/test_write_info_json.py @@ -33,6 +33,7 @@ TEST_ID = 'BaW_jenozKc' INFO_JSON_FILE = TEST_ID + '.info.json' DESCRIPTION_FILE = TEST_ID + '.mp4.description' EXPECTED_DESCRIPTION = u'''test chars: "'/\ä↭𝕐 +test URL: https://github.com/rg3/youtube-dl/issues/1892 This is a test video for youtube-dl. From bfb9f7bc4c5c6fd9b2d3d46be133988f70534d26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Wed, 4 Dec 2013 20:36:26 +0100 Subject: [PATCH 220/425] [hotnewhiphop] Update test's title --- youtube_dl/extractor/hotnewhiphop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/hotnewhiphop.py b/youtube_dl/extractor/hotnewhiphop.py index 3798118a7..0ee74fb38 100644 --- a/youtube_dl/extractor/hotnewhiphop.py +++ b/youtube_dl/extractor/hotnewhiphop.py @@ -11,7 +11,7 @@ class HotNewHipHopIE(InfoExtractor): u'file': u'1435540.mp3', u'md5': u'2c2cd2f76ef11a9b3b581e8b232f3d96', u'info_dict': { - u"title": u"Freddie Gibbs - Lay It Down" + u"title": u'Freddie Gibbs "Lay It Down"' } } From e9bf7479d209c2623753628201ca0daffa19f3cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Wed, 4 Dec 2013 23:28:40 +0100 Subject: [PATCH 221/425] Add an extractor for theplatform.com --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/theplatform.py | 69 +++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 youtube_dl/extractor/theplatform.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index bd996483b..900a6f02f 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -144,6 +144,7 @@ from .teamcoco import TeamcocoIE from .techtalks import TechTalksIE from .ted import TEDIE from .tf1 import TF1IE +from .theplatform import ThePlatformIE from .thisav import ThisAVIE from .toutv import TouTvIE from .traileraddict import TrailerAddictIE diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py new file mode 100644 index 000000000..d1d6a4c2c --- /dev/null +++ b/youtube_dl/extractor/theplatform.py @@ -0,0 +1,69 @@ +import re +import json + +from .common import InfoExtractor +from ..utils import ( + xpath_with_ns, + find_xpath_attr, +) + +_x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language'}) + + +class ThePlatformIE(InfoExtractor): + _VALID_URL = r'https?://link\.theplatform\.com/s/[^/]+/(?P<id>[^/\?]+)' + + _TEST = { + # from http://www.metacafe.com/watch/cb-e9I_cZgTgIPd/blackberrys_big_bold_z30/ + u'url': u'http://link.theplatform.com/s/dJ5BDC/e9I_cZgTgIPd/meta.smil?format=smil&Tracking=true&mbr=true', + u'info_dict': { + u'id': u'e9I_cZgTgIPd', + u'ext': u'flv', + u'title': u'Blackberry\'s big, bold Z30', + u'description': u'The Z30 is Blackberry\'s biggest, baddest mobile messaging device yet.', + u'duration': 247, + }, + u'params': { + # rtmp download + u'skip_download': True, + }, + } + + def _get_info(self, video_id): + smil_url = ('http://link.theplatform.com/s/dJ5BDC/{0}/meta.smil?' + 'format=smil&mbr=true'.format(video_id)) + meta = self._download_xml(smil_url, video_id) + info_url = 'http://link.theplatform.com/s/dJ5BDC/{0}?format=preview'.format(video_id) + info_json = self._download_webpage(info_url, video_id) + info = json.loads(info_json) + + head = meta.find(_x('smil:head')) + body = meta.find(_x('smil:body')) + base_url = head.find(_x('smil:meta')).attrib['base'] + switch = body.find(_x('smil:switch')) + formats = [] + for f in switch.findall(_x('smil:video')): + attr = f.attrib + formats.append({ + 'url': base_url, + 'play_path': 'mp4:' + attr['src'], + 'ext': 'flv', + 'width': int(attr['width']), + 'height': int(attr['height']), + 'vbr': int(attr['system-bitrate']), + }) + formats.sort(key=lambda f: (f['height'], f['width'], f['vbr'])) + + return { + 'id': video_id, + 'title': info['title'], + 'formats': formats, + 'description': info['description'], + 'thumbnail': info['defaultThumbnailUrl'], + 'duration': info['duration']//1000, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + return self._get_info(video_id) From b9a2c53833a3cebc32df908aad74f7c5a3537aa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Wed, 4 Dec 2013 23:43:50 +0100 Subject: [PATCH 222/425] [metacafe] Add support for cbs videos (fixes #1838) They use theplatform.com --- youtube_dl/extractor/metacafe.py | 29 +++++++++++++++++++++++++---- youtube_dl/extractor/theplatform.py | 2 +- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py index 91480ba87..e59bdd604 100644 --- a/youtube_dl/extractor/metacafe.py +++ b/youtube_dl/extractor/metacafe.py @@ -69,6 +69,21 @@ class MetacafeIE(InfoExtractor): u'age_limit': 18, }, }, + # cbs video + { + u'url': u'http://www.metacafe.com/watch/cb-0rOxMBabDXN6/samsung_galaxy_note_2_samsungs_next_generation_phablet/', + u'info_dict': { + u'id': u'0rOxMBabDXN6', + u'ext': u'flv', + u'title': u'Samsung Galaxy Note 2: Samsung\'s next-generation phablet', + u'description': u'md5:54d49fac53d26d5a0aaeccd061ada09d', + u'duration': 129, + }, + u'params': { + # rtmp download + u'skip_download': True, + }, + }, ] @@ -106,10 +121,16 @@ class MetacafeIE(InfoExtractor): video_id = mobj.group(1) - # Check if video comes from YouTube - mobj2 = re.match(r'^yt-(.*)$', video_id) - if mobj2 is not None: - return [self.url_result('http://www.youtube.com/watch?v=%s' % mobj2.group(1), 'Youtube')] + # the video may come from an external site + m_external = re.match('^(\w{2})-(.*)$', video_id) + if m_external is not None: + prefix, ext_id = m_external.groups() + # Check if video comes from YouTube + if prefix == 'yt': + return self.url_result('http://www.youtube.com/watch?v=%s' % ext_id, 'Youtube') + # CBS videos use theplatform.com + if prefix == 'cb': + return self.url_result('theplatform:%s' % ext_id, 'ThePlatform') # Retrieve video webpage to extract further information req = compat_urllib_request.Request('http://www.metacafe.com/watch/%s/' % video_id) diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py index d1d6a4c2c..920689511 100644 --- a/youtube_dl/extractor/theplatform.py +++ b/youtube_dl/extractor/theplatform.py @@ -11,7 +11,7 @@ _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language class ThePlatformIE(InfoExtractor): - _VALID_URL = r'https?://link\.theplatform\.com/s/[^/]+/(?P<id>[^/\?]+)' + _VALID_URL = r'(?:https?://link\.theplatform\.com/s/[^/]+/|theplatform:)(?P<id>[^/\?]+)' _TEST = { # from http://www.metacafe.com/watch/cb-e9I_cZgTgIPd/blackberrys_big_bold_z30/ From 673d1273ff6f6d3267728fbe6f79c9c801598fd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Thu, 5 Dec 2013 12:41:58 +0100 Subject: [PATCH 223/425] [vevo] Support '/watch/{id}' urls --- youtube_dl/extractor/vevo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index d8bfcd155..3eedcf7dd 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -15,7 +15,7 @@ class VevoIE(InfoExtractor): Accepts urls from vevo.com or in the format 'vevo:{id}' (currently used by MTVIE) """ - _VALID_URL = r'((http://www\.vevo\.com/watch/.*?/.*?/)|(vevo:))(?P<id>.*?)(\?|$)' + _VALID_URL = r'((http://www\.vevo\.com/watch/(?:[^/]+/[^/]+/)?)|(vevo:))(?P<id>.*?)(\?|$)' _TESTS = [{ u'url': u'http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280', u'file': u'GB1101300280.mp4', From 7fc3fa0545f8a07414e8c97be9862a3c2f79bb98 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Thu, 5 Dec 2013 14:29:08 +0100 Subject: [PATCH 224/425] [9gag] Add extractor --- youtube_dl/YoutubeDL.py | 34 +++++++++++++++++++++++--- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/ninegag.py | 41 ++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 youtube_dl/extractor/ninegag.py diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index b68b110a4..8ad7bd1da 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -405,7 +405,8 @@ class YoutubeDL(object): for key, value in extra_info.items(): info_dict.setdefault(key, value) - def extract_info(self, url, download=True, ie_key=None, extra_info={}): + def extract_info(self, url, download=True, ie_key=None, extra_info={}, + process=True): ''' Returns a list with a dictionary for each video we find. If 'download', also downloads the videos. @@ -441,7 +442,10 @@ class YoutubeDL(object): 'webpage_url': url, 'extractor_key': ie.ie_key(), }) - return self.process_ie_result(ie_result, download, extra_info) + if process: + return self.process_ie_result(ie_result, download, extra_info) + else: + return ie_result except ExtractorError as de: # An error we somewhat expected self.report_error(compat_str(de), de.format_traceback()) break @@ -474,8 +478,32 @@ class YoutubeDL(object): download, ie_key=ie_result.get('ie_key'), extra_info=extra_info) - elif result_type == 'playlist': + elif result_type == 'url_transparent': + # Use the information from the embedding page + info = self.extract_info( + ie_result['url'], ie_key=ie_result.get('ie_key'), + extra_info=extra_info, download=False, process=False) + def make_result(embedded_info): + new_result = ie_result.copy() + for f in ('_type', 'url', 'ext', 'player_url', 'formats', + 'entries', 'urlhandle', 'ie_key', 'duration', + 'subtitles', 'annotations', 'format'): + if f in new_result: + del new_result[f] + if f in embedded_info: + new_result[f] = embedded_info[f] + return new_result + new_result = make_result(info) + + assert new_result.get('_type') != 'url_transparent' + if new_result.get('_type') == 'compat_list': + new_result['entries'] = [ + make_result(e) for e in new_result['entries']] + + return self.process_ie_result( + new_result, download=download, extra_info=extra_info) + elif result_type == 'playlist': # We process each entry in the playlist playlist = ie_result.get('title', None) or ie_result.get('id', None) self.to_screen(u'[download] Downloading playlist: %s' % playlist) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index bd996483b..a77e98d49 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -102,6 +102,7 @@ from .nbc import NBCNewsIE from .newgrounds import NewgroundsIE from .nhl import NHLIE, NHLVideocenterIE from .niconico import NiconicoIE +from .ninegag import NineGagIE from .nowvideo import NowVideoIE from .ooyala import OoyalaIE from .orf import ORFIE diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py new file mode 100644 index 000000000..cc00ffbcc --- /dev/null +++ b/youtube_dl/extractor/ninegag.py @@ -0,0 +1,41 @@ +import json +import re + +from .common import InfoExtractor + + +class NineGagIE(InfoExtractor): + IE_NAME = '9gag' + _VALID_URL = r'^https?://(?:www\.)?9gag\.tv/v/(?P<id>[0-9]+)' + + _TEST = { + u"url": u"http://9gag.tv/v/1912", + u"file": u"1912.mp4", + u"info_dict": { + u"description": u"This 3-minute video will make you smile and then make you feel untalented and insignificant. Anyway, you should share this awesomeness. (Thanks, Dino!)", + u"title": u"\"People Are Awesome 2013\" Is Absolutely Awesome" + }, + u'add_ie': [u'Youtube'] + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + data_json = self._html_search_regex(r'''(?x) + <div\s*id="tv-video"\s*data-video-source="youtube"\s* + data-video-meta="([^"]+)"''', webpage, u'video metadata') + + data = json.loads(data_json) + + return { + '_type': 'url_transparent', + 'url': data['youtubeVideoId'], + 'ie_key': 'Youtube', + 'id': video_id, + 'title': data['title'], + 'description': data['description'], + 'view_count': int(data['view_count']), + 'thumbnail': data['thumbnail_url'], + } From a1ef7e85d6834d5e8a9a2171b220a9e3b93dd2cf Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Thu, 5 Dec 2013 14:31:54 +0100 Subject: [PATCH 225/425] Remove unused imports --- youtube_dl/extractor/smotri.py | 1 - youtube_dl/extractor/theplatform.py | 1 - youtube_dl/extractor/viddler.py | 3 --- youtube_dl/extractor/yahoo.py | 2 +- youtube_dl/utils.py | 1 - 5 files changed, 1 insertion(+), 7 deletions(-) diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py index f035a3214..5a28bc820 100644 --- a/youtube_dl/extractor/smotri.py +++ b/youtube_dl/extractor/smotri.py @@ -6,7 +6,6 @@ import hashlib from .common import InfoExtractor from ..utils import ( - determine_ext, ExtractorError ) diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py index 920689511..61452e47d 100644 --- a/youtube_dl/extractor/theplatform.py +++ b/youtube_dl/extractor/theplatform.py @@ -4,7 +4,6 @@ import json from .common import InfoExtractor from ..utils import ( xpath_with_ns, - find_xpath_attr, ) _x = lambda p: xpath_with_ns(p, {'smil': 'http://www.w3.org/2005/SMIL21/Language'}) diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py index 138a35b2a..9328ef4a2 100644 --- a/youtube_dl/extractor/viddler.py +++ b/youtube_dl/extractor/viddler.py @@ -2,9 +2,6 @@ import json import re from .common import InfoExtractor -from ..utils import ( - determine_ext, -) class ViddlerIE(InfoExtractor): diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py index e457c4707..5c9c361b9 100644 --- a/youtube_dl/extractor/yahoo.py +++ b/youtube_dl/extractor/yahoo.py @@ -47,7 +47,7 @@ class YahooIE(InfoExtractor): # The 'meta' field is not always in the video webpage, we request it # from another page long_id = info['id'] - return self._get_info(info['id'], video_id) + return self._get_info(long_id, video_id) def _get_info(self, long_id, video_id): query = ('SELECT * FROM yahoo.media.video.streams WHERE id="%s"' diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index c486ef8ec..77609f7ca 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -17,7 +17,6 @@ import ssl import socket import sys import traceback -import xml.etree.ElementTree import zlib try: From 19e3dfc9f8444a1341a6e71752a3235a0447a565 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Thu, 5 Dec 2013 18:29:07 +0100 Subject: [PATCH 226/425] [9gag] Like/dislike count (#1895) --- youtube_dl/extractor/common.py | 3 +++ youtube_dl/extractor/ninegag.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 1b049082d..92a0c5050 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -55,6 +55,9 @@ class InfoExtractor(object): subtitles: The subtitle file contents as a dictionary in the format {language: subtitles}. view_count: How many users have watched the video on the platform. + like_count: Number of positive ratings of the video + dislike_count: Number of negative ratings of the video + comment_count: Number of comments on the video urlhandle: [internal] The urlHandle to be used to download the file, like returned by urllib.request.urlopen age_limit: Age restriction for the video, as an integer (years) diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py index cc00ffbcc..ea986c00e 100644 --- a/youtube_dl/extractor/ninegag.py +++ b/youtube_dl/extractor/ninegag.py @@ -37,5 +37,7 @@ class NineGagIE(InfoExtractor): 'title': data['title'], 'description': data['description'], 'view_count': int(data['view_count']), + 'like_count': int(data['statistic']['like']), + 'dislike_count': int(data['statistic']['dislike']), 'thumbnail': data['thumbnail_url'], } From 9e6060208430cef6af5e1f6ae24feb65c35fc03c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Thu, 5 Dec 2013 20:45:53 +0100 Subject: [PATCH 227/425] =?UTF-8?q?[francetv]=20Add=20support=20for=20more?= =?UTF-8?q?=20channels:=203,=204,=205=20and=20=C3=94=20(#1898)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename the France2IE extractor to FranceTVIE --- youtube_dl/extractor/__init__.py | 2 +- youtube_dl/extractor/francetv.py | 103 +++++++++++++++++++++++++------ 2 files changed, 85 insertions(+), 20 deletions(-) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 2e1a8be14..a78dcad7f 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -56,7 +56,7 @@ from .flickr import FlickrIE from .francetv import ( PluzzIE, FranceTvInfoIE, - France2IE, + FranceTVIE, GenerationQuoiIE ) from .freesound import FreesoundIE diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py index 66aa3aa0d..290e650e0 100644 --- a/youtube_dl/extractor/francetv.py +++ b/youtube_dl/extractor/francetv.py @@ -21,7 +21,7 @@ class FranceTVBaseInfoExtractor(InfoExtractor): thumbnail_path = info.find('image').text return {'id': video_id, - 'ext': 'mp4', + 'ext': 'flv' if video_url.startswith('rtmp') else 'mp4', 'url': video_url, 'title': info.find('titre').text, 'thumbnail': compat_urlparse.urljoin('http://pluzz.francetv.fr', thumbnail_path), @@ -66,35 +66,100 @@ class FranceTvInfoIE(FranceTVBaseInfoExtractor): return self._extract_video(video_id) -class France2IE(FranceTVBaseInfoExtractor): - IE_NAME = u'france2.fr' - _VALID_URL = r'''(?x)https?://www\.france2\.fr/ +class FranceTVIE(FranceTVBaseInfoExtractor): + IE_NAME = u'francetv' + IE_DESC = u'France 2, 3, 4, 5 and Ô' + _VALID_URL = r'''(?x)https?://www\.france[2345o]\.fr/ (?: - emissions/.*?/videos/(?P<id>\d+) - | emission/(?P<key>[^/?]+) + emissions/.*?/(videos|emissions)/(?P<id>[^/?]+) + | (emission|jt)/(?P<key>[^/?]+) )''' - _TEST = { - u'url': u'http://www.france2.fr/emissions/13h15-le-samedi-le-dimanche/videos/75540104', - u'file': u'75540104.mp4', - u'info_dict': { - u'title': u'13h15, le samedi...', - u'description': u'md5:2e5b58ba7a2d3692b35c792be081a03d', + _TESTS = [ + # france2 + { + u'url': u'http://www.france2.fr/emissions/13h15-le-samedi-le-dimanche/videos/75540104', + u'file': u'75540104.mp4', + u'info_dict': { + u'title': u'13h15, le samedi...', + u'description': u'md5:2e5b58ba7a2d3692b35c792be081a03d', + }, + u'params': { + # m3u8 download + u'skip_download': True, + }, }, - u'params': { - u'skip_download': True, + # france3 + { + u'url': u'http://www.france3.fr/emissions/pieces-a-conviction/videos/rhozet_pac_ba_20131204_1933_03122013164521_F3', + u'info_dict': { + u'id': u'rhozet_pac_ba_20131204_1933_03122013164521_F3', + u'ext': u'flv', + u'title': u'Pièces à conviction du 04/12/2013', + u'description': u'md5:1cf14ea302ba5f10d992c9eb2bff30dd', + }, + u'params': { + # rtmp download + u'skip_download': True, + }, }, - } + # france4 + { + u'url': u'http://www.france4.fr/emissions/hero-corp/videos/rhozet_herocorp_bonus_1_20131106_1923_06112013172108_F4', + u'info_dict': { + u'id': u'rhozet_herocorp_bonus_1_20131106_1923_06112013172108_F4', + u'ext': u'flv', + u'title': u'Hero Corp Making of - Extrait 1', + u'description': u'md5:c87d54871b1790679aec1197e73d650a', + }, + u'params': { + # rtmp download + u'skip_download': True, + }, + }, + # france5 + { + u'url': u'http://www.france5.fr/emissions/c-a-dire/videos/92837968', + u'info_dict': { + u'id': u'92837968', + u'ext': u'mp4', + u'title': u'C à dire ?!', + u'description': u'md5:fb1db1cbad784dcce7c7a7bd177c8e2f', + }, + u'params': { + # m3u8 download + u'skip_download': True, + }, + }, + # franceo + { + u'url': u'http://www.franceo.fr/jt/info-afrique/04-12-2013', + u'info_dict': { + u'id': u'92327925', + u'ext': u'mp4', + u'title': u'Infô-Afrique', + u'description': u'md5:ebf346da789428841bee0fd2a935ea55', + }, + u'params': { + # m3u8 download + u'skip_download': True, + }, + u'skip': u'The id changes frequently', + }, + ] def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) if mobj.group('key'): webpage = self._download_webpage(url, mobj.group('key')) - video_id = self._html_search_regex( - r'''(?x)<div\s+class="video-player">\s* + id_res = [ + (r'''(?x)<div\s+class="video-player">\s* <a\s+href="http://videos.francetv.fr/video/([0-9]+)"\s+ - class="francetv-video-player">''', - webpage, u'video ID') + class="francetv-video-player">'''), + (r'<a id="player_direct" href="http://info\.francetelevisions' + '\.fr/\?id-video=([^"/&]+)'), + ] + video_id = self._html_search_regex(id_res, webpage, u'video ID') else: video_id = mobj.group('id') return self._extract_video(video_id) From 3514813d5b021c5595b212ba3b1801175840c5c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Thu, 5 Dec 2013 21:26:35 +0100 Subject: [PATCH 228/425] [francetv] Add support for urls in the format http://www.france3.fr/emissions/{program}/diffusions/{date} (fixes #1898) --- youtube_dl/extractor/francetv.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py index 290e650e0..ad85bc16d 100644 --- a/youtube_dl/extractor/francetv.py +++ b/youtube_dl/extractor/francetv.py @@ -72,7 +72,7 @@ class FranceTVIE(FranceTVBaseInfoExtractor): _VALID_URL = r'''(?x)https?://www\.france[2345o]\.fr/ (?: emissions/.*?/(videos|emissions)/(?P<id>[^/?]+) - | (emission|jt)/(?P<key>[^/?]+) + | (emissions?|jt)/(?P<key>[^/?]+) )''' _TESTS = [ @@ -91,12 +91,12 @@ class FranceTVIE(FranceTVBaseInfoExtractor): }, # france3 { - u'url': u'http://www.france3.fr/emissions/pieces-a-conviction/videos/rhozet_pac_ba_20131204_1933_03122013164521_F3', + u'url': u'http://www.france3.fr/emissions/pieces-a-conviction/diffusions/13-11-2013_145575', u'info_dict': { - u'id': u'rhozet_pac_ba_20131204_1933_03122013164521_F3', + u'id': u'000702326_CAPP_PicesconvictionExtrait313022013_120220131722_Au', u'ext': u'flv', - u'title': u'Pièces à conviction du 04/12/2013', - u'description': u'md5:1cf14ea302ba5f10d992c9eb2bff30dd', + u'title': u'Le scandale du prix des médicaments', + u'description': u'md5:1384089fbee2f04fc6c9de025ee2e9ce', }, u'params': { # rtmp download @@ -158,6 +158,7 @@ class FranceTVIE(FranceTVBaseInfoExtractor): class="francetv-video-player">'''), (r'<a id="player_direct" href="http://info\.francetelevisions' '\.fr/\?id-video=([^"/&]+)'), + (r'<a class="video" id="ftv_player_(.+?)"'), ] video_id = self._html_search_regex(id_res, webpage, u'video ID') else: From ef4fd848573b601502ba9142d5ce521294024356 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 6 Dec 2013 09:15:04 +0100 Subject: [PATCH 229/425] [wistia] Add extractor --- youtube_dl/YoutubeDL.py | 3 +- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/generic.py | 31 ++++++++++++------ youtube_dl/extractor/wistia.py | 55 ++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 10 deletions(-) create mode 100644 youtube_dl/extractor/wistia.py diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 8ad7bd1da..07b36a98e 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -488,7 +488,8 @@ class YoutubeDL(object): new_result = ie_result.copy() for f in ('_type', 'url', 'ext', 'player_url', 'formats', 'entries', 'urlhandle', 'ie_key', 'duration', - 'subtitles', 'annotations', 'format'): + 'subtitles', 'annotations', 'format', + 'thumbnail', 'thumbnails'): if f in new_result: del new_result[f] if f in embedded_info: diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index a78dcad7f..a7d37d48b 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -178,6 +178,7 @@ from .wat import WatIE from .websurg import WeBSurgIE from .weibo import WeiboIE from .wimp import WimpIE +from .wistia import WistiaIE from .worldstarhiphop import WorldStarHipHopIE from .xhamster import XHamsterIE from .xnxx import XNXXIE diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 10ae06263..216e03218 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -169,8 +169,13 @@ class GenericIE(InfoExtractor): # Site Name | Video Title # Video Title - Tagline | Site Name # and so on and so forth; it's just not practical - video_title = self._html_search_regex(r'<title>(.*)', - webpage, u'video title', default=u'video', flags=re.DOTALL) + video_title = self._html_search_regex( + r'(?s)(.*?)', webpage, u'video title', + default=u'video') + + # video uploader is domain name + video_uploader = self._search_regex( + r'^(?:https?://)?([^/]*)/.*', url, u'video uploader') # Look for BrightCove: bc_url = BrightcoveIE._extract_brightcove_url(webpage) @@ -188,7 +193,7 @@ class GenericIE(InfoExtractor): # Look for embedded YouTube player matches = re.findall( - r']+?src=(["\'])(?P(?:https?:)?//(?:www\.)?youtube.com/embed/.+?)\1', webpage) + r']+?src=(["\'])(?P(?:https?:)?//(?:www\.)?youtube\.com/embed/.+?)\1', webpage) if matches: urlrs = [self.url_result(unescapeHTML(tuppl[1]), 'Youtube') for tuppl in matches] @@ -197,13 +202,26 @@ class GenericIE(InfoExtractor): # Look for embedded Dailymotion player matches = re.findall( - r']+?src=(["\'])(?P(?:https?:)?//(?:www\.)?dailymotion.com/embed/video/.+?)\1', webpage) + r']+?src=(["\'])(?P(?:https?:)?//(?:www\.)?dailymotion\.com/embed/video/.+?)\1', webpage) if matches: urlrs = [self.url_result(unescapeHTML(tuppl[1]), 'Dailymotion') for tuppl in matches] return self.playlist_result( urlrs, playlist_id=video_id, playlist_title=video_title) + # Look for embedded Wistia player + match = re.search( + r']+?src=(["\'])(?P(?:https?:)?//(?:fast\.)?wistia\.net/embed/iframe/.+?)\1', webpage) + if match: + return { + '_type': 'url_transparent', + 'url': unescapeHTML(match.group('url')), + 'ie_key': 'Wistia', + 'uploader': video_uploader, + 'title': video_title, + 'id': video_id, + } + # Look for Bandcamp pages with custom domain mobj = re.search(r']*?content="(.*?bandcamp\.com.*?)"', webpage) if mobj is not None: @@ -247,14 +265,9 @@ class GenericIE(InfoExtractor): # here's a fun little line of code for you: video_id = os.path.splitext(video_id)[0] - # video uploader is domain name - video_uploader = self._search_regex(r'(?:https?://)?([^/]*)/.*', - url, u'video uploader') - return { 'id': video_id, 'url': video_url, 'uploader': video_uploader, - 'upload_date': None, 'title': video_title, } diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py new file mode 100644 index 000000000..e1748c261 --- /dev/null +++ b/youtube_dl/extractor/wistia.py @@ -0,0 +1,55 @@ +import json +import re + +from .common import InfoExtractor + + +class WistiaIE(InfoExtractor): + _VALID_URL = r'^https?://(?:fast\.)?wistia\.net/embed/iframe/(?P[a-z0-9]+)' + + _TEST = { + u"url": u"http://fast.wistia.net/embed/iframe/sh7fpupwlt", + u"file": u"sh7fpupwlt.mov", + u"md5": u"cafeb56ec0c53c18c97405eecb3133df", + u"info_dict": { + u"title": u"cfh_resourceful_zdkh_final_1" + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + data_json = self._html_search_regex( + r'Wistia.iframeInit\((.*?), {}\);', webpage, u'video data') + + data = json.loads(data_json) + + formats = [] + thumbnails = [] + for atype, a in data['assets'].items(): + if atype == 'still': + thumbnails.append({ + 'url': a['url'], + 'resolution': '%dx%d' % (a['width'], a['height']), + }) + continue + if atype == 'preview': + continue + formats.append({ + 'format_id': atype, + 'url': a['url'], + 'width': a['width'], + 'height': a['height'], + 'filesize': a['size'], + 'ext': a['ext'], + }) + formats.sort(key=lambda a: a['filesize']) + + return { + 'id': video_id, + 'title': data['name'], + 'formats': formats, + 'thumbnails': thumbnails, + } From 4e761794760ff5b281205838bf8a02ea496b89b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 6 Dec 2013 13:03:08 +0100 Subject: [PATCH 230/425] [vimeo] Extract views count, likes count and comments count (#1895) --- youtube_dl/extractor/vimeo.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index f27763ae2..ac956e673 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -196,6 +196,16 @@ class VimeoIE(InfoExtractor): if mobj is not None: video_upload_date = mobj.group(1) + mobj.group(2) + mobj.group(3) + try: + view_count = int(self._search_regex(r'UserPlays:(\d+)', webpage, u'view count')) + like_count = int(self._search_regex(r'UserLikes:(\d+)', webpage, u'like count')) + comment_count = int(self._search_regex(r'UserComments:(\d+)', webpage, u'comment count')) + except RegexNotFoundError: + # This info is only available in vimeo.com/{id} urls + view_count = None + like_count = None + comment_count = None + # Vimeo specific: extract request signature and timestamp sig = config['request']['signature'] timestamp = config['request']['timestamp'] @@ -242,6 +252,9 @@ class VimeoIE(InfoExtractor): 'description': video_description, 'formats': formats, 'webpage_url': url, + 'view_count': view_count, + 'like_count': like_count, + 'comment_count': comment_count, } From 336c3a69bd198130e2f65f14dfc83383fec7c5e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 6 Dec 2013 13:22:04 +0100 Subject: [PATCH 231/425] [youtube] Extract like and dislike count (#1895) --- youtube_dl/extractor/youtube.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 7fff761bd..52c8e7d04 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -388,10 +388,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): super(YoutubeIE, self).__init__(*args, **kwargs) self._player_cache = {} - def report_video_webpage_download(self, video_id): - """Report attempt to download video webpage.""" - self.to_screen(u'%s: Downloading video webpage' % video_id) - def report_video_info_webpage_download(self, video_id): """Report attempt to download video info webpage.""" self.to_screen(u'%s: Downloading video info webpage' % video_id) @@ -1258,15 +1254,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): video_id = self._extract_id(url) # Get video webpage - self.report_video_webpage_download(video_id) url = 'https://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id - request = compat_urllib_request.Request(url) - try: - video_webpage_bytes = compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to download video webpage: %s' % compat_str(err)) - - video_webpage = video_webpage_bytes.decode('utf-8', 'ignore') + video_webpage = self._download_webpage(url, video_id) # Attempt to extract SWF player URL mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage) @@ -1383,6 +1372,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): else: video_description = u'' + def _extract_count(klass): + count = self._search_regex(r'class="%s">([\d,]+)' % re.escape(klass), video_webpage, klass, fatal=False) + if count is not None: + return int(count.replace(',', '')) + return None + like_count = _extract_count(u'likes-count') + dislike_count = _extract_count(u'dislikes-count') + # subtitles video_subtitles = self.extract_subtitles(video_id, video_webpage) @@ -1515,6 +1512,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): 'annotations': video_annotations, 'webpage_url': 'https://www.youtube.com/watch?v=%s' % video_id, 'view_count': view_count, + 'like_count': like_count, + 'dislike_count': dislike_count, }) return results From f53c966a73df42a9a949912ef8ab99a64fb99466 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 6 Dec 2013 13:36:36 +0100 Subject: [PATCH 232/425] [dailymotion] Extract view count (#1895) --- test/test_utils.py | 5 +++++ youtube_dl/extractor/dailymotion.py | 5 +++++ youtube_dl/utils.py | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/test/test_utils.py b/test/test_utils.py index e9e590e74..0fa66beec 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -26,6 +26,7 @@ from youtube_dl.utils import ( unsmuggle_url, shell_quote, encodeFilename, + str_to_int, ) if sys.version_info < (3, 0): @@ -176,6 +177,10 @@ class TestUtil(unittest.TestCase): args = ['ffmpeg', '-i', encodeFilename(u'ñ€ß\'.mp4')] self.assertEqual(shell_quote(args), u"""ffmpeg -i 'ñ€ß'"'"'.mp4'""") + def test_str_to_int(self): + self.assertEqual(str_to_int('123,456'), 123456) + self.assertEqual(str_to_int('123.456'), 123456) + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py index 71f5e03ee..3756cf765 100644 --- a/youtube_dl/extractor/dailymotion.py +++ b/youtube_dl/extractor/dailymotion.py @@ -11,6 +11,7 @@ from ..utils import ( get_element_by_attribute, get_element_by_id, orderedSet, + str_to_int, ExtractorError, ) @@ -146,6 +147,9 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor): self._list_available_subtitles(video_id, webpage) return + view_count = str_to_int(self._search_regex( + r'video_views_value[^>]+>([\d\.]+)<', webpage, u'view count')) + return { 'id': video_id, 'formats': formats, @@ -155,6 +159,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor): 'subtitles': video_subtitles, 'thumbnail': info['thumbnail_url'], 'age_limit': age_limit, + 'view_count': view_count, } def _get_available_subtitles(self, video_id, webpage): diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 77609f7ca..7b5878830 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1020,3 +1020,7 @@ def format_bytes(bytes): suffix = [u'B', u'KiB', u'MiB', u'GiB', u'TiB', u'PiB', u'EiB', u'ZiB', u'YiB'][exponent] converted = float(bytes) / float(1024 ** exponent) return u'%.2f%s' % (converted, suffix) + +def str_to_int(int_str): + int_str = re.sub(r'[,\.]', u'', int_str) + return int(int_str) From 563e405411131628a6ea160c3fe2b2b4a883ac85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 6 Dec 2013 13:41:07 +0100 Subject: [PATCH 233/425] [dailymotion] Fix view count regex In some languages they can be in the format '123,456' instead of '123.456' --- youtube_dl/extractor/dailymotion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py index 3756cf765..3bd0b862c 100644 --- a/youtube_dl/extractor/dailymotion.py +++ b/youtube_dl/extractor/dailymotion.py @@ -148,7 +148,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor): return view_count = str_to_int(self._search_regex( - r'video_views_value[^>]+>([\d\.]+)<', webpage, u'view count')) + r'video_views_value[^>]+>([\d\.,]+)<', webpage, u'view count')) return { 'id': video_id, From 7d4afc557f88a05f4f45618c07443aee5aa2099e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 6 Dec 2013 19:48:54 +0100 Subject: [PATCH 234/425] [youtube:playlist] Support mix ids longer than 13 (#1295) --- youtube_dl/extractor/youtube.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 52c8e7d04..91f8028ff 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1553,7 +1553,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor): def _extract_mix(self, playlist_id): # The mixes are generated from a a single video # the id of the playlist is just 'RD' + video_id - url = 'https://youtube.com/watch?v=%s&list=%s' % (playlist_id[2:], playlist_id) + url = 'https://youtube.com/watch?v=%s&list=%s' % (playlist_id[-11:], playlist_id) webpage = self._download_webpage(url, playlist_id, u'Downloading Youtube mix') title_span = (get_element_by_attribute('class', 'title long-title', webpage) or get_element_by_attribute('class', 'title ', webpage)) @@ -1581,7 +1581,7 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor): else: self.to_screen(u'Downloading playlist PL%s - add --no-playlist to just download video %s' % (playlist_id, video_id)) - if len(playlist_id) == 13: # 'RD' + 11 characters for the video id + if playlist_id.startswith('RD'): # Mixes require a custom extraction process return self._extract_mix(playlist_id) From 715c8e7bdb219f30f83c7d76cbbbc77195366cb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 6 Dec 2013 19:52:41 +0100 Subject: [PATCH 235/425] [youtube:playlist] Recognize mix ids for direct use (fixes #1295) --- youtube_dl/extractor/youtube.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 91f8028ff..01715024c 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1528,10 +1528,10 @@ class YoutubePlaylistIE(YoutubeBaseInfoExtractor): \? (?:.*?&)*? (?:p|a|list)= | p/ ) - ((?:PL|EC|UU|FL)?[0-9A-Za-z-_]{10,}) + ((?:PL|EC|UU|FL|RD)?[0-9A-Za-z-_]{10,}) .* | - ((?:PL|EC|UU|FL)[0-9A-Za-z-_]{10,}) + ((?:PL|EC|UU|FL|RD)[0-9A-Za-z-_]{10,}) )""" _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s&page=%s' _MORE_PAGES_INDICATOR = r'data-link-type="next"' From 0b6a9f639f6447c7e09c38b88b42964e8fa05349 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 6 Dec 2013 20:14:29 +0100 Subject: [PATCH 236/425] [vevo] Update test video's duration --- youtube_dl/extractor/vevo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index 3eedcf7dd..4823992ef 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -24,7 +24,7 @@ class VevoIE(InfoExtractor): u"upload_date": u"20130624", u"uploader": u"Hurts", u"title": u"Somebody to Die For", - u"duration": 230, + u"duration": 230.12, u"width": 1920, u"height": 1080, } From d349cd22401648e88d57b6dcdd0c8bbb12aaa0d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 6 Dec 2013 20:26:55 +0100 Subject: [PATCH 237/425] [imdb] Fix extraction The paths to each format's page may have leading whitespace. The height and the duration can't be extracted. --- youtube_dl/extractor/imdb.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py index d8e9712a7..6fb373db2 100644 --- a/youtube_dl/extractor/imdb.py +++ b/youtube_dl/extractor/imdb.py @@ -21,7 +21,6 @@ class ImdbIE(InfoExtractor): u'ext': u'mp4', u'title': u'Ice Age: Continental Drift Trailer (No. 2) - IMDb', u'description': u'md5:9061c2219254e5d14e03c25c98e96a81', - u'duration': 151, } } @@ -35,6 +34,7 @@ class ImdbIE(InfoExtractor): flags=re.MULTILINE) formats = [] for f_id, f_path in available_formats: + f_path = f_path.strip() format_page = self._download_webpage( compat_urlparse.urljoin(url, f_path), u'Downloading info for %s format' % f_id) @@ -46,7 +46,6 @@ class ImdbIE(InfoExtractor): formats.append({ 'format_id': f_id, 'url': format_info['url'], - 'height': int(info['titleObject']['encoding']['selected'][:-1]), }) return { @@ -55,5 +54,4 @@ class ImdbIE(InfoExtractor): 'formats': formats, 'description': descr, 'thumbnail': format_info['slate'], - 'duration': int(info['titleObject']['title']['duration_seconds']), } From 5cc14c2fd74a721be0effd5bc06a76164a9c97a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 6 Dec 2013 21:47:32 +0100 Subject: [PATCH 238/425] [vimeo] Add an extractor for albums (closes #1911) --- test/test_playlists.py | 9 +++++++++ youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/vimeo.py | 23 ++++++++++++++++++++--- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/test/test_playlists.py b/test/test_playlists.py index 00c950109..6a5e0b780 100644 --- a/test/test_playlists.py +++ b/test/test_playlists.py @@ -16,6 +16,7 @@ from youtube_dl.extractor import ( DailymotionUserIE, VimeoChannelIE, VimeoUserIE, + VimeoAlbumIE, UstreamChannelIE, SoundcloudSetIE, SoundcloudUserIE, @@ -65,6 +66,14 @@ class TestPlaylists(unittest.TestCase): self.assertEqual(result['title'], u'Nki') self.assertTrue(len(result['entries']) > 65) + def test_vimeo_album(self): + dl = FakeYDL() + ie = VimeoAlbumIE(dl) + result = ie.extract('http://vimeo.com/album/2632481') + self.assertIsPlaylist(result) + self.assertEqual(result['title'], u'Staff Favorites: November 2013') + self.assertTrue(len(result['entries']) > 12) + def test_ustream_channel(self): dl = FakeYDL() ie = UstreamChannelIE(dl) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index a7d37d48b..ac0a11dfe 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -170,6 +170,7 @@ from .vimeo import ( VimeoIE, VimeoChannelIE, VimeoUserIE, + VimeoAlbumIE, ) from .vine import VineIE from .viki import VikiIE diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index ac956e673..293dad3c0 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -20,7 +20,7 @@ class VimeoIE(InfoExtractor): """Information extractor for vimeo.com.""" # _VALID_URL matches Vimeo URLs - _VALID_URL = r'(?Phttps?://)?(?:(?:www|(?Pplayer))\.)?vimeo(?Ppro)?\.com/(?:(?:(?:groups|album)/[^/]+)|(?:.*?)/)?(?Pplay_redirect_hls\?clip_id=)?(?:videos?/)?(?P[0-9]+)/?(?:[?].*)?(?:#.*)?$' + _VALID_URL = r'(?Phttps?://)?(?:(?:www|(?Pplayer))\.)?vimeo(?Ppro)?\.com/(?:(?:(?:groups)/[^/]+)|(?:.*?)/)?(?Pplay_redirect_hls\?clip_id=)?(?:videos?/)?(?P[0-9]+)/?(?:[?].*)?(?:#.*)?$' _NETRC_MACHINE = 'vimeo' IE_NAME = u'vimeo' _TESTS = [ @@ -264,11 +264,14 @@ class VimeoChannelIE(InfoExtractor): _MORE_PAGES_INDICATOR = r']+?title="(.*?)"' + def _page_url(self, base_url, pagenum): + return '%s/videos/page:%d/' % (base_url, pagenum) + def _extract_videos(self, list_id, base_url): video_ids = [] for pagenum in itertools.count(1): webpage = self._download_webpage( - '%s/videos/page:%d/' % (base_url, pagenum),list_id, + self._page_url(base_url, pagenum) ,list_id, u'Downloading page %s' % pagenum) video_ids.extend(re.findall(r'id="clip_(\d+?)"', webpage)) if re.search(self._MORE_PAGES_INDICATOR, webpage, re.DOTALL) is None: @@ -297,7 +300,7 @@ class VimeoUserIE(VimeoChannelIE): @classmethod def suitable(cls, url): - if VimeoChannelIE.suitable(url) or VimeoIE.suitable(url): + if VimeoChannelIE.suitable(url) or VimeoIE.suitable(url) or VimeoAlbumIE.suitable(url): return False return super(VimeoUserIE, cls).suitable(url) @@ -305,3 +308,17 @@ class VimeoUserIE(VimeoChannelIE): mobj = re.match(self._VALID_URL, url) name = mobj.group('name') return self._extract_videos(name, 'http://vimeo.com/%s' % name) + + +class VimeoAlbumIE(VimeoChannelIE): + IE_NAME = u'vimeo:album' + _VALID_URL = r'(?:https?://)?vimeo.\com/album/(?P\d+)' + _TITLE_RE = r'
\n\s*

(.*?)

' + + def _page_url(self, base_url, pagenum): + return '%s/page:%d/' % (base_url, pagenum) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + album_id = mobj.group('id') + return self._extract_videos(album_id, 'http://vimeo.com/album/%s' % album_id) From fb30ec22fd25e2dfc57ecf821d3072c2d8ab3ce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Fri, 6 Dec 2013 22:01:41 +0100 Subject: [PATCH 239/425] [vimeo] Add an extractor for groups --- test/test_playlists.py | 9 +++++++++ youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/vimeo.py | 24 +++++++++++++++++++----- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/test/test_playlists.py b/test/test_playlists.py index 6a5e0b780..87ca401e5 100644 --- a/test/test_playlists.py +++ b/test/test_playlists.py @@ -17,6 +17,7 @@ from youtube_dl.extractor import ( VimeoChannelIE, VimeoUserIE, VimeoAlbumIE, + VimeoGroupsIE, UstreamChannelIE, SoundcloudSetIE, SoundcloudUserIE, @@ -74,6 +75,14 @@ class TestPlaylists(unittest.TestCase): self.assertEqual(result['title'], u'Staff Favorites: November 2013') self.assertTrue(len(result['entries']) > 12) + def test_vimeo_groups(self): + dl = FakeYDL() + ie = VimeoGroupsIE(dl) + result = ie.extract('http://vimeo.com/groups/rolexawards') + self.assertIsPlaylist(result) + self.assertEqual(result['title'], u'Rolex Awards for Enterprise') + self.assertTrue(len(result['entries']) > 72) + def test_ustream_channel(self): dl = FakeYDL() ie = UstreamChannelIE(dl) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index ac0a11dfe..f6a23f663 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -171,6 +171,7 @@ from .vimeo import ( VimeoChannelIE, VimeoUserIE, VimeoAlbumIE, + VimeoGroupsIE, ) from .vine import VineIE from .viki import VikiIE diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index 293dad3c0..fb2bd225a 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -20,7 +20,7 @@ class VimeoIE(InfoExtractor): """Information extractor for vimeo.com.""" # _VALID_URL matches Vimeo URLs - _VALID_URL = r'(?Phttps?://)?(?:(?:www|(?Pplayer))\.)?vimeo(?Ppro)?\.com/(?:(?:(?:groups)/[^/]+)|(?:.*?)/)?(?Pplay_redirect_hls\?clip_id=)?(?:videos?/)?(?P[0-9]+)/?(?:[?].*)?(?:#.*)?$' + _VALID_URL = r'(?Phttps?://)?(?:(?:www|(?Pplayer))\.)?vimeo(?Ppro)?\.com/(?:.*?/)?(?Pplay_redirect_hls\?clip_id=)?(?:videos?/)?(?P[0-9]+)/?(?:[?].*)?(?:#.*)?$' _NETRC_MACHINE = 'vimeo' IE_NAME = u'vimeo' _TESTS = [ @@ -267,6 +267,9 @@ class VimeoChannelIE(InfoExtractor): def _page_url(self, base_url, pagenum): return '%s/videos/page:%d/' % (base_url, pagenum) + def _extract_list_title(self, webpage): + return self._html_search_regex(self._TITLE_RE, webpage, u'list title') + def _extract_videos(self, list_id, base_url): video_ids = [] for pagenum in itertools.count(1): @@ -279,11 +282,9 @@ class VimeoChannelIE(InfoExtractor): entries = [self.url_result('http://vimeo.com/%s' % video_id, 'Vimeo') for video_id in video_ids] - list_title = self._html_search_regex(self._TITLE_RE, webpage, - u'list title') return {'_type': 'playlist', 'id': list_id, - 'title': list_title, + 'title': self._extract_list_title(webpage), 'entries': entries, } @@ -300,7 +301,7 @@ class VimeoUserIE(VimeoChannelIE): @classmethod def suitable(cls, url): - if VimeoChannelIE.suitable(url) or VimeoIE.suitable(url) or VimeoAlbumIE.suitable(url): + if VimeoChannelIE.suitable(url) or VimeoIE.suitable(url) or VimeoAlbumIE.suitable(url) or VimeoGroupsIE.suitable(url): return False return super(VimeoUserIE, cls).suitable(url) @@ -322,3 +323,16 @@ class VimeoAlbumIE(VimeoChannelIE): mobj = re.match(self._VALID_URL, url) album_id = mobj.group('id') return self._extract_videos(album_id, 'http://vimeo.com/album/%s' % album_id) + + +class VimeoGroupsIE(VimeoAlbumIE): + IE_NAME = u'vimeo:group' + _VALID_URL = r'(?:https?://)?vimeo.\com/groups/(?P[^/]+)' + + def _extract_list_title(self, webpage): + return self._og_search_title(webpage) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + name = mobj.group('name') + return self._extract_videos(name, 'http://vimeo.com/groups/%s' % name) From ca9e02dc00fcfbc86fdf5d645417430ecb135537 Mon Sep 17 00:00:00 2001 From: Adam Glenn Date: Fri, 6 Dec 2013 21:11:01 -0800 Subject: [PATCH 240/425] Adding pyvideo support --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/pyvideo.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 youtube_dl/extractor/pyvideo.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index f6a23f663..d7cb6e463 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -111,6 +111,7 @@ from .photobucket import PhotobucketIE from .podomatic import PodomaticIE from .pornhub import PornHubIE from .pornotube import PornotubeIE +from .pyvideo import PyvideoIE from .rbmaradio import RBMARadioIE from .redtube import RedTubeIE from .ringtv import RingTVIE diff --git a/youtube_dl/extractor/pyvideo.py b/youtube_dl/extractor/pyvideo.py new file mode 100644 index 000000000..243dff806 --- /dev/null +++ b/youtube_dl/extractor/pyvideo.py @@ -0,0 +1,26 @@ +import re + +from .common import InfoExtractor +from ..utils import determine_ext + + +class PyvideoIE(InfoExtractor): + _VALID_URL = r'(?:http://)?(?:www\.)?break\.com/video/([^/]+)' + _VALID_URL = r'(?:http://)?(?:www\.)?pyvideo\.org/video/(\d+)/(.*)' + _TEST = { + u'url': u'http://pyvideo.org/video/1737/become-a-logging-expert-in-30-minutes', + u'file': u'Become a logging expert in 30 minutes-24_4WWkSmNo.mp4', + u'md5': u'bf08cae24e1601027f98ae1262c299ad', + u'info_dict': { + u"title": u"Become a logging expert in 30 minutes" + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group(2) + webpage = self._download_webpage(url, video_id) + m_youtube = re.search(r'(https?://www\.youtube\.com/watch\?v=.*)', webpage) + + if m_youtube is not None: + return self.url_result(m_youtube.group(1), 'Youtube') From f623530d6e1ac63e0e2056274cb9b59b8699ce0c Mon Sep 17 00:00:00 2001 From: Adam Glenn Date: Fri, 6 Dec 2013 21:12:10 -0800 Subject: [PATCH 241/425] removing bad VALID_URL --- youtube_dl/extractor/pyvideo.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/extractor/pyvideo.py b/youtube_dl/extractor/pyvideo.py index 243dff806..5c47993e7 100644 --- a/youtube_dl/extractor/pyvideo.py +++ b/youtube_dl/extractor/pyvideo.py @@ -5,7 +5,6 @@ from ..utils import determine_ext class PyvideoIE(InfoExtractor): - _VALID_URL = r'(?:http://)?(?:www\.)?break\.com/video/([^/]+)' _VALID_URL = r'(?:http://)?(?:www\.)?pyvideo\.org/video/(\d+)/(.*)' _TEST = { u'url': u'http://pyvideo.org/video/1737/become-a-logging-expert-in-30-minutes', From 0d9ec5d9630afa298f8b9acdfae0d85fbaa3fe58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sat, 7 Dec 2013 10:59:18 +0100 Subject: [PATCH 242/425] [pyvideo] Cleanup and fix test --- youtube_dl/extractor/pyvideo.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/pyvideo.py b/youtube_dl/extractor/pyvideo.py index 5c47993e7..e28ca3fb6 100644 --- a/youtube_dl/extractor/pyvideo.py +++ b/youtube_dl/extractor/pyvideo.py @@ -1,18 +1,22 @@ import re from .common import InfoExtractor -from ..utils import determine_ext class PyvideoIE(InfoExtractor): _VALID_URL = r'(?:http://)?(?:www\.)?pyvideo\.org/video/(\d+)/(.*)' _TEST = { u'url': u'http://pyvideo.org/video/1737/become-a-logging-expert-in-30-minutes', - u'file': u'Become a logging expert in 30 minutes-24_4WWkSmNo.mp4', - u'md5': u'bf08cae24e1601027f98ae1262c299ad', + u'file': u'24_4WWkSmNo.mp4', + u'md5': u'de317418c8bc76b1fd8633e4f32acbc6', u'info_dict': { - u"title": u"Become a logging expert in 30 minutes" - } + u"title": u"Become a logging expert in 30 minutes", + u"description": u"md5:9665350d466c67fb5b1598de379021f7", + u"upload_date": u"20130320", + u"uploader": u"NextDayVideo", + u"uploader_id": u"NextDayVideo", + }, + u'add_ie': ['Youtube'], } def _real_extract(self, url): From c4d9e6731a80cc1ce173d444920d49b424c62887 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sat, 7 Dec 2013 11:19:59 +0100 Subject: [PATCH 243/425] [pyvideo] add support for videos that don't come from Youtube --- youtube_dl/extractor/pyvideo.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/pyvideo.py b/youtube_dl/extractor/pyvideo.py index e28ca3fb6..33054591b 100644 --- a/youtube_dl/extractor/pyvideo.py +++ b/youtube_dl/extractor/pyvideo.py @@ -1,11 +1,12 @@ import re +import os from .common import InfoExtractor class PyvideoIE(InfoExtractor): - _VALID_URL = r'(?:http://)?(?:www\.)?pyvideo\.org/video/(\d+)/(.*)' - _TEST = { + _VALID_URL = r'(?:http://)?(?:www\.)?pyvideo\.org/video/(?P\d+)/(.*)' + _TESTS = [{ u'url': u'http://pyvideo.org/video/1737/become-a-logging-expert-in-30-minutes', u'file': u'24_4WWkSmNo.mp4', u'md5': u'de317418c8bc76b1fd8633e4f32acbc6', @@ -17,13 +18,34 @@ class PyvideoIE(InfoExtractor): u"uploader_id": u"NextDayVideo", }, u'add_ie': ['Youtube'], - } + }, + { + u'url': u'http://pyvideo.org/video/2542/gloriajw-spotifywitherikbernhardsson182m4v', + u'md5': u'5fe1c7e0a8aa5570330784c847ff6d12', + u'info_dict': { + u'id': u'2542', + u'ext': u'm4v', + u'title': u'Gloriajw-SpotifyWithErikBernhardsson182', + }, + }, + ] def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) - video_id = mobj.group(2) + video_id = mobj.group('id') webpage = self._download_webpage(url, video_id) m_youtube = re.search(r'(https?://www\.youtube\.com/watch\?v=.*)', webpage) if m_youtube is not None: return self.url_result(m_youtube.group(1), 'Youtube') + + title = self._html_search_regex(r'
.*?

([^>]+?)

', + webpage, u'title', flags=re.DOTALL) + video_url = self._search_regex([r'Download.*? Date: Sat, 7 Dec 2013 21:39:32 +0100 Subject: [PATCH 244/425] [XHamsterIE] Fix HD video detection --- youtube_dl/extractor/xhamster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index 279f75e7a..0dd5e805c 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -46,7 +46,7 @@ class XHamsterIE(InfoExtractor): return mobj.group('server')+'/key='+mobj.group('file') def is_hd(webpage): - return webpage.find('
') != -1 + return webpage.find('
Date: Sat, 7 Dec 2013 22:17:13 +0100 Subject: [PATCH 245/425] [XHamsterIE] Fix md5 in second test --- youtube_dl/extractor/xhamster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index 0dd5e805c..ef9997ee4 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -26,7 +26,7 @@ class XHamsterIE(InfoExtractor): { u'url': u'http://xhamster.com/movies/2221348/britney_spears_sexy_booty.html?hd', u'file': u'2221348.flv', - u'md5': u'970a94178ca4118c5aa3aaea21211b81', + u'md5': u'e767b9475de189320f691f49c679c4c7', u'info_dict': { u"upload_date": u"20130914", u"uploader_id": u"jojo747400", From a213880aaffb0c236cb0b2141f167bca307f2628 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 8 Dec 2013 05:49:35 +0100 Subject: [PATCH 246/425] Simplify status reporting (#1918) --- youtube_dl/FileDownloader.py | 39 ++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 3ff9716b3..ac6a6d8a0 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -204,11 +204,19 @@ class FileDownloader(object): """Report destination filename.""" self.to_screen(u'[download] Destination: ' + filename) + def _report_progress_status(self, msg, is_last_line=False): + clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'') + if self.params.get('progress_with_newline', False): + self.to_screen(u'[download] ' + msg) + else: + self.to_screen(u'\r%s[download] %s' % (clear_line, msg), + skip_eol=not is_last_line) + self.to_console_title(u'youtube-dl ' + msg) + def report_progress(self, percent, data_len_str, speed, eta): """Report download progress.""" if self.params.get('noprogress', False): return - clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'') if eta is not None: eta_str = self.format_eta(eta) else: @@ -218,14 +226,20 @@ class FileDownloader(object): else: percent_str = 'Unknown %' speed_str = self.format_speed(speed) - if self.params.get('progress_with_newline', False): - self.to_screen(u'[download] %s of %s at %s ETA %s' % - (percent_str, data_len_str, speed_str, eta_str)) + + msg = (u'%s of %s at %s ETA %s' % + (percent_str, data_len_str, speed_str, eta_str)) + self._report_progress_status(msg) + + def report_finish(self, data_len_str, tot_time): + """Report download finished.""" + if self.params.get('noprogress', False): + self.to_screen(u'[download] Download completed') else: - self.to_screen(u'\r%s[download] %s of %s at %s ETA %s' % - (clear_line, percent_str, data_len_str, speed_str, eta_str), skip_eol=True) - self.to_console_title(u'youtube-dl - %s of %s at %s ETA %s' % - (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip())) + self._report_progress_status( + (u'100%% of %s in %s' % + (data_len_str, self.format_seconds(tot_time))), + is_last_line=True) def report_resuming_byte(self, resume_len): """Report attempt to resume at given byte.""" @@ -246,15 +260,6 @@ class FileDownloader(object): """Report it was impossible to resume download.""" self.to_screen(u'[download] Unable to resume') - def report_finish(self, data_len_str, tot_time): - """Report download finished.""" - if self.params.get('noprogress', False): - self.to_screen(u'[download] Download completed') - else: - clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'') - self.to_screen(u'\r%s[download] 100%% of %s in %s' % - (clear_line, data_len_str, self.format_seconds(tot_time))) - def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url, live): def run_rtmpdump(args): start = time.time() From 4c5216064657b267de378a84dd8ea4989934f25c Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 8 Dec 2013 06:53:46 +0100 Subject: [PATCH 247/425] [FileDownloader] Fix progress report on Windows (Fixes #1918) --- youtube_dl/FileDownloader.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index ac6a6d8a0..8f9577512 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -205,12 +205,20 @@ class FileDownloader(object): self.to_screen(u'[download] Destination: ' + filename) def _report_progress_status(self, msg, is_last_line=False): - clear_line = (u'\x1b[K' if sys.stderr.isatty() and os.name != 'nt' else u'') + fullmsg = u'[download] ' + msg if self.params.get('progress_with_newline', False): - self.to_screen(u'[download] ' + msg) + self.to_screen(fullmsg) else: - self.to_screen(u'\r%s[download] %s' % (clear_line, msg), - skip_eol=not is_last_line) + if os.name == 'nt': + prev_len = getattr(self, '_report_progress_prev_line_length', + 0) + if prev_len > len(fullmsg): + fullmsg += u' ' * (prev_len - len(fullmsg)) + self._report_progress_prev_line_length = len(fullmsg) + clear_line = u'\r' + else: + clear_line = (u'\r\x1b[K' if sys.stderr.isatty() else u'\r') + self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line) self.to_console_title(u'youtube-dl ' + msg) def report_progress(self, percent, data_len_str, speed, eta): From 303b479e0a4b7db5b8e30be6ff3ffc1ea267e87c Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 8 Dec 2013 06:54:39 +0100 Subject: [PATCH 248/425] Automatically load SSL certs on Windows --- youtube_dl/utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 7b5878830..a84aa59c2 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -561,11 +561,14 @@ def make_HTTPS_handler(opts_no_check_certificate): return HTTPSHandlerV3() else: context = ssl.SSLContext(ssl.PROTOCOL_SSLv3) - context.set_default_verify_paths() - context.verify_mode = (ssl.CERT_NONE if opts_no_check_certificate else ssl.CERT_REQUIRED) + context.set_default_verify_paths() + try: + context.load_default_certs() + except AttributeError: + pass # Python < 3.4 return compat_urllib_request.HTTPSHandler(context=context) class ExtractorError(Exception): From d4df5ed14c1efb41856224c963307d5aace0b0cf Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 8 Dec 2013 06:54:52 +0100 Subject: [PATCH 249/425] release 2013.12.08 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 68b30bfd4..794611810 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.04' +__version__ = '2013.12.08' From 11bf848191e6a2859f2e53dd7639305e0446e821 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 8 Dec 2013 07:22:19 +0100 Subject: [PATCH 250/425] [wimp] simplify --- youtube_dl/extractor/wimp.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py index b9c3b13f9..3635691e7 100644 --- a/youtube_dl/extractor/wimp.py +++ b/youtube_dl/extractor/wimp.py @@ -11,7 +11,8 @@ class WimpIE(InfoExtractor): u'file': u'deerfence.flv', u'md5': u'8b215e2e0168c6081a1cf84b2846a2b5', u'info_dict': { - u"title": u"Watch Till End: Herd of deer jump over a fence." + u"title": u"Watch Till End: Herd of deer jump over a fence.", + u"description": u"These deer look as fluid as running water when they jump over this fence as a herd. This video is one that needs to be watched until the very end for the true majesty to be witnessed, but once it comes, it's sure to take your breath away.", } } @@ -19,18 +20,15 @@ class WimpIE(InfoExtractor): mobj = re.match(self._VALID_URL, url) video_id = mobj.group(1) webpage = self._download_webpage(url, video_id) - title = self._search_regex(r'',webpage, 'video title') - thumbnail_url = self._search_regex(r'', webpage,'video thumbnail') + title = self._html_search_meta('description', webpage, u'video title') googleString = self._search_regex("googleCode = '(.*?)'", webpage, 'file url') googleString = base64.b64decode(googleString).decode('ascii') final_url = self._search_regex('","(.*?)"', googleString,'final video url') - ext = final_url.rpartition(u'.')[2] - - return [{ - 'id': video_id, - 'url': final_url, - 'ext': ext, - 'title': title, - 'thumbnail': thumbnail_url, - }] + return { + 'id': video_id, + 'url': final_url, + 'title': self._og_search_title(webpage), + 'thumbnail': self._og_search_thumbnail(webpage), + 'description': self._og_search_description(webpage), + } From 31812a9e0e13925bc4d266a6d8eff779e10816c5 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 8 Dec 2013 07:30:42 +0100 Subject: [PATCH 251/425] [youtube:channel] Fix automated channel detection --- youtube_dl/extractor/youtube.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 01715024c..35121e6e5 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1631,10 +1631,11 @@ class YoutubeChannelIE(InfoExtractor): video_ids = [] url = 'https://www.youtube.com/channel/%s/videos' % channel_id channel_page = self._download_webpage(url, channel_id) - if re.search(r'channel-header-autogenerated-label', channel_page) is not None: - autogenerated = True - else: - autogenerated = False + autogenerated = re.search(r'''(?x) + class="[^"]*?(?: + channel-header-autogenerated-label| + yt-channel-title-autogenerated + )[^"]*"''', channel_page) is not None if autogenerated: # The videos are contained in a single page From 22686b91f0883e3686d039a5ddc104abd6cf3b26 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sun, 8 Dec 2013 07:32:25 +0100 Subject: [PATCH 252/425] release 2013.12.08.1 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 794611810..eaeba1cbf 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.08' +__version__ = '2013.12.08.1' From 56a8ab7d6073493856d7e0ab8c5d1f128e3d475b Mon Sep 17 00:00:00 2001 From: Camillo Dell'mour Date: Sun, 8 Dec 2013 14:02:14 +0100 Subject: [PATCH 253/425] added arte.tv extractor support for subdomain ddc - Mit offenen Karten(german) Le Dessous des Cartes(france) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/arte.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index d7cb6e463..7ecafb104 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -8,6 +8,7 @@ from .arte import ( ArteTVPlus7IE, ArteTVCreativeIE, ArteTVFutureIE, + ArteTVDDCIE, ) from .auengine import AUEngineIE from .bambuser import BambuserIE, BambuserChannelIE diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py index 56a5d009f..986643612 100644 --- a/youtube_dl/extractor/arte.py +++ b/youtube_dl/extractor/arte.py @@ -10,6 +10,7 @@ from ..utils import ( determine_ext, get_element_by_id, compat_str, + get_element_by_attribute, ) # There are different sources of video in arte.tv, the extraction process @@ -142,7 +143,9 @@ class ArteTVPlus7IE(InfoExtractor): def _extract_from_webpage(self, webpage, video_id, lang): json_url = self._html_search_regex(r'arte_vp_url="(.*?)"', webpage, 'json url') + return self._extract_from_json_url(json_url, video_id, lang) + def _extract_from_json_url(self, json_url, video_id, lang): json_info = self._download_webpage(json_url, video_id, 'Downloading info json') self.report_extraction(video_id) info = json.loads(json_info) @@ -257,3 +260,28 @@ class ArteTVFutureIE(ArteTVPlus7IE): webpage = self._download_webpage(url, anchor_id) row = get_element_by_id(anchor_id, webpage) return self._extract_from_webpage(row, anchor_id, lang) + +class ArteTVDDCIE(ArteTVPlus7IE): + IE_NAME = u'arte.tv:ddc' + _VALID_URL = r'http?://ddc\.arte\.tv/(?Pemission|folge)/(?P.+)' + + _TEST = { + u'url': u'http://ddc.arte.tv/folge/neues-aus-mauretanien', + u'file': u'Mit offenen Karten-049881-009_PLUS7-D.flv', + u'info_dict': { + u'title': u'neues-aus-mauretanien', + }, + } + + def _real_extract(self, url): + video_id, lang = self._extract_url_info(url) + if lang == 'folge': + lang = 'de' + elif lang == 'emission': + lang = 'fr' + webpage = self._download_webpage(url, video_id) + scriptElement = get_element_by_attribute('class', 'visu_video_block', webpage) + script_url = self._html_search_regex(r'src="(.*?)"', scriptElement, 'script url') + javascriptPlayerGenerator = self._download_webpage(script_url, video_id, 'Download javascript player generator') + json_url = self._search_regex(r"json_url=(.*)&rendering_place.*", javascriptPlayerGenerator, 'json url') + return self._extract_from_json_url(json_url, video_id, lang) From 52defb0c9bbf6716b303a06ce2cca7ffee7d8682 Mon Sep 17 00:00:00 2001 From: Camillo Dell'mour Date: Sun, 8 Dec 2013 16:22:31 +0100 Subject: [PATCH 254/425] made ddc.arte.tv test working --- youtube_dl/extractor/arte.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py index 986643612..2c472464e 100644 --- a/youtube_dl/extractor/arte.py +++ b/youtube_dl/extractor/arte.py @@ -267,9 +267,9 @@ class ArteTVDDCIE(ArteTVPlus7IE): _TEST = { u'url': u'http://ddc.arte.tv/folge/neues-aus-mauretanien', - u'file': u'Mit offenen Karten-049881-009_PLUS7-D.flv', + u'file': u'049881-009_PLUS7-D.flv', u'info_dict': { - u'title': u'neues-aus-mauretanien', + u'title': u'Mit offenen Karten', }, } From ac5118bcb92a489eec0e9997e55e07945dc720db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sun, 8 Dec 2013 16:34:57 +0100 Subject: [PATCH 255/425] [arte.tv:ddc] Add fields to the test and skip download (rtmp) --- youtube_dl/extractor/arte.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py index 2c472464e..4b7bef775 100644 --- a/youtube_dl/extractor/arte.py +++ b/youtube_dl/extractor/arte.py @@ -261,6 +261,7 @@ class ArteTVFutureIE(ArteTVPlus7IE): row = get_element_by_id(anchor_id, webpage) return self._extract_from_webpage(row, anchor_id, lang) + class ArteTVDDCIE(ArteTVPlus7IE): IE_NAME = u'arte.tv:ddc' _VALID_URL = r'http?://ddc\.arte\.tv/(?Pemission|folge)/(?P.+)' @@ -270,6 +271,12 @@ class ArteTVDDCIE(ArteTVPlus7IE): u'file': u'049881-009_PLUS7-D.flv', u'info_dict': { u'title': u'Mit offenen Karten', + u'description': u'md5:57929b0eaeddeb8a0c983f58e9ebd3b6', + u'upload_date': u'20131207', + }, + u'params': { + # rtmp download + u'skip_download': True, }, } From baa7b1978bc1d77858458c2b31aec3ff819a3e6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sun, 8 Dec 2013 22:24:55 +0100 Subject: [PATCH 256/425] Remove the calls to 'compat_urllib_request.urlopen' in a few extractors --- youtube_dl/extractor/bliptv.py | 39 +++++++++++++-------------- youtube_dl/extractor/metacafe.py | 18 +++---------- youtube_dl/extractor/mixcloud.py | 11 ++++---- youtube_dl/extractor/stanfordoc.py | 18 +++---------- youtube_dl/extractor/youtube.py | 42 +++++++++--------------------- 5 files changed, 42 insertions(+), 86 deletions(-) diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py index 493504f75..5e33a69df 100644 --- a/youtube_dl/extractor/bliptv.py +++ b/youtube_dl/extractor/bliptv.py @@ -51,8 +51,7 @@ class BlipTVIE(InfoExtractor): url = 'http://blip.tv/play/g_%s' % api_mobj.group('video_id') urlp = compat_urllib_parse_urlparse(url) if urlp.path.startswith('/play/'): - request = compat_urllib_request.Request(url) - response = compat_urllib_request.urlopen(request) + response = self._request_webpage(url, None, False) redirecturl = response.geturl() rurlp = compat_urllib_parse_urlparse(redirecturl) file_id = compat_parse_qs(rurlp.fragment)['file'][0].rpartition('/')[2] @@ -69,25 +68,23 @@ class BlipTVIE(InfoExtractor): request.add_header('User-Agent', 'iTunes/10.6.1') self.report_extraction(mobj.group(1)) info = None - try: - urlh = compat_urllib_request.urlopen(request) - if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download - basename = url.split('/')[-1] - title,ext = os.path.splitext(basename) - title = title.decode('UTF-8') - ext = ext.replace('.', '') - self.report_direct_download(title) - info = { - 'id': title, - 'url': url, - 'uploader': None, - 'upload_date': None, - 'title': title, - 'ext': ext, - 'urlhandle': urlh - } - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'ERROR: unable to download video info webpage: %s' % compat_str(err)) + urlh = self._request_webpage(request, None, False, + u'unable to download video info webpage') + if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download + basename = url.split('/')[-1] + title,ext = os.path.splitext(basename) + title = title.decode('UTF-8') + ext = ext.replace('.', '') + self.report_direct_download(title) + info = { + 'id': title, + 'url': url, + 'uploader': None, + 'upload_date': None, + 'title': title, + 'ext': ext, + 'urlhandle': urlh + } if info is None: # Regular URL try: json_code_bytes = urlh.read() diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py index e59bdd604..bd044fb60 100644 --- a/youtube_dl/extractor/metacafe.py +++ b/youtube_dl/extractor/metacafe.py @@ -1,11 +1,8 @@ import re -import socket from .common import InfoExtractor from ..utils import ( - compat_http_client, compat_parse_qs, - compat_urllib_error, compat_urllib_parse, compat_urllib_request, compat_str, @@ -93,12 +90,8 @@ class MetacafeIE(InfoExtractor): def _real_initialize(self): # Retrieve disclaimer - request = compat_urllib_request.Request(self._DISCLAIMER) - try: - self.report_disclaimer() - compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to retrieve disclaimer: %s' % compat_str(err)) + self.report_disclaimer() + self._download_webpage(self._DISCLAIMER, None, False, u'Unable to retrieve disclaimer') # Confirm age disclaimer_form = { @@ -107,11 +100,8 @@ class MetacafeIE(InfoExtractor): } request = compat_urllib_request.Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form)) request.add_header('Content-Type', 'application/x-www-form-urlencoded') - try: - self.report_age_confirmation() - compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to confirm age: %s' % compat_str(err)) + self.report_age_confirmation() + self._download_webpage(request, None, False, u'Unable to confirm age') def _real_extract(self, url): # Extract id and simplified title from URL diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py index e2baf44d7..04fa3ac7a 100644 --- a/youtube_dl/extractor/mixcloud.py +++ b/youtube_dl/extractor/mixcloud.py @@ -1,13 +1,10 @@ import json import re -import socket from .common import InfoExtractor from ..utils import ( - compat_http_client, - compat_urllib_error, - compat_urllib_request, unified_strdate, + ExtractorError, ) @@ -31,9 +28,11 @@ class MixcloudIE(InfoExtractor): """Returns 1st active url from list""" for url in url_list: try: - compat_urllib_request.urlopen(url) + # We only want to know if the request succeed + # don't download the whole file + self._request_webpage(url, None, False) return url - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error): + except ExtractorError: url = None return None diff --git a/youtube_dl/extractor/stanfordoc.py b/youtube_dl/extractor/stanfordoc.py index d54e01a12..d0d0989f0 100644 --- a/youtube_dl/extractor/stanfordoc.py +++ b/youtube_dl/extractor/stanfordoc.py @@ -1,13 +1,8 @@ import re -import socket -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( - compat_http_client, compat_str, - compat_urllib_error, - compat_urllib_request, ExtractorError, orderedSet, @@ -45,11 +40,7 @@ class StanfordOpenClassroomIE(InfoExtractor): self.report_extraction(info['id']) baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/' xmlUrl = baseUrl + video + '.xml' - try: - metaXml = compat_urllib_request.urlopen(xmlUrl).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to download video info XML: %s' % compat_str(err)) - mdoc = xml.etree.ElementTree.fromstring(metaXml) + mdoc = self._download_xml(xmlUrl, info['id']) try: info['title'] = mdoc.findall('./title')[0].text info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text @@ -95,12 +86,9 @@ class StanfordOpenClassroomIE(InfoExtractor): 'upload_date': None, } - self.report_download_webpage(info['id']) rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php' - try: - rootpage = compat_urllib_request.urlopen(rootURL).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to download course info page: ' + compat_str(err)) + rootpage = self._download_webpage(rootURL, info['id'], + errnote=u'Unable to download course info page') info['title'] = info['id'] diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 35121e6e5..1d211c450 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -7,7 +7,6 @@ import itertools import json import os.path import re -import socket import string import struct import traceback @@ -17,9 +16,7 @@ from .common import InfoExtractor, SearchInfoExtractor from .subtitles import SubtitlesInfoExtractor from ..utils import ( compat_chr, - compat_http_client, compat_parse_qs, - compat_urllib_error, compat_urllib_parse, compat_urllib_request, compat_urlparse, @@ -53,9 +50,9 @@ class YoutubeBaseInfoExtractor(InfoExtractor): request = compat_urllib_request.Request(self._LANG_URL) try: self.report_lang() - compat_urllib_request.urlopen(request).read() - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.report_warning(u'unable to set language: %s' % compat_str(err)) + self._download_webpage(self._LANG_URL, None, False) + except ExtractorError as err: + self._downloader.report_warning(u'unable to set language: %s' % compat_str(err.cause)) return False return True @@ -67,12 +64,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor): raise ExtractorError(u'No login info available, needed for using %s.' % self.IE_NAME, expected=True) return False - request = compat_urllib_request.Request(self._LOGIN_URL) - try: - login_page = compat_urllib_request.urlopen(request).read().decode('utf-8') - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.report_warning(u'unable to fetch login page: %s' % compat_str(err)) - return False + login_page = self._download_webpage(self._LOGIN_URL, None, False, + u'Unable to fetch login page') galx = self._search_regex(r'(?s)]* id="gaia_loginform"', login_results) is not None: self._downloader.report_warning(u'unable to log in: bad username or password') return False - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self._downloader.report_warning(u'unable to log in: %s' % compat_str(err)) + except ExtractorError as err: + self._downloader.report_warning(u'unable to log in: %s' % compat_str(err.cause)) return False return True @@ -120,11 +113,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor): 'action_confirm': 'Confirm', } request = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form)) - try: - self.report_age_confirmation() - compat_urllib_request.urlopen(request).read().decode('utf-8') - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to confirm age: %s' % compat_str(err)) + self.report_age_confirmation() + self._download_webpage(request, None, False, u'Unable to confirm age') return True def _real_initialize(self): @@ -1737,10 +1727,6 @@ class YoutubeSearchIE(SearchInfoExtractor): IE_NAME = u'youtube:search' _SEARCH_KEY = 'ytsearch' - def report_download_page(self, query, pagenum): - """Report attempt to download search page with given number.""" - self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum)) - def _get_n_results(self, query, n): """Get a specified number of results for a query""" @@ -1749,13 +1735,9 @@ class YoutubeSearchIE(SearchInfoExtractor): limit = n while (50 * pagenum) < limit: - self.report_download_page(query, pagenum+1) result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1) - request = compat_urllib_request.Request(result_url) - try: - data = compat_urllib_request.urlopen(request).read().decode('utf-8') - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to download API page: %s' % compat_str(err)) + data = self._download_webpage(result_url, u'query "%s"' % query, + u'Downloading page %s' % pagenum, u'Unable to download API page') api_response = json.loads(data)['data'] if not 'items' in api_response: From 7cc3570e53a22dd7f8329ecb4105c5738dcf76c9 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 9 Dec 2013 01:49:01 +0100 Subject: [PATCH 257/425] Add fatal=False parameter to _download_* functions. This allows us to simplify the calls in the youtube extractor even further. --- youtube_dl/extractor/common.py | 30 +++++++++++---- youtube_dl/extractor/youtube.py | 67 +++++++++++++++++---------------- 2 files changed, 57 insertions(+), 40 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 92a0c5050..534908a2b 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -154,27 +154,38 @@ class InfoExtractor(object): def IE_NAME(self): return type(self).__name__[:-2] - def _request_webpage(self, url_or_request, video_id, note=None, errnote=None): + def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True): """ Returns the response handle """ if note is None: self.report_download_webpage(video_id) elif note is not False: - self.to_screen(u'%s: %s' % (video_id, note)) + if video_id is None: + self.to_screen(u'%s' % (note,)) + else: + self.to_screen(u'%s: %s' % (video_id, note)) try: return self._downloader.urlopen(url_or_request) except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: if errnote is None: errnote = u'Unable to download webpage' - raise ExtractorError(u'%s: %s' % (errnote, compat_str(err)), sys.exc_info()[2], cause=err) + errmsg = u'%s: %s' % (errnote, compat_str(err)) + if fatal: + raise ExtractorError(errmsg, sys.exc_info()[2], cause=err) + else: + self._downloader.report_warning(errmsg) + return False - def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None): + def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True): """ Returns a tuple (page content as string, URL handle) """ # Strip hashes from the URL (#1038) if isinstance(url_or_request, (compat_str, str)): url_or_request = url_or_request.partition('#')[0] - urlh = self._request_webpage(url_or_request, video_id, note, errnote) + urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal) + if urlh is False: + assert not fatal + return False content_type = urlh.headers.get('Content-Type', '') webpage_bytes = urlh.read() m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type) @@ -209,9 +220,14 @@ class InfoExtractor(object): content = webpage_bytes.decode(encoding, 'replace') return (content, urlh) - def _download_webpage(self, url_or_request, video_id, note=None, errnote=None): + def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True): """ Returns the data of the page as a string """ - return self._download_webpage_handle(url_or_request, video_id, note, errnote)[0] + res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal) + if res is False: + return res + else: + content, _ = res + return content def _download_xml(self, url_or_request, video_id, note=u'Downloading XML', errnote=u'Unable to download XML'): diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 1d211c450..7f7508c74 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -42,19 +42,11 @@ class YoutubeBaseInfoExtractor(InfoExtractor): # If True it will raise an error if no login info is provided _LOGIN_REQUIRED = False - def report_lang(self): - """Report attempt to set language.""" - self.to_screen(u'Setting language') - def _set_language(self): - request = compat_urllib_request.Request(self._LANG_URL) - try: - self.report_lang() - self._download_webpage(self._LANG_URL, None, False) - except ExtractorError as err: - self._downloader.report_warning(u'unable to set language: %s' % compat_str(err.cause)) - return False - return True + return bool(self._download_webpage( + self._LANG_URL, None, + note=u'Setting language', errnote='unable to set language', + fatal=False)) def _login(self): (username, password) = self._get_login_info() @@ -64,8 +56,12 @@ class YoutubeBaseInfoExtractor(InfoExtractor): raise ExtractorError(u'No login info available, needed for using %s.' % self.IE_NAME, expected=True) return False - login_page = self._download_webpage(self._LOGIN_URL, None, False, - u'Unable to fetch login page') + login_page = self._download_webpage( + self._LOGIN_URL, None, + note=u'Downloading login page', + errnote=u'unable to fetch login page', fatal=False) + if login_page is False: + return galx = self._search_regex(r'(?s)]* id="gaia_loginform"', login_results) is not None: - self._downloader.report_warning(u'unable to log in: bad username or password') - return False - except ExtractorError as err: - self._downloader.report_warning(u'unable to log in: %s' % compat_str(err.cause)) + + req = compat_urllib_request.Request(self._LOGIN_URL, login_data) + login_results = self._download_webpage( + req, None, + note=u'Logging in', errnote=u'unable to log in', fatal=False) + if login_results is False: + return False + if re.search(r'(?i)]* id="gaia_loginform"', login_results) is not None: + self._downloader.report_warning(u'unable to log in: bad username or password') return False return True def _confirm_age(self): age_form = { - 'next_url': '/', - 'action_confirm': 'Confirm', - } - request = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form)) - self.report_age_confirmation() - self._download_webpage(request, None, False, u'Unable to confirm age') + 'next_url': '/', + 'action_confirm': 'Confirm', + } + req = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form)) + + self._download_webpage( + req, None, + note=u'Confirming age', errnote=u'Unable to confirm age') return True def _real_initialize(self): @@ -1736,11 +1734,14 @@ class YoutubeSearchIE(SearchInfoExtractor): while (50 * pagenum) < limit: result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1) - data = self._download_webpage(result_url, u'query "%s"' % query, - u'Downloading page %s' % pagenum, u'Unable to download API page') - api_response = json.loads(data)['data'] + data_json = self._download_webpage( + result_url, video_id=u'query "%s"' % query, + note=u'Downloading page %s' % (pagenum + 1), + errnote=u'Unable to download API page') + data = json.loads(data_json) + api_response = data['data'] - if not 'items' in api_response: + if 'items' not in api_response: raise ExtractorError(u'[youtube] No video results') new_ids = list(video['id'] for video in api_response['items']) From ac79fa02b80f91a71b3c1f6d9a750c79d2ff378a Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 9 Dec 2013 03:02:54 +0100 Subject: [PATCH 258/425] Restore Python 2.6.<6 compatibility (Fixes #1860) --- youtube_dl/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index a84aa59c2..5ba06d965 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -547,7 +547,7 @@ def make_HTTPS_handler(opts_no_check_certificate): def connect(self): sock = socket.create_connection((self.host, self.port), self.timeout) - if self._tunnel_host: + if getattr(self, '_tunnel_host', False): self.sock = sock self._tunnel() try: From ffe62508e4ea5a7ef58f1634dd29fb0fa9ad7837 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 9 Dec 2013 03:03:01 +0100 Subject: [PATCH 259/425] release 2013.12.09 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index eaeba1cbf..4f9e5e12b 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.08.1' +__version__ = '2013.12.09' From 0783b09b9253b8dc55d592de98877f25ec7c3b90 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 9 Dec 2013 04:08:51 +0100 Subject: [PATCH 260/425] Add a workaround for terminals without bidi support (Fixes #1912) --- youtube_dl/YoutubeDL.py | 69 ++++++++++++++++++++++++++++++++--------- youtube_dl/__init__.py | 4 +++ 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 07b36a98e..fc36be56f 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -133,6 +133,8 @@ class YoutubeDL(object): nocheckcertificate:Do not verify SSL certificates proxy: URL of the proxy server to use socket_timeout: Time to wait for unresponsive hosts, in seconds + bidi_workaround: Work around buggy terminals without bidirectional text + support, using fridibi The following parameters are not used by YoutubeDL itself, they are used by the FileDownloader: @@ -156,8 +158,45 @@ class YoutubeDL(object): self._download_retcode = 0 self._num_downloads = 0 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] + self._err_file = sys.stderr self.params = {} if params is None else params + # Pipe messsages through fribidi + if params.get('bidi_workaround', False): + # fribidi does not support ungetting, so force newlines + params['progress_with_newline'] = True + + for fid in ['_screen_file', '_err_file']: + class FribidiOut(object): + def __init__(self, outfile, errfile): + self.outfile = outfile + self.process = subprocess.Popen( + ['fribidi'], + stdin=subprocess.PIPE, + stdout=outfile, + stderr=errfile) + + def write(self, s): + res = self.process.stdin.write(s) + self.flush() + return res + + def flush(self): + return self.process.stdin.flush() + + def isatty(self): + return self.outfile.isatty() + + try: + vout = FribidiOut(getattr(self, fid), self._err_file) + setattr(self, fid, vout) + except OSError as ose: + if ose.errno == 2: + self.report_warning(u'Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.') + break + else: + raise + if (sys.version_info >= (3,) and sys.platform != 'win32' and sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and not params['restrictfilenames']): @@ -206,10 +245,14 @@ class YoutubeDL(object): pp.set_downloader(self) def to_screen(self, message, skip_eol=False): + """Print message to stdout if not in quiet mode.""" + return self.to_stdout(message, skip_eol, check_quiet=True) + + def to_stdout(self, message, skip_eol=False, check_quiet=False): """Print message to stdout if not in quiet mode.""" if self.params.get('logger'): self.params['logger'].debug(message) - elif not self.params.get('quiet', False): + elif not check_quiet or not self.params.get('quiet', False): terminator = [u'\n', u''][skip_eol] output = message + terminator write_string(output, self._screen_file) @@ -221,9 +264,7 @@ class YoutubeDL(object): self.params['logger'].error(message) else: output = message + u'\n' - if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr - output = output.encode(preferredencoding()) - sys.stderr.write(output) + write_string(output, self._err_file) def to_console_title(self, message): if not self.params.get('consoletitle', False): @@ -294,7 +335,7 @@ class YoutubeDL(object): Print the message to stderr, it will be prefixed with 'WARNING:' If stderr is a tty file the 'WARNING:' will be colored ''' - if sys.stderr.isatty() and os.name != 'nt': + if self._err_file.isatty() and os.name != 'nt': _msg_header = u'\033[0;33mWARNING:\033[0m' else: _msg_header = u'WARNING:' @@ -306,7 +347,7 @@ class YoutubeDL(object): Do the same as trouble, but prefixes the message with 'ERROR:', colored in red if stderr is a tty file. ''' - if sys.stderr.isatty() and os.name != 'nt': + if self._err_file.isatty() and os.name != 'nt': _msg_header = u'\033[0;31mERROR:\033[0m' else: _msg_header = u'ERROR:' @@ -695,22 +736,22 @@ class YoutubeDL(object): # Forced printings if self.params.get('forcetitle', False): - compat_print(info_dict['fulltitle']) + self.to_stdout(info_dict['fulltitle']) if self.params.get('forceid', False): - compat_print(info_dict['id']) + self.to_stdout(info_dict['id']) if self.params.get('forceurl', False): # For RTMP URLs, also include the playpath - compat_print(info_dict['url'] + info_dict.get('play_path', u'')) + self.to_stdout(info_dict['url'] + info_dict.get('play_path', u'')) if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None: - compat_print(info_dict['thumbnail']) + self.to_stdout(info_dict['thumbnail']) if self.params.get('forcedescription', False) and info_dict.get('description') is not None: - compat_print(info_dict['description']) + self.to_stdout(info_dict['description']) if self.params.get('forcefilename', False) and filename is not None: - compat_print(filename) + self.to_stdout(filename) if self.params.get('forceformat', False): - compat_print(info_dict['format']) + self.to_stdout(info_dict['format']) if self.params.get('forcejson', False): - compat_print(json.dumps(info_dict)) + self.to_stdout(json.dumps(info_dict)) # Do nothing else if in simulate mode if self.params.get('simulate', False): diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index d2446b670..2e3f96919 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -204,6 +204,9 @@ def parseOpts(overrideArguments=None): general.add_option( '--socket-timeout', dest='socket_timeout', type=float, default=None, help=optparse.SUPPRESS_HELP) + general.add_option( + '--bidi-workaround', dest='bidi_workaround', action='store_true', + help=u'Work around terminals that lack bidirectional text support. Requires fribidi executable in PATH') selection.add_option('--playlist-start', @@ -684,6 +687,7 @@ def _real_main(argv=None): 'nocheckcertificate': opts.no_check_certificate, 'proxy': opts.proxy, 'socket_timeout': opts.socket_timeout, + 'bidi_workaround': opts.bidi_workaround, } with YoutubeDL(ydl_opts) as ydl: From eaa1a7bde32fc3e5ea103b87cc54e915cd70c5e8 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 9 Dec 2013 04:09:06 +0100 Subject: [PATCH 261/425] release 2013.12.09.1 --- README.md | 2 ++ youtube_dl/version.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 029c418d1..27cfd7080 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,8 @@ which means you can modify it, redistribute it or use it however you like. default $XDG_CACHE_HOME/youtube-dl or ~/.cache /youtube-dl . --no-cache-dir Disable filesystem caching + --bidi-workaround Work around terminals that lack bidirectional + text support. Requires fribidi executable in PATH ## Video Selection: --playlist-start NUMBER playlist video to start at (default is 1) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 4f9e5e12b..f7f658f49 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.09' +__version__ = '2013.12.09.1' From 2a18bc9a4b2b0304a234091e2169283181892b4d Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 9 Dec 2013 04:20:14 +0100 Subject: [PATCH 262/425] Add some bug reporting hints --- README.md | 56 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 27cfd7080..9d4835053 100644 --- a/README.md +++ b/README.md @@ -276,14 +276,54 @@ This README file was originally written by Daniel Bolton ( +Bugs and suggestions should be reported at: . Unless you were prompted so or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. -Please include: - -* Your exact command line, like `youtube-dl -t "http://www.youtube.com/watch?v=uHlDtZ6Oc3s&feature=channel_video_title"`. A common mistake is not to escape the `&`. Putting URLs in quotes should solve this problem. -* If possible re-run the command with `--verbose`, and include the full output, it is really helpful to us. -* The output of `youtube-dl --version` -* The output of `python --version` -* The name and version of your Operating System ("Ubuntu 11.04 x64" or "Windows 7 x64" is usually enough). +Please include the full output of the command when run with `--verbose`. The output (including the first lines) contain important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever. For discussions, join us in the irc channel #youtube-dl on freenode. + +When you submit a request, please re-read it once to avoid a couple of mistakes (you can and should use this as a checklist): + +### Is the description of the issue itself sufficient? + +We often get issue reports that we cannot really decipher. While in most cases we eventually get the required information after asking back multiple times, this poses an unnecessary drain on our resources. Many contributors, including myself, are also not native speakers, so we may misread some parts. + +So please elaborate on what feature you are requesting, or what bug you want to be fixed. Make sure that it's obvious + +- What the problem is +- How it could be fixed +- How your proposed solution would look like + +If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a commiter myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over. + +For bug reports, this means that your report should contain the *complete* output of youtube-dl when called with the -v flag. The error message you get for (most) bugs even says so, but you would not believe how many of our bug reports do not contain this information. + +Site support requests must contain an example URL. An example URL is a URL you might want to download, like http://www.youtube.com/watch?v=BaW_jenozKc . There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. http://www.youtube.com/ ) is *not* an example URL. + +### Are you using the latest version? + +Before reporting any issue, type youtube-dl -U. This should report that you're up-to-date. Ábout 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well. + +### Is the issue already documented? + +Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or at https://github.com/rg3/youtube-dl/search?type=Issues . If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity. + +### Why are existing options not enough? + +Before requesting a new feature, please have a quick peek at [the list of supported options](https://github.com/rg3/youtube-dl/blob/master/README.md#synopsis). Many feature requests are for features that actually exist already! Please, absolutely do show off your work in the issue report and detail how the existing similar options do *not* solve your problem. + +### Is there enough context in your bug report? + +People want to solve problems, and often think they do us a favor by breaking down their larger problems (e.g. wanting to skip already downloaded files) to a specific request (e.g. requesting us to look whether the file exists before downloading the info page). However, what often happens is that they break down the problem into two steps: One simple, and one impossible (or extremely complicated one). + +We are then presented with a very complicated request when the original problem could be solved far easier, e.g. by recording the downloaded video IDs in a separate file. To avoid this, you must include the greater context where it is non-obvious. In particular, every feature request that does not consist of adding support for a new site should contain a use case scenario that explains in what situation the missing feature would be useful. + +### Does the issue involve one problem, and one problem only? + +Some of our users seem to think there is a limit of issues they can or should open. There is no limit of issues they can or should open. While it may seem appealing to be able to dump all your issues into one ticket, that means that someone who solves one of your issues cannot mark the issue as closed. Typically, reporting a bunch of issues leads to the ticket lingering since nobody wants to attack that behemoth, until someone mercifully splits the issue into multiple ones. + +In particular, every site support request issue should only pertain to services at one site (generally under a common domain, but always using the same backend technology). Do not request support for vimeo user videos, Whitehouse podcasts, and Google Plus pages in the same issue. Also, make sure that you don't post bug reports alongside feature requests. As a rule of thumb, a feature request does not include outputs of youtube-dl that are not immediately related to the feature at hand. Do not post reports of a network error alongside the request for a new video service. + +### Is anyone going to need the feature? + +Only post features that you (or an incapicated friend you can personally talk to) require. Do not post features because they seem like a good idea. If they are really useful, they will be requested by someone who requires them. From a0d96c9843da915bee28c29993d6e1a4e2d21d9f Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 9 Dec 2013 04:31:18 +0100 Subject: [PATCH 263/425] Add filename to --dump-json output (Fixes #1908) --- youtube_dl/YoutubeDL.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index fc36be56f..3fbbf3ba0 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -751,6 +751,7 @@ class YoutubeDL(object): if self.params.get('forceformat', False): self.to_stdout(info_dict['format']) if self.params.get('forcejson', False): + info_dict['_filename'] = filename self.to_stdout(json.dumps(info_dict)) # Do nothing else if in simulate mode From 977887469c3ebb7a6d6a7fee3bcd7bcc046f21e3 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 9 Dec 2013 04:50:48 +0100 Subject: [PATCH 264/425] Lower number of expected entries in top list --- test/test_youtube_lists.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py index 33db09f43..d9fe5af4e 100644 --- a/test/test_youtube_lists.py +++ b/test/test_youtube_lists.py @@ -122,7 +122,7 @@ class TestYoutubeLists(unittest.TestCase): ie = YoutubeTopListIE(dl) result = ie.extract('yttoplist:music:Top Tracks') entries = result['entries'] - self.assertTrue(len(entries) >= 9) + self.assertTrue(len(entries) >= 5) if __name__ == '__main__': unittest.main() From 2101830c0d27e6d9ad2f564ff004565e71da1ec5 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 9 Dec 2013 04:53:23 +0100 Subject: [PATCH 265/425] Remove unused imports --- youtube_dl/YoutubeDL.py | 1 - youtube_dl/extractor/metacafe.py | 1 - youtube_dl/extractor/stanfordoc.py | 2 -- youtube_dl/extractor/wimp.py | 3 +-- 4 files changed, 1 insertion(+), 6 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 3fbbf3ba0..17b3827f2 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -22,7 +22,6 @@ if os.name == 'nt': from .utils import ( compat_cookiejar, compat_http_client, - compat_print, compat_str, compat_urllib_error, compat_urllib_request, diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py index bd044fb60..99d3c83a5 100644 --- a/youtube_dl/extractor/metacafe.py +++ b/youtube_dl/extractor/metacafe.py @@ -5,7 +5,6 @@ from ..utils import ( compat_parse_qs, compat_urllib_parse, compat_urllib_request, - compat_str, determine_ext, ExtractorError, ) diff --git a/youtube_dl/extractor/stanfordoc.py b/youtube_dl/extractor/stanfordoc.py index d0d0989f0..44c52c718 100644 --- a/youtube_dl/extractor/stanfordoc.py +++ b/youtube_dl/extractor/stanfordoc.py @@ -2,8 +2,6 @@ import re from .common import InfoExtractor from ..utils import ( - compat_str, - ExtractorError, orderedSet, unescapeHTML, diff --git a/youtube_dl/extractor/wimp.py b/youtube_dl/extractor/wimp.py index 3635691e7..82a626e0e 100644 --- a/youtube_dl/extractor/wimp.py +++ b/youtube_dl/extractor/wimp.py @@ -20,10 +20,9 @@ class WimpIE(InfoExtractor): mobj = re.match(self._VALID_URL, url) video_id = mobj.group(1) webpage = self._download_webpage(url, video_id) - title = self._html_search_meta('description', webpage, u'video title') googleString = self._search_regex("googleCode = '(.*?)'", webpage, 'file url') googleString = base64.b64decode(googleString).decode('ascii') - final_url = self._search_regex('","(.*?)"', googleString,'final video url') + final_url = self._search_regex('","(.*?)"', googleString, u'final video url') return { 'id': video_id, From 395293a88956a030f1be637748d50d216ff317a5 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 9 Dec 2013 04:59:50 +0100 Subject: [PATCH 266/425] [--load-info] Always read file as UTF-8 This allows editing the file (and not escaping non-ASCII characters) and reloading it in. --- youtube_dl/YoutubeDL.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 79d5c7e5e..6538fc06c 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -883,8 +883,7 @@ class YoutubeDL(object): return self._download_retcode def download_with_info_file(self, info_filename): - with open(info_filename, 'r') as f: - # TODO: Check for errors + with io.open(info_filename, 'r', encoding='utf-8') as f: info = json.load(f) try: self.process_ie_result(info, download=True) From de2dd4c502c5d1b7a51546d52bde12169a4f9d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Mon, 9 Dec 2013 17:08:58 +0100 Subject: [PATCH 267/425] [soundcloud] add support for private links (fixes #1927) --- youtube_dl/extractor/soundcloud.py | 38 +++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index cb6dedab7..0571b36ac 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -1,3 +1,4 @@ +# encoding: utf-8 import json import re import itertools @@ -23,7 +24,9 @@ class SoundcloudIE(InfoExtractor): """ _VALID_URL = r'''^(?:https?://)? - (?:(?:(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)/?(?:[?].*)?$) + (?:(?:(?:www\.)?soundcloud\.com/ + (?P[\w\d-]+)/(?P[\w\d-]+)/? + (?P<token>[^?]+?)?(?:[?].*)?$) |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)) |(?P<widget>w\.soundcloud\.com/player/?.*?url=.*) ) @@ -56,6 +59,19 @@ class SoundcloudIE(InfoExtractor): u'skip_download': True, }, }, + # private link + { + u'url': u'https://soundcloud.com/jaimemf/youtube-dl-test-video-a-y-baw/s-8Pjrp', + u'md5': u'aa0dd32bfea9b0c5ef4f02aacd080604', + u'info_dict': { + u'id': u'123998367', + u'ext': u'mp3', + u'title': u'Youtube - Dl Test Video \'\' Ä↭', + u'uploader': u'jaimeMF', + u'description': u'test chars: \"\'/\\ä↭', + u'upload_date': u'20131209', + }, + }, ] _CLIENT_ID = 'b45b1aa10f1ac2941910a7f0d10f8e28' @@ -73,7 +89,7 @@ class SoundcloudIE(InfoExtractor): def _resolv_url(cls, url): return 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=' + cls._CLIENT_ID - def _extract_info_dict(self, info, full_title=None, quiet=False): + def _extract_info_dict(self, info, full_title=None, quiet=False, secret_token=None): track_id = compat_str(info['id']) name = full_title or track_id if quiet: @@ -104,8 +120,10 @@ class SoundcloudIE(InfoExtractor): }] else: # We have to retrieve the url + streams_url = ('http://api.soundcloud.com/i1/tracks/{0}/streams?' + 'client_id={1}&secret_token={2}'.format(track_id, self._IPHONE_CLIENT_ID, secret_token)) stream_json = self._download_webpage( - 'http://api.soundcloud.com/i1/tracks/{0}/streams?client_id={1}'.format(track_id, self._IPHONE_CLIENT_ID), + streams_url, track_id, u'Downloading track url') formats = [] @@ -157,6 +175,7 @@ class SoundcloudIE(InfoExtractor): raise ExtractorError(u'Invalid URL: %s' % url) track_id = mobj.group('track_id') + token = None if track_id is not None: info_json_url = 'http://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID full_title = track_id @@ -165,19 +184,22 @@ class SoundcloudIE(InfoExtractor): return self.url_result(query['url'][0], ie='Soundcloud') else: # extract uploader (which is in the url) - uploader = mobj.group(1) + uploader = mobj.group('uploader') # extract simple title (uploader + slug of song title) - slug_title = mobj.group(2) - full_title = '%s/%s' % (uploader, slug_title) + slug_title = mobj.group('title') + token = mobj.group('token') + full_title = resolve_title = '%s/%s' % (uploader, slug_title) + if token: + resolve_title += '/%s' % token self.report_resolve(full_title) - url = 'http://soundcloud.com/%s/%s' % (uploader, slug_title) + url = 'http://soundcloud.com/%s' % resolve_title info_json_url = self._resolv_url(url) info_json = self._download_webpage(info_json_url, full_title, u'Downloading info JSON') info = json.loads(info_json) - return self._extract_info_dict(info, full_title) + return self._extract_info_dict(info, full_title, secret_token=token) class SoundcloudSetIE(SoundcloudIE): _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/sets/([\w\d-]+)(?:[?].*)?$' From 1c088fa89ddf1e7065334e9063c378d90c668cdb Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 9 Dec 2013 18:29:07 +0100 Subject: [PATCH 268/425] Improve --bidi-workaround support --- youtube_dl/YoutubeDL.py | 69 +++++++++++++++++++++-------------------- youtube_dl/__init__.py | 17 ++-------- youtube_dl/utils.py | 18 +++++++++++ 3 files changed, 55 insertions(+), 49 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 6538fc06c..2dd7e4907 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -33,6 +33,7 @@ from .utils import ( encodeFilename, ExtractorError, format_bytes, + get_term_width, locked_file, make_HTTPS_handler, MaxDownloadsReached, @@ -160,41 +161,26 @@ class YoutubeDL(object): self._err_file = sys.stderr self.params = {} if params is None else params - # Pipe messsages through fribidi if params.get('bidi_workaround', False): - # fribidi does not support ungetting, so force newlines - params['progress_with_newline'] = True - - for fid in ['_screen_file', '_err_file']: - class FribidiOut(object): - def __init__(self, outfile, errfile): - self.outfile = outfile - self.process = subprocess.Popen( - ['fribidi'], - stdin=subprocess.PIPE, - stdout=outfile, - stderr=errfile) - - def write(self, s): - res = self.process.stdin.write(s) - self.flush() - return res - - def flush(self): - return self.process.stdin.flush() - - def isatty(self): - return self.outfile.isatty() - - try: - vout = FribidiOut(getattr(self, fid), self._err_file) - setattr(self, fid, vout) - except OSError as ose: - if ose.errno == 2: - self.report_warning(u'Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.') - break - else: - raise + try: + import pty + master, slave = pty.openpty() + width = get_term_width() + if width is None: + width_args = [] + else: + width_args = ['-w', str(width)] + self._fribidi = subprocess.Popen( + ['fribidi', '-c', 'UTF-8'] + width_args, + stdin=subprocess.PIPE, + stdout=slave, + stderr=self._err_file) + self._fribidi_channel = os.fdopen(master, 'rb') + except OSError as ose: + if ose.errno == 2: + self.report_warning(u'Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.') + else: + raise if (sys.version_info >= (3,) and sys.platform != 'win32' and sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] @@ -243,6 +229,18 @@ class YoutubeDL(object): self._pps.append(pp) pp.set_downloader(self) + def _bidi_workaround(self, message): + if not hasattr(self, '_fribidi_channel'): + return message + + assert type(message) == type(u'') + line_count = message.count(u'\n') + 1 + self._fribidi.stdin.write((message + u'\n').encode('utf-8')) + self._fribidi.stdin.flush() + res = u''.join(self._fribidi_channel.readline().decode('utf-8') + for _ in range(line_count)) + return res[:-len(u'\n')] + def to_screen(self, message, skip_eol=False): """Print message to stdout if not in quiet mode.""" return self.to_stdout(message, skip_eol, check_quiet=True) @@ -252,8 +250,10 @@ class YoutubeDL(object): if self.params.get('logger'): self.params['logger'].debug(message) elif not check_quiet or not self.params.get('quiet', False): + message = self._bidi_workaround(message) terminator = [u'\n', u''][skip_eol] output = message + terminator + write_string(output, self._screen_file) def to_stderr(self, message): @@ -262,6 +262,7 @@ class YoutubeDL(object): if self.params.get('logger'): self.params['logger'].error(message) else: + message = self._bidi_workaround(message) output = message + u'\n' write_string(output, self._err_file) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 6e9dd68c4..3e82cd637 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -48,7 +48,6 @@ import os import random import re import shlex -import subprocess import sys @@ -57,6 +56,7 @@ from .utils import ( DateRange, decodeOption, determine_ext, + get_term_width, DownloadError, get_cachedir, MaxDownloadsReached, @@ -113,19 +113,6 @@ def parseOpts(overrideArguments=None): def _comma_separated_values_options_callback(option, opt_str, value, parser): setattr(parser.values, option.dest, value.split(',')) - def _find_term_columns(): - columns = os.environ.get('COLUMNS', None) - if columns: - return int(columns) - - try: - sp = subprocess.Popen(['stty', 'size'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out,err = sp.communicate() - return int(out.split()[1]) - except: - pass - return None - def _hide_login_info(opts): opts = list(opts) for private_opt in ['-p', '--password', '-u', '--username', '--video-password']: @@ -140,7 +127,7 @@ def parseOpts(overrideArguments=None): max_help_position = 80 # No need to wrap help messages if we're on a wide console - columns = _find_term_columns() + columns = get_term_width() if columns: max_width = columns fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 5ba06d965..64300d8e0 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -15,6 +15,7 @@ import platform import re import ssl import socket +import subprocess import sys import traceback import zlib @@ -1024,6 +1025,23 @@ def format_bytes(bytes): converted = float(bytes) / float(1024 ** exponent) return u'%.2f%s' % (converted, suffix) + def str_to_int(int_str): int_str = re.sub(r'[,\.]', u'', int_str) return int(int_str) + + +def get_term_width(): + columns = os.environ.get('COLUMNS', None) + if columns: + return int(columns) + + try: + sp = subprocess.Popen( + ['stty', 'size'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = sp.communicate() + return int(out.split()[1]) + except: + pass + return None From df8ae1e3a2222acf189c7b82278a773e93887cec Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 9 Dec 2013 18:31:31 +0100 Subject: [PATCH 269/425] release 2013.12.09.2 --- README.md | 2 ++ youtube_dl/version.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d4835053..68b2e1ae7 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,8 @@ which means you can modify it, redistribute it or use it however you like. --restrict-filenames Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames -a, --batch-file FILE file containing URLs to download ('-' for stdin) + --load-info FILE json file containing the video information + (created with the "--write-json" option -w, --no-overwrites do not overwrite files -c, --continue force resume of partially downloaded files. By default, youtube-dl will resume downloads if diff --git a/youtube_dl/version.py b/youtube_dl/version.py index f7f658f49..0b19fa9c5 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.09.1' +__version__ = '2013.12.09.2' From 1d87e3a1c64e3183d17a6dc6fbc4353a86aa5f99 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 9 Dec 2013 18:55:54 +0100 Subject: [PATCH 270/425] [rtlnow] Allow double slashes after domain name (Fixes #1928) --- youtube_dl/extractor/rtlnow.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py index 2f238de35..511674d8d 100644 --- a/youtube_dl/extractor/rtlnow.py +++ b/youtube_dl/extractor/rtlnow.py @@ -7,14 +7,15 @@ from ..utils import ( ExtractorError, ) + class RTLnowIE(InfoExtractor): """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW""" - _VALID_URL = r'(?:http://)?(?P<url>(?P<base_url>rtl-now\.rtl\.de/|rtl2now\.rtl2\.de/|(?:www\.)?voxnow\.de/|(?:www\.)?rtlnitronow\.de/|(?:www\.)?superrtlnow\.de/|(?:www\.)?n-tvnow\.de/)[a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.php\?(?:container_id|film_id)=(?P<video_id>[0-9]+)&player=1(?:&season=[0-9]+)?(?:&.*)?)' + _VALID_URL = r'(?:http://)?(?P<url>(?P<base_url>rtl-now\.rtl\.de|rtl2now\.rtl2\.de|(?:www\.)?voxnow\.de|(?:www\.)?rtlnitronow\.de|(?:www\.)?superrtlnow\.de|(?:www\.)?n-tvnow\.de)/+[a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.php\?(?:container_id|film_id)=(?P<video_id>[0-9]+)&player=1(?:&season=[0-9]+)?(?:&.*)?)' _TESTS = [{ u'url': u'http://rtl-now.rtl.de/ahornallee/folge-1.php?film_id=90419&player=1&season=1', u'file': u'90419.flv', u'info_dict': { - u'upload_date': u'20070416', + u'upload_date': u'20070416', u'title': u'Ahornallee - Folge 1 - Der Einzug', u'description': u'Folge 1 - Der Einzug', }, From 1e1f84dac9b4fe8cc32f3d5aefb5fceb5d65232a Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 9 Dec 2013 18:56:17 +0100 Subject: [PATCH 271/425] release 2013.12.09.3 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 0b19fa9c5..17c0e5542 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.09.2' +__version__ = '2013.12.09.3' From caefb1de877e163fa3ece44757cb2fae6adf47e4 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 9 Dec 2013 19:39:41 +0100 Subject: [PATCH 272/425] [ndtv] Add extractor (Fixes #1924) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/ndtv.py | 66 ++++++++++++++++++++++++++++++++ youtube_dl/utils.py | 12 ++++++ 3 files changed, 79 insertions(+) create mode 100644 youtube_dl/extractor/ndtv.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 3f740baa1..1149dc1ec 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -100,6 +100,7 @@ from .myvideo import MyVideoIE from .naver import NaverIE from .nba import NBAIE from .nbc import NBCNewsIE +from .ndtv import NDTVIE from .newgrounds import NewgroundsIE from .nhl import NHLIE, NHLVideocenterIE from .niconico import NiconicoIE diff --git a/youtube_dl/extractor/ndtv.py b/youtube_dl/extractor/ndtv.py new file mode 100644 index 000000000..2e8501f99 --- /dev/null +++ b/youtube_dl/extractor/ndtv.py @@ -0,0 +1,66 @@ +import json +import re +import time + +from .common import InfoExtractor +from ..utils import month_by_name + + +class NDTVIE(InfoExtractor): + _VALID_URL = r'^https?://(?:www\.)?ndtv\.com/video/player/[^/]*/[^/]*/(?P<id>[a-z0-9]+)' + + _TEST = { + u"url": u"http://www.ndtv.com/video/player/news/ndtv-exclusive-don-t-need-character-certificate-from-rahul-gandhi-says-arvind-kejriwal/300710", + u"file": u"300710.mp4", + u"md5": u"39f992dbe5fb531c395d8bbedb1e5e88", + u"info_dict": { + u"title": u"NDTV exclusive: Don't need character certificate from Rahul Gandhi, says Arvind Kejriwal", + u"description": u"In an exclusive interview to NDTV, Aam Aadmi Party's Arvind Kejriwal says it makes no difference to him that Rahul Gandhi said the Congress needs to learn from his party.", + u"upload_date": u"20131208", + u"duration": 1327, + u"thumbnail": u"http://i.ndtvimg.com/video/images/vod/medium/2013-12/big_300710_1386518307.jpg", + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + + filename = self._search_regex( + r"__filename='([^']+)'", webpage, u'video filename') + video_url = (u'http://bitcast-b.bitgravity.com/ndtvod/23372/ndtv/%s' % + filename) + + duration_str = filename = self._search_regex( + r"__duration='([^']+)'", webpage, u'duration', fatal=False) + duration = None if duration_str is None else int(duration_str) + + date_m = re.search(r'''(?x) + <p\s+class="vod_dateline">\s* + Published\s+On:\s* + (?P<monthname>[A-Za-z]+)\s+(?P<day>[0-9]+),\s*(?P<year>[0-9]+) + ''', webpage) + upload_date = None + assert date_m + if date_m is not None: + month = month_by_name(date_m.group('monthname')) + if month is not None: + upload_date = '%s%02d%02d' % ( + date_m.group('year'), month, int(date_m.group('day'))) + + description = self._og_search_description(webpage) + READ_MORE = u' (Read more)' + if description.endswith(READ_MORE): + description = description[:-len(READ_MORE)] + + return { + 'id': video_id, + 'url': video_url, + 'title': self._og_search_title(webpage), + 'description': description, + 'thumbnail': self._og_search_thumbnail(webpage), + 'duration': duration, + 'upload_date': upload_date, + } diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 64300d8e0..0dab9fcc5 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1045,3 +1045,15 @@ def get_term_width(): except: pass return None + + +def month_by_name(name): + """ Return the number of a month by (locale-independently) English name """ + + ENGLISH_NAMES = [ + u'Januar', u'February', u'March', u'April', u'May', u'June', + u'July', u'August', u'September', u'October', u'November', u'December'] + try: + return ENGLISH_NAMES.index(name) + 1 + except ValueError: + return None From 4ff50ef84607ee60ff813f7d7f3d35c8b497bf07 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 9 Dec 2013 19:57:00 +0100 Subject: [PATCH 273/425] [soundcloud] Do not match sets (Fixes #1930) --- test/test_all_urls.py | 3 +++ youtube_dl/extractor/soundcloud.py | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/test_all_urls.py b/test/test_all_urls.py index 6b9764c67..e9458b2e3 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -110,6 +110,9 @@ class TestAllURLsMatching(unittest.TestCase): self.assertMatch('http://vimeo.com/channels/tributes', ['vimeo:channel']) self.assertMatch('http://vimeo.com/user7108434', ['vimeo:user']) + # https://github.com/rg3/youtube-dl/issues/1930 + def test_soundcloud_not_matching_sets(self): + self.assertMatch('http://soundcloud.com/floex/sets/gone-ep', ['soundcloud:set']) if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 0571b36ac..5c026c0b8 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -25,7 +25,8 @@ class SoundcloudIE(InfoExtractor): _VALID_URL = r'''^(?:https?://)? (?:(?:(?:www\.)?soundcloud\.com/ - (?P<uploader>[\w\d-]+)/(?P<title>[\w\d-]+)/? + (?P<uploader>[\w\d-]+)/ + (?!sets/)(?P<title>[\w\d-]+)/? (?P<token>[^?]+?)?(?:[?].*)?$) |(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+)) |(?P<widget>w\.soundcloud\.com/player/?.*?url=.*) From 77526143e789368a15a0e89bfed5dbc52f8db9e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Mon, 9 Dec 2013 20:01:43 +0100 Subject: [PATCH 274/425] [brightcove] Use the original url (usually the player) as the default referer (fixes #1929) --- youtube_dl/extractor/brightcove.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py index 66fe0ac9a..63577a3cb 100644 --- a/youtube_dl/extractor/brightcove.py +++ b/youtube_dl/extractor/brightcove.py @@ -55,6 +55,18 @@ class BrightcoveIE(InfoExtractor): u'uploader': u'Mashable', }, }, + { + # test that the default referer works + # from http://national.ballet.ca/interact/video/Lost_in_Motion_II/ + u'url': u'http://link.brightcove.com/services/player/bcpid756015033001?bckey=AQ~~,AAAApYJi_Ck~,GxhXCegT1Dp39ilhXuxMJxasUhVNZiil&bctid=2878862109001', + u'info_dict': { + u'id': u'2878862109001', + u'ext': u'mp4', + u'title': u'Lost in Motion II', + u'description': u'md5:363109c02998fee92ec02211bd8000df', + u'uploader': u'National Ballet of Canada', + }, + }, ] @classmethod @@ -118,17 +130,21 @@ class BrightcoveIE(InfoExtractor): videoPlayer = query.get('@videoPlayer') if videoPlayer: - return self._get_video_info(videoPlayer[0], query_str, query) + return self._get_video_info(videoPlayer[0], query_str, query, + # We set the original url as the default 'Referer' header + referer=url) else: player_key = query['playerKey'] return self._get_playlist_info(player_key[0]) - def _get_video_info(self, video_id, query_str, query): + def _get_video_info(self, video_id, query_str, query, referer=None): request_url = self._FEDERATED_URL_TEMPLATE % query_str req = compat_urllib_request.Request(request_url) linkBase = query.get('linkBaseURL') if linkBase is not None: - req.add_header('Referer', linkBase[0]) + referer = linkBase[0] + if referer is not None: + req.add_header('Referer', referer) webpage = self._download_webpage(req, video_id) self.report_extraction(video_id) From f8bd0194a7723bcb702c34816e8fa522fce4e723 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 9 Dec 2013 20:05:10 +0100 Subject: [PATCH 275/425] Remove superfluous spaces --- youtube_dl/extractor/brightcove.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py index 63577a3cb..b1b7526ca 100644 --- a/youtube_dl/extractor/brightcove.py +++ b/youtube_dl/extractor/brightcove.py @@ -64,7 +64,7 @@ class BrightcoveIE(InfoExtractor): u'ext': u'mp4', u'title': u'Lost in Motion II', u'description': u'md5:363109c02998fee92ec02211bd8000df', - u'uploader': u'National Ballet of Canada', + u'uploader': u'National Ballet of Canada', }, }, ] From 49929a20a7166199307b9d8eda623f8b81540bdb Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 9 Dec 2013 20:05:27 +0100 Subject: [PATCH 276/425] release 2013.12.09.4 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 17c0e5542..8906d6090 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.09.3' +__version__ = '2013.12.09.4' From 26e6393134b35121ab956a408250c565596dd2a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Mon, 9 Dec 2013 22:00:42 +0100 Subject: [PATCH 277/425] Set 'NA' as the default value for missing fields in the output template (fixes #1931) Remove the `except KeyError` clause, it won't get raised anymore --- test/test_YoutubeDL.py | 15 +++++++++++++++ youtube_dl/YoutubeDL.py | 5 ++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index 58cf9c313..3100c362a 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -7,6 +7,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import FakeYDL +from youtube_dl import YoutubeDL class YDL(FakeYDL): @@ -140,6 +141,20 @@ class TestFormatSelection(unittest.TestCase): self.assertEqual(test_dict['extractor'], 'Foo') self.assertEqual(test_dict['playlist'], 'funny videos') + def test_prepare_filename(self): + info = { + u'id': u'1234', + u'ext': u'mp4', + u'width': None, + } + def fname(templ): + ydl = YoutubeDL({'outtmpl': templ}) + return ydl.prepare_filename(info) + self.assertEqual(fname(u'%(id)s.%(ext)s'), u'1234.mp4') + self.assertEqual(fname(u'%(id)s-%(width)s.%(ext)s'), u'1234-NA.mp4') + # Replace missing fields with 'NA' + self.assertEqual(fname(u'%(uploader_date)s-%(id)s.%(ext)s'), u'NA-1234.mp4') + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 2dd7e4907..11d4972dd 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -3,6 +3,7 @@ from __future__ import absolute_import +import collections import errno import io import json @@ -401,13 +402,11 @@ class YoutubeDL(object): is_id=(k == u'id')) template_dict = dict((k, sanitize(k, v)) for k, v in template_dict.items()) + template_dict = collections.defaultdict(lambda: u'NA', template_dict) tmpl = os.path.expanduser(self.params['outtmpl']) filename = tmpl % template_dict return filename - except KeyError as err: - self.report_error(u'Erroneous output template') - return None except ValueError as err: self.report_error(u'Error in output template: ' + str(err) + u' (encoding: ' + repr(preferredencoding()) + ')') return None From 45598aab0873c31dd6200899192ddeb07bd2c472 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 10 Dec 2013 11:23:35 +0100 Subject: [PATCH 278/425] [YoutubeDL] Simplify filename preparation --- youtube_dl/YoutubeDL.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 11d4972dd..cabe30980 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -397,11 +397,12 @@ class YoutubeDL(object): template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index'] sanitize = lambda k, v: sanitize_filename( - u'NA' if v is None else compat_str(v), + compat_str(v), restricted=self.params.get('restrictfilenames'), is_id=(k == u'id')) template_dict = dict((k, sanitize(k, v)) - for k, v in template_dict.items()) + for k, v in template_dict.items() + if v is not None) template_dict = collections.defaultdict(lambda: u'NA', template_dict) tmpl = os.path.expanduser(self.params['outtmpl']) From 475700acfe3ee1604b884450c6a35c709b6410d5 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 10 Dec 2013 11:45:13 +0100 Subject: [PATCH 279/425] [soundcloud] Do not mistake original_format for ext (Fixes #1934) --- youtube_dl/extractor/soundcloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 5c026c0b8..cc5f14593 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -99,7 +99,7 @@ class SoundcloudIE(InfoExtractor): thumbnail = info['artwork_url'] if thumbnail is not None: thumbnail = thumbnail.replace('-large', '-t500x500') - ext = info.get('original_format', u'mp3') + ext = u'mp3' result = { 'id': track_id, 'uploader': info['user']['username'], From 5a3ea17c9470c93be27f4b177f2e4114bb06d0e3 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 10 Dec 2013 11:52:10 +0100 Subject: [PATCH 280/425] [zdf] Correct order of unknown formats (#1936) --- youtube_dl/extractor/zdf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py index 689f19735..35ece354a 100644 --- a/youtube_dl/extractor/zdf.py +++ b/youtube_dl/extractor/zdf.py @@ -73,14 +73,14 @@ class ZDFIE(InfoExtractor): try: proto_pref = -PROTO_ORDER.index(format_m.group('proto')) except ValueError: - proto_pref = 999 + proto_pref = -999 quality = fnode.find('./quality').text QUALITY_ORDER = ['veryhigh', '300', 'high', 'med', 'low'] try: quality_pref = -QUALITY_ORDER.index(quality) except ValueError: - quality_pref = 999 + quality_pref = -999 abr = int(fnode.find('./audioBitrate').text) // 1000 vbr = int(fnode.find('./videoBitrate').text) // 1000 From a30a60d8eb027a55ec14c912bad4359b3128997e Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 10 Dec 2013 11:54:59 +0100 Subject: [PATCH 281/425] release 2013.12.10 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 8906d6090..4da3026b3 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.09.4' +__version__ = '2013.12.10' From e2b38da93112c97d46d612bf89c329b22ac2d00d Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 10 Dec 2013 12:45:22 +0100 Subject: [PATCH 282/425] [mtv] Fixup incorrectly encoded XML documents --- youtube_dl/extractor/common.py | 5 ++++- youtube_dl/extractor/mtv.py | 9 +++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 534908a2b..69a083b68 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -230,9 +230,12 @@ class InfoExtractor(object): return content def _download_xml(self, url_or_request, video_id, - note=u'Downloading XML', errnote=u'Unable to download XML'): + note=u'Downloading XML', errnote=u'Unable to download XML', + transform_source=None): """Return the xml as an xml.etree.ElementTree.Element""" xml_string = self._download_webpage(url_or_request, video_id, note, errnote) + if transform_source: + xml_string = transform_source(xml_string) return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8')) def to_screen(self, msg): diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py index 6b3feb560..5b2bd9633 100644 --- a/youtube_dl/extractor/mtv.py +++ b/youtube_dl/extractor/mtv.py @@ -82,8 +82,13 @@ class MTVServicesInfoExtractor(InfoExtractor): def _get_videos_info(self, uri): video_id = self._id_from_uri(uri) data = compat_urllib_parse.urlencode({'uri': uri}) - idoc = self._download_xml(self._FEED_URL +'?' + data, video_id, - u'Downloading info') + + def fix_ampersand(s): + """ Fix unencoded ampersand in XML """ + return s.replace(u'& ', '& ') + idoc = self._download_xml( + self._FEED_URL + '?' + data, video_id, + u'Downloading info', transform_source=fix_ampersand) return [self._get_video_info(item) for item in idoc.findall('.//item')] From f67ca84d4a77bb15b959bcc2c0a88ad5c4efc9af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Tue, 10 Dec 2013 13:04:21 +0100 Subject: [PATCH 283/425] [soundcloud] Fix the extension for 'downloadable' songs In this case the 'original_format' field must be used. --- youtube_dl/extractor/soundcloud.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index cc5f14593..cbba4094b 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -73,6 +73,19 @@ class SoundcloudIE(InfoExtractor): u'upload_date': u'20131209', }, }, + # downloadable song + { + u'url': u'https://soundcloud.com/simgretina/just-your-problem-baby-1', + u'md5': u'56a8b69568acaa967b4c49f9d1d52d19', + u'info_dict': { + u'id': u'105614606', + u'ext': u'wav', + u'title': u'Just Your Problem Baby (Acapella)', + u'description': u'Vocals', + u'uploader': u'Sim Gretina', + u'upload_date': u'20130815', + }, + }, ] _CLIENT_ID = 'b45b1aa10f1ac2941910a7f0d10f8e28' @@ -115,7 +128,7 @@ class SoundcloudIE(InfoExtractor): track_id, self._CLIENT_ID)) result['formats'] = [{ 'format_id': 'download', - 'ext': ext, + 'ext': info.get('original_format', u'mp3'), 'url': format_url, 'vcodec': 'none', }] From cbfc4702280dcf26b76a1e2be151cdb97a293f4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Tue, 10 Dec 2013 13:42:41 +0100 Subject: [PATCH 284/425] [mixcloud] Try to get the m4a url if the mp3 url fails to download (fixes #1939) --- youtube_dl/extractor/mixcloud.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py index 04fa3ac7a..125d81551 100644 --- a/youtube_dl/extractor/mixcloud.py +++ b/youtube_dl/extractor/mixcloud.py @@ -37,6 +37,9 @@ class MixcloudIE(InfoExtractor): return None + def _get_url(self, template_url): + return self.check_urls(template_url % i for i in range(30)) + def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) @@ -52,13 +55,18 @@ class MixcloudIE(InfoExtractor): preview_url = self._search_regex(r'data-preview-url="(.+?)"', webpage, u'preview url') song_url = preview_url.replace('/previews/', '/cloudcasts/originals/') template_url = re.sub(r'(stream\d*)', 'stream%d', song_url) - final_song_url = self.check_urls(template_url % i for i in range(30)) + final_song_url = self._get_url(template_url) + if final_song_url is None: + self.to_screen('Trying with m4a extension') + template_url = template_url.replace('.mp3', '.m4a').replace('originals/', 'm4a/64/') + final_song_url = self._get_url(template_url) + if final_song_url is None: + raise ExtractorError(u'Unable to extract track url') return { 'id': track_id, 'title': info['name'], 'url': final_song_url, - 'ext': 'mp3', 'description': info.get('description'), 'thumbnail': info['pictures'].get('extra_large'), 'uploader': info['user']['name'], From df1d7da2afe748a9bc7be3ef43f96914dc0d2576 Mon Sep 17 00:00:00 2001 From: mc2avr <mc2avr@googlemail.com> Date: Tue, 10 Dec 2013 18:40:50 +0100 Subject: [PATCH 285/425] add MDRIE --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/mdr.py | 78 ++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 youtube_dl/extractor/mdr.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index d7cb6e463..8a063648c 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -86,6 +86,7 @@ from .kickstarter import KickStarterIE from .keek import KeekIE from .liveleak import LiveLeakIE from .livestream import LivestreamIE, LivestreamOriginalIE +from .mdr import MDRIE from .metacafe import MetacafeIE from .metacritic import MetacriticIE from .mit import TechTVMITIE, MITIE diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py new file mode 100644 index 000000000..8b096a78a --- /dev/null +++ b/youtube_dl/extractor/mdr.py @@ -0,0 +1,78 @@ +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, +) + +class MDRIE(InfoExtractor): + _VALID_URL = r'^(?P<domain>(?:https?://)?(?:www\.)?mdr\.de)/mediathek/(?:.*)/(?P<type>video|audio)(?P<video_id>[^/_]+)_.*' + _TITLE = r'<h2>(?P<title1>[^<]+)<span>(?P<title2>[^<]+)</span></h2>' + + _MEDIA_XML = r'(?P<xmlurl>/mediathek/(.+)/(video|audio)([0-9]+)-avCustom.xml)' + _MEDIA_STREAM_VIDEO = r'<asset>.*<frameWidth>(?P<frameWidth>[0-9]+)</frameWidth>.*<flashMediaServerApplicationURL>(?P<flashMediaServerApplicationURL>[^<]+)</flashMediaServerApplicationURL><flashMediaServerURL>(?P<flashMediaServerURL>[^<]+)</flashMediaServerURL>.*<progressiveDownloadUrl>(?P<progressiveDownloadUrl>[^<]+)</progressiveDownloadUrl></asset>' + _MEDIA_STREAM_AUDIO = r'<asset>.*<mediaType>(?P<mediaType>[A-Z0-9]+)</mediaType><bitrateAudio>(?P<bitrateAudio>[0-9]+)</bitrateAudio>.*<flashMediaServerApplicationURL>(?P<flashMediaServerApplicationURL>[^<]+)</flashMediaServerApplicationURL><flashMediaServerURL>(?P<flashMediaServerURL>[^<]+)</flashMediaServerURL>.*<progressiveDownloadUrl>(?P<progressiveDownloadUrl>[^<]+)</progressiveDownloadUrl></asset>' + _TESTS = [{ + u'url': u'http://www.mdr.de/mediathek/themen/nachrichten/video165624_zc-c5c7de76_zs-3795826d.html', + u'file': u'165624.mp4', + u'md5': u'95165945756198b8fa2dea10f0b04614', + u'info_dict': { + u"title": u"MDR aktuell Eins30 09.12.2013, 22:48 Uhr" + }, + #u'skip': u'Requires rtmpdump' # rtmp is optional + }, + { + u'url': u' http://www.mdr.de/mediathek/radio/mdr1-radio-sachsen/audio718370_zc-67b21197_zs-1b9b2483.html', + u'file': u'718370.mp4', + u'md5': u'4a5b1fbb5519fb0d929c384b6ff7cb8b', + u'info_dict': { + u"title": u"MDR 1 RADIO SACHSEN 10.12.2013, 05:00 Uhr" + }, + #u'skip': u'Requires rtmpdump' # rtmp is optional + }] + + def _real_extract(self, url): + + # determine video id from url + m = re.match(self._VALID_URL, url) + video_id = m.group('video_id') + domain = m.group('domain') + mediatype = m.group('type') + + # determine title and media streams from webpage + html = self._download_webpage(url, video_id) + t = re.search(self._TITLE, html) + if not t: + raise ExtractorError(u'no title found') + title = t.group('title1') + t.group('title2') + m = re.search(self._MEDIA_XML, html) + if not m: + raise ExtractorError(u'no xml found') + xmlurl = m.group('xmlurl') + xml = self._download_webpage(domain+xmlurl, video_id, 'download XML').replace('\n','').replace('\r','').replace('<asset>','\n<asset>').replace('</asset>','</asset>\n') + if(mediatype == "video"): + streams = [mo.groupdict() for mo in re.finditer(self._MEDIA_STREAM_VIDEO, xml)] + if not streams: + raise ExtractorError(u'no media found') + # choose default media type and highest quality for now + stream = max([s for s in streams if s["progressiveDownloadUrl"].startswith("http://") ], + key=lambda s: int(s["frameWidth"])) + else: + streams = [mo.groupdict() for mo in re.finditer(self._MEDIA_STREAM_AUDIO, xml)] + if not streams: + raise ExtractorError(u'no media found') + # choose default media type (MP4) and highest quality for now + stream = max([s for s in streams if s["progressiveDownloadUrl"].startswith("http://") and s["mediaType"] == "MP4" ], + key=lambda s: int(s["bitrateAudio"])) + + # there's two possibilities: RTMP stream or HTTP download + info = {'id': video_id, 'title': title, 'ext': 'mp4'} + if not stream["progressiveDownloadUrl"]: + self.to_screen(u'RTMP download detected') + assert stream['flashMediaServerURL'].startswith('mp4:') + info["url"] = stream["flashMediaServerApplicationURL"] + info["play_path"] = stream['flashMediaServerURL'] + else: + assert stream["progressiveDownloadUrl"].endswith('.mp4') + info["url"] = stream["progressiveDownloadUrl"] + return [info] From 7c86cd5ab1c8e9091c42eaae7354520a708f7431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Tue, 10 Dec 2013 19:44:16 +0100 Subject: [PATCH 286/425] [dailymotion] Fix uploader extraction Now it looks directly in the info dictionary --- youtube_dl/extractor/dailymotion.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py index 3bd0b862c..7f2f0d6a4 100644 --- a/youtube_dl/extractor/dailymotion.py +++ b/youtube_dl/extractor/dailymotion.py @@ -101,10 +101,6 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor): self.to_screen(u'Vevo video detected: %s' % vevo_id) return self.url_result(u'vevo:%s' % vevo_id, ie='Vevo') - video_uploader = self._search_regex([r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>', - # Looking for official user - r'<(?:span|a) .*?rel="author".*?>([^<]+?)</'], - webpage, 'video uploader', fatal=False) age_limit = self._rta_search(webpage) video_upload_date = None @@ -153,7 +149,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor): return { 'id': video_id, 'formats': formats, - 'uploader': video_uploader, + 'uploader': info['owner_screenname'], 'upload_date': video_upload_date, 'title': self._og_search_title(webpage), 'subtitles': video_subtitles, From 5458b4cefbfff4b419f22ad33fbaffda25b83382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Tue, 10 Dec 2013 19:47:00 +0100 Subject: [PATCH 287/425] [dailymotion] Fix view count extraction and make it non fatal (fixes #1940) --- youtube_dl/extractor/dailymotion.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py index 7f2f0d6a4..aea7e557e 100644 --- a/youtube_dl/extractor/dailymotion.py +++ b/youtube_dl/extractor/dailymotion.py @@ -143,8 +143,10 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor): self._list_available_subtitles(video_id, webpage) return - view_count = str_to_int(self._search_regex( - r'video_views_value[^>]+>([\d\.,]+)<', webpage, u'view count')) + view_count = self._search_regex( + r'video_views_count[^>]+>\s+([\d\.,]+)', webpage, u'view count', fatal=False) + if view_count is not None: + view_count = str_to_int(view_count) return { 'id': video_id, From 48ad51b243b1fbca7f1e72e209f38f5ca90335ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Tue, 10 Dec 2013 20:28:12 +0100 Subject: [PATCH 288/425] [vimeo] Fix the extraction for some 'player' or 'pro' videos The variable the config dict is assigned to can change, now we try to detect it or fallback to a, b or c --- youtube_dl/extractor/vimeo.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index fb2bd225a..7c7f537d8 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -151,8 +151,14 @@ class VimeoIE(InfoExtractor): config = json.loads(config_json) except RegexNotFoundError: # For pro videos or player.vimeo.com urls - config = self._search_regex([r' = {config:({.+?}),assets:', r'(?:c|b)=({.+?});'], - webpage, u'info section', flags=re.DOTALL) + # We try to find out to which variable is assigned the config dic + m_variable_name = re.search('(\w)\.video\.id', webpage) + if m_variable_name is not None: + config_re = r'%s=({.+?});' % re.escape(m_variable_name.group(1)) + else: + config_re = [r' = {config:({.+?}),assets:', r'(?:[abc])=({.+?});'] + config = self._search_regex(config_re, webpage, u'info section', + flags=re.DOTALL) config = json.loads(config) except Exception as e: if re.search('The creator of this video has not given you permission to embed it on this domain.', webpage): From a0088bdf9342408a1fc5033a0f4599bae3b9aa0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Tue, 10 Dec 2013 20:43:16 +0100 Subject: [PATCH 289/425] [vimeo] Fix unused argument of the `_real_extract` method --- youtube_dl/extractor/vimeo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index 7c7f537d8..ea4409528 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -115,7 +115,7 @@ class VimeoIE(InfoExtractor): def _real_initialize(self): self._login() - def _real_extract(self, url, new_video=True): + def _real_extract(self, url): url, data = unsmuggle_url(url) headers = std_headers if data is not None: From 182583623583c8e71af9b4e24acf8c409fcff197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Tue, 10 Dec 2013 21:03:53 +0100 Subject: [PATCH 290/425] Use `_download_xml` in more extractors --- youtube_dl/extractor/appletrailers.py | 23 +++++++++++------------ youtube_dl/extractor/clipsyndicate.py | 10 ++++------ youtube_dl/extractor/metacritic.py | 9 +++++---- youtube_dl/utils.py | 5 +++++ 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py index a527f10de..ef5644aa5 100644 --- a/youtube_dl/extractor/appletrailers.py +++ b/youtube_dl/extractor/appletrailers.py @@ -1,5 +1,4 @@ import re -import xml.etree.ElementTree import json from .common import InfoExtractor @@ -65,18 +64,18 @@ class AppleTrailersIE(InfoExtractor): uploader_id = mobj.group('company') playlist_url = compat_urlparse.urljoin(url, u'includes/playlists/itunes.inc') - playlist_snippet = self._download_webpage(playlist_url, movie) - playlist_cleaned = re.sub(r'(?s)<script[^<]*?>.*?</script>', u'', playlist_snippet) - playlist_cleaned = re.sub(r'<img ([^<]*?)>', r'<img \1/>', playlist_cleaned) - # The ' in the onClick attributes are not escaped, it couldn't be parsed - # with xml.etree.ElementTree.fromstring - # like: http://trailers.apple.com/trailers/wb/gravity/ - def _clean_json(m): - return u'iTunes.playURL(%s);' % m.group(1).replace('\'', ''') - playlist_cleaned = re.sub(self._JSON_RE, _clean_json, playlist_cleaned) - playlist_html = u'<html>' + playlist_cleaned + u'</html>' + def fix_html(s): + s = re.sub(r'(?s)<script[^<]*?>.*?</script>', u'', s) + s = re.sub(r'<img ([^<]*?)>', r'<img \1/>', s) + # The ' in the onClick attributes are not escaped, it couldn't be parsed + # like: http://trailers.apple.com/trailers/wb/gravity/ + def _clean_json(m): + return u'iTunes.playURL(%s);' % m.group(1).replace('\'', ''') + s = re.sub(self._JSON_RE, _clean_json, s) + s = u'<html>' + s + u'</html>' + return s + doc = self._download_xml(playlist_url, movie, transform_source=fix_html) - doc = xml.etree.ElementTree.fromstring(playlist_html) playlist = [] for li in doc.findall('./div/ul/li'): on_click = li.find('.//a').attrib['onClick'] diff --git a/youtube_dl/extractor/clipsyndicate.py b/youtube_dl/extractor/clipsyndicate.py index d4fc86973..c60089ad3 100644 --- a/youtube_dl/extractor/clipsyndicate.py +++ b/youtube_dl/extractor/clipsyndicate.py @@ -1,9 +1,9 @@ import re -import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( find_xpath_attr, + fix_xml_all_ampersand, ) @@ -30,12 +30,10 @@ class ClipsyndicateIE(InfoExtractor): # it includes a required token flvars = self._search_regex(r'flvars: "(.*?)"', js_player, u'flvars') - playlist_page = self._download_webpage( + pdoc = self._download_xml( 'http://eplayer.clipsyndicate.com/osmf/playlist?%s' % flvars, - video_id, u'Downloading video info') - # Fix broken xml - playlist_page = re.sub('&', '&', playlist_page) - pdoc = xml.etree.ElementTree.fromstring(playlist_page.encode('utf-8')) + video_id, u'Downloading video info', + transform_source=fix_xml_all_ampersand) track_doc = pdoc.find('trackList/track') def find_param(name): diff --git a/youtube_dl/extractor/metacritic.py b/youtube_dl/extractor/metacritic.py index 6b95b4998..e560c1d35 100644 --- a/youtube_dl/extractor/metacritic.py +++ b/youtube_dl/extractor/metacritic.py @@ -1,8 +1,10 @@ import re -import xml.etree.ElementTree import operator from .common import InfoExtractor +from ..utils import ( + fix_xml_all_ampersand, +) class MetacriticIE(InfoExtractor): @@ -23,9 +25,8 @@ class MetacriticIE(InfoExtractor): video_id = mobj.group('id') webpage = self._download_webpage(url, video_id) # The xml is not well formatted, there are raw '&' - info_xml = self._download_webpage('http://www.metacritic.com/video_data?video=' + video_id, - video_id, u'Downloading info xml').replace('&', '&') - info = xml.etree.ElementTree.fromstring(info_xml.encode('utf-8')) + info = self._download_xml('http://www.metacritic.com/video_data?video=' + video_id, + video_id, u'Downloading info xml', transform_source=fix_xml_all_ampersand) clip = next(c for c in info.findall('playList/clip') if c.find('id').text == video_id) formats = [] diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 0dab9fcc5..4593488ce 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1057,3 +1057,8 @@ def month_by_name(name): return ENGLISH_NAMES.index(name) + 1 except ValueError: return None + + +def fix_xml_all_ampersand(xml_str): + """Replace all the '&' by '&' in XML""" + return xml_str.replace(u'&', u'&') From 08d03235f9a5a7d28d7992a3caf27a8b88b69e12 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Wed, 11 Dec 2013 08:45:51 +0100 Subject: [PATCH 291/425] release 2013.12.11 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 4da3026b3..a4a89317c 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.10' +__version__ = '2013.12.11' From 357ddadbf53b48721d43061edc67d37c73d1ac95 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Wed, 11 Dec 2013 08:54:48 +0100 Subject: [PATCH 292/425] Fix thumbnail filename determination (Fixes #1945) --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index cabe30980..c77777ba0 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -827,7 +827,7 @@ class YoutubeDL(object): if self.params.get('writethumbnail', False): if info_dict.get('thumbnail') is not None: thumb_format = determine_ext(info_dict['thumbnail'], u'jpg') - thumb_filename = filename.rpartition('.')[0] + u'.' + thumb_format + thumb_filename = os.path.splitext(filename)[0] + u'.' + thumb_format self.to_screen(u'[%s] %s: Downloading thumbnail ...' % (info_dict['extractor'], info_dict['id'])) try: From fca1ef19c1dfc5cd841424700816e1ad3b3112eb Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Wed, 11 Dec 2013 08:54:54 +0100 Subject: [PATCH 293/425] release 2013.12.11.1 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index a4a89317c..a3ac65579 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.11' +__version__ = '2013.12.11.1' From 00381b4ccbf0504f15f48aeddc242fd8535b5c2d Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Wed, 11 Dec 2013 09:22:08 +0100 Subject: [PATCH 294/425] [pornhub] Fix URL regexp --- youtube_dl/extractor/pornhub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py index 8b3471919..d9135c6b9 100644 --- a/youtube_dl/extractor/pornhub.py +++ b/youtube_dl/extractor/pornhub.py @@ -12,7 +12,7 @@ from ..aes import ( ) class PornHubIE(InfoExtractor): - _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>pornhub\.com/view_video\.php\?viewkey=(?P<videoid>[0-9]+))' + _VALID_URL = r'^(?:https?://)?(?:www\.)?(?P<url>pornhub\.com/view_video\.php\?viewkey=(?P<videoid>[0-9a-f]+))' _TEST = { u'url': u'http://www.pornhub.com/view_video.php?viewkey=648719015', u'file': u'648719015.mp4', From f2c36ee43e06f81cd0ac77e5b1d0c238c0057623 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Wed, 11 Dec 2013 09:22:25 +0100 Subject: [PATCH 295/425] release 2013.12.11.2 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index a3ac65579..b9a52fcfa 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.11.1' +__version__ = '2013.12.11.2' From 8ab470f1b26b6c42ab74ceb345994c201743bed8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Mon, 23 Sep 2013 18:09:28 +0200 Subject: [PATCH 296/425] Now a new FileDownloader is created when downloading a video The progress hooks can be added using the method "add_downloader_progress_hook" --- test/test_download.py | 2 +- youtube_dl/YoutubeDL.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/test/test_download.py b/test/test_download.py index dd5818dba..f242b3528 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -90,7 +90,7 @@ def generator(test_case): def _hook(status): if status['status'] == 'finished': finished_hook_called.add(status['filename']) - ydl.fd.add_progress_hook(_hook) + ydl.add_downloader_progress_hook(_hook) def get_tc_filename(tc): return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {})) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index c77777ba0..6a34f16d5 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -155,7 +155,7 @@ class YoutubeDL(object): self._ies = [] self._ies_instances = {} self._pps = [] - self._progress_hooks = [] + self._fd_progress_hooks = [] self._download_retcode = 0 self._num_downloads = 0 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] @@ -193,8 +193,6 @@ class YoutubeDL(object): u'Set the LC_ALL environment variable to fix this.') self.params['restrictfilenames'] = True - self.fd = FileDownloader(self, self.params) - if '%(stitle)s' in self.params.get('outtmpl', ''): self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.') @@ -230,6 +228,10 @@ class YoutubeDL(object): self._pps.append(pp) pp.set_downloader(self) + def add_downloader_progress_hook(self, ph): + """Add the progress hook to the file downloader""" + self._fd_progress_hooks.append(ph) + def _bidi_workaround(self, message): if not hasattr(self, '_fribidi_channel'): return message @@ -845,7 +847,10 @@ class YoutubeDL(object): success = True else: try: - success = self.fd._do_download(filename, info_dict) + fd = FileDownloader(self, self.params) + for ph in self._fd_progress_hooks: + fd.add_progress_hook(ph) + success = fd._do_download(filename, info_dict) except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self.report_error(u'unable to download video data: %s' % str(err)) return From 3bc2ddccc8622379ec11e802dff30a635285a9c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Mon, 23 Sep 2013 17:59:27 +0200 Subject: [PATCH 297/425] Move FileDownloader to its own module and create a new class for each download process A suitable downloader can be found using the 'get_suitable_downloader' function. Each subclass implements 'real_download', for downloading an info dict you call the 'download' method, which first checks if the video has already been downloaded --- setup.py | 2 +- youtube_dl/FileDownloader.py | 728 +----------------------------- youtube_dl/YoutubeDL.py | 6 +- youtube_dl/downloader/__init__.py | 23 + youtube_dl/downloader/common.py | 321 +++++++++++++ youtube_dl/downloader/hls.py | 44 ++ youtube_dl/downloader/http.py | 191 ++++++++ youtube_dl/downloader/mplayer.py | 39 ++ youtube_dl/downloader/rtmp.py | 178 ++++++++ 9 files changed, 808 insertions(+), 724 deletions(-) create mode 100644 youtube_dl/downloader/__init__.py create mode 100644 youtube_dl/downloader/common.py create mode 100644 youtube_dl/downloader/hls.py create mode 100644 youtube_dl/downloader/http.py create mode 100644 youtube_dl/downloader/mplayer.py create mode 100644 youtube_dl/downloader/rtmp.py diff --git a/setup.py b/setup.py index 8e24fe679..653ca9a73 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ setup( author_email='ytdl@yt-dl.org', maintainer='Philipp Hagemeister', maintainer_email='phihag@phihag.de', - packages=['youtube_dl', 'youtube_dl.extractor'], + packages=['youtube_dl', 'youtube_dl.extractor', 'youtube_dl.downloader'], # Provokes warning on most systems (why?!) # test_suite = 'nose.collector', diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 47124932f..5c8e676a2 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -1,724 +1,12 @@ -import os -import re -import subprocess -import sys -import time - -from .utils import ( - compat_urllib_error, - compat_urllib_request, - ContentTooShortError, - determine_ext, - encodeFilename, - format_bytes, - sanitize_open, - timeconvert, -) - - -class FileDownloader(object): - """File Downloader class. - - File downloader objects are the ones responsible of downloading the - actual video file and writing it to disk. - - File downloaders accept a lot of parameters. In order not to saturate - the object constructor with arguments, it receives a dictionary of - options instead. - - Available options: - - verbose: Print additional info to stdout. - quiet: Do not print messages to stdout. - ratelimit: Download speed limit, in bytes/sec. - retries: Number of times to retry for HTTP error 5xx - buffersize: Size of download buffer in bytes. - noresizebuffer: Do not automatically resize the download buffer. - continuedl: Try to continue downloads if possible. - noprogress: Do not print the progress bar. - logtostderr: Log messages to stderr instead of stdout. - consoletitle: Display progress in console window's titlebar. - nopart: Do not use temporary .part files. - updatetime: Use the Last-modified header to set output file timestamps. - test: Download only first bytes to test the downloader. - min_filesize: Skip files smaller than this size - max_filesize: Skip files larger than this size - """ - - params = None - - def __init__(self, ydl, params): - """Create a FileDownloader object with the given options.""" - self.ydl = ydl - self._progress_hooks = [] - self.params = params - - @staticmethod - def format_seconds(seconds): - (mins, secs) = divmod(seconds, 60) - (hours, mins) = divmod(mins, 60) - if hours > 99: - return '--:--:--' - if hours == 0: - return '%02d:%02d' % (mins, secs) - else: - return '%02d:%02d:%02d' % (hours, mins, secs) - - @staticmethod - def calc_percent(byte_counter, data_len): - if data_len is None: - return None - return float(byte_counter) / float(data_len) * 100.0 - - @staticmethod - def format_percent(percent): - if percent is None: - return '---.-%' - return '%6s' % ('%3.1f%%' % percent) - - @staticmethod - def calc_eta(start, now, total, current): - if total is None: - return None - dif = now - start - if current == 0 or dif < 0.001: # One millisecond - return None - rate = float(current) / dif - return int((float(total) - float(current)) / rate) - - @staticmethod - def format_eta(eta): - if eta is None: - return '--:--' - return FileDownloader.format_seconds(eta) - - @staticmethod - def calc_speed(start, now, bytes): - dif = now - start - if bytes == 0 or dif < 0.001: # One millisecond - return None - return float(bytes) / dif - - @staticmethod - def format_speed(speed): - if speed is None: - return '%10s' % '---b/s' - return '%10s' % ('%s/s' % format_bytes(speed)) - - @staticmethod - def best_block_size(elapsed_time, bytes): - new_min = max(bytes / 2.0, 1.0) - new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB - if elapsed_time < 0.001: - return int(new_max) - rate = bytes / elapsed_time - if rate > new_max: - return int(new_max) - if rate < new_min: - return int(new_min) - return int(rate) - - @staticmethod - def parse_bytes(bytestr): - """Parse a string indicating a byte quantity into an integer.""" - matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr) - if matchobj is None: - return None - number = float(matchobj.group(1)) - multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower()) - return int(round(number * multiplier)) - - def to_screen(self, *args, **kargs): - self.ydl.to_screen(*args, **kargs) - - def to_stderr(self, message): - self.ydl.to_screen(message) - - def to_console_title(self, message): - self.ydl.to_console_title(message) - - def trouble(self, *args, **kargs): - self.ydl.trouble(*args, **kargs) - - def report_warning(self, *args, **kargs): - self.ydl.report_warning(*args, **kargs) - - def report_error(self, *args, **kargs): - self.ydl.report_error(*args, **kargs) - - def slow_down(self, start_time, byte_counter): - """Sleep if the download speed is over the rate limit.""" - rate_limit = self.params.get('ratelimit', None) - if rate_limit is None or byte_counter == 0: - return - now = time.time() - elapsed = now - start_time - if elapsed <= 0.0: - return - speed = float(byte_counter) / elapsed - if speed > rate_limit: - time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) - - def temp_name(self, filename): - """Returns a temporary filename for the given filename.""" - if self.params.get('nopart', False) or filename == u'-' or \ - (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))): - return filename - return filename + u'.part' - - def undo_temp_name(self, filename): - if filename.endswith(u'.part'): - return filename[:-len(u'.part')] - return filename - - def try_rename(self, old_filename, new_filename): - try: - if old_filename == new_filename: - return - os.rename(encodeFilename(old_filename), encodeFilename(new_filename)) - except (IOError, OSError): - self.report_error(u'unable to rename file') - - def try_utime(self, filename, last_modified_hdr): - """Try to set the last-modified time of the given file.""" - if last_modified_hdr is None: - return - if not os.path.isfile(encodeFilename(filename)): - return - timestr = last_modified_hdr - if timestr is None: - return - filetime = timeconvert(timestr) - if filetime is None: - return filetime - # Ignore obviously invalid dates - if filetime == 0: - return - try: - os.utime(filename, (time.time(), filetime)) - except: - pass - return filetime - - def report_destination(self, filename): - """Report destination filename.""" - self.to_screen(u'[download] Destination: ' + filename) - - def _report_progress_status(self, msg, is_last_line=False): - fullmsg = u'[download] ' + msg - if self.params.get('progress_with_newline', False): - self.to_screen(fullmsg) - else: - if os.name == 'nt': - prev_len = getattr(self, '_report_progress_prev_line_length', - 0) - if prev_len > len(fullmsg): - fullmsg += u' ' * (prev_len - len(fullmsg)) - self._report_progress_prev_line_length = len(fullmsg) - clear_line = u'\r' - else: - clear_line = (u'\r\x1b[K' if sys.stderr.isatty() else u'\r') - self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line) - self.to_console_title(u'youtube-dl ' + msg) - - def report_progress(self, percent, data_len_str, speed, eta): - """Report download progress.""" - if self.params.get('noprogress', False): - return - if eta is not None: - eta_str = self.format_eta(eta) - else: - eta_str = 'Unknown ETA' - if percent is not None: - percent_str = self.format_percent(percent) - else: - percent_str = 'Unknown %' - speed_str = self.format_speed(speed) - - msg = (u'%s of %s at %s ETA %s' % - (percent_str, data_len_str, speed_str, eta_str)) - self._report_progress_status(msg) - - def report_progress_live_stream(self, downloaded_data_len, speed, elapsed): - if self.params.get('noprogress', False): - return - downloaded_str = format_bytes(downloaded_data_len) - speed_str = self.format_speed(speed) - elapsed_str = FileDownloader.format_seconds(elapsed) - msg = u'%s at %s (%s)' % (downloaded_str, speed_str, elapsed_str) - self._report_progress_status(msg) - - def report_finish(self, data_len_str, tot_time): - """Report download finished.""" - if self.params.get('noprogress', False): - self.to_screen(u'[download] Download completed') - else: - self._report_progress_status( - (u'100%% of %s in %s' % - (data_len_str, self.format_seconds(tot_time))), - is_last_line=True) - - def report_resuming_byte(self, resume_len): - """Report attempt to resume at given byte.""" - self.to_screen(u'[download] Resuming download at byte %s' % resume_len) - - def report_retry(self, count, retries): - """Report retry in case of HTTP error 5xx""" - self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries)) - - def report_file_already_downloaded(self, file_name): - """Report file has already been fully downloaded.""" - try: - self.to_screen(u'[download] %s has already been downloaded' % file_name) - except UnicodeEncodeError: - self.to_screen(u'[download] The file has already been downloaded') - - def report_unable_to_resume(self): - """Report it was impossible to resume download.""" - self.to_screen(u'[download] Unable to resume') - - def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url, live, conn): - def run_rtmpdump(args): - start = time.time() - resume_percent = None - resume_downloaded_data_len = None - proc = subprocess.Popen(args, stderr=subprocess.PIPE) - cursor_in_new_line = True - proc_stderr_closed = False - while not proc_stderr_closed: - # read line from stderr - line = u'' - while True: - char = proc.stderr.read(1) - if not char: - proc_stderr_closed = True - break - if char in [b'\r', b'\n']: - break - line += char.decode('ascii', 'replace') - if not line: - # proc_stderr_closed is True - continue - mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec \(([0-9]{1,2}\.[0-9])%\)', line) - if mobj: - downloaded_data_len = int(float(mobj.group(1))*1024) - percent = float(mobj.group(2)) - if not resume_percent: - resume_percent = percent - resume_downloaded_data_len = downloaded_data_len - eta = self.calc_eta(start, time.time(), 100-resume_percent, percent-resume_percent) - speed = self.calc_speed(start, time.time(), downloaded_data_len-resume_downloaded_data_len) - data_len = None - if percent > 0: - data_len = int(downloaded_data_len * 100 / percent) - data_len_str = u'~' + format_bytes(data_len) - self.report_progress(percent, data_len_str, speed, eta) - cursor_in_new_line = False - self._hook_progress({ - 'downloaded_bytes': downloaded_data_len, - 'total_bytes': data_len, - 'tmpfilename': tmpfilename, - 'filename': filename, - 'status': 'downloading', - 'eta': eta, - 'speed': speed, - }) - else: - # no percent for live streams - mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec', line) - if mobj: - downloaded_data_len = int(float(mobj.group(1))*1024) - time_now = time.time() - speed = self.calc_speed(start, time_now, downloaded_data_len) - self.report_progress_live_stream(downloaded_data_len, speed, time_now - start) - cursor_in_new_line = False - self._hook_progress({ - 'downloaded_bytes': downloaded_data_len, - 'tmpfilename': tmpfilename, - 'filename': filename, - 'status': 'downloading', - 'speed': speed, - }) - elif self.params.get('verbose', False): - if not cursor_in_new_line: - self.to_screen(u'') - cursor_in_new_line = True - self.to_screen(u'[rtmpdump] '+line) - proc.wait() - if not cursor_in_new_line: - self.to_screen(u'') - return proc.returncode - - self.report_destination(filename) - tmpfilename = self.temp_name(filename) - test = self.params.get('test', False) - - # Check for rtmpdump first - try: - subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) - except (OSError, IOError): - self.report_error(u'RTMP download detected but "rtmpdump" could not be run') - return False - - # Download using rtmpdump. rtmpdump returns exit code 2 when - # the connection was interrumpted and resuming appears to be - # possible. This is part of rtmpdump's normal usage, AFAIK. - basic_args = ['rtmpdump', '--verbose', '-r', url, '-o', tmpfilename] - if player_url is not None: - basic_args += ['--swfVfy', player_url] - if page_url is not None: - basic_args += ['--pageUrl', page_url] - if play_path is not None: - basic_args += ['--playpath', play_path] - if tc_url is not None: - basic_args += ['--tcUrl', url] - if test: - basic_args += ['--stop', '1'] - if live: - basic_args += ['--live'] - if conn: - basic_args += ['--conn', conn] - args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)] - - if sys.platform == 'win32' and sys.version_info < (3, 0): - # Windows subprocess module does not actually support Unicode - # on Python 2.x - # See http://stackoverflow.com/a/9951851/35070 - subprocess_encoding = sys.getfilesystemencoding() - args = [a.encode(subprocess_encoding, 'ignore') for a in args] - else: - subprocess_encoding = None - - if self.params.get('verbose', False): - if subprocess_encoding: - str_args = [ - a.decode(subprocess_encoding) if isinstance(a, bytes) else a - for a in args] - else: - str_args = args - try: - import pipes - shell_quote = lambda args: ' '.join(map(pipes.quote, str_args)) - except ImportError: - shell_quote = repr - self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(str_args)) - - retval = run_rtmpdump(args) - - while (retval == 2 or retval == 1) and not test: - prevsize = os.path.getsize(encodeFilename(tmpfilename)) - self.to_screen(u'[rtmpdump] %s bytes' % prevsize) - time.sleep(5.0) # This seems to be needed - retval = run_rtmpdump(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) - cursize = os.path.getsize(encodeFilename(tmpfilename)) - if prevsize == cursize and retval == 1: - break - # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those - if prevsize == cursize and retval == 2 and cursize > 1024: - self.to_screen(u'[rtmpdump] Could not download the whole video. This can happen for some advertisements.') - retval = 0 - break - if retval == 0 or (test and retval == 2): - fsize = os.path.getsize(encodeFilename(tmpfilename)) - self.to_screen(u'[rtmpdump] %s bytes' % fsize) - self.try_rename(tmpfilename, filename) - self._hook_progress({ - 'downloaded_bytes': fsize, - 'total_bytes': fsize, - 'filename': filename, - 'status': 'finished', - }) - return True - else: - self.to_stderr(u"\n") - self.report_error(u'rtmpdump exited with code %d' % retval) - return False - - def _download_with_mplayer(self, filename, url): - self.report_destination(filename) - tmpfilename = self.temp_name(filename) - - args = ['mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy', '-dumpstream', '-dumpfile', tmpfilename, url] - # Check for mplayer first - try: - subprocess.call(['mplayer', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) - except (OSError, IOError): - self.report_error(u'MMS or RTSP download detected but "%s" could not be run' % args[0] ) - return False - - # Download using mplayer. - retval = subprocess.call(args) - if retval == 0: - fsize = os.path.getsize(encodeFilename(tmpfilename)) - self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize)) - self.try_rename(tmpfilename, filename) - self._hook_progress({ - 'downloaded_bytes': fsize, - 'total_bytes': fsize, - 'filename': filename, - 'status': 'finished', - }) - return True - else: - self.to_stderr(u"\n") - self.report_error(u'mplayer exited with code %d' % retval) - return False - - def _download_m3u8_with_ffmpeg(self, filename, url): - self.report_destination(filename) - tmpfilename = self.temp_name(filename) - - args = ['-y', '-i', url, '-f', 'mp4', '-c', 'copy', - '-bsf:a', 'aac_adtstoasc', tmpfilename] - - for program in ['avconv', 'ffmpeg']: - try: - subprocess.call([program, '-version'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) - break - except (OSError, IOError): - pass - else: - self.report_error(u'm3u8 download detected but ffmpeg or avconv could not be found') - cmd = [program] + args - - retval = subprocess.call(cmd) - if retval == 0: - fsize = os.path.getsize(encodeFilename(tmpfilename)) - self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize)) - self.try_rename(tmpfilename, filename) - self._hook_progress({ - 'downloaded_bytes': fsize, - 'total_bytes': fsize, - 'filename': filename, - 'status': 'finished', - }) - return True - else: - self.to_stderr(u"\n") - self.report_error(u'ffmpeg exited with code %d' % retval) - return False +# Legacy file for backwards compatibility, use youtube_dl.downloader instead! +from .downloader import FileDownloader as RealFileDownloader +from .downloader import get_suitable_downloader +# This class reproduces the old behaviour of FileDownloader +class FileDownloader(RealFileDownloader): def _do_download(self, filename, info_dict): - url = info_dict['url'] - - # Check file already present - if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False): - self.report_file_already_downloaded(filename) - self._hook_progress({ - 'filename': filename, - 'status': 'finished', - 'total_bytes': os.path.getsize(encodeFilename(filename)), - }) - return True - - # Attempt to download using rtmpdump - if url.startswith('rtmp'): - return self._download_with_rtmpdump(filename, url, - info_dict.get('player_url', None), - info_dict.get('page_url', None), - info_dict.get('play_path', None), - info_dict.get('tc_url', None), - info_dict.get('rtmp_live', False), - info_dict.get('rtmp_conn', None)) - - # Attempt to download using mplayer - if url.startswith('mms') or url.startswith('rtsp'): - return self._download_with_mplayer(filename, url) - - # m3u8 manifest are downloaded with ffmpeg - if determine_ext(url) == u'm3u8': - return self._download_m3u8_with_ffmpeg(filename, url) - - tmpfilename = self.temp_name(filename) - stream = None - - # Do not include the Accept-Encoding header - headers = {'Youtubedl-no-compression': 'True'} - if 'user_agent' in info_dict: - headers['Youtubedl-user-agent'] = info_dict['user_agent'] - basic_request = compat_urllib_request.Request(url, None, headers) - request = compat_urllib_request.Request(url, None, headers) - - if self.params.get('test', False): - request.add_header('Range','bytes=0-10240') - - # Establish possible resume length - if os.path.isfile(encodeFilename(tmpfilename)): - resume_len = os.path.getsize(encodeFilename(tmpfilename)) - else: - resume_len = 0 - - open_mode = 'wb' - if resume_len != 0: - if self.params.get('continuedl', False): - self.report_resuming_byte(resume_len) - request.add_header('Range','bytes=%d-' % resume_len) - open_mode = 'ab' - else: - resume_len = 0 - - count = 0 - retries = self.params.get('retries', 0) - while count <= retries: - # Establish connection - try: - if count == 0 and 'urlhandle' in info_dict: - data = info_dict['urlhandle'] - data = compat_urllib_request.urlopen(request) - break - except (compat_urllib_error.HTTPError, ) as err: - if (err.code < 500 or err.code >= 600) and err.code != 416: - # Unexpected HTTP error - raise - elif err.code == 416: - # Unable to resume (requested range not satisfiable) - try: - # Open the connection again without the range header - data = compat_urllib_request.urlopen(basic_request) - content_length = data.info()['Content-Length'] - except (compat_urllib_error.HTTPError, ) as err: - if err.code < 500 or err.code >= 600: - raise - else: - # Examine the reported length - if (content_length is not None and - (resume_len - 100 < int(content_length) < resume_len + 100)): - # The file had already been fully downloaded. - # Explanation to the above condition: in issue #175 it was revealed that - # YouTube sometimes adds or removes a few bytes from the end of the file, - # changing the file size slightly and causing problems for some users. So - # I decided to implement a suggested change and consider the file - # completely downloaded if the file size differs less than 100 bytes from - # the one in the hard drive. - self.report_file_already_downloaded(filename) - self.try_rename(tmpfilename, filename) - self._hook_progress({ - 'filename': filename, - 'status': 'finished', - }) - return True - else: - # The length does not match, we start the download over - self.report_unable_to_resume() - open_mode = 'wb' - break - # Retry - count += 1 - if count <= retries: - self.report_retry(count, retries) - - if count > retries: - self.report_error(u'giving up after %s retries' % retries) - return False - - data_len = data.info().get('Content-length', None) - if data_len is not None: - data_len = int(data_len) + resume_len - min_data_len = self.params.get("min_filesize", None) - max_data_len = self.params.get("max_filesize", None) - if min_data_len is not None and data_len < min_data_len: - self.to_screen(u'\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len)) - return False - if max_data_len is not None and data_len > max_data_len: - self.to_screen(u'\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len)) - return False - - data_len_str = format_bytes(data_len) - byte_counter = 0 + resume_len - block_size = self.params.get('buffersize', 1024) - start = time.time() - while True: - # Download and write - before = time.time() - data_block = data.read(block_size) - after = time.time() - if len(data_block) == 0: - break - byte_counter += len(data_block) - - # Open file just in time - if stream is None: - try: - (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode) - assert stream is not None - filename = self.undo_temp_name(tmpfilename) - self.report_destination(filename) - except (OSError, IOError) as err: - self.report_error(u'unable to open for writing: %s' % str(err)) - return False - try: - stream.write(data_block) - except (IOError, OSError) as err: - self.to_stderr(u"\n") - self.report_error(u'unable to write data: %s' % str(err)) - return False - if not self.params.get('noresizebuffer', False): - block_size = self.best_block_size(after - before, len(data_block)) - - # Progress message - speed = self.calc_speed(start, time.time(), byte_counter - resume_len) - if data_len is None: - eta = percent = None - else: - percent = self.calc_percent(byte_counter, data_len) - eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len) - self.report_progress(percent, data_len_str, speed, eta) - - self._hook_progress({ - 'downloaded_bytes': byte_counter, - 'total_bytes': data_len, - 'tmpfilename': tmpfilename, - 'filename': filename, - 'status': 'downloading', - 'eta': eta, - 'speed': speed, - }) - - # Apply rate limit - self.slow_down(start, byte_counter - resume_len) - - if stream is None: - self.to_stderr(u"\n") - self.report_error(u'Did not get any data blocks') - return False - stream.close() - self.report_finish(data_len_str, (time.time() - start)) - if data_len is not None and byte_counter != data_len: - raise ContentTooShortError(byte_counter, int(data_len)) - self.try_rename(tmpfilename, filename) - - # Update file modification time - if self.params.get('updatetime', True): - info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None)) - - self._hook_progress({ - 'downloaded_bytes': byte_counter, - 'total_bytes': byte_counter, - 'filename': filename, - 'status': 'finished', - }) - - return True - - def _hook_progress(self, status): + real_fd = get_suitable_downloader(info_dict)(self.ydl, self.params) for ph in self._progress_hooks: - ph(status) - - def add_progress_hook(self, ph): - """ ph gets called on download progress, with a dictionary with the entries - * filename: The final filename - * status: One of "downloading" and "finished" - - It can also have some of the following entries: - - * downloaded_bytes: Bytes on disks - * total_bytes: Total bytes, None if unknown - * tmpfilename: The filename we're currently writing to - * eta: The estimated time in seconds, None if unknown - * speed: The download speed in bytes/second, None if unknown - - Hooks are guaranteed to be called at least once (with status "finished") - if the download is successful. - """ - self._progress_hooks.append(ph) + real_fd.add_progress_hook(ph) + return real_fd.download(filename, info_dict) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 6a34f16d5..414aa5a80 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -51,7 +51,7 @@ from .utils import ( YoutubeDLHandler, ) from .extractor import get_info_extractor, gen_extractors -from .FileDownloader import FileDownloader +from .downloader import get_suitable_downloader from .version import __version__ @@ -847,10 +847,10 @@ class YoutubeDL(object): success = True else: try: - fd = FileDownloader(self, self.params) + fd = get_suitable_downloader(info_dict)(self, self.params) for ph in self._fd_progress_hooks: fd.add_progress_hook(ph) - success = fd._do_download(filename, info_dict) + success = fd.download(filename, info_dict) except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self.report_error(u'unable to download video data: %s' % str(err)) return diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py new file mode 100644 index 000000000..f19b490f1 --- /dev/null +++ b/youtube_dl/downloader/__init__.py @@ -0,0 +1,23 @@ +from .common import FileDownloader +from .hls import HlsFD +from .http import HttpFD +from .mplayer import MplayerFD +from .rtmp import RtmpFD + +from ..utils import ( + determine_ext, +) + +def get_suitable_downloader(info_dict): + """Get the downloader class that can handle the info dict.""" + url = info_dict['url'] + + if url.startswith('rtmp'): + return RtmpFD + if determine_ext(url) == u'm3u8': + return HlsFD + if url.startswith('mms') or url.startswith('rtsp'): + return MplayerFD + else: + return HttpFD + diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py new file mode 100644 index 000000000..e9c9b4473 --- /dev/null +++ b/youtube_dl/downloader/common.py @@ -0,0 +1,321 @@ +import math +import os +import re +import subprocess +import sys +import time + +from ..utils import ( + encodeFilename, + timeconvert, + format_bytes, +) + + +class FileDownloader(object): + """File Downloader class. + + File downloader objects are the ones responsible of downloading the + actual video file and writing it to disk. + + File downloaders accept a lot of parameters. In order not to saturate + the object constructor with arguments, it receives a dictionary of + options instead. + + Available options: + + verbose: Print additional info to stdout. + quiet: Do not print messages to stdout. + ratelimit: Download speed limit, in bytes/sec. + retries: Number of times to retry for HTTP error 5xx + buffersize: Size of download buffer in bytes. + noresizebuffer: Do not automatically resize the download buffer. + continuedl: Try to continue downloads if possible. + noprogress: Do not print the progress bar. + logtostderr: Log messages to stderr instead of stdout. + consoletitle: Display progress in console window's titlebar. + nopart: Do not use temporary .part files. + updatetime: Use the Last-modified header to set output file timestamps. + test: Download only first bytes to test the downloader. + min_filesize: Skip files smaller than this size + max_filesize: Skip files larger than this size + + Subclasses of this one must re-define the real_download method. + """ + + params = None + + def __init__(self, ydl, params): + """Create a FileDownloader object with the given options.""" + self.ydl = ydl + self._progress_hooks = [] + self.params = params + + @staticmethod + def format_seconds(seconds): + (mins, secs) = divmod(seconds, 60) + (hours, mins) = divmod(mins, 60) + if hours > 99: + return '--:--:--' + if hours == 0: + return '%02d:%02d' % (mins, secs) + else: + return '%02d:%02d:%02d' % (hours, mins, secs) + + @staticmethod + def calc_percent(byte_counter, data_len): + if data_len is None: + return None + return float(byte_counter) / float(data_len) * 100.0 + + @staticmethod + def format_percent(percent): + if percent is None: + return '---.-%' + return '%6s' % ('%3.1f%%' % percent) + + @staticmethod + def calc_eta(start, now, total, current): + if total is None: + return None + dif = now - start + if current == 0 or dif < 0.001: # One millisecond + return None + rate = float(current) / dif + return int((float(total) - float(current)) / rate) + + @staticmethod + def format_eta(eta): + if eta is None: + return '--:--' + return FileDownloader.format_seconds(eta) + + @staticmethod + def calc_speed(start, now, bytes): + dif = now - start + if bytes == 0 or dif < 0.001: # One millisecond + return None + return float(bytes) / dif + + @staticmethod + def format_speed(speed): + if speed is None: + return '%10s' % '---b/s' + return '%10s' % ('%s/s' % format_bytes(speed)) + + @staticmethod + def best_block_size(elapsed_time, bytes): + new_min = max(bytes / 2.0, 1.0) + new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB + if elapsed_time < 0.001: + return int(new_max) + rate = bytes / elapsed_time + if rate > new_max: + return int(new_max) + if rate < new_min: + return int(new_min) + return int(rate) + + @staticmethod + def parse_bytes(bytestr): + """Parse a string indicating a byte quantity into an integer.""" + matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr) + if matchobj is None: + return None + number = float(matchobj.group(1)) + multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower()) + return int(round(number * multiplier)) + + def to_screen(self, *args, **kargs): + self.ydl.to_screen(*args, **kargs) + + def to_stderr(self, message): + self.ydl.to_screen(message) + + def to_console_title(self, message): + self.ydl.to_console_title(message) + + def trouble(self, *args, **kargs): + self.ydl.trouble(*args, **kargs) + + def report_warning(self, *args, **kargs): + self.ydl.report_warning(*args, **kargs) + + def report_error(self, *args, **kargs): + self.ydl.report_error(*args, **kargs) + + def slow_down(self, start_time, byte_counter): + """Sleep if the download speed is over the rate limit.""" + rate_limit = self.params.get('ratelimit', None) + if rate_limit is None or byte_counter == 0: + return + now = time.time() + elapsed = now - start_time + if elapsed <= 0.0: + return + speed = float(byte_counter) / elapsed + if speed > rate_limit: + time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) + + def temp_name(self, filename): + """Returns a temporary filename for the given filename.""" + if self.params.get('nopart', False) or filename == u'-' or \ + (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))): + return filename + return filename + u'.part' + + def undo_temp_name(self, filename): + if filename.endswith(u'.part'): + return filename[:-len(u'.part')] + return filename + + def try_rename(self, old_filename, new_filename): + try: + if old_filename == new_filename: + return + os.rename(encodeFilename(old_filename), encodeFilename(new_filename)) + except (IOError, OSError) as err: + self.report_error(u'unable to rename file') + + def try_utime(self, filename, last_modified_hdr): + """Try to set the last-modified time of the given file.""" + if last_modified_hdr is None: + return + if not os.path.isfile(encodeFilename(filename)): + return + timestr = last_modified_hdr + if timestr is None: + return + filetime = timeconvert(timestr) + if filetime is None: + return filetime + # Ignore obviously invalid dates + if filetime == 0: + return + try: + os.utime(filename, (time.time(), filetime)) + except: + pass + return filetime + + def report_destination(self, filename): + """Report destination filename.""" + self.to_screen(u'[download] Destination: ' + filename) + + def _report_progress_status(self, msg, is_last_line=False): + fullmsg = u'[download] ' + msg + if self.params.get('progress_with_newline', False): + self.to_screen(fullmsg) + else: + if os.name == 'nt': + prev_len = getattr(self, '_report_progress_prev_line_length', + 0) + if prev_len > len(fullmsg): + fullmsg += u' ' * (prev_len - len(fullmsg)) + self._report_progress_prev_line_length = len(fullmsg) + clear_line = u'\r' + else: + clear_line = (u'\r\x1b[K' if sys.stderr.isatty() else u'\r') + self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line) + self.to_console_title(u'youtube-dl ' + msg) + + def report_progress(self, percent, data_len_str, speed, eta): + """Report download progress.""" + if self.params.get('noprogress', False): + return + if eta is not None: + eta_str = self.format_eta(eta) + else: + eta_str = 'Unknown ETA' + if percent is not None: + percent_str = self.format_percent(percent) + else: + percent_str = 'Unknown %' + speed_str = self.format_speed(speed) + + msg = (u'%s of %s at %s ETA %s' % + (percent_str, data_len_str, speed_str, eta_str)) + self._report_progress_status(msg) + + def report_progress_live_stream(self, downloaded_data_len, speed, elapsed): + if self.params.get('noprogress', False): + return + downloaded_str = format_bytes(downloaded_data_len) + speed_str = self.format_speed(speed) + elapsed_str = FileDownloader.format_seconds(elapsed) + msg = u'%s at %s (%s)' % (downloaded_str, speed_str, elapsed_str) + self._report_progress_status(msg) + + def report_finish(self, data_len_str, tot_time): + """Report download finished.""" + if self.params.get('noprogress', False): + self.to_screen(u'[download] Download completed') + else: + self._report_progress_status( + (u'100%% of %s in %s' % + (data_len_str, self.format_seconds(tot_time))), + is_last_line=True) + + def report_resuming_byte(self, resume_len): + """Report attempt to resume at given byte.""" + self.to_screen(u'[download] Resuming download at byte %s' % resume_len) + + def report_retry(self, count, retries): + """Report retry in case of HTTP error 5xx""" + self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries)) + + def report_file_already_downloaded(self, file_name): + """Report file has already been fully downloaded.""" + try: + self.to_screen(u'[download] %s has already been downloaded' % file_name) + except UnicodeEncodeError: + self.to_screen(u'[download] The file has already been downloaded') + + def report_unable_to_resume(self): + """Report it was impossible to resume download.""" + self.to_screen(u'[download] Unable to resume') + + def download(self, filename, info_dict): + """Download to a filename using the info from info_dict + Return True on success and False otherwise + """ + url = info_dict['url'] + + # Check file already present + if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False): + self.report_file_already_downloaded(filename) + self._hook_progress({ + 'filename': filename, + 'status': 'finished', + 'total_bytes': os.path.getsize(encodeFilename(filename)), + }) + return True + else: + return self.real_download(filename, info_dict) + + def real_download(self, filename, info_dict): + """Real download process. Redefine in subclasses.""" + raise NotImplementedError(u'This method must be implemented by sublcasses') + + def _hook_progress(self, status): + for ph in self._progress_hooks: + ph(status) + + def add_progress_hook(self, ph): + """ ph gets called on download progress, with a dictionary with the entries + * filename: The final filename + * status: One of "downloading" and "finished" + + It can also have some of the following entries: + + * downloaded_bytes: Bytes on disks + * total_bytes: Total bytes, None if unknown + * tmpfilename: The filename we're currently writing to + * eta: The estimated time in seconds, None if unknown + * speed: The download speed in bytes/second, None if unknown + + Hooks are guaranteed to be called at least once (with status "finished") + if the download is successful. + """ + self._progress_hooks.append(ph) + diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py new file mode 100644 index 000000000..51e8c4778 --- /dev/null +++ b/youtube_dl/downloader/hls.py @@ -0,0 +1,44 @@ +import os +import subprocess + +from .common import FileDownloader +from ..utils import ( + encodeFilename, +) + + +class HlsFD(FileDownloader): + def real_download(self, filename, info_dict): + url = info_dict['url'] + self.report_destination(filename) + tmpfilename = self.temp_name(filename) + + args = ['-y', '-i', url, '-f', 'mp4', '-c', 'copy', + '-bsf:a', 'aac_adtstoasc', tmpfilename] + + for program in ['avconv', 'ffmpeg']: + try: + subprocess.call([program, '-version'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) + break + except (OSError, IOError): + pass + else: + self.report_error(u'm3u8 download detected but ffmpeg or avconv could not be found') + cmd = [program] + args + + retval = subprocess.call(cmd) + if retval == 0: + fsize = os.path.getsize(encodeFilename(tmpfilename)) + self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize)) + self.try_rename(tmpfilename, filename) + self._hook_progress({ + 'downloaded_bytes': fsize, + 'total_bytes': fsize, + 'filename': filename, + 'status': 'finished', + }) + return True + else: + self.to_stderr(u"\n") + self.report_error(u'ffmpeg exited with code %d' % retval) + return False diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py new file mode 100644 index 000000000..54dc7616b --- /dev/null +++ b/youtube_dl/downloader/http.py @@ -0,0 +1,191 @@ +import os +import re +import subprocess +import sys +import time + +from .common import FileDownloader +from ..utils import ( + compat_urllib_request, + compat_urllib_error, + ContentTooShortError, + + encodeFilename, + sanitize_open, + format_bytes, +) + + +class HttpFD(FileDownloader): + def real_download(self, filename, info_dict): + url = info_dict['url'] + tmpfilename = self.temp_name(filename) + stream = None + + # Do not include the Accept-Encoding header + headers = {'Youtubedl-no-compression': 'True'} + if 'user_agent' in info_dict: + headers['Youtubedl-user-agent'] = info_dict['user_agent'] + basic_request = compat_urllib_request.Request(url, None, headers) + request = compat_urllib_request.Request(url, None, headers) + + if self.params.get('test', False): + request.add_header('Range','bytes=0-10240') + + # Establish possible resume length + if os.path.isfile(encodeFilename(tmpfilename)): + resume_len = os.path.getsize(encodeFilename(tmpfilename)) + else: + resume_len = 0 + + open_mode = 'wb' + if resume_len != 0: + if self.params.get('continuedl', False): + self.report_resuming_byte(resume_len) + request.add_header('Range','bytes=%d-' % resume_len) + open_mode = 'ab' + else: + resume_len = 0 + + count = 0 + retries = self.params.get('retries', 0) + while count <= retries: + # Establish connection + try: + if count == 0 and 'urlhandle' in info_dict: + data = info_dict['urlhandle'] + data = compat_urllib_request.urlopen(request) + break + except (compat_urllib_error.HTTPError, ) as err: + if (err.code < 500 or err.code >= 600) and err.code != 416: + # Unexpected HTTP error + raise + elif err.code == 416: + # Unable to resume (requested range not satisfiable) + try: + # Open the connection again without the range header + data = compat_urllib_request.urlopen(basic_request) + content_length = data.info()['Content-Length'] + except (compat_urllib_error.HTTPError, ) as err: + if err.code < 500 or err.code >= 600: + raise + else: + # Examine the reported length + if (content_length is not None and + (resume_len - 100 < int(content_length) < resume_len + 100)): + # The file had already been fully downloaded. + # Explanation to the above condition: in issue #175 it was revealed that + # YouTube sometimes adds or removes a few bytes from the end of the file, + # changing the file size slightly and causing problems for some users. So + # I decided to implement a suggested change and consider the file + # completely downloaded if the file size differs less than 100 bytes from + # the one in the hard drive. + self.report_file_already_downloaded(filename) + self.try_rename(tmpfilename, filename) + self._hook_progress({ + 'filename': filename, + 'status': 'finished', + }) + return True + else: + # The length does not match, we start the download over + self.report_unable_to_resume() + open_mode = 'wb' + break + # Retry + count += 1 + if count <= retries: + self.report_retry(count, retries) + + if count > retries: + self.report_error(u'giving up after %s retries' % retries) + return False + + data_len = data.info().get('Content-length', None) + if data_len is not None: + data_len = int(data_len) + resume_len + min_data_len = self.params.get("min_filesize", None) + max_data_len = self.params.get("max_filesize", None) + if min_data_len is not None and data_len < min_data_len: + self.to_screen(u'\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len)) + return False + if max_data_len is not None and data_len > max_data_len: + self.to_screen(u'\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len)) + return False + + data_len_str = format_bytes(data_len) + byte_counter = 0 + resume_len + block_size = self.params.get('buffersize', 1024) + start = time.time() + while True: + # Download and write + before = time.time() + data_block = data.read(block_size) + after = time.time() + if len(data_block) == 0: + break + byte_counter += len(data_block) + + # Open file just in time + if stream is None: + try: + (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode) + assert stream is not None + filename = self.undo_temp_name(tmpfilename) + self.report_destination(filename) + except (OSError, IOError) as err: + self.report_error(u'unable to open for writing: %s' % str(err)) + return False + try: + stream.write(data_block) + except (IOError, OSError): + self.to_stderr(u"\n") + self.report_error(u'unable to write data: %s' % str(err)) + return False + if not self.params.get('noresizebuffer', False): + block_size = self.best_block_size(after - before, len(data_block)) + + # Progress message + speed = self.calc_speed(start, time.time(), byte_counter - resume_len) + if data_len is None: + eta = percent = None + else: + percent = self.calc_percent(byte_counter, data_len) + eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len) + self.report_progress(percent, data_len_str, speed, eta) + + self._hook_progress({ + 'downloaded_bytes': byte_counter, + 'total_bytes': data_len, + 'tmpfilename': tmpfilename, + 'filename': filename, + 'status': 'downloading', + 'eta': eta, + 'speed': speed, + }) + + # Apply rate limit + self.slow_down(start, byte_counter - resume_len) + + if stream is None: + self.to_stderr(u"\n") + self.report_error(u'Did not get any data blocks') + return False + stream.close() + self.report_finish(data_len_str, (time.time() - start)) + if data_len is not None and byte_counter != data_len: + raise ContentTooShortError(byte_counter, int(data_len)) + self.try_rename(tmpfilename, filename) + + # Update file modification time + if self.params.get('updatetime', True): + info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None)) + + self._hook_progress({ + 'downloaded_bytes': byte_counter, + 'total_bytes': byte_counter, + 'filename': filename, + 'status': 'finished', + }) + + return True diff --git a/youtube_dl/downloader/mplayer.py b/youtube_dl/downloader/mplayer.py new file mode 100644 index 000000000..7dd6e76d0 --- /dev/null +++ b/youtube_dl/downloader/mplayer.py @@ -0,0 +1,39 @@ +import os +import subprocess + +from .common import FileDownloader +from ..utils import ( + encodeFilename, +) + + +class MplayerFD(FileDownloader): + def real_download(self, filename, info_dict): + self.report_destination(filename) + tmpfilename = self.temp_name(filename) + + args = ['mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy', '-dumpstream', '-dumpfile', tmpfilename, url] + # Check for mplayer first + try: + subprocess.call(['mplayer', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) + except (OSError, IOError): + self.report_error(u'MMS or RTSP download detected but "%s" could not be run' % args[0] ) + return False + + # Download using mplayer. + retval = subprocess.call(args) + if retval == 0: + fsize = os.path.getsize(encodeFilename(tmpfilename)) + self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize)) + self.try_rename(tmpfilename, filename) + self._hook_progress({ + 'downloaded_bytes': fsize, + 'total_bytes': fsize, + 'filename': filename, + 'status': 'finished', + }) + return True + else: + self.to_stderr(u"\n") + self.report_error(u'mplayer exited with code %d' % retval) + return False diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py new file mode 100644 index 000000000..b165e396f --- /dev/null +++ b/youtube_dl/downloader/rtmp.py @@ -0,0 +1,178 @@ +import os +import re +import subprocess +import sys +import time + +from .common import FileDownloader +from ..utils import ( + encodeFilename, + format_bytes, +) + + +class RtmpFD(FileDownloader): + def real_download(self, filename, info_dict): + def run_rtmpdump(args): + start = time.time() + resume_percent = None + resume_downloaded_data_len = None + proc = subprocess.Popen(args, stderr=subprocess.PIPE) + cursor_in_new_line = True + proc_stderr_closed = False + while not proc_stderr_closed: + # read line from stderr + line = u'' + while True: + char = proc.stderr.read(1) + if not char: + proc_stderr_closed = True + break + if char in [b'\r', b'\n']: + break + line += char.decode('ascii', 'replace') + if not line: + # proc_stderr_closed is True + continue + mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec \(([0-9]{1,2}\.[0-9])%\)', line) + if mobj: + downloaded_data_len = int(float(mobj.group(1))*1024) + percent = float(mobj.group(2)) + if not resume_percent: + resume_percent = percent + resume_downloaded_data_len = downloaded_data_len + eta = self.calc_eta(start, time.time(), 100-resume_percent, percent-resume_percent) + speed = self.calc_speed(start, time.time(), downloaded_data_len-resume_downloaded_data_len) + data_len = None + if percent > 0: + data_len = int(downloaded_data_len * 100 / percent) + data_len_str = u'~' + format_bytes(data_len) + self.report_progress(percent, data_len_str, speed, eta) + cursor_in_new_line = False + self._hook_progress({ + 'downloaded_bytes': downloaded_data_len, + 'total_bytes': data_len, + 'tmpfilename': tmpfilename, + 'filename': filename, + 'status': 'downloading', + 'eta': eta, + 'speed': speed, + }) + else: + # no percent for live streams + mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec', line) + if mobj: + downloaded_data_len = int(float(mobj.group(1))*1024) + time_now = time.time() + speed = self.calc_speed(start, time_now, downloaded_data_len) + self.report_progress_live_stream(downloaded_data_len, speed, time_now - start) + cursor_in_new_line = False + self._hook_progress({ + 'downloaded_bytes': downloaded_data_len, + 'tmpfilename': tmpfilename, + 'filename': filename, + 'status': 'downloading', + 'speed': speed, + }) + elif self.params.get('verbose', False): + if not cursor_in_new_line: + self.to_screen(u'') + cursor_in_new_line = True + self.to_screen(u'[rtmpdump] '+line) + proc.wait() + if not cursor_in_new_line: + self.to_screen(u'') + return proc.returncode + + url = info_dict['url'] + player_url = info_dict.get('player_url', None) + page_url = info_dict.get('page_url', None) + play_path = info_dict.get('play_path', None) + tc_url = info_dict.get('tc_url', None) + live = info_dict.get('rtmp_live', False) + conn = info_dict.get('rtmp_conn', None) + + self.report_destination(filename) + tmpfilename = self.temp_name(filename) + test = self.params.get('test', False) + + # Check for rtmpdump first + try: + subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) + except (OSError, IOError): + self.report_error(u'RTMP download detected but "rtmpdump" could not be run') + return False + + # Download using rtmpdump. rtmpdump returns exit code 2 when + # the connection was interrumpted and resuming appears to be + # possible. This is part of rtmpdump's normal usage, AFAIK. + basic_args = ['rtmpdump', '--verbose', '-r', url, '-o', tmpfilename] + if player_url is not None: + basic_args += ['--swfVfy', player_url] + if page_url is not None: + basic_args += ['--pageUrl', page_url] + if play_path is not None: + basic_args += ['--playpath', play_path] + if tc_url is not None: + basic_args += ['--tcUrl', url] + if test: + basic_args += ['--stop', '1'] + if live: + basic_args += ['--live'] + if conn: + basic_args += ['--conn', conn] + args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)] + + if sys.platform == 'win32' and sys.version_info < (3, 0): + # Windows subprocess module does not actually support Unicode + # on Python 2.x + # See http://stackoverflow.com/a/9951851/35070 + subprocess_encoding = sys.getfilesystemencoding() + args = [a.encode(subprocess_encoding, 'ignore') for a in args] + else: + subprocess_encoding = None + + if self.params.get('verbose', False): + if subprocess_encoding: + str_args = [ + a.decode(subprocess_encoding) if isinstance(a, bytes) else a + for a in args] + else: + str_args = args + try: + import pipes + shell_quote = lambda args: ' '.join(map(pipes.quote, str_args)) + except ImportError: + shell_quote = repr + self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(str_args)) + + retval = run_rtmpdump(args) + + while (retval == 2 or retval == 1) and not test: + prevsize = os.path.getsize(encodeFilename(tmpfilename)) + self.to_screen(u'[rtmpdump] %s bytes' % prevsize) + time.sleep(5.0) # This seems to be needed + retval = run_rtmpdump(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) + cursize = os.path.getsize(encodeFilename(tmpfilename)) + if prevsize == cursize and retval == 1: + break + # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those + if prevsize == cursize and retval == 2 and cursize > 1024: + self.to_screen(u'[rtmpdump] Could not download the whole video. This can happen for some advertisements.') + retval = 0 + break + if retval == 0 or (test and retval == 2): + fsize = os.path.getsize(encodeFilename(tmpfilename)) + self.to_screen(u'[rtmpdump] %s bytes' % fsize) + self.try_rename(tmpfilename, filename) + self._hook_progress({ + 'downloaded_bytes': fsize, + 'total_bytes': fsize, + 'filename': filename, + 'status': 'finished', + }) + return True + else: + self.to_stderr(u"\n") + self.report_error(u'rtmpdump exited with code %d' % retval) + return False From df5374743615fe54178b3942b115e7168ce0cd97 Mon Sep 17 00:00:00 2001 From: dst <dstftw@gmail.com> Date: Thu, 12 Dec 2013 15:13:45 +0700 Subject: [PATCH 298/425] [channel9] Initial implementation (#1885) --- youtube_dl/extractor/channel9.py | 351 +++++++++++++++++++++++++++++++ 1 file changed, 351 insertions(+) create mode 100644 youtube_dl/extractor/channel9.py diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py new file mode 100644 index 000000000..016c4497a --- /dev/null +++ b/youtube_dl/extractor/channel9.py @@ -0,0 +1,351 @@ +# encoding: utf-8 + +import re + +from .common import InfoExtractor +from ..utils import ( + format_bytes, + ExtractorError, +) + +class Channel9IE(InfoExtractor): + ''' + Common extractor for channel9.msdn.com. + + The type of provided URL (video or playlist) is determined according to + meta Search.PageType from web page HTML rather than URL itself, as it is + not always possible to do. + ''' + IE_DESC = u'Channel 9' + IE_NAME = u'channel9' + _VALID_URL = r'^https?://(?:www\.)?channel9\.msdn\.com/(?P<contentpath>.+)/?' + + _TESTS = [ + { + u'url': u'http://channel9.msdn.com/Events/TechEd/Australia/2013/KOS002', + u'file': u'Events_TechEd_Australia_2013_KOS002.mp4', + u'md5': u'bbd75296ba47916b754e73c3a4bbdf10', + u'info_dict': { + u'title': u'Developer Kick-Off Session: Stuff We Love', + u'description': u'md5:c08d72240b7c87fcecafe2692f80e35f', + u'duration': 4576, + u'thumbnail': u'http://media.ch9.ms/ch9/9d51/03902f2d-fc97-4d3c-b195-0bfe15a19d51/KOS002_220.jpg', + u'session_code': u'KOS002', + u'session_day': u'Day 1', + u'session_room': u'Arena 1A', + u'session_speakers': [ u'Ed Blankenship', u'Andrew Coates', u'Brady Gaster', u'Patrick Klug', u'Mads Kristensen' ], + }, + }, + { + u'url': u'http://channel9.msdn.com/posts/Self-service-BI-with-Power-BI-nuclear-testing', + u'file': u'posts_Self-service-BI-with-Power-BI-nuclear-testing.mp4', + u'md5': u'b43ee4529d111bc37ba7ee4f34813e68', + u'info_dict': { + u'title': u'Self-service BI with Power BI - nuclear testing', + u'description': u'md5:a6d5cfd9ee46d1851cf6e40ea61cfc10', + u'duration': 1540, + u'thumbnail': u'http://media.ch9.ms/ch9/87e1/0300391f-a455-4c72-bec3-4422f19287e1/selfservicenuk_512.jpg', + u'authors': [ u'Mike Wilmot' ], + }, + } + ] + + _RSS_URL = 'http://channel9.msdn.com/%s/RSS' + _EXTRACT_ENTRY_ITEMS_FROM_RSS = False + + # Sorted by quality + _known_formats = ['MP3', 'MP4', 'Mid Quality WMV', 'Mid Quality MP4', 'High Quality WMV', 'High Quality MP4'] + + def _restore_bytes(self, formatted_size): + if not formatted_size: + return 0 + m = re.match(r'^(?P<size>\d+(?:\.\d+)?)\s+(?P<units>[a-zA-Z]+)', formatted_size) + if not m: + return 0 + units = m.group('units') + try: + exponent = [u'B', u'KB', u'MB', u'GB', u'TB', u'PB', u'EB', u'ZB', u'YB'].index(units.upper()) + except ValueError: + return 0 + size = float(m.group('size')) + return int(size * (1024 ** exponent)) + + def _formats_from_html(self, html): + FORMAT_REGEX = r''' + (?x) + <a\s+href="(?P<url>[^"]+)">(?P<quality>[^<]+)</a>\s* + <span\s+class="usage">\((?P<note>[^\)]+)\)</span>\s* + (?:<div\s+class="popup\s+rounded">\s* + <h3>File\s+size</h3>\s*(?P<filesize>.*?)\s* + </div>)? # File size part may be missing + ''' + # Extract known formats + formats = [{'url': x.group('url'), + 'format_id': x.group('quality'), + 'format_note': x.group('note'), + 'format': '%s (%s)' % (x.group('quality'), x.group('note')), + 'filesize': self._restore_bytes(x.group('filesize')), # File size is approximate + } for x in list(re.finditer(FORMAT_REGEX, html)) if x.group('quality') in self._known_formats] + # Sort according to known formats list + formats.sort(key=lambda fmt: self._known_formats.index(fmt['format_id'])) + return formats + + def _formats_from_rss_item(self, item): + + def process_formats(elem): + formats = [] + for media_content in elem.findall('./{http://search.yahoo.com/mrss/}content'): + url = media_content.attrib['url'] + # Ignore unrelated media + if url.endswith('.ism/manifest'): + continue + format_note = media_content.attrib['type'] + filesize = int(media_content.attrib['fileSize']) + formats.append({'url': url, + 'format_note': format_note, + 'format': '%s %s' % (format_note, format_bytes(filesize)), + 'filesize': filesize, + }) + return formats + + formats = [] + + for media_group in item.findall('./{http://search.yahoo.com/mrss/}group'): + formats.extend(process_formats(media_group)) + + # Sometimes there are no media:groups in item, but there is media:content + # right in item (usually when there is the only media source) + formats.extend(process_formats(item)) + + # Sort by file size + formats.sort(key=lambda fmt: fmt['filesize']) + return formats + + def _extract_title(self, html): + title = self._html_search_meta(u'title', html, u'title') + if title is None: + title = self._og_search_title(html) + TITLE_SUFFIX = u' (Channel 9)' + if title is not None and title.endswith(TITLE_SUFFIX): + title = title[:-len(TITLE_SUFFIX)] + return title + + def _extract_description(self, html): + DESCRIPTION_REGEX = r'''(?sx) + <div\s+class="entry-content">\s* + <div\s+id="entry-body">\s* + (?P<description>.+?)\s* + </div>\s* + </div> + ''' + m = re.search(DESCRIPTION_REGEX, html) + if m is not None: + return m.group('description') + return self._html_search_meta(u'description', html, u'description') + + def _extract_duration(self, html): + m = re.search(r'data-video_duration="(?P<hours>\d{2}):(?P<minutes>\d{2}):(?P<seconds>\d{2})"', html) + return ((int(m.group('hours')) * 60 * 60) + (int(m.group('minutes')) * 60) + int(m.group('seconds'))) if m else None + + def _extract_slides(self, html): + m = re.search(r'<a href="(?P<slidesurl>[^"]+)" class="slides">Slides</a>', html) + return m.group('slidesurl') if m is not None else None + + def _extract_zip(self, html): + m = re.search(r'<a href="(?P<zipurl>[^"]+)" class="zip">Zip</a>', html) + return m.group('zipurl') if m is not None else None + + def _extract_avg_rating(self, html): + m = re.search(r'<p class="avg-rating">Avg Rating: <span>(?P<avgrating>[^<]+)</span></p>', html) + return float(m.group('avgrating')) if m is not None else 0 + + def _extract_rating_count(self, html): + m = re.search(r'<div class="rating-count">\((?P<ratingcount>[^<]+)\)</div>', html) + return int(self._fix_count(m.group('ratingcount'))) if m is not None else 0 + + def _extract_view_count(self, html): + m = re.search(r'<li class="views">\s*<span class="count">(?P<viewcount>[^<]+)</span> Views\s*</li>', html) + return int(self._fix_count(m.group('viewcount'))) if m is not None else 0 + + def _extract_comment_count(self, html): + m = re.search(r'<li class="comments">\s*<a href="#comments">\s*<span class="count">(?P<commentcount>[^<]+)</span> Comments\s*</a>\s*</li>', html) + return int(self._fix_count(m.group('commentcount'))) if m is not None else 0 + + def _fix_count(self, count): + return int(str(count).replace(',', '')) if count is not None else None + + def _extract_authors(self, html): + m = re.search(r'(?s)<li class="author">(.*?)</li>', html) + if m is None: + return None + return re.findall(r'<a href="/Niners/[^"]+">([^<]+)</a>', m.group(1)) + + def _extract_session_code(self, html): + m = re.search(r'<li class="code">\s*(?P<code>.+?)\s*</li>', html) + return m.group('code') if m is not None else None + + def _extract_session_day(self, html): + m = re.search(r'<li class="day">\s*<a href="/Events/[^"]+">(?P<day>[^<]+)</a>\s*</li>', html) + return m.group('day') if m is not None else None + + def _extract_session_room(self, html): + m = re.search(r'<li class="room">\s*(?P<room>.+?)\s*</li>', html) + return m.group('room') if m is not None else None + + def _extract_session_speakers(self, html): + return re.findall(r'<a href="/Events/Speakers/[^"]+">([^<]+)</a>', html) + + def _extract_content(self, html, content_path): + # Look for downloadable content + formats = self._formats_from_html(html) + slides = self._extract_slides(html) + zip_ = self._extract_zip(html) + + # Nothing to download + if len(formats) == 0 and slides is None and zip_ is None: + self._downloader.report_warning(u'None of recording, slides or zip are available for %s' % content_path) + return + + # Extract meta + title = self._extract_title(html) + description = self._extract_description(html) + thumbnail = self._og_search_thumbnail(html) + duration = self._extract_duration(html) + avg_rating = self._extract_avg_rating(html) + rating_count = self._extract_rating_count(html) + view_count = self._extract_view_count(html) + comment_count = self._extract_comment_count(html) + + common = {'_type': 'video', + 'id': content_path, + 'description': description, + 'thumbnail': thumbnail, + 'duration': duration, + 'avg_rating': avg_rating, + 'rating_count': rating_count, + 'view_count': view_count, + 'comment_count': comment_count, + } + + result = [] + + if slides is not None: + d = common.copy() + d.update({ 'title': title + '-Slides', 'url': slides }) + result.append(d) + + if zip_ is not None: + d = common.copy() + d.update({ 'title': title + '-Zip', 'url': zip_ }) + result.append(d) + + if len(formats) > 0: + d = common.copy() + d.update({ 'title': title, 'formats': formats }) + result.append(d) + + return result + + def _extract_entry_item(self, html, content_path): + contents = self._extract_content(html, content_path) + if contents is None: + return contents + + authors = self._extract_authors(html) + + for content in contents: + content['authors'] = authors + + return contents + + def _extract_session(self, html, content_path): + contents = self._extract_content(html, content_path) + if contents is None: + return contents + + session_meta = {'session_code': self._extract_session_code(html), + 'session_day': self._extract_session_day(html), + 'session_room': self._extract_session_room(html), + 'session_speakers': self._extract_session_speakers(html), + } + + for content in contents: + content.update(session_meta) + + return contents + + def _extract_content_rss(self, rss): + ''' + Extracts links to entry items right out of RSS feed. + This approach is faster than extracting from web pages + one by one, but suffers from some problems. + Pros: + - no need to download additional pages + - provides more media links + - accurate file size + Cons: + - fewer meta data provided + - links to media files have no appropriate data that may be used as format_id + - RSS does not contain links to presentation materials (slides, zip) + ''' + entries = [] + for item in rss.findall('./channel/item'): + url = item.find('./link').text + video_id = url.split('/')[-1] + formats = self._formats_from_rss_item(item) + + if len(formats) == 0: + self._downloader.report_warning(u'The recording for session %s is not yet available' % video_id) + continue + + title = item.find('./title').text + description = item.find('./description').text + + thumbnail = item.find('./{http://search.yahoo.com/mrss/}thumbnail').text + + duration_e = item.find('./{http://www.itunes.com/dtds/podcast-1.0.dtd}duration') + duration = duration_e.text if duration_e is not None else 0 + + speakers_e = item.find('./{http://purl.org/dc/elements/1.1/}creator') + speakers = speakers_e.text.split(', ') if speakers_e is not None and speakers_e.text else [] + + entries.append({'_type': 'video', + 'id': video_id, + 'formats': formats, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + 'duration': duration, + 'session_speakers': speakers, + }) + return entries + + def _extract_list(self, content_path): + rss = self._download_xml(self._RSS_URL % content_path, content_path, u'Downloading RSS') + if self._EXTRACT_ENTRY_ITEMS_FROM_RSS: + return self._extract_content_rss(rss) + else: + entries = [self.url_result(session_url.text, 'Channel9') + for session_url in rss.findall('./channel/item/link')] + title_text = rss.find('./channel/title').text + return self.playlist_result(entries, content_path, title_text) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + content_path = mobj.group('contentpath') + + webpage = self._download_webpage(url, content_path, u'Downloading web page') + + page_type_m = re.search(r'<meta name="Search.PageType" content="(?P<pagetype>[^"]+)"/>', webpage) + if page_type_m is None: + raise ExtractorError(u'Search.PageType not found, don\'t know how to process this page', expected=True) + + page_type = page_type_m.group('pagetype') + if page_type == 'List': # List page, may contain list of 'item'-like objects + return self._extract_list(content_path) + elif page_type == 'Entry.Item': # Any 'item'-like page, may contain downloadable content + return self._extract_entry_item(webpage, content_path) + elif page_type == 'Session': # Event session page, may contain downloadable content + return self._extract_session(webpage, content_path) + else: + raise ExtractorError(u'Unexpected Search.PageType %s' % page_type, expected=True) \ No newline at end of file From 4d2ebb6bd771edaa26ebb5e76f9ee8e880a4d152 Mon Sep 17 00:00:00 2001 From: dst <dstftw@gmail.com> Date: Thu, 12 Dec 2013 15:19:23 +0700 Subject: [PATCH 299/425] [channel9] Cleanup --- youtube_dl/extractor/channel9.py | 94 ++------------------------------ 1 file changed, 5 insertions(+), 89 deletions(-) diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py index 016c4497a..b4b586c12 100644 --- a/youtube_dl/extractor/channel9.py +++ b/youtube_dl/extractor/channel9.py @@ -3,10 +3,7 @@ import re from .common import InfoExtractor -from ..utils import ( - format_bytes, - ExtractorError, -) +from ..utils import ExtractorError class Channel9IE(InfoExtractor): ''' @@ -51,7 +48,6 @@ class Channel9IE(InfoExtractor): ] _RSS_URL = 'http://channel9.msdn.com/%s/RSS' - _EXTRACT_ENTRY_ITEMS_FROM_RSS = False # Sorted by quality _known_formats = ['MP3', 'MP4', 'Mid Quality WMV', 'Mid Quality MP4', 'High Quality WMV', 'High Quality MP4'] @@ -90,37 +86,6 @@ class Channel9IE(InfoExtractor): formats.sort(key=lambda fmt: self._known_formats.index(fmt['format_id'])) return formats - def _formats_from_rss_item(self, item): - - def process_formats(elem): - formats = [] - for media_content in elem.findall('./{http://search.yahoo.com/mrss/}content'): - url = media_content.attrib['url'] - # Ignore unrelated media - if url.endswith('.ism/manifest'): - continue - format_note = media_content.attrib['type'] - filesize = int(media_content.attrib['fileSize']) - formats.append({'url': url, - 'format_note': format_note, - 'format': '%s %s' % (format_note, format_bytes(filesize)), - 'filesize': filesize, - }) - return formats - - formats = [] - - for media_group in item.findall('./{http://search.yahoo.com/mrss/}group'): - formats.extend(process_formats(media_group)) - - # Sometimes there are no media:groups in item, but there is media:content - # right in item (usually when there is the only media source) - formats.extend(process_formats(item)) - - # Sort by file size - formats.sort(key=lambda fmt: fmt['filesize']) - return formats - def _extract_title(self, html): title = self._html_search_meta(u'title', html, u'title') if title is None: @@ -274,61 +239,12 @@ class Channel9IE(InfoExtractor): return contents - def _extract_content_rss(self, rss): - ''' - Extracts links to entry items right out of RSS feed. - This approach is faster than extracting from web pages - one by one, but suffers from some problems. - Pros: - - no need to download additional pages - - provides more media links - - accurate file size - Cons: - - fewer meta data provided - - links to media files have no appropriate data that may be used as format_id - - RSS does not contain links to presentation materials (slides, zip) - ''' - entries = [] - for item in rss.findall('./channel/item'): - url = item.find('./link').text - video_id = url.split('/')[-1] - formats = self._formats_from_rss_item(item) - - if len(formats) == 0: - self._downloader.report_warning(u'The recording for session %s is not yet available' % video_id) - continue - - title = item.find('./title').text - description = item.find('./description').text - - thumbnail = item.find('./{http://search.yahoo.com/mrss/}thumbnail').text - - duration_e = item.find('./{http://www.itunes.com/dtds/podcast-1.0.dtd}duration') - duration = duration_e.text if duration_e is not None else 0 - - speakers_e = item.find('./{http://purl.org/dc/elements/1.1/}creator') - speakers = speakers_e.text.split(', ') if speakers_e is not None and speakers_e.text else [] - - entries.append({'_type': 'video', - 'id': video_id, - 'formats': formats, - 'title': title, - 'description': description, - 'thumbnail': thumbnail, - 'duration': duration, - 'session_speakers': speakers, - }) - return entries - def _extract_list(self, content_path): rss = self._download_xml(self._RSS_URL % content_path, content_path, u'Downloading RSS') - if self._EXTRACT_ENTRY_ITEMS_FROM_RSS: - return self._extract_content_rss(rss) - else: - entries = [self.url_result(session_url.text, 'Channel9') - for session_url in rss.findall('./channel/item/link')] - title_text = rss.find('./channel/title').text - return self.playlist_result(entries, content_path, title_text) + entries = [self.url_result(session_url.text, 'Channel9') + for session_url in rss.findall('./channel/item/link')] + title_text = rss.find('./channel/title').text + return self.playlist_result(entries, content_path, title_text) def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) From 211f555d4cb0bc142d5b9304a919e27aba237d40 Mon Sep 17 00:00:00 2001 From: dst <dstftw@gmail.com> Date: Thu, 12 Dec 2013 15:55:31 +0700 Subject: [PATCH 300/425] [channel9] Missing import in __init__ --- youtube_dl/extractor/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 1149dc1ec..fb206a742 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -20,6 +20,7 @@ from .brightcove import BrightcoveIE from .c56 import C56IE from .canalplus import CanalplusIE from .canalc2 import Canalc2IE +from .channel9 import Channel9IE from .cinemassacre import CinemassacreIE from .clipfish import ClipfishIE from .clipsyndicate import ClipsyndicateIE From 9b17ba0fa512a9882a22bb2407ec433038bc8327 Mon Sep 17 00:00:00 2001 From: dst <dstftw@gmail.com> Date: Thu, 12 Dec 2013 16:10:17 +0700 Subject: [PATCH 301/425] [channel9] Fix test description md5 --- youtube_dl/extractor/channel9.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py index b4b586c12..ae70ea229 100644 --- a/youtube_dl/extractor/channel9.py +++ b/youtube_dl/extractor/channel9.py @@ -39,7 +39,7 @@ class Channel9IE(InfoExtractor): u'md5': u'b43ee4529d111bc37ba7ee4f34813e68', u'info_dict': { u'title': u'Self-service BI with Power BI - nuclear testing', - u'description': u'md5:a6d5cfd9ee46d1851cf6e40ea61cfc10', + u'description': u'md5:d1e6ecaafa7fb52a2cacdf9599829f5b', u'duration': 1540, u'thumbnail': u'http://media.ch9.ms/ch9/87e1/0300391f-a455-4c72-bec3-4422f19287e1/selfservicenuk_512.jpg', u'authors': [ u'Mike Wilmot' ], From 24b173fa5c4abf740cd5700722bc9162f1dc4c3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Thu, 12 Dec 2013 13:04:02 +0100 Subject: [PATCH 302/425] [naver] Recognize mobile urls (fixes #1951) --- youtube_dl/extractor/naver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/naver.py b/youtube_dl/extractor/naver.py index c012ec0cf..4cab30631 100644 --- a/youtube_dl/extractor/naver.py +++ b/youtube_dl/extractor/naver.py @@ -9,7 +9,7 @@ from ..utils import ( class NaverIE(InfoExtractor): - _VALID_URL = r'https?://tvcast\.naver\.com/v/(?P<id>\d+)' + _VALID_URL = r'https?://(?:m\.)?tvcast\.naver\.com/v/(?P<id>\d+)' _TEST = { u'url': u'http://tvcast.naver.com/v/81652', From 7a563df90ac93cf206d53f1af82f57a36343d589 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Thu, 12 Dec 2013 13:05:38 +0100 Subject: [PATCH 303/425] [daum] Recognize mobile urls (#1952) --- youtube_dl/extractor/daum.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py index d418ce4a8..4876ecb48 100644 --- a/youtube_dl/extractor/daum.py +++ b/youtube_dl/extractor/daum.py @@ -9,7 +9,7 @@ from ..utils import ( class DaumIE(InfoExtractor): - _VALID_URL = r'https?://tvpot\.daum\.net/.*?clipid=(?P<id>\d+)' + _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/.*?clipid=(?P<id>\d+)' IE_NAME = u'daum.net' _TEST = { From dadb8184e4374c278f5c66c36b13c08d8474e30f Mon Sep 17 00:00:00 2001 From: dst <dstftw@gmail.com> Date: Fri, 13 Dec 2013 22:27:37 +0700 Subject: [PATCH 304/425] Fix typo in month name --- youtube_dl/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 0dab9fcc5..d9bf6c24c 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1051,7 +1051,7 @@ def month_by_name(name): """ Return the number of a month by (locale-independently) English name """ ENGLISH_NAMES = [ - u'Januar', u'February', u'March', u'April', u'May', u'June', + u'January', u'February', u'March', u'April', u'May', u'June', u'July', u'August', u'September', u'October', u'November', u'December'] try: return ENGLISH_NAMES.index(name) + 1 From 5d574e143f41e1b38a25c814e2bf573b19f022da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Fri, 13 Dec 2013 17:04:40 +0100 Subject: [PATCH 305/425] [ign] Update one of test video's title --- youtube_dl/extractor/ign.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py index 57b79a336..381af91e4 100644 --- a/youtube_dl/extractor/ign.py +++ b/youtube_dl/extractor/ign.py @@ -44,7 +44,7 @@ class IGNIE(InfoExtractor): { u'file': u'638672ee848ae4ff108df2a296418ee2.mp4', u'info_dict': { - u'title': u'GTA 5\'s Twisted Beauty in Super Slow Motion', + u'title': u'26 Twisted Moments from GTA 5 in Slow Motion', u'description': u'The twisted beauty of GTA 5 in stunning slow motion.', }, }, From 8e05c870b4650599acfa35050df0ba7143bbe6ad Mon Sep 17 00:00:00 2001 From: Michael Orlitzky <michael@orlitzky.com> Date: Fri, 13 Dec 2013 22:22:53 -0500 Subject: [PATCH 306/425] Add support for pornhd.com. --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/pornhd.py | 42 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 youtube_dl/extractor/pornhd.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 1149dc1ec..bd16f0f3e 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -111,6 +111,7 @@ from .orf import ORFIE from .pbs import PBSIE from .photobucket import PhotobucketIE from .podomatic import PodomaticIE +from .pornhd import PornHdIE from .pornhub import PornHubIE from .pornotube import PornotubeIE from .pyvideo import PyvideoIE diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py new file mode 100644 index 000000000..c56740b8a --- /dev/null +++ b/youtube_dl/extractor/pornhd.py @@ -0,0 +1,42 @@ +import re + +from .common import InfoExtractor +from ..utils import compat_urllib_parse + +class PornHdIE(InfoExtractor): + _VALID_URL = r'(?:http://)?(?:www\.)?pornhd\.com/videos/(?P<video_id>[0-9]+)/(?P<video_title>.+)' + _TEST = { + u'id': u'1962', + u'url': u'http://www.pornhd.com/videos/1962/sierra-day-gets-his-cum-all-over-herself-hd-porn-video', + u'md5': u'4fe06e5108e8b524c35896f4c54c7155', + u'info_dict': { + u"title": u"sierra-day-gets-his-cum-all-over-herself-hd-porn-video", + u"age_limit": 18, + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + + video_id = mobj.group('video_id') + video_title = mobj.group('video_title') + video_extension = 'flv' + + webpage = self._download_webpage(url, video_id) + + + self.report_extraction(video_id) + + video_url = self._html_search_regex( + r'&hd=(http.+?)&', webpage, u'video URL') + video_url = compat_urllib_parse.unquote(video_url) + + age_limit = 18 + + return { + 'id': video_id, + 'url': video_url, + 'ext': video_extension, + 'title': video_title, + 'age_limit': age_limit, + } From 9ee859b683cd19bcdc01c2b6271ad7380a402ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Sat, 14 Dec 2013 14:20:12 +0100 Subject: [PATCH 307/425] [daylimotion] Add support for urls from the mobile site (fixes #1953) It uses the 'touch' subdomain and adds a '#' before 'video' --- youtube_dl/extractor/dailymotion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py index aea7e557e..6685c94a3 100644 --- a/youtube_dl/extractor/dailymotion.py +++ b/youtube_dl/extractor/dailymotion.py @@ -28,7 +28,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor): class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor): """Information Extractor for Dailymotion""" - _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/(?:embed/)?video/([^/]+)' + _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(embed|#)/)?video/(?P<id>[^/?_]+)' IE_NAME = u'dailymotion' _FORMATS = [ @@ -81,7 +81,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor): # Extract id and simplified title from URL mobj = re.match(self._VALID_URL, url) - video_id = mobj.group(1).split('_')[0].split('?')[0] + video_id = mobj.group('id') url = 'http://www.dailymotion.com/video/%s' % video_id From dca02c80bc5c370bb3b15c9343d255dd124d2a93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Sun, 15 Dec 2013 11:42:38 +0100 Subject: [PATCH 308/425] Fix detection of the extension if the 'extractaudio' is given and improve the error message (#1969) Using 'foo.mp4' shouldn't raise an error. If 'foo' is given suggest using 'foo.%(ext)s' for the template --- youtube_dl/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 3e82cd637..55e59814b 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -604,10 +604,10 @@ def _real_main(argv=None): or (opts.useid and u'%(id)s.%(ext)s') or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s') or u'%(title)s-%(id)s.%(ext)s') - if '%(ext)s' not in outtmpl and opts.extractaudio: + if not os.path.splitext(outtmpl)[1] and opts.extractaudio: parser.error(u'Cannot download a video and extract audio into the same' - u' file! Use "%%(ext)s" instead of %r' % - determine_ext(outtmpl, u'')) + u' file! Use "{0}.%(ext)s" instead of "{0}" as the output' + u' template'.format(outtmpl)) ydl_opts = { 'usenetrc': opts.usenetrc, From 5fe18bdbde749267c35bb360a3a0699e555a3fcf Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 03:09:49 +0100 Subject: [PATCH 309/425] Add --min-views / --max-views (Fixes #1979) --- youtube_dl/YoutubeDL.py | 27 ++++++++++++++++++++++----- youtube_dl/__init__.py | 10 ++++++++++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index c77777ba0..ab68d013c 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -127,7 +127,16 @@ class YoutubeDL(object): noplaylist: Download single video instead of a playlist if in doubt. age_limit: An integer representing the user's age in years. Unsuitable videos for the given age are skipped. - download_archive: File name of a file where all downloads are recorded. + min_views: An integer representing the minimum view count the video + must have in order to not be skipped. + Videos without view count information are always + downloaded. None for no limit. + max_views: An integer representing the maximum view count. + Videos that are more popular than that are not + downloaded. + Videos without view count information are always + downloaded. None for no limit. + download_archive: File name of a file where all downloads are recorded. Videos already present in the file are not downloaded again. cookiefile: File name where cookies should be read from and dumped to. @@ -415,13 +424,14 @@ class YoutubeDL(object): def _match_entry(self, info_dict): """ Returns None iff the file should be downloaded """ + video_title = info_dict.get('title', info_dict.get('id', u'video')) if 'title' in info_dict: # This can happen when we're just evaluating the playlist title = info_dict['title'] matchtitle = self.params.get('matchtitle', False) if matchtitle: if not re.search(matchtitle, title, re.IGNORECASE): - return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"' + return u'"' + title + '" title did not match pattern "' + matchtitle + '"' rejecttitle = self.params.get('rejecttitle', False) if rejecttitle: if re.search(rejecttitle, title, re.IGNORECASE): @@ -430,14 +440,21 @@ class YoutubeDL(object): if date is not None: dateRange = self.params.get('daterange', DateRange()) if date not in dateRange: - return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange) + return u'%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange) + view_count = info_dict.get('view_count', None) + if view_count is not None: + min_views = self.params.get('min_views') + if min_views is not None and view_count < min_views: + return u'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views) + max_views = self.params.get('max_views') + if max_views is not None and view_count > max_views: + return u'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views) age_limit = self.params.get('age_limit') if age_limit is not None: if age_limit < info_dict.get('age_limit', 0): return u'Skipping "' + title + '" because it is age restricted' if self.in_download_archive(info_dict): - return (u'%s has already been recorded in archive' - % info_dict.get('title', info_dict.get('id', u'video'))) + return u'%s has already been recorded in archive' % video_title return None @staticmethod diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 55e59814b..437c37541 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -210,6 +210,14 @@ def parseOpts(overrideArguments=None): selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None) selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None) selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None) + selection.add_option( + '--min-views', metavar='COUNT', dest='min_views', + default=None, type=int, + help="Do not download any videos with less than COUNT views",) + selection.add_option( + '--max-views', metavar='COUNT', dest='max_views', + default=None, type=int, + help="Do not download any videos with more than COUNT views",) selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False) selection.add_option('--age-limit', metavar='YEARS', dest='age_limit', help='download only videos suitable for the given age', @@ -668,6 +676,8 @@ def _real_main(argv=None): 'keepvideo': opts.keepvideo, 'min_filesize': opts.min_filesize, 'max_filesize': opts.max_filesize, + 'min_views': opts.min_views, + 'max_views': opts.max_views, 'daterange': date, 'cachedir': opts.cachedir, 'youtube_print_sig_code': opts.youtube_print_sig_code, From f8b56e95b82d9cb783c5a789b6b2770efff57b73 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 03:34:46 +0100 Subject: [PATCH 310/425] [theplatform] Detect geoblocked content --- youtube_dl/extractor/theplatform.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py index 61452e47d..650d7517f 100644 --- a/youtube_dl/extractor/theplatform.py +++ b/youtube_dl/extractor/theplatform.py @@ -3,6 +3,7 @@ import json from .common import InfoExtractor from ..utils import ( + ExtractorError, xpath_with_ns, ) @@ -32,6 +33,17 @@ class ThePlatformIE(InfoExtractor): smil_url = ('http://link.theplatform.com/s/dJ5BDC/{0}/meta.smil?' 'format=smil&mbr=true'.format(video_id)) meta = self._download_xml(smil_url, video_id) + + try: + error_msg = next( + n.attrib['abstract'] + for n in meta.findall(_x('.//smil:ref')) + if n.attrib['title'] == u'Geographic Restriction') + except StopIteration: + pass + else: + raise ExtractorError(error_msg, expected=True) + info_url = 'http://link.theplatform.com/s/dJ5BDC/{0}?format=preview'.format(video_id) info_json = self._download_webpage(info_url, video_id) info = json.loads(info_json) From 48462108f3b1937bcaa19a4752246748a5fc9360 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 03:43:45 +0100 Subject: [PATCH 311/425] [theplatform] Fix geographic restriction check --- youtube_dl/extractor/theplatform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/theplatform.py b/youtube_dl/extractor/theplatform.py index 650d7517f..cec65261b 100644 --- a/youtube_dl/extractor/theplatform.py +++ b/youtube_dl/extractor/theplatform.py @@ -38,7 +38,7 @@ class ThePlatformIE(InfoExtractor): error_msg = next( n.attrib['abstract'] for n in meta.findall(_x('.//smil:ref')) - if n.attrib['title'] == u'Geographic Restriction') + if n.attrib.get('title') == u'Geographic Restriction') except StopIteration: pass else: From fa3ae234e04f631a0754b6b474417ddffc2faee2 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 03:53:43 +0100 Subject: [PATCH 312/425] [cbs] Add extractor (Fixes #1977) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/cbs.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 youtube_dl/extractor/cbs.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 1149dc1ec..37bf8e306 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -20,6 +20,7 @@ from .brightcove import BrightcoveIE from .c56 import C56IE from .canalplus import CanalplusIE from .canalc2 import Canalc2IE +from .cbs import CBSIE from .cinemassacre import CinemassacreIE from .clipfish import ClipfishIE from .clipsyndicate import ClipsyndicateIE diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py new file mode 100644 index 000000000..ac0315853 --- /dev/null +++ b/youtube_dl/extractor/cbs.py @@ -0,0 +1,30 @@ +import re + +from .common import InfoExtractor + + +class CBSIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?cbs\.com/shows/[^/]+/video/(?P<id>[^/]+)/.*' + + _TEST = { + u'url': u'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/', + u'file': u'4JUVEwq3wUT7.flv', + u'info_dict': { + u'title': u'Connect Chat feat. Garth Brooks', + u'description': u'Connect with country music singer Garth Brooks, as he chats with fans on Wednesday November 27, 2013. Be sure to tune in to Garth Brooks: Live from Las Vegas, Friday November 29, at 9/8c on CBS!', + u'duration': 1495, + }, + u'params': { + # rtmp download + u'skip_download': True, + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + webpage = self._download_webpage(url, video_id) + real_id = self._search_regex( + r"video\.settings\.pid\s*=\s*'([^']+)';", + webpage, u'real video ID') + return self.url_result(u'theplatform:%s' % real_id) From b466b7029d0af24dc06ecd8bc1f147a39be91c62 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 04:09:05 +0100 Subject: [PATCH 313/425] [youtube] Make duration an integer or None --- youtube_dl/extractor/youtube.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 874429b78..a68a214ca 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1377,9 +1377,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): if 'length_seconds' not in video_info: self._downloader.report_warning(u'unable to extract video duration') - video_duration = '' + video_duration = None else: - video_duration = compat_urllib_parse.unquote_plus(video_info['length_seconds'][0]) + video_duration = int(compat_urllib_parse.unquote_plus(video_info['length_seconds'][0])) # annotations video_annotations = None From c0ba0f485914330d5b4d7b175fb1da3724d4bda0 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 04:09:30 +0100 Subject: [PATCH 314/425] Document duration field --- youtube_dl/extractor/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 69a083b68..fe8ce9e6c 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -34,8 +34,8 @@ class InfoExtractor(object): The dictionaries must include the following fields: id: Video identifier. - url: Final video URL. title: Video title, unescaped. + url: Final video URL. ext: Video filename extension. Instead of url and ext, formats can also specified. @@ -54,6 +54,7 @@ class InfoExtractor(object): player_url: SWF Player URL (used for rtmpdump). subtitles: The subtitle file contents as a dictionary in the format {language: subtitles}. + duration: Length of the video in seconds, as an integer. view_count: How many users have watched the video on the platform. like_count: Number of positive ratings of the video dislike_count: Number of negative ratings of the video From 525ef9227f178a965a2010971256a1929af8cdd3 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 04:15:10 +0100 Subject: [PATCH 315/425] Add --get-duration (Fixes #859) --- youtube_dl/YoutubeDL.py | 4 ++++ youtube_dl/__init__.py | 10 ++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index ab68d013c..52bd8e0e3 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -34,6 +34,7 @@ from .utils import ( encodeFilename, ExtractorError, format_bytes, + formatSeconds, get_term_width, locked_file, make_HTTPS_handler, @@ -94,6 +95,7 @@ class YoutubeDL(object): forcethumbnail: Force printing thumbnail URL. forcedescription: Force printing description. forcefilename: Force printing final filename. + forceduration: Force printing duration. forcejson: Force printing info_dict as JSON. simulate: Do not download the video files. format: Video format code. @@ -765,6 +767,8 @@ class YoutubeDL(object): self.to_stdout(info_dict['description']) if self.params.get('forcefilename', False) and filename is not None: self.to_stdout(filename) + if self.params.get('forceduration', False) and info_dict.get('duration') is not None: + self.to_stdout(formatSeconds(info_dict['duration'])) if self.params.get('forceformat', False): self.to_stdout(info_dict['format']) if self.params.get('forcejson', False): diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 437c37541..d5c0a3643 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -298,6 +298,9 @@ def parseOpts(overrideArguments=None): verbosity.add_option('--get-description', action='store_true', dest='getdescription', help='simulate, quiet but print video description', default=False) + verbosity.add_option('--get-duration', + action='store_true', dest='getduration', + help='simulate, quiet but print video length', default=False) verbosity.add_option('--get-filename', action='store_true', dest='getfilename', help='simulate, quiet but print output filename', default=False) @@ -617,22 +620,25 @@ def _real_main(argv=None): u' file! Use "{0}.%(ext)s" instead of "{0}" as the output' u' template'.format(outtmpl)) + any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson + ydl_opts = { 'usenetrc': opts.usenetrc, 'username': opts.username, 'password': opts.password, 'videopassword': opts.videopassword, - 'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.dumpjson), + 'quiet': (opts.quiet or any_printing), 'forceurl': opts.geturl, 'forcetitle': opts.gettitle, 'forceid': opts.getid, 'forcethumbnail': opts.getthumbnail, 'forcedescription': opts.getdescription, + 'forceduration': opts.getduration, 'forcefilename': opts.getfilename, 'forceformat': opts.getformat, 'forcejson': opts.dumpjson, 'simulate': opts.simulate, - 'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.dumpjson), + 'skip_download': (opts.skip_download or opts.simulate or any_printing), 'format': opts.format, 'format_limit': opts.format_limit, 'listformats': opts.listformats, From 7b6fefc9d4df92dca11ea536bdaef633f7a4229c Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 04:39:04 +0100 Subject: [PATCH 316/425] Apply --no-overwrites for --write-* files as well (Fixes #1980) --- youtube_dl/YoutubeDL.py | 114 ++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 52bd8e0e3..2a4ab674d 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -366,22 +366,6 @@ class YoutubeDL(object): error_message = u'%s %s' % (_msg_header, message) self.trouble(error_message, tb) - def report_writedescription(self, descfn): - """ Report that the description file is being written """ - self.to_screen(u'[info] Writing video description to: ' + descfn) - - def report_writesubtitles(self, sub_filename): - """ Report that the subtitles file is being written """ - self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename) - - def report_writeinfojson(self, infofn): - """ Report that the metadata file has been written """ - self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn) - - def report_writeannotations(self, annofn): - """ Report that the annotations file has been written. """ - self.to_screen(u'[info] Writing video annotations to: ' + annofn) - def report_file_already_downloaded(self, file_name): """Report file has already been fully downloaded.""" try: @@ -791,28 +775,34 @@ class YoutubeDL(object): return if self.params.get('writedescription', False): - try: - descfn = filename + u'.description' - self.report_writedescription(descfn) - with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile: - descfile.write(info_dict['description']) - except (KeyError, TypeError): - self.report_warning(u'There\'s no description to write.') - except (OSError, IOError): - self.report_error(u'Cannot write description file ' + descfn) - return + descfn = filename + u'.description' + if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)): + self.to_screen(u'[info] Video description is already present') + else: + try: + self.to_screen(u'[info] Writing video description to: ' + descfn) + with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile: + descfile.write(info_dict['description']) + except (KeyError, TypeError): + self.report_warning(u'There\'s no description to write.') + except (OSError, IOError): + self.report_error(u'Cannot write description file ' + descfn) + return if self.params.get('writeannotations', False): - try: - annofn = filename + u'.annotations.xml' - self.report_writeannotations(annofn) - with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile: - annofile.write(info_dict['annotations']) - except (KeyError, TypeError): - self.report_warning(u'There are no annotations to write.') - except (OSError, IOError): - self.report_error(u'Cannot write annotations file: ' + annofn) - return + annofn = filename + u'.annotations.xml' + if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)): + self.to_screen(u'[info] Video annotations are already present') + else: + try: + self.to_screen(u'[info] Writing video annotations to: ' + annofn) + with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile: + annofile.write(info_dict['annotations']) + except (KeyError, TypeError): + self.report_warning(u'There are no annotations to write.') + except (OSError, IOError): + self.report_error(u'Cannot write annotations file: ' + annofn) + return subtitles_are_requested = any([self.params.get('writesubtitles', False), self.params.get('writeautomaticsub')]) @@ -828,38 +818,48 @@ class YoutubeDL(object): continue try: sub_filename = subtitles_filename(filename, sub_lang, sub_format) - self.report_writesubtitles(sub_filename) - with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile: - subfile.write(sub) + if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)): + self.to_screen(u'[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format)) + else: + self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename) + with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile: + subfile.write(sub) except (OSError, IOError): self.report_error(u'Cannot write subtitles file ' + descfn) return if self.params.get('writeinfojson', False): infofn = os.path.splitext(filename)[0] + u'.info.json' - self.report_writeinfojson(infofn) - try: - json_info_dict = dict((k, v) for k, v in info_dict.items() if not k in ['urlhandle']) - write_json_file(json_info_dict, encodeFilename(infofn)) - except (OSError, IOError): - self.report_error(u'Cannot write metadata to JSON file ' + infofn) - return + if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)): + self.to_screen(u'[info] Video description metadata is already present') + else: + self.to_screen(u'[info] Writing video description metadata as JSON to: ' + infofn) + try: + json_info_dict = dict((k, v) for k, v in info_dict.items() if not k in ['urlhandle']) + write_json_file(json_info_dict, encodeFilename(infofn)) + except (OSError, IOError): + self.report_error(u'Cannot write metadata to JSON file ' + infofn) + return if self.params.get('writethumbnail', False): if info_dict.get('thumbnail') is not None: thumb_format = determine_ext(info_dict['thumbnail'], u'jpg') thumb_filename = os.path.splitext(filename)[0] + u'.' + thumb_format - self.to_screen(u'[%s] %s: Downloading thumbnail ...' % - (info_dict['extractor'], info_dict['id'])) - try: - uf = compat_urllib_request.urlopen(info_dict['thumbnail']) - with open(thumb_filename, 'wb') as thumbf: - shutil.copyfileobj(uf, thumbf) - self.to_screen(u'[%s] %s: Writing thumbnail to: %s' % - (info_dict['extractor'], info_dict['id'], thumb_filename)) - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self.report_warning(u'Unable to download thumbnail "%s": %s' % - (info_dict['thumbnail'], compat_str(err))) + if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)): + self.to_screen(u'[%s] %s: Thumbnail is already present' % + (info_dict['extractor'], info_dict['id'])) + else: + self.to_screen(u'[%s] %s: Downloading thumbnail ...' % + (info_dict['extractor'], info_dict['id'])) + try: + uf = compat_urllib_request.urlopen(info_dict['thumbnail']) + with open(thumb_filename, 'wb') as thumbf: + shutil.copyfileobj(uf, thumbf) + self.to_screen(u'[%s] %s: Writing thumbnail to: %s' % + (info_dict['extractor'], info_dict['id'], thumb_filename)) + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + self.report_warning(u'Unable to download thumbnail "%s": %s' % + (info_dict['thumbnail'], compat_str(err))) if not self.params.get('skip_download', False): if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)): From 8863d0de91ee0b7f7bac28fe9b6ba27f986e656c Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 04:45:32 +0100 Subject: [PATCH 317/425] release 2013.12.16 --- README.md | 5 +++++ youtube_dl/version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 68b2e1ae7..caed94846 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,10 @@ which means you can modify it, redistribute it or use it however you like. --date DATE download only videos uploaded in this date --datebefore DATE download only videos uploaded before this date --dateafter DATE download only videos uploaded after this date + --min-views COUNT Do not download any videos with less than COUNT + views + --max-views COUNT Do not download any videos with more than COUNT + views --no-playlist download only the currently playing video --age-limit YEARS download only videos suitable for the given age --download-archive FILE Download only videos not listed in the archive @@ -127,6 +131,7 @@ which means you can modify it, redistribute it or use it however you like. --get-id simulate, quiet but print id --get-thumbnail simulate, quiet but print thumbnail URL --get-description simulate, quiet but print video description + --get-duration simulate, quiet but print video length --get-filename simulate, quiet but print output filename --get-format simulate, quiet but print output format -j, --dump-json simulate, quiet but print JSON information diff --git a/youtube_dl/version.py b/youtube_dl/version.py index b9a52fcfa..5bc7fd774 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.11.2' +__version__ = '2013.12.16' From e3946f989ea53c14fb7ae0afd29f80a74512a3a3 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 05:04:12 +0100 Subject: [PATCH 318/425] Set process title to youtube-dl This allows killing all youtube-dl processes with killall youtube-dl, and shows up nicer in some programs. --- youtube_dl/__init__.py | 4 ++++ youtube_dl/utils.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index d5c0a3643..09a99f0db 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -62,6 +62,7 @@ from .utils import ( MaxDownloadsReached, preferredencoding, SameFileError, + setproctitle, std_headers, write_string, ) @@ -471,12 +472,15 @@ def parseOpts(overrideArguments=None): return parser, opts, args + def _real_main(argv=None): # Compatibility fixes for Windows if sys.platform == 'win32': # https://github.com/rg3/youtube-dl/issues/820 codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None) + setproctitle(u'youtube-dl') + parser, opts, args = parseOpts(argv) # Set user agent diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 4e8a84a56..bd46a2da2 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import ctypes import datetime import email.utils import errno @@ -1062,3 +1063,17 @@ def month_by_name(name): def fix_xml_all_ampersand(xml_str): """Replace all the '&' by '&' in XML""" return xml_str.replace(u'&', u'&') + + +def setproctitle(title): + try: + libc = ctypes.cdll.LoadLibrary("libc.so.6") + except OSError: + return + title = title + buf = ctypes.create_string_buffer(len(title) + 1) + buf.value = title + try: + libc.prctl(15, ctypes.byref(buf), 0, 0, 0) + except AttributeError: + return # Strange libc, just skip this From 6f5dcd4eee10a291b89b70a68e1a955cd94d341f Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 05:10:42 +0100 Subject: [PATCH 319/425] [pornhd] Simplify --- youtube_dl/extractor/pornhd.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py index c56740b8a..71abd5013 100644 --- a/youtube_dl/extractor/pornhd.py +++ b/youtube_dl/extractor/pornhd.py @@ -3,12 +3,13 @@ import re from .common import InfoExtractor from ..utils import compat_urllib_parse + class PornHdIE(InfoExtractor): _VALID_URL = r'(?:http://)?(?:www\.)?pornhd\.com/videos/(?P<video_id>[0-9]+)/(?P<video_title>.+)' _TEST = { - u'id': u'1962', u'url': u'http://www.pornhd.com/videos/1962/sierra-day-gets-his-cum-all-over-herself-hd-porn-video', - u'md5': u'4fe06e5108e8b524c35896f4c54c7155', + u'file': u'1962.flv', + u'md5': u'35272469887dca97abd30abecc6cdf75', u'info_dict': { u"title": u"sierra-day-gets-his-cum-all-over-herself-hd-porn-video", u"age_limit": 18, @@ -20,23 +21,18 @@ class PornHdIE(InfoExtractor): video_id = mobj.group('video_id') video_title = mobj.group('video_title') - video_extension = 'flv' webpage = self._download_webpage(url, video_id) - - self.report_extraction(video_id) - video_url = self._html_search_regex( r'&hd=(http.+?)&', webpage, u'video URL') video_url = compat_urllib_parse.unquote(video_url) - age_limit = 18 return { - 'id': video_id, - 'url': video_url, - 'ext': video_extension, - 'title': video_title, + 'id': video_id, + 'url': video_url, + 'ext': 'flv', + 'title': video_title, 'age_limit': age_limit, } From b83be81d27e252eed4585fcce14205266b51b1d5 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 05:11:19 +0100 Subject: [PATCH 320/425] Credit @mjorlitzky for pornhd (#1961) --- youtube_dl/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 09a99f0db..0775b72fd 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -37,6 +37,7 @@ __authors__ = ( 'Anton Larionov', 'Takuya Tsuchida', 'Sergey M.', + 'Michael Orlitzky', ) __license__ = 'Public Domain' From 09dacfa57fa92643e181f039bd7fdf5513921d0f Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 05:44:34 +0100 Subject: [PATCH 321/425] [mdr] Simplify --- youtube_dl/extractor/mdr.py | 93 +++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 46 deletions(-) diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py index 8b096a78a..366352dbe 100644 --- a/youtube_dl/extractor/mdr.py +++ b/youtube_dl/extractor/mdr.py @@ -5,35 +5,28 @@ from ..utils import ( ExtractorError, ) + class MDRIE(InfoExtractor): _VALID_URL = r'^(?P<domain>(?:https?://)?(?:www\.)?mdr\.de)/mediathek/(?:.*)/(?P<type>video|audio)(?P<video_id>[^/_]+)_.*' - _TITLE = r'<h2>(?P<title1>[^<]+)<span>(?P<title2>[^<]+)</span></h2>' - _MEDIA_XML = r'(?P<xmlurl>/mediathek/(.+)/(video|audio)([0-9]+)-avCustom.xml)' - _MEDIA_STREAM_VIDEO = r'<asset>.*<frameWidth>(?P<frameWidth>[0-9]+)</frameWidth>.*<flashMediaServerApplicationURL>(?P<flashMediaServerApplicationURL>[^<]+)</flashMediaServerApplicationURL><flashMediaServerURL>(?P<flashMediaServerURL>[^<]+)</flashMediaServerURL>.*<progressiveDownloadUrl>(?P<progressiveDownloadUrl>[^<]+)</progressiveDownloadUrl></asset>' - _MEDIA_STREAM_AUDIO = r'<asset>.*<mediaType>(?P<mediaType>[A-Z0-9]+)</mediaType><bitrateAudio>(?P<bitrateAudio>[0-9]+)</bitrateAudio>.*<flashMediaServerApplicationURL>(?P<flashMediaServerApplicationURL>[^<]+)</flashMediaServerApplicationURL><flashMediaServerURL>(?P<flashMediaServerURL>[^<]+)</flashMediaServerURL>.*<progressiveDownloadUrl>(?P<progressiveDownloadUrl>[^<]+)</progressiveDownloadUrl></asset>' _TESTS = [{ u'url': u'http://www.mdr.de/mediathek/themen/nachrichten/video165624_zc-c5c7de76_zs-3795826d.html', u'file': u'165624.mp4', - u'md5': u'95165945756198b8fa2dea10f0b04614', + u'md5': u'ae785f36ecbf2f19b42edf1bc9c85815', u'info_dict': { u"title": u"MDR aktuell Eins30 09.12.2013, 22:48 Uhr" }, - #u'skip': u'Requires rtmpdump' # rtmp is optional }, { - u'url': u' http://www.mdr.de/mediathek/radio/mdr1-radio-sachsen/audio718370_zc-67b21197_zs-1b9b2483.html', - u'file': u'718370.mp4', - u'md5': u'4a5b1fbb5519fb0d929c384b6ff7cb8b', + u'url': u'http://www.mdr.de/mediathek/radio/mdr1-radio-sachsen/audio718370_zc-67b21197_zs-1b9b2483.html', + u'file': u'718370.mp3', + u'md5': u'a9d21345a234c7b45dee612f290fd8d7', u'info_dict': { u"title": u"MDR 1 RADIO SACHSEN 10.12.2013, 05:00 Uhr" }, - #u'skip': u'Requires rtmpdump' # rtmp is optional }] def _real_extract(self, url): - - # determine video id from url m = re.match(self._VALID_URL, url) video_id = m.group('video_id') domain = m.group('domain') @@ -41,38 +34,46 @@ class MDRIE(InfoExtractor): # determine title and media streams from webpage html = self._download_webpage(url, video_id) - t = re.search(self._TITLE, html) - if not t: - raise ExtractorError(u'no title found') - title = t.group('title1') + t.group('title2') - m = re.search(self._MEDIA_XML, html) - if not m: - raise ExtractorError(u'no xml found') - xmlurl = m.group('xmlurl') - xml = self._download_webpage(domain+xmlurl, video_id, 'download XML').replace('\n','').replace('\r','').replace('<asset>','\n<asset>').replace('</asset>','</asset>\n') - if(mediatype == "video"): - streams = [mo.groupdict() for mo in re.finditer(self._MEDIA_STREAM_VIDEO, xml)] - if not streams: - raise ExtractorError(u'no media found') - # choose default media type and highest quality for now - stream = max([s for s in streams if s["progressiveDownloadUrl"].startswith("http://") ], - key=lambda s: int(s["frameWidth"])) - else: - streams = [mo.groupdict() for mo in re.finditer(self._MEDIA_STREAM_AUDIO, xml)] - if not streams: - raise ExtractorError(u'no media found') - # choose default media type (MP4) and highest quality for now - stream = max([s for s in streams if s["progressiveDownloadUrl"].startswith("http://") and s["mediaType"] == "MP4" ], - key=lambda s: int(s["bitrateAudio"])) - # there's two possibilities: RTMP stream or HTTP download - info = {'id': video_id, 'title': title, 'ext': 'mp4'} - if not stream["progressiveDownloadUrl"]: - self.to_screen(u'RTMP download detected') - assert stream['flashMediaServerURL'].startswith('mp4:') - info["url"] = stream["flashMediaServerApplicationURL"] - info["play_path"] = stream['flashMediaServerURL'] - else: - assert stream["progressiveDownloadUrl"].endswith('.mp4') - info["url"] = stream["progressiveDownloadUrl"] - return [info] + title = self._html_search_regex(r'<h2>(.*?)</h2>', html, u'title') + xmlurl = self._search_regex( + r'(/mediathek/(?:.+)/(?:video|audio)[0-9]+-avCustom.xml)', html, u'XML URL') + + doc = self._download_xml(domain + xmlurl, video_id) + formats = [] + for a in doc.findall('./assets/asset'): + url_el = a.find('.//progressiveDownloadUrl') + if url_el is None: + continue + abr = int(a.find('bitrateAudio').text) // 1000 + media_type = a.find('mediaType').text + format = { + 'abr': abr, + 'filesize': int(a.find('fileSize').text), + 'url': url_el.text, + } + + vbr_el = a.find('bitrateVideo') + if vbr_el is None: + format.update({ + 'vcodec': 'none', + 'format_id': u'%s-%d' % (media_type, abr), + }) + else: + vbr = int(vbr_el.text) // 1000 + format.update({ + 'vbr': vbr, + 'width': int(a.find('frameWidth').text), + 'height': int(a.find('frameHeight').text), + 'format_id': u'%s-%d' % (media_type, vbr), + }) + formats.append(format) + formats.sort(key=lambda f: (f.get('vbr'), f['abr'])) + if not formats: + raise ValueError('Could not find any valid formats') + + return { + 'id': video_id, + 'title': title, + 'formats': formats, + } From 6888a874a1dcdc916bafa5aa55864b60514f5fa4 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 05:45:15 +0100 Subject: [PATCH 322/425] release 2013.12.16.1 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 5bc7fd774..15ac46d17 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.16' +__version__ = '2013.12.16.1' From 8c5f0c9fbc09b361517cd5045d856fec96381696 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 08:16:11 +0100 Subject: [PATCH 323/425] [mdr] Clean up --- youtube_dl/extractor/mdr.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py index 366352dbe..d29cf2c07 100644 --- a/youtube_dl/extractor/mdr.py +++ b/youtube_dl/extractor/mdr.py @@ -30,7 +30,6 @@ class MDRIE(InfoExtractor): m = re.match(self._VALID_URL, url) video_id = m.group('video_id') domain = m.group('domain') - mediatype = m.group('type') # determine title and media streams from webpage html = self._download_webpage(url, video_id) @@ -70,7 +69,7 @@ class MDRIE(InfoExtractor): formats.append(format) formats.sort(key=lambda f: (f.get('vbr'), f['abr'])) if not formats: - raise ValueError('Could not find any valid formats') + raise ExtractorError(u'Could not find any valid formats') return { 'id': video_id, From d66152a8982261f930c2a324c1fc3ba3b3324134 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 08:16:38 +0100 Subject: [PATCH 324/425] [ndtv] Remove unused imports --- youtube_dl/extractor/ndtv.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/youtube_dl/extractor/ndtv.py b/youtube_dl/extractor/ndtv.py index 2e8501f99..d81df3c10 100644 --- a/youtube_dl/extractor/ndtv.py +++ b/youtube_dl/extractor/ndtv.py @@ -1,6 +1,4 @@ -import json import re -import time from .common import InfoExtractor from ..utils import month_by_name From a19fd00cc4f799215a942c92fd5c722b3ea499cd Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 13:16:20 +0100 Subject: [PATCH 325/425] Simplify --playlist-start / --playlist-end interface --- youtube_dl/YoutubeDL.py | 12 ++++++------ youtube_dl/__init__.py | 28 ++++++++++++---------------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 2a4ab674d..2fa34ebc9 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -557,16 +557,16 @@ class YoutubeDL(object): n_all_entries = len(ie_result['entries']) playliststart = self.params.get('playliststart', 1) - 1 - playlistend = self.params.get('playlistend', -1) - + playlistend = self.params.get('playlistend', None) + # For backwards compatibility, interpret -1 as whole list if playlistend == -1: - entries = ie_result['entries'][playliststart:] - else: - entries = ie_result['entries'][playliststart:playlistend] + playlistend = None + entries = ie_result['entries'][playliststart:playlistend] n_entries = len(entries) - self.to_screen(u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" % + self.to_screen( + u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" % (ie_result['extractor'], playlist, n_all_entries, n_entries)) for i, entry in enumerate(entries, 1): diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 0775b72fd..6df44020b 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -198,10 +198,14 @@ def parseOpts(overrideArguments=None): help=u'Work around terminals that lack bidirectional text support. Requires fribidi executable in PATH') - selection.add_option('--playlist-start', - dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1) - selection.add_option('--playlist-end', - dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1) + selection.add_option( + '--playlist-start', + dest='playliststart', metavar='NUMBER', default=1, type=int, + help='playlist video to start at (default is %default)') + selection.add_option( + '--playlist-end', + dest='playlistend', metavar='NUMBER', default=None, type=int, + help='playlist video to end at (default is last)') selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)') selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)') selection.add_option('--max-downloads', metavar='NUMBER', @@ -576,18 +580,10 @@ def _real_main(argv=None): if numeric_buffersize is None: parser.error(u'invalid buffer size specified') opts.buffersize = numeric_buffersize - try: - opts.playliststart = int(opts.playliststart) - if opts.playliststart <= 0: - raise ValueError(u'Playlist start must be positive') - except (TypeError, ValueError): - parser.error(u'invalid playlist start number specified') - try: - opts.playlistend = int(opts.playlistend) - if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart): - raise ValueError(u'Playlist end must be greater than playlist start') - except (TypeError, ValueError): - parser.error(u'invalid playlist end number specified') + if opts.playliststart <= 0: + raise ValueError(u'Playlist start must be positive') + if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart: + raise ValueError(u'Playlist end must be greater than playlist start') if opts.extractaudio: if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']: parser.error(u'invalid audio format specified') From d7dda1688886284b17c35be200d980f24c2546d2 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 13:56:13 +0100 Subject: [PATCH 326/425] [blinkx] Add extractor (Fixes #1972) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/blinkx.py | 86 ++++++++++++++++++++++++++++++++ youtube_dl/utils.py | 6 +++ 3 files changed, 93 insertions(+) create mode 100644 youtube_dl/extractor/blinkx.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index cebb8717f..b8ff750d0 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -13,6 +13,7 @@ from .arte import ( from .auengine import AUEngineIE from .bambuser import BambuserIE, BambuserChannelIE from .bandcamp import BandcampIE, BandcampAlbumIE +from .blinkx import BlinkxIE from .bliptv import BlipTVIE, BlipTVUserIE from .bloomberg import BloombergIE from .breakcom import BreakIE diff --git a/youtube_dl/extractor/blinkx.py b/youtube_dl/extractor/blinkx.py new file mode 100644 index 000000000..48f16b692 --- /dev/null +++ b/youtube_dl/extractor/blinkx.py @@ -0,0 +1,86 @@ +import datetime +import json +import re + +from .common import InfoExtractor +from ..utils import ( + remove_start, +) + + +class BlinkxIE(InfoExtractor): + _VALID_URL = r'^(?:https?://(?:www\.)blinkx\.com/ce/|blinkx:)(?P<id>[^?]+)' + _IE_NAME = u'blinkx' + + _TEST = { + u'url': u'http://www.blinkx.com/ce/8aQUy7GVFYgFzpKhT0oqsilwOGFRVXk3R1ZGWWdGenBLaFQwb3FzaWx3OGFRVXk3R1ZGWWdGenB', + u'file': u'8aQUy7GV.mp4', + u'md5': u'2e9a07364af40163a908edbf10bb2492', + u'info_dict': { + u"title": u"Police Car Rolls Away", + u"uploader": u"stupidvideos.com", + u"upload_date": u"20131215", + u"description": u"A police car gently rolls away from a fight. Maybe it felt weird being around a confrontation and just had to get out of there!", + u"duration": 14.886, + u"thumbnails": [{ + "width": 100, + "height": 76, + "url": "http://cdn.blinkx.com/stream/b/41/StupidVideos/20131215/1873969261/1873969261_tn_0.jpg", + }], + }, + } + + def _real_extract(self, url): + m = re.match(self._VALID_URL, url) + video_id = m.group('id') + display_id = video_id[:8] + + api_url = (u'https://apib4.blinkx.com/api.php?action=play_video&' + + u'video=%s' % video_id) + data_json = self._download_webpage(api_url, display_id) + data = json.loads(data_json)['api']['results'][0] + dt = datetime.datetime.fromtimestamp(data['pubdate_epoch']) + upload_date = dt.strftime('%Y%m%d') + + duration = None + thumbnails = [] + formats = [] + for m in data['media']: + if m['type'] == 'jpg': + thumbnails.append({ + 'url': m['link'], + 'width': int(m['w']), + 'height': int(m['h']), + }) + elif m['type'] == 'original': + duration = m['d'] + elif m['type'] in ('flv', 'mp4'): + vcodec = remove_start(m['vcodec'], 'ff') + acodec = remove_start(m['acodec'], 'ff') + format_id = (u'%s-%sk-%s' % + (vcodec, + (int(m['vbr']) + int(m['abr'])) // 1000, + m['w'])) + formats.append({ + 'format_id': format_id, + 'url': m['link'], + 'vcodec': vcodec, + 'acodec': acodec, + 'abr': int(m['abr']) // 1000, + 'vbr': int(m['vbr']) // 1000, + 'width': int(m['w']), + 'height': int(m['h']), + }) + formats.sort(key=lambda f: (f['width'], f['vbr'], f['abr'])) + + return { + 'id': display_id, + 'fullid': video_id, + 'title': data['title'], + 'formats': formats, + 'uploader': data['channel_name'], + 'upload_date': upload_date, + 'description': data.get('description'), + 'thumbnails': thumbnails, + 'duration': duration, + } diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index bd46a2da2..f3ad47422 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1077,3 +1077,9 @@ def setproctitle(title): libc.prctl(15, ctypes.byref(buf), 0, 0, 0) except AttributeError: return # Strange libc, just skip this + + +def remove_start(s, start): + if s.startswith(start): + return s[len(start):] + return s From d67b0b15964c3f335e47a291b274e07129ac199d Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 14:13:40 +0100 Subject: [PATCH 327/425] Reorder info_dict documentation --- youtube_dl/extractor/common.py | 52 ++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index fe8ce9e6c..11b6ed524 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -35,14 +35,38 @@ class InfoExtractor(object): id: Video identifier. title: Video title, unescaped. + + Additionally, it must contain either a formats entry or url and ext: + + formats: A list of dictionaries for each format available, it must + be ordered from worst to best quality. Potential fields: + * url Mandatory. The URL of the video file + * ext Will be calculated from url if missing + * format A human-readable description of the format + ("mp4 container with h264/opus"). + Calculated from the format_id, width, height. + and format_note fields if missing. + * format_id A short description of the format + ("mp4_h264_opus" or "19") + * format_note Additional info about the format + ("3D" or "DASH video") + * width Width of the video, if known + * height Height of the video, if known + * abr Average audio bitrate in KBit/s + * acodec Name of the audio codec in use + * vbr Average video bitrate in KBit/s + * vcodec Name of the video codec in use + * filesize The number of bytes, if known in advance + * player_url SWF Player URL (used for rtmpdump). url: Final video URL. ext: Video filename extension. - - Instead of url and ext, formats can also specified. + format: The video format, defaults to ext (used for --get-format) + player_url: SWF Player URL (used for rtmpdump). + urlhandle: [internal] The urlHandle to be used to download the file, + like returned by urllib.request.urlopen The following fields are optional: - format: The video format, defaults to ext (used for --get-format) thumbnails: A list of dictionaries (with the entries "resolution" and "url") for the varying thumbnails thumbnail: Full URL to a video thumbnail image. @@ -51,7 +75,6 @@ class InfoExtractor(object): upload_date: Video upload date (YYYYMMDD). uploader_id: Nickname or id of the video uploader. location: Physical location of the video. - player_url: SWF Player URL (used for rtmpdump). subtitles: The subtitle file contents as a dictionary in the format {language: subtitles}. duration: Length of the video in seconds, as an integer. @@ -59,28 +82,7 @@ class InfoExtractor(object): like_count: Number of positive ratings of the video dislike_count: Number of negative ratings of the video comment_count: Number of comments on the video - urlhandle: [internal] The urlHandle to be used to download the file, - like returned by urllib.request.urlopen age_limit: Age restriction for the video, as an integer (years) - formats: A list of dictionaries for each format available, it must - be ordered from worst to best quality. Potential fields: - * url Mandatory. The URL of the video file - * ext Will be calculated from url if missing - * format A human-readable description of the format - ("mp4 container with h264/opus"). - Calculated from the format_id, width, height. - and format_note fields if missing. - * format_id A short description of the format - ("mp4_h264_opus" or "19") - * format_note Additional info about the format - ("3D" or "DASH video") - * width Width of the video, if known - * height Height of the video, if known - * abr Average audio bitrate in KBit/s - * acodec Name of the audio codec in use - * vbr Average video bitrate in KBit/s - * vcodec Name of the video codec in use - * filesize The number of bytes, if known in advance webpage_url: The url to the video webpage, if given to youtube-dl it should allow to get the same result again. (It will be set by YoutubeDL if it's missing) From 00902cd60139c919871aa30c43d0ae21c130fb86 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 14:13:51 +0100 Subject: [PATCH 328/425] release 2013.12.16.2 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 15ac46d17..76b86c976 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.16.1' +__version__ = '2013.12.16.2' From 780603027f8e57b0e4ec9ea1bf74b6a1afa24cb4 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 14:42:07 +0100 Subject: [PATCH 329/425] [videopremium] Skip test --- youtube_dl/extractor/videopremium.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/videopremium.py b/youtube_dl/extractor/videopremium.py index acae81448..65463c733 100644 --- a/youtube_dl/extractor/videopremium.py +++ b/youtube_dl/extractor/videopremium.py @@ -15,6 +15,7 @@ class VideoPremiumIE(InfoExtractor): u'params': { u'skip_download': True, }, + u'skip': u'Test file has been deleted.', } def _real_extract(self, url): From e64eaaa97dd00b15ff0ebde17d6d6e99e6a7394e Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 14:44:17 +0100 Subject: [PATCH 330/425] Fix execution under Python 3 --- youtube_dl/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index f3ad47422..dbfac0f43 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1066,13 +1066,14 @@ def fix_xml_all_ampersand(xml_str): def setproctitle(title): + assert isinstance(title, type(u'')) try: libc = ctypes.cdll.LoadLibrary("libc.so.6") except OSError: return title = title buf = ctypes.create_string_buffer(len(title) + 1) - buf.value = title + buf.value = title.encode('utf-8') try: libc.prctl(15, ctypes.byref(buf), 0, 0, 0) except AttributeError: From 4b2da48ea794178236975eaeef1be120b8a95f2a Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 14:44:29 +0100 Subject: [PATCH 331/425] release 2013.12.16.3 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 76b86c976..76691fb46 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.16.2' +__version__ = '2013.12.16.3' From 0a9ce268bad2d2bfaba1498e3843784a2f14e856 Mon Sep 17 00:00:00 2001 From: alimirjamali <ali.mirjamali@gmail.com> Date: Mon, 16 Dec 2013 20:14:28 +0330 Subject: [PATCH 332/425] Incorrect variable is used to check whether thumbnail exists Dear @phihag I believe in line 848, the correct variable to check is 'thumb_filename' rather than 'infofn' Kindly advise Mit freundlichen Gruessen Ali --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 2fa34ebc9..b1f87415b 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -845,7 +845,7 @@ class YoutubeDL(object): if info_dict.get('thumbnail') is not None: thumb_format = determine_ext(info_dict['thumbnail'], u'jpg') thumb_filename = os.path.splitext(filename)[0] + u'.' + thumb_format - if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)): + if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)): self.to_screen(u'[%s] %s: Thumbnail is already present' % (info_dict['extractor'], info_dict['id'])) else: From ee3e63e477c12591bbfcb6f13382397139900d44 Mon Sep 17 00:00:00 2001 From: rzhxeo <rzhxeot7z81b4700@mailcatch.com> Date: Mon, 16 Dec 2013 20:08:23 +0100 Subject: [PATCH 333/425] [GenericIE] Add support for embedded blip.tv --- youtube_dl/extractor/generic.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 216e03218..a7ea9f733 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -222,6 +222,18 @@ class GenericIE(InfoExtractor): 'id': video_id, } + # Look for embedded blip.tv player + mobj = re.search(r'<meta\s[^>]*https?://api.blip.tv/\w+/redirect/\w+/(\d+)', webpage) + if mobj: + return self.url_result('http://blip.tv/seo/-'+mobj.group(1), 'BlipTV') + mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*https?://(?:\w+\.)?blip.tv/(?:play/|api\.swf#)([a-zA-Z0-9]+)', webpage) + if mobj: + player_url = 'http://blip.tv/play/%s.x?p=1' % mobj.group(1) + player_page = self._download_webpage(player_url, mobj.group(1)) + blip_video_id = self._search_regex(r'data-episode-id="(\d+)', player_page, u'blip_video_id', fatal=False) + if blip_video_id: + return self.url_result('http://blip.tv/seo/-'+blip_video_id, 'BlipTV') + # Look for Bandcamp pages with custom domain mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage) if mobj is not None: From 8c8e3eec793d04cc4dc884a65c119902d1ec7793 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 21:10:06 +0100 Subject: [PATCH 334/425] [facebook] Recognize #! URLs (Fixes #1988) --- test/test_all_urls.py | 6 +++++- youtube_dl/extractor/facebook.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/test/test_all_urls.py b/test/test_all_urls.py index e9458b2e3..bd77b7c30 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -10,6 +10,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import get_testcases from youtube_dl.extractor import ( + FacebookIE, gen_extractors, JustinTVIE, YoutubeIE, @@ -87,12 +88,15 @@ class TestAllURLsMatching(unittest.TestCase): assertExtractId('http://www.youtube.com/watch?v=BaW_jenozKcsharePLED17F32AD9753930', 'BaW_jenozKc') assertExtractId('BaW_jenozKc', 'BaW_jenozKc') + def test_facebook_matching(self): + self.assertTrue(FacebookIE.suitable(u'https://www.facebook.com/Shiniknoh#!/photo.php?v=10153317450565268')) + def test_no_duplicates(self): ies = gen_extractors() for tc in get_testcases(): url = tc['url'] for ie in ies: - if type(ie).__name__ in ['GenericIE', tc['name'] + 'IE']: + if type(ie).__name__ in ('GenericIE', tc['name'] + 'IE'): self.assertTrue(ie.suitable(url), '%s should match URL %r' % (type(ie).__name__, url)) else: self.assertFalse(ie.suitable(url), '%s should not match URL %r' % (type(ie).__name__, url)) diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py index 3b210710e..4556079c8 100644 --- a/youtube_dl/extractor/facebook.py +++ b/youtube_dl/extractor/facebook.py @@ -17,7 +17,7 @@ from ..utils import ( class FacebookIE(InfoExtractor): """Information Extractor for Facebook""" - _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)' + _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:[^#?]*#!/)?(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)' _LOGIN_URL = 'https://www.facebook.com/login.php?next=http%3A%2F%2Ffacebook.com%2Fhome.php&login_attempt=1' _CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1' _NETRC_MACHINE = 'facebook' @@ -27,7 +27,7 @@ class FacebookIE(InfoExtractor): u'file': u'120708114770723.mp4', u'md5': u'48975a41ccc4b7a581abd68651c1a5a8', u'info_dict': { - u"duration": 279, + u"duration": 279, u"title": u"PEOPLE ARE AWESOME 2013" } } From 24050dd11c0fe46136344cdcccedfef0d0b260c3 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 21:10:18 +0100 Subject: [PATCH 335/425] release 2013.12.16.4 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 76691fb46..6c1fb95dc 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.16.3' +__version__ = '2013.12.16.4' From 0e2a436dcebc7dc17c5848bada4adcad4248491d Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 21:34:41 +0100 Subject: [PATCH 336/425] [radiofrance] Add support (Fixes #1942) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/radiofrance.py | 60 +++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 youtube_dl/extractor/radiofrance.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index b8ff750d0..2761b5439 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -119,6 +119,7 @@ from .pornhd import PornHdIE from .pornhub import PornHubIE from .pornotube import PornotubeIE from .pyvideo import PyvideoIE +from .radiofrance import RadioFranceIE from .rbmaradio import RBMARadioIE from .redtube import RedTubeIE from .ringtv import RingTVIE diff --git a/youtube_dl/extractor/radiofrance.py b/youtube_dl/extractor/radiofrance.py new file mode 100644 index 000000000..bb33b50bc --- /dev/null +++ b/youtube_dl/extractor/radiofrance.py @@ -0,0 +1,60 @@ +# coding: utf-8 +import datetime +import json +import re + +from .common import InfoExtractor +from ..utils import ( + remove_start, +) + + +class RadioFranceIE(InfoExtractor): + _VALID_URL = r'^https?://maison\.radiofrance\.fr/radiovisions/(?P<id>[^?#]+)' + IE_NAME = u'radiofrance' + + _TEST = { + u'url': u'http://maison.radiofrance.fr/radiovisions/one-one', + u'file': u'one-one.mp4', + u'md5': u'todo', + u'info_dict': { + u"title": u"One to one", + u"description": u"Plutôt que d'imaginer la radio de demain comme technologie ou comme création de contenu, je veux montrer que quelles que soient ses évolutions, j'ai l'intime conviction que la radio continuera d'être un grand média de proximité pour les auditeurs.", + u"uploader": u"ferdi", + }, + } + + def _real_extract(self, url): + m = re.match(self._VALID_URL, url) + video_id = m.group('id') + + webpage = self._download_webpage(url, video_id) + title = self._html_search_regex(r'<h1>(.*?)</h1>', webpage, u'title') + description = self._html_search_regex( + r'<div class="bloc_page_wrapper"><div class="text">(.*?)</div>', + webpage, u'description', fatal=False) + uploader = self._html_search_regex( + r'<div class="credit">  © (.*?)</div>', + webpage, u'uploader', fatal=False) + + formats_str = self._html_search_regex( + r'class="jp-jplayer[^"]*" data-source="([^"]+)">', + webpage, u'audio URLs') + formats = [ + { + 'format_id': m[0], + 'url': m[1], + 'vcodec': 'none', + } + for m in + re.findall(r"([a-z0-9]+)\s*:\s*'([^']+)'", formats_str) + ] + # No sorting, we don't know any more about these formats + + return { + 'id': video_id, + 'title': title, + 'formats': formats, + 'description': description, + 'uploader': uploader, + } From 8fe56478f879a21e24695a4e27ba8546d0822f7b Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 21:34:47 +0100 Subject: [PATCH 337/425] release 2013.12.16.5 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 6c1fb95dc..2ce0bc9d1 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.16.4' +__version__ = '2013.12.16.5' From f7a68925724eda913c96519f2a5644aebb3688c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Mon, 16 Dec 2013 21:42:41 +0100 Subject: [PATCH 338/425] [arte:ddc] Remove test video seems to expire in 7 days, as arte+7 --- youtube_dl/extractor/arte.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py index 4b7bef775..9254fbfe0 100644 --- a/youtube_dl/extractor/arte.py +++ b/youtube_dl/extractor/arte.py @@ -266,20 +266,6 @@ class ArteTVDDCIE(ArteTVPlus7IE): IE_NAME = u'arte.tv:ddc' _VALID_URL = r'http?://ddc\.arte\.tv/(?P<lang>emission|folge)/(?P<id>.+)' - _TEST = { - u'url': u'http://ddc.arte.tv/folge/neues-aus-mauretanien', - u'file': u'049881-009_PLUS7-D.flv', - u'info_dict': { - u'title': u'Mit offenen Karten', - u'description': u'md5:57929b0eaeddeb8a0c983f58e9ebd3b6', - u'upload_date': u'20131207', - }, - u'params': { - # rtmp download - u'skip_download': True, - }, - } - def _real_extract(self, url): video_id, lang = self._extract_url_info(url) if lang == 'folge': From f25571ffbf67c14bb91798e445a2690e03481937 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 21:45:21 +0100 Subject: [PATCH 339/425] Add support for embedded vevo player (Fixes #1957) --- youtube_dl/extractor/generic.py | 6 ++++++ youtube_dl/extractor/vevo.py | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index a7ea9f733..fca23b6c4 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -241,6 +241,12 @@ class GenericIE(InfoExtractor): # Don't set the extractor because it can be a track url or an album return self.url_result(burl) + # Look for embedded Vevo player + mobj = re.search( + r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1', webpage) + if mobj is not None: + return self.url_result(mobj.group('url')) + # Start with something easy: JW Player in SWFObject mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage) if mobj is None: diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index 4823992ef..7eda9469b 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -15,7 +15,11 @@ class VevoIE(InfoExtractor): Accepts urls from vevo.com or in the format 'vevo:{id}' (currently used by MTVIE) """ - _VALID_URL = r'((http://www\.vevo\.com/watch/(?:[^/]+/[^/]+/)?)|(vevo:))(?P<id>.*?)(\?|$)' + _VALID_URL = r'''(?x) + (?:https?://www\.vevo\.com/watch/(?:[^/]+/[^/]+/)?| + https?://cache\.vevo\.com/m/html/embed\.html\?video=| + vevo:) + (?P<id>[^&?#]+)''' _TESTS = [{ u'url': u'http://www.vevo.com/watch/hurts/somebody-to-die-for/GB1101300280', u'file': u'GB1101300280.mp4', From 83c632dc439cbf475e34f788d81e6308359c5c52 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 21:46:16 +0100 Subject: [PATCH 340/425] release 2013.12.16.6 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 2ce0bc9d1..712b6c4e1 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.16.5' +__version__ = '2013.12.16.6' From ebce53b3d85bc5c099b4fd0ca27d03034d7024ea Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 21:48:38 +0100 Subject: [PATCH 341/425] [vevo] Add suppor for videoplayer. URLs (#1957) --- youtube_dl/extractor/vevo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index 7eda9469b..a4b26a26f 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -18,6 +18,7 @@ class VevoIE(InfoExtractor): _VALID_URL = r'''(?x) (?:https?://www\.vevo\.com/watch/(?:[^/]+/[^/]+/)?| https?://cache\.vevo\.com/m/html/embed\.html\?video=| + https?://videoplayer\.vevo\.com/embed/embedded\?videoId=| vevo:) (?P<id>[^&?#]+)''' _TESTS = [{ From 87a28127d225f698c3cbf77a7f943338a3499d6e Mon Sep 17 00:00:00 2001 From: Itay Brandes <Brandes.Itay@gmail.com> Date: Sat, 14 Dec 2013 17:40:51 +0200 Subject: [PATCH 342/425] _search_regex's "isatty" call fails with Py2exe's _search_regex calls the sys.stderr.isatty() function for unix systems. Py2exe uses a custom Stderr() stream which doesn't have an `isatty()` function, leading to it's crash. Fixes easily with checking that it's a unix system first. --- youtube_dl/extractor/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 11b6ed524..1fc0624a3 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -295,7 +295,7 @@ class InfoExtractor(object): mobj = re.search(p, string, flags) if mobj: break - if sys.stderr.isatty() and os.name != 'nt': + if os.name != 'nt' and sys.stderr.isatty(): _name = u'\033[0;34m%s\033[0m' % name else: _name = name From 5c541b2cb79113952efbdb72be72db568c8132a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Mon, 16 Dec 2013 22:05:28 +0100 Subject: [PATCH 343/425] [mtv] Add support for urls from the mobile site (fixes #1959) --- youtube_dl/extractor/mtv.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py index 5b2bd9633..ed11f521a 100644 --- a/youtube_dl/extractor/mtv.py +++ b/youtube_dl/extractor/mtv.py @@ -93,7 +93,9 @@ class MTVServicesInfoExtractor(InfoExtractor): class MTVIE(MTVServicesInfoExtractor): - _VALID_URL = r'^https?://(?:www\.)?mtv\.com/videos/.+?/(?P<videoid>[0-9]+)/[^/]+$' + _VALID_URL = r'''(?x)^https?:// + (?:(?:www\.)?mtv\.com/videos/.+?/(?P<videoid>[0-9]+)/[^/]+$| + m\.mtv\.com/videos/video\.rbml\?.*?id=(?P<mgid>[^&]+))''' _FEED_URL = 'http://www.mtv.com/player/embed/AS3/rss/' @@ -127,16 +129,17 @@ class MTVIE(MTVServicesInfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('videoid') - - webpage = self._download_webpage(url, video_id) - - # Some videos come from Vevo.com - m_vevo = re.search(r'isVevoVideo = true;.*?vevoVideoId = "(.*?)";', - webpage, re.DOTALL) - if m_vevo: - vevo_id = m_vevo.group(1); - self.to_screen(u'Vevo video detected: %s' % vevo_id) - return self.url_result('vevo:%s' % vevo_id, ie='Vevo') - - uri = self._html_search_regex(r'/uri/(.*?)\?', webpage, u'uri') + uri = mobj.group('mgid') + if uri is None: + webpage = self._download_webpage(url, video_id) + + # Some videos come from Vevo.com + m_vevo = re.search(r'isVevoVideo = true;.*?vevoVideoId = "(.*?)";', + webpage, re.DOTALL) + if m_vevo: + vevo_id = m_vevo.group(1); + self.to_screen(u'Vevo video detected: %s' % vevo_id) + return self.url_result('vevo:%s' % vevo_id, ie='Vevo') + + uri = self._html_search_regex(r'/uri/(.*?)\?', webpage, u'uri') return self._get_videos_info(uri) From d90df974c3164ea377f2ce2b04742e6ff21379e8 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 22:18:27 +0100 Subject: [PATCH 344/425] [academicearth] Add support for courses (#1976) --- test/test_playlists.py | 12 +++++++++ youtube_dl/extractor/__init__.py | 3 ++- youtube_dl/extractor/academicearth.py | 36 +++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 youtube_dl/extractor/academicearth.py diff --git a/test/test_playlists.py b/test/test_playlists.py index 87ca401e5..b7c6850fd 100644 --- a/test/test_playlists.py +++ b/test/test_playlists.py @@ -12,6 +12,7 @@ from test.helper import FakeYDL from youtube_dl.extractor import ( + AcademicEarthCourseIE, DailymotionPlaylistIE, DailymotionUserIE, VimeoChannelIE, @@ -158,5 +159,16 @@ class TestPlaylists(unittest.TestCase): self.assertEqual(result['title'], u'Inspector') self.assertTrue(len(result['entries']) >= 9) + def test_AcademicEarthCourse(self): + dl = FakeYDL() + ie = AcademicEarthCourseIE(dl) + result = ie.extract(u'http://academicearth.org/courses/building-dynamic-websites/') + self.assertIsPlaylist(result) + self.assertEqual(result['id'], u'building-dynamic-websites') + self.assertEqual(result['title'], u'Building Dynamic Websites') + self.assertEqual(result['description'], "Today's websites are increasingly dynamic. Pages are no longer static HTML files but instead generated by scripts and database calls. User interfaces are more seamless, with technologies like Ajax replacing traditional page reloads. This course teaches students how to build dynamic websites with Ajax and with Linux, Apache, MySQL, and PHP (LAMP), one of today's most popular frameworks. Students learn how to set up domain names with DNS, how to structure pages with XHTML and CSS, how to program in JavaScript and PHP, how to configure Apache and MySQL, how to design and query databases with SQL, how to use Ajax with both XML and JSON, and how to build mashups. The course explores issues of security, scalability, and cross-browser support and also discusses enterprise-level deployments of websites, including third-party hosting, virtualization, colocation in data centers, firewalling, and load-balancing.") + self.assertEqual(len(result['entries']), 10) + + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 2761b5439..7f2f8806e 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -1,6 +1,7 @@ -from .appletrailers import AppleTrailersIE +from .academicearth import AcademicEarthCourseIE from .addanime import AddAnimeIE from .anitube import AnitubeIE +from .appletrailers import AppleTrailersIE from .archiveorg import ArchiveOrgIE from .ard import ARDIE from .arte import ( diff --git a/youtube_dl/extractor/academicearth.py b/youtube_dl/extractor/academicearth.py new file mode 100644 index 000000000..5045e7332 --- /dev/null +++ b/youtube_dl/extractor/academicearth.py @@ -0,0 +1,36 @@ +import datetime +import json +import re + +from .common import InfoExtractor +from ..utils import ( + remove_start, +) + + +class AcademicEarthCourseIE(InfoExtractor): + _VALID_URL = r'^https?://(?:www\.)?academicearth\.org/courses/(?P<id>[^?#/]+)' + IE_NAME = u'AcademicEarth:Course' + + def _real_extract(self, url): + m = re.match(self._VALID_URL, url) + playlist_id = m.group('id') + + webpage = self._download_webpage(url, playlist_id) + title = self._html_search_regex( + r'<h1 class="playlist-name">(.*?)</h1>', webpage, u'title') + description = self._html_search_regex( + r'<p class="excerpt">(.*?)</p>', + webpage, u'description', fatal=False) + urls = re.findall( + r'<h3 class="lecture-title"><a target="_blank" href="([^"]+)">', + webpage) + entries = [self.url_result(u) for u in urls] + + return { + '_type': 'playlist', + 'id': playlist_id, + 'title': title, + 'description': description, + 'entries': entries, + } From 11b68f6e1be60a6faa61fc7df3460bb58ed95d70 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 22:18:58 +0100 Subject: [PATCH 345/425] release 2013.12.16.7 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 712b6c4e1..f5bcc67ce 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.16.6' +__version__ = '2013.12.16.7' From d6756d375807f84e86daca7168f12da03f7572f4 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 22:25:02 +0100 Subject: [PATCH 346/425] [playlist-test] require a string --- test/test_playlists.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_playlists.py b/test/test_playlists.py index b7c6850fd..5004d0464 100644 --- a/test/test_playlists.py +++ b/test/test_playlists.py @@ -166,7 +166,7 @@ class TestPlaylists(unittest.TestCase): self.assertIsPlaylist(result) self.assertEqual(result['id'], u'building-dynamic-websites') self.assertEqual(result['title'], u'Building Dynamic Websites') - self.assertEqual(result['description'], "Today's websites are increasingly dynamic. Pages are no longer static HTML files but instead generated by scripts and database calls. User interfaces are more seamless, with technologies like Ajax replacing traditional page reloads. This course teaches students how to build dynamic websites with Ajax and with Linux, Apache, MySQL, and PHP (LAMP), one of today's most popular frameworks. Students learn how to set up domain names with DNS, how to structure pages with XHTML and CSS, how to program in JavaScript and PHP, how to configure Apache and MySQL, how to design and query databases with SQL, how to use Ajax with both XML and JSON, and how to build mashups. The course explores issues of security, scalability, and cross-browser support and also discusses enterprise-level deployments of websites, including third-party hosting, virtualization, colocation in data centers, firewalling, and load-balancing.") + self.assertEqual(result['description'], u"Today's websites are increasingly dynamic. Pages are no longer static HTML files but instead generated by scripts and database calls. User interfaces are more seamless, with technologies like Ajax replacing traditional page reloads. This course teaches students how to build dynamic websites with Ajax and with Linux, Apache, MySQL, and PHP (LAMP), one of today's most popular frameworks. Students learn how to set up domain names with DNS, how to structure pages with XHTML and CSS, how to program in JavaScript and PHP, how to configure Apache and MySQL, how to design and query databases with SQL, how to use Ajax with both XML and JSON, and how to build mashups. The course explores issues of security, scalability, and cross-browser support and also discusses enterprise-level deployments of websites, including third-party hosting, virtualization, colocation in data centers, firewalling, and load-balancing.") self.assertEqual(len(result['entries']), 10) From 8b4e27461075ec246906487f000b2424e0603a93 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 22:28:52 +0100 Subject: [PATCH 347/425] [rtlnow] Fix URL calculation (Closes #1989) --- youtube_dl/extractor/rtlnow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py index 511674d8d..ccf0b1546 100644 --- a/youtube_dl/extractor/rtlnow.py +++ b/youtube_dl/extractor/rtlnow.py @@ -10,7 +10,7 @@ from ..utils import ( class RTLnowIE(InfoExtractor): """Information Extractor for RTL NOW, RTL2 NOW, RTL NITRO, SUPER RTL NOW, VOX NOW and n-tv NOW""" - _VALID_URL = r'(?:http://)?(?P<url>(?P<base_url>rtl-now\.rtl\.de|rtl2now\.rtl2\.de|(?:www\.)?voxnow\.de|(?:www\.)?rtlnitronow\.de|(?:www\.)?superrtlnow\.de|(?:www\.)?n-tvnow\.de)/+[a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.php\?(?:container_id|film_id)=(?P<video_id>[0-9]+)&player=1(?:&season=[0-9]+)?(?:&.*)?)' + _VALID_URL = r'(?:http://)?(?P<url>(?P<domain>rtl-now\.rtl\.de|rtl2now\.rtl2\.de|(?:www\.)?voxnow\.de|(?:www\.)?rtlnitronow\.de|(?:www\.)?superrtlnow\.de|(?:www\.)?n-tvnow\.de)/+[a-zA-Z0-9-]+/[a-zA-Z0-9-]+\.php\?(?:container_id|film_id)=(?P<video_id>[0-9]+)&player=1(?:&season=[0-9]+)?(?:&.*)?)' _TESTS = [{ u'url': u'http://rtl-now.rtl.de/ahornallee/folge-1.php?film_id=90419&player=1&season=1', u'file': u'90419.flv', @@ -82,7 +82,7 @@ class RTLnowIE(InfoExtractor): mobj = re.match(self._VALID_URL, url) webpage_url = u'http://' + mobj.group('url') - video_page_url = u'http://' + mobj.group('base_url') + video_page_url = u'http://' + mobj.group('domain') + u'/' video_id = mobj.group(u'video_id') webpage = self._download_webpage(webpage_url, video_id) From fa77b742ac54c2125e7dca931f533cd3e945b8f7 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 16 Dec 2013 23:07:57 +0100 Subject: [PATCH 348/425] [radiofrance] Fill in test details --- youtube_dl/extractor/radiofrance.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/radiofrance.py b/youtube_dl/extractor/radiofrance.py index bb33b50bc..2866115ef 100644 --- a/youtube_dl/extractor/radiofrance.py +++ b/youtube_dl/extractor/radiofrance.py @@ -15,12 +15,12 @@ class RadioFranceIE(InfoExtractor): _TEST = { u'url': u'http://maison.radiofrance.fr/radiovisions/one-one', - u'file': u'one-one.mp4', - u'md5': u'todo', + u'file': u'one-one.ogg', + u'md5': u'bdbb28ace95ed0e04faab32ba3160daf', u'info_dict': { u"title": u"One to one", u"description": u"Plutôt que d'imaginer la radio de demain comme technologie ou comme création de contenu, je veux montrer que quelles que soient ses évolutions, j'ai l'intime conviction que la radio continuera d'être un grand média de proximité pour les auditeurs.", - u"uploader": u"ferdi", + u"uploader": u"Thomas Hercouët", }, } From ec98946ef9ae19f5218ac66d72d61883709982ca Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 17 Dec 2013 02:41:34 +0100 Subject: [PATCH 349/425] [academicearth] Support playlists (Closes #1976) --- youtube_dl/extractor/academicearth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/academicearth.py b/youtube_dl/extractor/academicearth.py index 5045e7332..c450c30ca 100644 --- a/youtube_dl/extractor/academicearth.py +++ b/youtube_dl/extractor/academicearth.py @@ -9,7 +9,7 @@ from ..utils import ( class AcademicEarthCourseIE(InfoExtractor): - _VALID_URL = r'^https?://(?:www\.)?academicearth\.org/courses/(?P<id>[^?#/]+)' + _VALID_URL = r'^https?://(?:www\.)?academicearth\.org/(?:courses|playlists)/(?P<id>[^?#/]+)' IE_NAME = u'AcademicEarth:Course' def _real_extract(self, url): From 46374a56b214cae9f66ef3c01cf3d62a71544030 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 17 Dec 2013 02:49:54 +0100 Subject: [PATCH 350/425] [youtube] Do not warn for videos with allow_rating=0 This fixes #1982 Test video: http://www.youtube.com/watch?v=gi2uH3YxohU --- youtube_dl/extractor/common.py | 7 ++++--- youtube_dl/extractor/xtube.py | 2 +- youtube_dl/extractor/youtube.py | 4 +++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 1fc0624a3..939249d7b 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -18,6 +18,7 @@ from ..utils import ( sanitize_filename, unescapeHTML, ) +_NO_DEFAULT = object() class InfoExtractor(object): @@ -281,7 +282,7 @@ class InfoExtractor(object): video_info['title'] = playlist_title return video_info - def _search_regex(self, pattern, string, name, default=None, fatal=True, flags=0): + def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0): """ Perform a regex search on the given string, using a single or a list of patterns returning the first matching group. @@ -303,7 +304,7 @@ class InfoExtractor(object): if mobj: # return the first matching group return next(g for g in mobj.groups() if g is not None) - elif default is not None: + elif default is not _NO_DEFAULT: return default elif fatal: raise RegexNotFoundError(u'Unable to extract %s' % _name) @@ -312,7 +313,7 @@ class InfoExtractor(object): u'please report this issue on http://yt-dl.org/bug' % _name) return None - def _html_search_regex(self, pattern, string, name, default=None, fatal=True, flags=0): + def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0): """ Like _search_regex, but strips HTML tags and unescapes entities. """ diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py index e3458d2bd..1a6a7688d 100644 --- a/youtube_dl/extractor/xtube.py +++ b/youtube_dl/extractor/xtube.py @@ -32,7 +32,7 @@ class XTubeIE(InfoExtractor): video_title = self._html_search_regex(r'<div class="p_5px[^>]*>([^<]+)', webpage, u'title') video_uploader = self._html_search_regex(r'so_s\.addVariable\("owner_u", "([^"]+)', webpage, u'uploader', fatal=False) - video_description = self._html_search_regex(r'<p class="video_description">([^<]+)', webpage, u'description', default=None) + video_description = self._html_search_regex(r'<p class="video_description">([^<]+)', webpage, u'description', fatal=False) video_url= self._html_search_regex(r'var videoMp4 = "([^"]+)', webpage, u'video_url').replace('\\/', '/') path = compat_urllib_parse_urlparse(video_url).path extension = os.path.splitext(path)[1][1:] diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index a68a214ca..c860eedda 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1361,7 +1361,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): video_description = u'' def _extract_count(klass): - count = self._search_regex(r'class="%s">([\d,]+)</span>' % re.escape(klass), video_webpage, klass, fatal=False) + count = self._search_regex( + r'class="%s">([\d,]+)</span>' % re.escape(klass), + video_webpage, klass, default=None) if count is not None: return int(count.replace(',', '')) return None From 44c471c3b873473157adb8ba8a55667ab54b2602 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 17 Dec 2013 02:51:22 +0100 Subject: [PATCH 351/425] release 2013.12.17 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index f5bcc67ce..9ccfa6a81 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.16.7' +__version__ = '2013.12.17' From 29eb5174031cfc0b5de556da3da7761ac377de4e Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 17 Dec 2013 04:13:36 +0100 Subject: [PATCH 352/425] Add webpage_url_basename info_dict field (Fixes #1938) --- test/test_utils.py | 25 ++++++++++++++++--------- youtube_dl/YoutubeDL.py | 4 ++++ youtube_dl/utils.py | 7 +++++++ 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/test/test_utils.py b/test/test_utils.py index 0fa66beec..5f4fdb771 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -13,20 +13,21 @@ import xml.etree.ElementTree #from youtube_dl.utils import htmlentity_transform from youtube_dl.utils import ( - timeconvert, - sanitize_filename, - unescapeHTML, - orderedSet, DateRange, - unified_strdate, + encodeFilename, find_xpath_attr, get_meta_content, - xpath_with_ns, - smuggle_url, - unsmuggle_url, + orderedSet, + sanitize_filename, shell_quote, - encodeFilename, + smuggle_url, str_to_int, + timeconvert, + unescapeHTML, + unified_strdate, + unsmuggle_url, + url_basename, + xpath_with_ns, ) if sys.version_info < (3, 0): @@ -181,6 +182,12 @@ class TestUtil(unittest.TestCase): self.assertEqual(str_to_int('123,456'), 123456) self.assertEqual(str_to_int('123.456'), 123456) + def test_url_basename(self): + self.assertEqual(url_basename(u'http://foo.de/'), u'') + self.assertEqual(url_basename(u'http://foo.de/bar/baz'), u'baz') + self.assertEqual(url_basename(u'http://foo.de/bar/baz?x=y'), u'baz') + self.assertEqual(url_basename(u'http://foo.de/bar/baz#x=y'), u'baz') + self.assertEqual(url_basename(u'http://foo.de/bar/baz/'), u'baz') if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index b1f87415b..2a078adfb 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -47,6 +47,7 @@ from .utils import ( subtitles_filename, takewhile_inclusive, UnavailableVideoError, + url_basename, write_json_file, write_string, YoutubeDLHandler, @@ -484,6 +485,7 @@ class YoutubeDL(object): { 'extractor': ie.IE_NAME, 'webpage_url': url, + 'webpage_url_basename': url_basename(url), 'extractor_key': ie.ie_key(), }) if process: @@ -576,6 +578,7 @@ class YoutubeDL(object): 'playlist_index': i + playliststart, 'extractor': ie_result['extractor'], 'webpage_url': ie_result['webpage_url'], + 'webpage_url_basename': url_basename(ie_result['webpage_url']), 'extractor_key': ie_result['extractor_key'], } @@ -596,6 +599,7 @@ class YoutubeDL(object): { 'extractor': ie_result['extractor'], 'webpage_url': ie_result['webpage_url'], + 'webpage_url_basename': url_basename(ie_result['webpage_url']), 'extractor_key': ie_result['extractor_key'], }) return r diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index dbfac0f43..a249c7ec1 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1084,3 +1084,10 @@ def remove_start(s, start): if s.startswith(start): return s[len(start):] return s + + +def url_basename(url): + m = re.match(r'(?:https?:|)//[^/]+/(?:[^/?#]+/)?([^/?#]+)/?(?:[?#]|$)', url) + if not m: + return u'' + return m.group(1) From f09828b4e175997adce92f6ccecdcc30bb8060b6 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 17 Dec 2013 04:13:41 +0100 Subject: [PATCH 353/425] release 2013.12.17.1 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 9ccfa6a81..910c5a34f 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.17' +__version__ = '2013.12.17.1' From cecaaf3f58ad9f544dbb79af1e565d9353fa2b2d Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 17 Dec 2013 12:04:33 +0100 Subject: [PATCH 354/425] [generic] Do not use compatibility result fallback --- youtube_dl/extractor/generic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index fca23b6c4..da933067a 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -148,7 +148,7 @@ class GenericIE(InfoExtractor): try: new_url = self._test_redirect(url) if new_url: - return [self.url_result(new_url)] + return self.url_result(new_url) except compat_urllib_error.HTTPError: # This may be a stupid server that doesn't like HEAD, our UA, or so pass From d6c7a367e88096bb17e323954002c084477fe908 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 17 Dec 2013 12:32:58 +0100 Subject: [PATCH 355/425] [utils] Fix url_basename --- test/test_utils.py | 3 +++ youtube_dl/utils.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_utils.py b/test/test_utils.py index 5f4fdb771..e5778cd83 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -188,6 +188,9 @@ class TestUtil(unittest.TestCase): self.assertEqual(url_basename(u'http://foo.de/bar/baz?x=y'), u'baz') self.assertEqual(url_basename(u'http://foo.de/bar/baz#x=y'), u'baz') self.assertEqual(url_basename(u'http://foo.de/bar/baz/'), u'baz') + self.assertEqual( + url_basename(u'http://media.w3.org/2010/05/sintel/trailer.mp4'), + u'trailer.mp4') if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index a249c7ec1..2d12e2df9 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1087,7 +1087,7 @@ def remove_start(s, start): def url_basename(url): - m = re.match(r'(?:https?:|)//[^/]+/(?:[^/?#]+/)?([^/?#]+)/?(?:[?#]|$)', url) + m = re.match(r'(?:https?:|)//[^/]+/(?:[^?#]+/)?([^/?#]+)/?(?:[?#]|$)', url) if not m: return u'' return m.group(1) From 42393ce234c651aaae244e1546e1803101765acc Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 17 Dec 2013 12:33:55 +0100 Subject: [PATCH 356/425] Add support for direct links to a video (#1973) --- youtube_dl/extractor/generic.py | 52 ++++++++++++++++++++++++++------- youtube_dl/utils.py | 5 ++++ 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index da933067a..209f68204 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -13,6 +13,8 @@ from ..utils import ( ExtractorError, smuggle_url, unescapeHTML, + unified_strdate, + url_basename, ) from .brightcove import BrightcoveIE @@ -71,6 +73,17 @@ class GenericIE(InfoExtractor): u'skip_download': True, }, }, + # Direct link to a video + { + u'url': u'http://media.w3.org/2010/05/sintel/trailer.mp4', + u'file': u'trailer.mp4', + u'md5': u'67d406c2bcb6af27fa886f31aa934bbe', + u'info_dict': { + u'id': u'trailer', + u'title': u'trailer', + u'upload_date': u'20100513', + } + } ] def report_download_webpage(self, video_id): @@ -83,7 +96,7 @@ class GenericIE(InfoExtractor): """Report information extraction.""" self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url) - def _test_redirect(self, url): + def _send_head(self, url): """Check if it is a redirect, like url shorteners, in case return the new url.""" class HeadRequest(compat_urllib_request.Request): def get_method(self): @@ -131,29 +144,46 @@ class GenericIE(InfoExtractor): response = opener.open(HeadRequest(url)) if response is None: raise ExtractorError(u'Invalid URL protocol') - new_url = response.geturl() - - if url == new_url: - return False - - self.report_following_redirect(new_url) - return new_url + return response def _real_extract(self, url): parsed_url = compat_urlparse.urlparse(url) if not parsed_url.scheme: self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http') return self.url_result('http://' + url) + video_id = os.path.splitext(url.split('/')[-1])[0] try: - new_url = self._test_redirect(url) - if new_url: + response = self._send_head(url) + + # Check for redirect + new_url = response.geturl() + if url != new_url: + self.report_following_redirect(new_url) return self.url_result(new_url) + + # Check for direct link to a video + content_type = response.headers.get('Content-Type', '') + m = re.match(r'^(?:audio|video)/(?P<format_id>.+)$', content_type) + if m: + upload_date = response.headers.get('Last-Modified') + if upload_date: + upload_date = unified_strdate(upload_date) + assert (url_basename(url) == 'trailer.mp4') + return { + 'id': video_id, + 'title': os.path.splitext(url_basename(url))[0], + 'formats': [{ + 'format_id': m.group('format_id'), + 'url': url, + }], + 'upload_date': upload_date, + } + except compat_urllib_error.HTTPError: # This may be a stupid server that doesn't like HEAD, our UA, or so pass - video_id = url.split('/')[-1] try: webpage = self._download_webpage(url, video_id) except ValueError: diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 2d12e2df9..d5069dcca 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -761,12 +761,17 @@ def unified_strdate(date_str): '%Y-%m-%dT%H:%M:%S.%fZ', '%Y-%m-%dT%H:%M:%S.%f0Z', '%Y-%m-%dT%H:%M:%S', + '%Y-%m-%dT%H:%M:%S', ] for expression in format_expressions: try: upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d') except: pass + if upload_date is None: + timetuple = email.utils.parsedate_tz(date_str) + if timetuple: + upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d') return upload_date def determine_ext(url, default_ext=u'unknown_video'): From 946135aa2ab846a367cde5b3b86a0224ac696fdd Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 17 Dec 2013 12:34:30 +0100 Subject: [PATCH 357/425] [academicearth] remove unused imports --- youtube_dl/extractor/academicearth.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/youtube_dl/extractor/academicearth.py b/youtube_dl/extractor/academicearth.py index c450c30ca..ac05f8246 100644 --- a/youtube_dl/extractor/academicearth.py +++ b/youtube_dl/extractor/academicearth.py @@ -1,11 +1,6 @@ -import datetime -import json import re from .common import InfoExtractor -from ..utils import ( - remove_start, -) class AcademicEarthCourseIE(InfoExtractor): From 7de6e075b4b8738fed185d9924375100ae0d8425 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 17 Dec 2013 12:35:16 +0100 Subject: [PATCH 358/425] [radiofrance] remove unused imports --- youtube_dl/extractor/radiofrance.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/youtube_dl/extractor/radiofrance.py b/youtube_dl/extractor/radiofrance.py index 2866115ef..34652f6c1 100644 --- a/youtube_dl/extractor/radiofrance.py +++ b/youtube_dl/extractor/radiofrance.py @@ -1,12 +1,7 @@ # coding: utf-8 -import datetime -import json import re from .common import InfoExtractor -from ..utils import ( - remove_start, -) class RadioFranceIE(InfoExtractor): @@ -42,11 +37,11 @@ class RadioFranceIE(InfoExtractor): webpage, u'audio URLs') formats = [ { - 'format_id': m[0], - 'url': m[1], + 'format_id': fm[0], + 'url': fm[1], 'vcodec': 'none', } - for m in + for fm in re.findall(r"([a-z0-9]+)\s*:\s*'([^']+)'", formats_str) ] # No sorting, we don't know any more about these formats From 6086d121cbc3e49c30e3ef64151cc2c4b22ed713 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 17 Dec 2013 12:35:57 +0100 Subject: [PATCH 359/425] release 2013.12.17.2 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 910c5a34f..7cbee7335 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.17.1' +__version__ = '2013.12.17.2' From 9b8aaeed856f189fef016563a10f7f4e46a1590e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Tue, 17 Dec 2013 14:56:29 +0100 Subject: [PATCH 360/425] Simplify url_basename Use urlparse from the standard library. --- youtube_dl/utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index d5069dcca..4c7ad89c0 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1092,7 +1092,5 @@ def remove_start(s, start): def url_basename(url): - m = re.match(r'(?:https?:|)//[^/]+/(?:[^?#]+/)?([^/?#]+)/?(?:[?#]|$)', url) - if not m: - return u'' - return m.group(1) + path = compat_urlparse.urlparse(url).path + return path.strip(u'/').split(u'/')[-1] From e029b8bd43a30211e1ed71ee8a123c9b081d4c22 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 17 Dec 2013 16:12:14 +0100 Subject: [PATCH 361/425] [utils] Remove duplicated line This line was added by accident in 42393ce234c651aaae244e1546e1803101765acc --- youtube_dl/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 4c7ad89c0..cc391bddd 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -761,7 +761,6 @@ def unified_strdate(date_str): '%Y-%m-%dT%H:%M:%S.%fZ', '%Y-%m-%dT%H:%M:%S.%f0Z', '%Y-%m-%dT%H:%M:%S', - '%Y-%m-%dT%H:%M:%S', ] for expression in format_expressions: try: From 3e78514568ddb516d23de052f694c8de7ab469d7 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 17 Dec 2013 16:26:32 +0100 Subject: [PATCH 362/425] [generic] Support application/ogg for direct links Also remove some debugging code. --- youtube_dl/extractor/generic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 209f68204..fd32370c2 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -164,18 +164,18 @@ class GenericIE(InfoExtractor): # Check for direct link to a video content_type = response.headers.get('Content-Type', '') - m = re.match(r'^(?:audio|video)/(?P<format_id>.+)$', content_type) + m = re.match(r'^(?P<type>audio|video|application(?=/ogg$))/(?P<format_id>.+)$', content_type) if m: upload_date = response.headers.get('Last-Modified') if upload_date: upload_date = unified_strdate(upload_date) - assert (url_basename(url) == 'trailer.mp4') return { 'id': video_id, 'title': os.path.splitext(url_basename(url))[0], 'formats': [{ 'format_id': m.group('format_id'), 'url': url, + 'vcodec': u'none' if m.group('type') == 'audio' else None }], 'upload_date': upload_date, } From 4ea3be0a5c91398ff7032557bbee7b2a28757764 Mon Sep 17 00:00:00 2001 From: rzhxeo <rzhxeot7z81b4700@mailcatch.com> Date: Wed, 18 Dec 2013 03:30:55 +0100 Subject: [PATCH 363/425] [YoutubeIE] Externalize format selection --- youtube_dl/YoutubeDL.py | 2 +- youtube_dl/extractor/youtube.py | 125 +++++++++----------------------- 2 files changed, 34 insertions(+), 93 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 2a078adfb..74c0db4fc 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -636,7 +636,7 @@ class YoutubeDL(object): info_dict['playlist_index'] = None # This extractors handle format selection themselves - if info_dict['extractor'] in [u'youtube', u'Youku']: + if info_dict['extractor'] in [u'Youku']: if download: self.process_info(info_dict) return info_dict diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index c860eedda..4feac13d1 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -162,23 +162,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): # Dash audio '141', '172', '140', '171', '139', ] - _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '36', '17', '13', - # Apple HTTP Live Streaming - '96', '95', '94', '93', '92', '132', '151', - # 3D - '85', '102', '84', '101', '83', '100', '82', - # Dash video - '138', '248', '137', '247', '136', '246', '245', - '244', '135', '243', '134', '242', '133', '160', - # Dash audio - '172', '141', '171', '140', '139', - ] - _video_formats_map = { - 'flv': ['35', '34', '6', '5'], - '3gp': ['36', '17', '13'], - 'mp4': ['38', '37', '22', '18'], - 'webm': ['46', '45', '44', '43'], - } _video_extensions = { '13': '3gp', '17': '3gp', @@ -1153,13 +1136,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): self._downloader.report_warning(err_msg) return {} - def _print_formats(self, formats): - print('Available formats:') - for x in formats: - print('%s\t:\t%s\t[%s]%s' %(x, self._video_extensions.get(x, 'flv'), - self._video_dimensions.get(x, '???'), - ' ('+self._special_itags[x]+')' if x in self._special_itags else '')) - def _extract_id(self, url): mobj = re.match(self._VALID_URL, url, re.VERBOSE) if mobj is None: @@ -1172,48 +1148,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): Transform a dictionary in the format {itag:url} to a list of (itag, url) with the requested formats. """ - req_format = self._downloader.params.get('format', None) - format_limit = self._downloader.params.get('format_limit', None) - available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats - if format_limit is not None and format_limit in available_formats: - format_list = available_formats[available_formats.index(format_limit):] - else: - format_list = available_formats - existing_formats = [x for x in format_list if x in url_map] + existing_formats = [x for x in self._available_formats if x in url_map] if len(existing_formats) == 0: raise ExtractorError(u'no known formats available for video') - if self._downloader.params.get('listformats', None): - self._print_formats(existing_formats) - return - if req_format is None or req_format == 'best': - video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality - elif req_format == 'worst': - video_url_list = [(existing_formats[-1], url_map[existing_formats[-1]])] # worst quality - elif req_format in ('-1', 'all'): - video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats - else: - # Specific formats. We pick the first in a slash-delimeted sequence. - # Format can be specified as itag or 'mp4' or 'flv' etc. We pick the highest quality - # available in the specified format. For example, - # if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'. - # if '1/mp4/3/4' is requested and '1' and '5' (is a mp4) are available, we pick '1'. - # if '1/mp4/3/4' is requested and '4' and '5' (is a mp4) are available, we pick '5'. - req_formats = req_format.split('/') - video_url_list = None - for rf in req_formats: - if rf in url_map: - video_url_list = [(rf, url_map[rf])] - break - if rf in self._video_formats_map: - for srf in self._video_formats_map[rf]: - if srf in url_map: - video_url_list = [(srf, url_map[srf])] - break - else: - continue - break - if video_url_list is None: - raise ExtractorError(u'requested format not available') + video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats + video_url_list.reverse() # order worst to best return video_url_list def _extract_from_m3u8(self, manifest_url, video_id): @@ -1462,19 +1401,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): url += '&ratebypass=yes' url_map[url_data['itag'][0]] = url video_url_list = self._get_video_url_list(url_map) - if not video_url_list: - return elif video_info.get('hlsvp'): manifest_url = video_info['hlsvp'][0] url_map = self._extract_from_m3u8(manifest_url, video_id) video_url_list = self._get_video_url_list(url_map) - if not video_url_list: - return - else: raise ExtractorError(u'no conn, hlsvp or url_encoded_fmt_stream_map information found in video info') - results = [] + formats = [] for itag, video_real_url in video_url_list: # Extension video_extension = self._video_extensions.get(itag, 'flv') @@ -1482,30 +1416,37 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): video_format = '{0} - {1}{2}'.format(itag if itag else video_extension, self._video_dimensions.get(itag, '???'), ' ('+self._special_itags[itag]+')' if itag in self._special_itags else '') + note = self._special_itags.get(itag, None) + resolution = self._video_dimensions.get(itag, None) - results.append({ - 'id': video_id, - 'url': video_real_url, - 'uploader': video_uploader, - 'uploader_id': video_uploader_id, - 'upload_date': upload_date, - 'title': video_title, - 'ext': video_extension, - 'format': video_format, - 'format_id': itag, - 'thumbnail': video_thumbnail, - 'description': video_description, - 'player_url': player_url, - 'subtitles': video_subtitles, - 'duration': video_duration, - 'age_limit': 18 if age_gate else 0, - 'annotations': video_annotations, - 'webpage_url': 'https://www.youtube.com/watch?v=%s' % video_id, - 'view_count': view_count, - 'like_count': like_count, - 'dislike_count': dislike_count, + formats.append({ + 'url': video_real_url, + 'ext': video_extension, + 'format': video_format, + 'format_id': itag, + 'player_url': player_url, + '_resolution': resolution, + 'format_note': note, }) - return results + + return { + 'id': video_id, + 'uploader': video_uploader, + 'uploader_id': video_uploader_id, + 'upload_date': upload_date, + 'title': video_title, + 'thumbnail': video_thumbnail, + 'description': video_description, + 'subtitles': video_subtitles, + 'duration': video_duration, + 'age_limit': 18 if age_gate else 0, + 'annotations': video_annotations, + 'webpage_url': 'https://www.youtube.com/watch?v=%s' % video_id, + 'view_count': view_count, + 'like_count': like_count, + 'dislike_count': dislike_count, + 'formats': formats, + } class YoutubePlaylistIE(YoutubeBaseInfoExtractor): IE_DESC = u'YouTube.com playlists' From dbd1988ed9eb2c5eeda9d16f33f4f0387ee22922 Mon Sep 17 00:00:00 2001 From: rzhxeo <rzhxeot7z81b4700@mailcatch.com> Date: Wed, 18 Dec 2013 21:21:25 +0100 Subject: [PATCH 364/425] [YoutubeIE] Add width and height to format dict --- youtube_dl/extractor/youtube.py | 106 +++++++++++++++++--------------- 1 file changed, 55 insertions(+), 51 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 4feac13d1..793df4881 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -219,54 +219,54 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): '248': 'webm', } _video_dimensions = { - '5': '400x240', - '6': '???', - '13': '???', - '17': '176x144', - '18': '640x360', - '22': '1280x720', - '34': '640x360', - '35': '854x480', - '36': '320x240', - '37': '1920x1080', - '38': '4096x3072', - '43': '640x360', - '44': '854x480', - '45': '1280x720', - '46': '1920x1080', - '82': '360p', - '83': '480p', - '84': '720p', - '85': '1080p', - '92': '240p', - '93': '360p', - '94': '480p', - '95': '720p', - '96': '1080p', - '100': '360p', - '101': '480p', - '102': '720p', - '132': '240p', - '151': '72p', - '133': '240p', - '134': '360p', - '135': '480p', - '136': '720p', - '137': '1080p', - '138': '>1080p', - '139': '48k', - '140': '128k', - '141': '256k', - '160': '192p', - '171': '128k', - '172': '256k', - '242': '240p', - '243': '360p', - '244': '480p', - '245': '480p', - '246': '480p', - '247': '720p', - '248': '1080p', + '5': {'width': 400, 'height': 240}, + '6': {}, + '13': {}, + '17': {'width': 176, 'height': 144}, + '18': {'width': 640, 'height': 360}, + '22': {'width': 1280, 'height': 720}, + '34': {'width': 640, 'height': 360}, + '35': {'width': 854, 'height': 480}, + '36': {'width': 320, 'height': 240}, + '37': {'width': 1920, 'height': 1080}, + '38': {'width': 4096, 'height': 3072}, + '43': {'width': 640, 'height': 360}, + '44': {'width': 854, 'height': 480}, + '45': {'width': 1280, 'height': 720}, + '46': {'width': 1920, 'height': 1080}, + '82': {'height': 360, 'display': '360p'}, + '83': {'height': 480, 'display': '480p'}, + '84': {'height': 720, 'display': '720p'}, + '85': {'height': 1080, 'display': '1080p'}, + '92': {'height': 240, 'display': '240p'}, + '93': {'height': 360, 'display': '360p'}, + '94': {'height': 480, 'display': '480p'}, + '95': {'height': 720, 'display': '720p'}, + '96': {'height': 1080, 'display': '1080p'}, + '100': {'height': 360, 'display': '360p'}, + '101': {'height': 480, 'display': '480p'}, + '102': {'height': 720, 'display': '720p'}, + '132': {'height': 240, 'display': '240p'}, + '151': {'height': 72, 'display': '72p'}, + '133': {'height': 240, 'display': '240p'}, + '134': {'height': 360, 'display': '360p'}, + '135': {'height': 480, 'display': '480p'}, + '136': {'height': 720, 'display': '720p'}, + '137': {'height': 1080, 'display': '1080p'}, + '138': {'height': 1081, 'display': '>1080p'}, + '139': {'display': '48k'}, + '140': {'display': '128k'}, + '141': {'display': '256k'}, + '160': {'height': 192, 'display': '192p'}, + '171': {'display': '128k'}, + '172': {'display': '256k'}, + '242': {'height': 240, 'display': '240p'}, + '243': {'height': 360, 'display': '360p'}, + '244': {'height': 480, 'display': '480p'}, + '245': {'height': 480, 'display': '480p'}, + '246': {'height': 480, 'display': '480p'}, + '247': {'height': 720, 'display': '720p'}, + '248': {'height': 1080, 'display': '1080p'}, } _special_itags = { '82': '3D', @@ -1412,12 +1412,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): for itag, video_real_url in video_url_list: # Extension video_extension = self._video_extensions.get(itag, 'flv') + resolution = self._video_dimensions.get(itag, {}).get('display') + width = self._video_dimensions.get(itag, {}).get('width') + height = self._video_dimensions.get(itag, {}).get('height') + note = self._special_itags.get(itag) video_format = '{0} - {1}{2}'.format(itag if itag else video_extension, - self._video_dimensions.get(itag, '???'), + '%dx%d' % (width, height) if width is not None and height is not None else (resolution if resolution is not None else '???'), ' ('+self._special_itags[itag]+')' if itag in self._special_itags else '') - note = self._special_itags.get(itag, None) - resolution = self._video_dimensions.get(itag, None) formats.append({ 'url': video_real_url, @@ -1426,6 +1428,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): 'format_id': itag, 'player_url': player_url, '_resolution': resolution, + 'width': width, + 'height': height, 'format_note': note, }) From e56f22ae203d10ab6d3715bc2dc1c9b927294e50 Mon Sep 17 00:00:00 2001 From: rzhxeo <rzhxeot7z81b4700@mailcatch.com> Date: Wed, 18 Dec 2013 21:22:37 +0100 Subject: [PATCH 365/425] [YoutubeIE] Sort formats by resolution --- youtube_dl/extractor/youtube.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 793df4881..58d274970 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1432,6 +1432,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): 'height': height, 'format_note': note, }) + def _formats_key(f): + return (f.get('height') if f.get('height') is not None else -1, + f.get('width') if f.get('width') is not None else -1) + formats = sorted(formats, key=_formats_key) return { 'id': video_id, From bfaae0a768d455112e003f6200edfdafb22affd6 Mon Sep 17 00:00:00 2001 From: rzhxeo <rzhxeot7z81b4700@mailcatch.com> Date: Wed, 18 Dec 2013 21:24:39 +0100 Subject: [PATCH 366/425] Filter and sort videos before calling list_formats --- youtube_dl/YoutubeDL.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 74c0db4fc..fd26f00b7 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -662,10 +662,6 @@ class YoutubeDL(object): if 'ext' not in format: format['ext'] = determine_ext(format['url']) - if self.params.get('listformats', None): - self.list_formats(info_dict) - return - format_limit = self.params.get('format_limit', None) if format_limit: formats = list(takewhile_inclusive( @@ -681,6 +677,11 @@ class YoutubeDL(object): return (f.get('height'), f.get('width'), ext_ord) formats = sorted(formats, key=_free_formats_key) + info_dict['formats'] = formats + if self.params.get('listformats', None): + self.list_formats(info_dict) + return + req_format = self.params.get('format', 'best') if req_format is None: req_format = 'best' From 62d68c43ed76fab391eacfd52f0391936ef0a4dc Mon Sep 17 00:00:00 2001 From: rzhxeo <rzhxeot7z81b4700@mailcatch.com> Date: Wed, 18 Dec 2013 21:25:13 +0100 Subject: [PATCH 367/425] Make prefer_free_formats sorting more robust --- youtube_dl/YoutubeDL.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index fd26f00b7..a93dd41a3 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -674,7 +674,9 @@ class YoutubeDL(object): except ValueError: ext_ord = -1 # We only compare the extension if they have the same height and width - return (f.get('height'), f.get('width'), ext_ord) + return (f.get('height') if f.get('height') is not None else -1, + f.get('width') if f.get('width') is not None else -1, + ext_ord) formats = sorted(formats, key=_free_formats_key) info_dict['formats'] = formats From 77aa6b329da84335ac3f183e0ed04198b318a990 Mon Sep 17 00:00:00 2001 From: dst <dstftw@gmail.com> Date: Thu, 19 Dec 2013 05:28:16 +0700 Subject: [PATCH 368/425] [ivi] Add support for ivi.ru --- youtube_dl/extractor/__init__.py | 4 + youtube_dl/extractor/ivi.py | 154 +++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 youtube_dl/extractor/ivi.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 7f2f8806e..677a894b3 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -82,6 +82,10 @@ from .ina import InaIE from .infoq import InfoQIE from .instagram import InstagramIE from .internetvideoarchive import InternetVideoArchiveIE +from .ivi import ( + IviIE, + IviCompilationIE +) from .jeuxvideo import JeuxVideoIE from .jukebox import JukeboxIE from .justintv import JustinTVIE diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py new file mode 100644 index 000000000..aa8b3b8a3 --- /dev/null +++ b/youtube_dl/extractor/ivi.py @@ -0,0 +1,154 @@ +# encoding: utf-8 + +import re +import json + +from .common import InfoExtractor +from ..utils import ( + compat_urllib_request, + ExtractorError, +) + + +class IviIE(InfoExtractor): + IE_DESC = u'ivi.ru' + IE_NAME = u'ivi' + _VALID_URL = r'^https?://(?:www\.)?(?P<url>ivi\.ru/watch(?:/(?P<compilationid>[^/]+))?/(?P<videoid>\d+))' + + _TESTS = [ + # Single movie + { + u'url': u'http://www.ivi.ru/watch/53141', + u'file': u'53141.mp4', + u'md5': u'6ff5be2254e796ed346251d117196cf4', + u'info_dict': { + u'title': u'Иван Васильевич меняет профессию', + u'description': u'md5:14d8eda24e9d93d29b5857012c6d6346', + u'duration': 5498, + u'thumbnail': u'http://thumbs.ivi.ru/f20.vcp.digitalaccess.ru/contents/d/1/c3c885163a082c29bceeb7b5a267a6.jpg', + }, + }, + # Serial's serie + { + u'url': u'http://www.ivi.ru/watch/dezhurnyi_angel/74791', + u'file': u'74791.mp4', + u'md5': u'3e6cc9a848c1d2ebcc6476444967baa9', + u'info_dict': { + u'title': u'Дежурный ангел - 1 серия', + u'duration': 2490, + u'thumbnail': u'http://thumbs.ivi.ru/f7.vcp.digitalaccess.ru/contents/8/e/bc2f6c2b6e5d291152fdd32c059141.jpg', + }, + } + ] + + # Sorted by quality + _known_formats = ['MP4-low-mobile', 'MP4-mobile', 'FLV-lo', 'MP4-lo', 'FLV-hi', 'MP4-hi', 'MP4-SHQ'] + + # Sorted by size + _known_thumbnails = ['Thumb-120x90', 'Thumb-160', 'Thumb-640x480'] + + def _extract_description(self, html): + m = re.search(r'<meta name="description" content="(?P<description>[^"]+)"/>', html) + return m.group('description') if m is not None else None + + def _extract_comment_count(self, html): + m = re.search(u'(?s)<a href="#" id="view-comments" class="action-button dim gradient">\s*Комментарии:\s*(?P<commentcount>\d+)\s*</a>', html) + return int(m.group('commentcount')) if m is not None else 0 + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('videoid') + + api_url = 'http://api.digitalaccess.ru/api/json/' + + data = {u'method': u'da.content.get', + u'params': [video_id, {u'site': u's183', + u'referrer': u'http://www.ivi.ru/watch/%s' % video_id, + u'contentid': video_id + } + ] + } + + request = compat_urllib_request.Request(api_url, json.dumps(data)) + + video_json_page = self._download_webpage(request, video_id, u'Downloading video JSON') + video_json = json.loads(video_json_page) + + if u'error' in video_json: + error = video_json[u'error'] + if error[u'origin'] == u'NoRedisValidData': + raise ExtractorError(u'Video %s does not exist' % video_id, expected=True) + raise ExtractorError(u'Unable to download video %s: %s' % (video_id, error[u'message']), expected=True) + + result = video_json[u'result'] + + formats = [{'url': x[u'url'], + 'format_id': x[u'content_format'] + } for x in result[u'files'] if x[u'content_format'] in self._known_formats] + formats.sort(key=lambda fmt: self._known_formats.index(fmt['format_id'])) + + if len(formats) == 0: + self._downloader.report_warning(u'No media links available for %s' % video_id) + return + + duration = result[u'duration'] + compilation = result[u'compilation'] + title = result[u'title'] + + title = '%s - %s' % (compilation, title) if compilation is not None else title + + previews = result[u'preview'] + previews.sort(key=lambda fmt: self._known_thumbnails.index(fmt['content_format'])) + thumbnail = previews[-1][u'url'] if len(previews) > 0 else None + + video_page_url = 'http://' + mobj.group('url') + video_page = self._download_webpage(video_page_url, video_id, u'Downloading video page') + + description = self._extract_description(video_page) + comment_count = self._extract_comment_count(video_page) + + return { + 'id': video_id, + 'title': title, + 'thumbnail': thumbnail, + 'description': description, + 'duration': duration, + 'comment_count': comment_count, + 'formats': formats, + } + + +class IviCompilationIE(InfoExtractor): + IE_DESC = u'ivi.ru compilations' + IE_NAME = u'ivi:compilation' + _VALID_URL = r'^https?://(?:www\.)?ivi\.ru/watch/(?!\d+)(?P<compilationid>[a-z\d_-]+)(?:/season(?P<seasonid>\d+))?$' + + def _extract_entries(self, html, compilation_id): + return [self.url_result('http://www.ivi.ru/watch/%s/%s' % (compilation_id, serie), 'Ivi') + for serie in re.findall(r'<strong><a href="/watch/%s/(\d+)">(?:[^<]+)</a></strong>' % compilation_id, html)] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + compilation_id = mobj.group('compilationid') + season_id = mobj.group('seasonid') + + if season_id is not None: # Season link + season_page = self._download_webpage(url, compilation_id, u'Downloading season %s web page' % season_id) + playlist_id = '%s/season%s' % (compilation_id, season_id) + playlist_title = self._html_search_meta(u'title', season_page, u'title') + entries = self._extract_entries(season_page, compilation_id) + else: # Compilation link + compilation_page = self._download_webpage(url, compilation_id, u'Downloading compilation web page') + playlist_id = compilation_id + playlist_title = self._html_search_meta(u'title', compilation_page, u'title') + seasons = re.findall(r'<a href="/watch/%s/season(\d+)">[^<]+</a>' % compilation_id, compilation_page) + if len(seasons) == 0: # No seasons in this compilation + entries = self._extract_entries(compilation_page, compilation_id) + else: + entries = [] + for season_id in seasons: + season_page = self._download_webpage('http://www.ivi.ru/watch/%s/season%s' % (compilation_id, season_id), + compilation_id, u'Downloading season %s web page' % season_id) + entries.extend(self._extract_entries(season_page, compilation_id)) + + return self.playlist_result(entries, playlist_id, playlist_title) \ No newline at end of file From 8c21b7c647d5328388cb5be6af6cbe9f6143485c Mon Sep 17 00:00:00 2001 From: dst <dstftw@gmail.com> Date: Thu, 19 Dec 2013 05:39:22 +0700 Subject: [PATCH 369/425] [ivi] Add playlist tests --- test/test_playlists.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/test_playlists.py b/test/test_playlists.py index 5004d0464..576f7fb4e 100644 --- a/test/test_playlists.py +++ b/test/test_playlists.py @@ -6,6 +6,7 @@ import os import sys import unittest +from youtube_dl.extractor.ivi import IviCompilationIE sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import FakeYDL @@ -168,6 +169,24 @@ class TestPlaylists(unittest.TestCase): self.assertEqual(result['title'], u'Building Dynamic Websites') self.assertEqual(result['description'], u"Today's websites are increasingly dynamic. Pages are no longer static HTML files but instead generated by scripts and database calls. User interfaces are more seamless, with technologies like Ajax replacing traditional page reloads. This course teaches students how to build dynamic websites with Ajax and with Linux, Apache, MySQL, and PHP (LAMP), one of today's most popular frameworks. Students learn how to set up domain names with DNS, how to structure pages with XHTML and CSS, how to program in JavaScript and PHP, how to configure Apache and MySQL, how to design and query databases with SQL, how to use Ajax with both XML and JSON, and how to build mashups. The course explores issues of security, scalability, and cross-browser support and also discusses enterprise-level deployments of websites, including third-party hosting, virtualization, colocation in data centers, firewalling, and load-balancing.") self.assertEqual(len(result['entries']), 10) + + def test_ivi_compilation(self): + dl = FakeYDL() + ie = IviCompilationIE(dl) + result = ie.extract('http://www.ivi.ru/watch/dezhurnyi_angel') + self.assertIsPlaylist(result) + self.assertEqual(result['id'], u'dezhurnyi_angel') + self.assertEqual(result['title'], u'Дежурный ангел (2010 - 2012)') + self.assertTrue(len(result['entries']) >= 36) + + def test_ivi_compilation_season(self): + dl = FakeYDL() + ie = IviCompilationIE(dl) + result = ie.extract('http://www.ivi.ru/watch/dezhurnyi_angel/season2') + self.assertIsPlaylist(result) + self.assertEqual(result['id'], u'dezhurnyi_angel/season2') + self.assertEqual(result['title'], u'Дежурный ангел (2010 - 2012) 2 сезон') + self.assertTrue(len(result['entries']) >= 20) if __name__ == '__main__': From 5ce54a8205df1eea7e38ed86d3540f601f300a7e Mon Sep 17 00:00:00 2001 From: dst <dstftw@gmail.com> Date: Thu, 19 Dec 2013 05:53:34 +0700 Subject: [PATCH 370/425] [ivi] Neat import --- test/test_playlists.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_playlists.py b/test/test_playlists.py index 576f7fb4e..1b7b4e3d8 100644 --- a/test/test_playlists.py +++ b/test/test_playlists.py @@ -6,7 +6,6 @@ import os import sys import unittest -from youtube_dl.extractor.ivi import IviCompilationIE sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import FakeYDL @@ -28,7 +27,8 @@ from youtube_dl.extractor import ( BambuserChannelIE, BandcampAlbumIE, SmotriCommunityIE, - SmotriUserIE + SmotriUserIE, + IviCompilationIE ) From 0cc83dc54b619376800f9983cc1d7338a6b414ec Mon Sep 17 00:00:00 2001 From: dst <dstftw@gmail.com> Date: Thu, 19 Dec 2013 05:56:48 +0700 Subject: [PATCH 371/425] [smotri] Fix duration field name --- youtube_dl/extractor/smotri.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py index 4ea89bf85..a589a893b 100644 --- a/youtube_dl/extractor/smotri.py +++ b/youtube_dl/extractor/smotri.py @@ -202,7 +202,7 @@ class SmotriIE(InfoExtractor): 'uploader': video_uploader, 'upload_date': video_upload_date, 'uploader_id': video_uploader_id, - 'video_duration': video_duration, + 'duration': video_duration, 'view_count': video_view_count, 'age_limit': 18 if adult_content else 0, 'video_page_url': video_page_url From 6c6db72ed4505520d80002e37b523e4177146979 Mon Sep 17 00:00:00 2001 From: dst <dstftw@gmail.com> Date: Thu, 19 Dec 2013 06:19:41 +0700 Subject: [PATCH 372/425] [ivi] Skip tests for travis build --- youtube_dl/extractor/ivi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py index aa8b3b8a3..10279478b 100644 --- a/youtube_dl/extractor/ivi.py +++ b/youtube_dl/extractor/ivi.py @@ -27,6 +27,7 @@ class IviIE(InfoExtractor): u'duration': 5498, u'thumbnail': u'http://thumbs.ivi.ru/f20.vcp.digitalaccess.ru/contents/d/1/c3c885163a082c29bceeb7b5a267a6.jpg', }, + u'skip': u'Only works from Russia', }, # Serial's serie { @@ -38,6 +39,7 @@ class IviIE(InfoExtractor): u'duration': 2490, u'thumbnail': u'http://thumbs.ivi.ru/f7.vcp.digitalaccess.ru/contents/8/e/bc2f6c2b6e5d291152fdd32c059141.jpg', }, + u'skip': u'Only works from Russia', } ] From a51e37af6242a3fa49ad258a63d3f1a40c0ef9f2 Mon Sep 17 00:00:00 2001 From: dst <dstftw@gmail.com> Date: Thu, 19 Dec 2013 10:53:38 +0700 Subject: [PATCH 373/425] [ivi] Simplify --- youtube_dl/extractor/ivi.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/ivi.py b/youtube_dl/extractor/ivi.py index 10279478b..4bdf55f93 100644 --- a/youtube_dl/extractor/ivi.py +++ b/youtube_dl/extractor/ivi.py @@ -13,7 +13,7 @@ from ..utils import ( class IviIE(InfoExtractor): IE_DESC = u'ivi.ru' IE_NAME = u'ivi' - _VALID_URL = r'^https?://(?:www\.)?(?P<url>ivi\.ru/watch(?:/(?P<compilationid>[^/]+))?/(?P<videoid>\d+))' + _VALID_URL = r'^https?://(?:www\.)?ivi\.ru/watch(?:/(?P<compilationid>[^/]+))?/(?P<videoid>\d+)' _TESTS = [ # Single movie @@ -103,9 +103,7 @@ class IviIE(InfoExtractor): previews.sort(key=lambda fmt: self._known_thumbnails.index(fmt['content_format'])) thumbnail = previews[-1][u'url'] if len(previews) > 0 else None - video_page_url = 'http://' + mobj.group('url') - video_page = self._download_webpage(video_page_url, video_id, u'Downloading video page') - + video_page = self._download_webpage(url, video_id, u'Downloading video page') description = self._extract_description(video_page) comment_count = self._extract_comment_count(video_page) From 71507a11c899ceba571c0e2d0b2d8e632bcd0c08 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Thu, 19 Dec 2013 16:39:01 +0100 Subject: [PATCH 374/425] [soundcloud] Support mobile URLs (Fixes #2009) --- youtube_dl/extractor/soundcloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index cbba4094b..e22ff9c38 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -24,7 +24,7 @@ class SoundcloudIE(InfoExtractor): """ _VALID_URL = r'''^(?:https?://)? - (?:(?:(?:www\.)?soundcloud\.com/ + (?:(?:(?:www\.|m\.)?soundcloud\.com/ (?P<uploader>[\w\d-]+)/ (?!sets/)(?P<title>[\w\d-]+)/? (?P<token>[^?]+?)?(?:[?].*)?$) From 97e302a419d559a9837eeefb46ad05066540cc66 Mon Sep 17 00:00:00 2001 From: dst <dstftw@gmail.com> Date: Fri, 20 Dec 2013 00:21:04 +0700 Subject: [PATCH 375/425] [imdb] Add support for mobile site URLs --- youtube_dl/extractor/imdb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/imdb.py b/youtube_dl/extractor/imdb.py index 6fb373db2..e5332cce8 100644 --- a/youtube_dl/extractor/imdb.py +++ b/youtube_dl/extractor/imdb.py @@ -11,7 +11,7 @@ from ..utils import ( class ImdbIE(InfoExtractor): IE_NAME = u'imdb' IE_DESC = u'Internet Movie Database trailers' - _VALID_URL = r'http://www\.imdb\.com/video/imdb/vi(?P<id>\d+)' + _VALID_URL = r'http://(?:www|m)\.imdb\.com/video/imdb/vi(?P<id>\d+)' _TEST = { u'url': u'http://www.imdb.com/video/imdb/vi2524815897', @@ -27,7 +27,7 @@ class ImdbIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') - webpage = self._download_webpage(url,video_id) + webpage = self._download_webpage('http://www.imdb.com/video/imdb/vi%s' % video_id, video_id) descr = get_element_by_attribute('itemprop', 'description', webpage) available_formats = re.findall( r'case \'(?P<f_id>.*?)\' :$\s+url = \'(?P<path>.*?)\'', webpage, From c0d0b01f0e12ce23f7a751ef05e52dabd3e4c1e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Thu, 19 Dec 2013 20:28:52 +0100 Subject: [PATCH 376/425] [generic] Detect ooyala videos (fixes #2013) --- youtube_dl/extractor/common.py | 6 ++++-- youtube_dl/extractor/generic.py | 18 +++++++++++++++++- youtube_dl/extractor/ooyala.py | 5 +++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 939249d7b..f89df57ed 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -263,7 +263,8 @@ class InfoExtractor(object): self.to_screen(u'Logging in') #Methods for following #608 - def url_result(self, url, ie=None, video_id=None): + @staticmethod + def url_result(url, ie=None, video_id=None): """Returns a url that points to a page that should be processed""" #TODO: ie should be the class used for getting the info video_info = {'_type': 'url', @@ -272,7 +273,8 @@ class InfoExtractor(object): if video_id is not None: video_info['id'] = video_id return video_info - def playlist_result(self, entries, playlist_id=None, playlist_title=None): + @staticmethod + def playlist_result(entries, playlist_id=None, playlist_title=None): """Returns a playlist""" video_info = {'_type': 'playlist', 'entries': entries} diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index fd32370c2..3c56daa02 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -17,6 +17,7 @@ from ..utils import ( url_basename, ) from .brightcove import BrightcoveIE +from .ooyala import OoyalaIE class GenericIE(InfoExtractor): @@ -83,7 +84,17 @@ class GenericIE(InfoExtractor): u'title': u'trailer', u'upload_date': u'20100513', } - } + }, + # ooyala video + { + u'url': u'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219', + u'md5': u'5644c6ca5d5782c1d0d350dad9bd840c', + u'info_dict': { + u'id': u'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ', + u'ext': u'mp4', + u'title': u'2cc213299525360.mov', #that's what we get + }, + }, ] def report_download_webpage(self, video_id): @@ -277,6 +288,11 @@ class GenericIE(InfoExtractor): if mobj is not None: return self.url_result(mobj.group('url')) + # Look for Ooyala videos + mobj = re.search(r'player.ooyala.com/[^"?]+\?[^"]*?(?:embedCode|ec)=([^"&]+)', webpage) + if mobj is not None: + return OoyalaIE._build_url_result(mobj.group(1)) + # Start with something easy: JW Player in SWFObject mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage) if mobj is None: diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py index 1f7b4d2e7..d08e47734 100644 --- a/youtube_dl/extractor/ooyala.py +++ b/youtube_dl/extractor/ooyala.py @@ -22,6 +22,11 @@ class OoyalaIE(InfoExtractor): def _url_for_embed_code(embed_code): return 'http://player.ooyala.com/player.js?embedCode=%s' % embed_code + @classmethod + def _build_url_result(cls, embed_code): + return cls.url_result(cls._url_for_embed_code(embed_code), + ie=cls.ie_key()) + def _extract_result(self, info, more_info): return {'id': info['embedCode'], 'ext': 'mp4', From 1f9da9049b5b6dcede3d274acd58aa1b6dea5d2c Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Thu, 19 Dec 2013 20:44:30 +0100 Subject: [PATCH 377/425] [generic] Support YouTube swf embed (Fixes #2010) --- youtube_dl/extractor/generic.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 3c56daa02..75cb96eb7 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -233,8 +233,11 @@ class GenericIE(InfoExtractor): return self.url_result(surl, 'Vimeo') # Look for embedded YouTube player - matches = re.findall( - r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?youtube\.com/embed/.+?)\1', webpage) + matches = re.findall(r'''(?x) + (?:<iframe[^>]+?src=|embedSWF\(\s*) + (["\'])(?P<url>(?:https?:)?//(?:www\.)?youtube\.com/ + (?:embed|v)/.+?) + \1''', webpage) if matches: urlrs = [self.url_result(unescapeHTML(tuppl[1]), 'Youtube') for tuppl in matches] From 768df745385a283f4df3a38ee4734feec518ec87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Thu, 19 Dec 2013 21:02:25 +0100 Subject: [PATCH 378/425] [blinkxx] Add support for youtube videos --- youtube_dl/extractor/blinkx.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/youtube_dl/extractor/blinkx.py b/youtube_dl/extractor/blinkx.py index 48f16b692..e118f2e9f 100644 --- a/youtube_dl/extractor/blinkx.py +++ b/youtube_dl/extractor/blinkx.py @@ -54,6 +54,10 @@ class BlinkxIE(InfoExtractor): }) elif m['type'] == 'original': duration = m['d'] + elif m['type'] == 'youtube': + yt_id = m['link'] + self.to_screen(u'Youtube video detected: %s' % yt_id) + return self.url_result(yt_id, 'Youtube', video_id=yt_id) elif m['type'] in ('flv', 'mp4'): vcodec = remove_start(m['vcodec'], 'ff') acodec = remove_start(m['acodec'], 'ff') From aa94a6d3159af8333b56d16f3ed0bc3a164a882a Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 20 Dec 2013 17:05:28 +0100 Subject: [PATCH 379/425] [aparat] Add support (Fixes #2012) --- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/aparat.py | 56 ++++++++++++++++++++++++++++++++ youtube_dl/extractor/common.py | 2 ++ youtube_dl/extractor/generic.py | 15 +++++---- youtube_dl/utils.py | 5 +++ 5 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 youtube_dl/extractor/aparat.py diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 7f2f8806e..7de9d594a 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -1,6 +1,7 @@ from .academicearth import AcademicEarthCourseIE from .addanime import AddAnimeIE from .anitube import AnitubeIE +from .aparat import AparatIE from .appletrailers import AppleTrailersIE from .archiveorg import ArchiveOrgIE from .ard import ARDIE diff --git a/youtube_dl/extractor/aparat.py b/youtube_dl/extractor/aparat.py new file mode 100644 index 000000000..7e93bc4df --- /dev/null +++ b/youtube_dl/extractor/aparat.py @@ -0,0 +1,56 @@ +#coding: utf-8 + +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + HEADRequest, +) + + +class AparatIE(InfoExtractor): + _VALID_URL = r'^https?://(?:www\.)?aparat\.com/(?:v/|video/video/embed/videohash/)(?P<id>[a-zA-Z0-9]+)' + + _TEST = { + u'url': u'http://www.aparat.com/v/wP8On', + u'file': u'wP8On.mp4', + u'md5': u'6714e0af7e0d875c5a39c4dc4ab46ad1', + u'info_dict': { + u"title": u"تیم گلکسی 11 - زومیت", + }, + #u'skip': u'Extremely unreliable', + } + + def _real_extract(self, url): + m = re.match(self._VALID_URL, url) + video_id = m.group('id') + + # Note: There is an easier-to-parse configuration at + # http://www.aparat.com/video/video/config/videohash/%video_id + # but the URL in there does not work + embed_url = (u'http://www.aparat.com/video/video/embed/videohash/' + + video_id + u'/vt/frame') + webpage = self._download_webpage(embed_url, video_id) + + video_urls = re.findall(r'fileList\[[0-9]+\]\s*=\s*"([^"]+)"', webpage) + for i, video_url in enumerate(video_urls): + req = HEADRequest(video_url) + res = self._request_webpage( + req, video_id, note=u'Testing video URL %d' % i, errnote=False) + if res: + break + else: + raise ExtractorError(u'No working video URLs found') + + title = self._search_regex(r'\s+title:\s*"([^"]+)"', webpage, u'title') + thumbnail = self._search_regex( + r'\s+image:\s*"([^"]+)"', webpage, u'thumbnail', fatal=False) + + return { + 'id': video_id, + 'title': title, + 'url': video_url, + 'ext': 'mp4', + 'thumbnail': thumbnail, + } diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index f89df57ed..ba46a7bc7 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -170,6 +170,8 @@ class InfoExtractor(object): try: return self._downloader.urlopen(url_or_request) except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + if errnote is False: + return False if errnote is None: errnote = u'Unable to download webpage' errmsg = u'%s: %s' % (errnote, compat_str(err)) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 75cb96eb7..bdb4f58d6 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -11,6 +11,7 @@ from ..utils import ( compat_urlparse, ExtractorError, + HEADRequest, smuggle_url, unescapeHTML, unified_strdate, @@ -109,21 +110,18 @@ class GenericIE(InfoExtractor): def _send_head(self, url): """Check if it is a redirect, like url shorteners, in case return the new url.""" - class HeadRequest(compat_urllib_request.Request): - def get_method(self): - return "HEAD" class HEADRedirectHandler(compat_urllib_request.HTTPRedirectHandler): """ Subclass the HTTPRedirectHandler to make it use our - HeadRequest also on the redirected URL + HEADRequest also on the redirected URL """ def redirect_request(self, req, fp, code, msg, headers, newurl): if code in (301, 302, 303, 307): newurl = newurl.replace(' ', '%20') newheaders = dict((k,v) for k,v in req.headers.items() if k.lower() not in ("content-length", "content-type")) - return HeadRequest(newurl, + return HEADRequest(newurl, headers=newheaders, origin_req_host=req.get_origin_req_host(), unverifiable=True) @@ -152,7 +150,7 @@ class GenericIE(InfoExtractor): compat_urllib_request.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]: opener.add_handler(handler()) - response = opener.open(HeadRequest(url)) + response = opener.open(HEADRequest(url)) if response is None: raise ExtractorError(u'Invalid URL protocol') return response @@ -296,6 +294,11 @@ class GenericIE(InfoExtractor): if mobj is not None: return OoyalaIE._build_url_result(mobj.group(1)) + # Look for Aparat videos + mobj = re.search(r'<iframe src="(http://www.aparat.com/video/[^"]+)"', webpage) + if mobj is not None: + return self.url_result(mobj.group(1), 'Aparat') + # Start with something easy: JW Player in SWFObject mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage) if mobj is None: diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index cc391bddd..2e48f187e 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1093,3 +1093,8 @@ def remove_start(s, start): def url_basename(url): path = compat_urlparse.urlparse(url).path return path.strip(u'/').split(u'/')[-1] + + +class HEADRequest(compat_urllib_request.Request): + def get_method(self): + return "HEAD" From f65c1d2be020b8078a6a5e166bf4fd17af981387 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 20 Dec 2013 17:08:16 +0100 Subject: [PATCH 380/425] release 2013.12.20 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 7cbee7335..c13af8abd 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.17.2' +__version__ = '2013.12.20' From 79fed2a4dfb9be350671632b8e0364cbed9dc83c Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 20 Dec 2013 17:20:39 +0100 Subject: [PATCH 381/425] [crunchyroll] Fix test (#1721) --- youtube_dl/extractor/crunchyroll.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py index 4bd366079..2b66bddbb 100644 --- a/youtube_dl/extractor/crunchyroll.py +++ b/youtube_dl/extractor/crunchyroll.py @@ -24,7 +24,7 @@ class CrunchyrollIE(InfoExtractor): u'file': u'645513.flv', #u'md5': u'b1639fd6ddfaa43788c85f6d1dddd412', u'info_dict': { - u'title': u'Wanna be the Strongest in the World – Episode 1 – An Idol-Wrestler is Born!', + u'title': u'Wanna be the Strongest in the World Episode 1 – An Idol-Wrestler is Born!', u'description': u'md5:2d17137920c64f2f49981a7797d275ef', u'thumbnail': u'http://img1.ak.crunchyroll.com/i/spire1-tmb/20c6b5e10f1a47b10516877d3c039cae1380951166_full.jpg', u'uploader': u'Yomiuri Telecasting Corporation (YTV)', @@ -37,9 +37,9 @@ class CrunchyrollIE(InfoExtractor): }] _FORMAT_IDS = { - u'360': (u'60', u'106'), - u'480': (u'61', u'106'), - u'720': (u'62', u'106'), + u'360': (u'60', u'106'), + u'480': (u'61', u'106'), + u'720': (u'62', u'106'), u'1080': (u'80', u'108'), } @@ -102,7 +102,7 @@ class CrunchyrollIE(InfoExtractor): raise ExtractorError(note_m) video_title = self._html_search_regex(r'<h1[^>]*>(.+?)</h1>', webpage, u'video_title', flags=re.DOTALL) - video_title = re.sub(r' {5} *–? *', u' – ', video_title) + video_title = re.sub(r' {2,}', u' ', video_title) video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, u'video_description', default=u'') if not video_description: video_description = None From bd1488ae647d735abc42db078804cf93c333912f Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 20 Dec 2013 17:23:59 +0100 Subject: [PATCH 382/425] [mdr] Remove test For context, refer to the http://de.wikipedia.org/wiki/Depublizieren --- youtube_dl/extractor/mdr.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py index d29cf2c07..08ce0647f 100644 --- a/youtube_dl/extractor/mdr.py +++ b/youtube_dl/extractor/mdr.py @@ -8,23 +8,8 @@ from ..utils import ( class MDRIE(InfoExtractor): _VALID_URL = r'^(?P<domain>(?:https?://)?(?:www\.)?mdr\.de)/mediathek/(?:.*)/(?P<type>video|audio)(?P<video_id>[^/_]+)_.*' - - _TESTS = [{ - u'url': u'http://www.mdr.de/mediathek/themen/nachrichten/video165624_zc-c5c7de76_zs-3795826d.html', - u'file': u'165624.mp4', - u'md5': u'ae785f36ecbf2f19b42edf1bc9c85815', - u'info_dict': { - u"title": u"MDR aktuell Eins30 09.12.2013, 22:48 Uhr" - }, - }, - { - u'url': u'http://www.mdr.de/mediathek/radio/mdr1-radio-sachsen/audio718370_zc-67b21197_zs-1b9b2483.html', - u'file': u'718370.mp3', - u'md5': u'a9d21345a234c7b45dee612f290fd8d7', - u'info_dict': { - u"title": u"MDR 1 RADIO SACHSEN 10.12.2013, 05:00 Uhr" - }, - }] + + # No tests, MDR regularily deletes its videos def _real_extract(self, url): m = re.match(self._VALID_URL, url) From 147e4aece0cfa6ca5d584b57785dc99f31d1e4eb Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 20 Dec 2013 17:27:43 +0100 Subject: [PATCH 383/425] [vbox7] New video checksum --- youtube_dl/extractor/vbox7.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py index 4f803bcd3..5a136a952 100644 --- a/youtube_dl/extractor/vbox7.py +++ b/youtube_dl/extractor/vbox7.py @@ -15,7 +15,7 @@ class Vbox7IE(InfoExtractor): _TEST = { u'url': u'http://vbox7.com/play:249bb972c2', u'file': u'249bb972c2.flv', - u'md5': u'9c70d6d956f888bdc08c124acc120cfe', + u'md5': u'99f65c0c9ef9b682b97313e052734c3f', u'info_dict': { u"title": u"\u0421\u043c\u044f\u0445! \u0427\u0443\u0434\u043e - \u0447\u0438\u0441\u0442 \u0437\u0430 \u0441\u0435\u043a\u0443\u043d\u0434\u0438 - \u0421\u043a\u0440\u0438\u0442\u0430 \u043a\u0430\u043c\u0435\u0440\u0430" } From c4d55a33fce91d846a6ae270ac3e4c69b7f45ba4 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Fri, 20 Dec 2013 17:28:50 +0100 Subject: [PATCH 384/425] [brightcove] Test checksum changed --- youtube_dl/extractor/brightcove.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py index b1b7526ca..f7f0041c0 100644 --- a/youtube_dl/extractor/brightcove.py +++ b/youtube_dl/extractor/brightcove.py @@ -26,7 +26,7 @@ class BrightcoveIE(InfoExtractor): # From http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/ u'url': u'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1654948606001&flashID=myExperience&%40videoPlayer=2371591881001', u'file': u'2371591881001.mp4', - u'md5': u'8eccab865181d29ec2958f32a6a754f5', + u'md5': u'5423e113865d26e40624dce2e4b45d95', u'note': u'Test Brightcove downloads and detection in GenericIE', u'info_dict': { u'title': u'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”', From bbafbe20c233d00e86fc87a1b1ccab8cf9e88232 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 22 Dec 2013 03:17:56 +0100 Subject: [PATCH 385/425] [vimeo] Better formatting for regexp --- youtube_dl/extractor/vimeo.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index ea4409528..4e8fef165 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -16,11 +16,20 @@ from ..utils import ( unsmuggle_url, ) + class VimeoIE(InfoExtractor): """Information extractor for vimeo.com.""" # _VALID_URL matches Vimeo URLs - _VALID_URL = r'(?P<proto>https?://)?(?:(?:www|(?P<player>player))\.)?vimeo(?P<pro>pro)?\.com/(?:.*?/)?(?P<direct_link>play_redirect_hls\?clip_id=)?(?:videos?/)?(?P<id>[0-9]+)/?(?:[?].*)?(?:#.*)?$' + _VALID_URL = r'''(?x) + (?P<proto>https?://)? + (?:(?:www|(?P<player>player))\.)? + vimeo(?P<pro>pro)?\.com/ + (?:.*?/)? + (?P<direct_link>play_redirect_hls\?clip_id=)? + (?:videos?/)? + (?P<id>[0-9]+) + /?(?:[?].*)?(?:[#].*)?$''' _NETRC_MACHINE = 'vimeo' IE_NAME = u'vimeo' _TESTS = [ From 04ff34ab8977ae14654089f5bf9956ffa8f23c5f Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 22 Dec 2013 03:25:55 +0100 Subject: [PATCH 386/425] Show all matching URLs --- youtube_dl/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 6df44020b..7e156d4d4 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -525,7 +525,6 @@ def _real_main(argv=None): for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()): compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '')) matchedUrls = [url for url in all_urls if ie.suitable(url)] - all_urls = [url for url in all_urls if url not in matchedUrls] for mu in matchedUrls: compat_print(u' ' + mu) sys.exit(0) From 7115ca84aacf8eca3016c9a27cb6411ae5f52305 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 22 Dec 2013 03:34:13 +0100 Subject: [PATCH 387/425] [vimeo/generic] Add support for embedded SWF vimeo videos --- youtube_dl/extractor/generic.py | 8 +++++++- youtube_dl/extractor/vimeo.py | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index bdb4f58d6..7a14c98f9 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -222,7 +222,7 @@ class GenericIE(InfoExtractor): self.to_screen(u'Brightcove video detected.') return self.url_result(bc_url, 'Brightcove') - # Look for embedded Vimeo player + # Look for embedded (iframe) Vimeo player mobj = re.search( r'<iframe[^>]+?src="(https?://player.vimeo.com/video/.+?)"', webpage) if mobj: @@ -230,6 +230,12 @@ class GenericIE(InfoExtractor): surl = smuggle_url(player_url, {'Referer': url}) return self.url_result(surl, 'Vimeo') + # Look for embedded (swf embed) Vimeo player + mobj = re.search( + r'<embed[^>]+?src="(https?://(?:www\.)?vimeo.com/moogaloop.swf.+?)"', webpage) + if mobj: + return self.url_result(mobj.group(1), 'Vimeo') + # Look for embedded YouTube player matches = re.findall(r'''(?x) (?:<iframe[^>]+?src=|embedSWF\(\s*) diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index 4e8fef165..c3623fcbe 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -26,10 +26,10 @@ class VimeoIE(InfoExtractor): (?:(?:www|(?P<player>player))\.)? vimeo(?P<pro>pro)?\.com/ (?:.*?/)? - (?P<direct_link>play_redirect_hls\?clip_id=)? + (?:(?:play_redirect_hls|moogaloop\.swf)\?clip_id=)? (?:videos?/)? (?P<id>[0-9]+) - /?(?:[?].*)?(?:[#].*)?$''' + /?(?:[?&].*)?(?:[#].*)?$''' _NETRC_MACHINE = 'vimeo' IE_NAME = u'vimeo' _TESTS = [ From 5a94982abe4b9824c8e886c8176f930541714adc Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 22 Dec 2013 03:52:12 +0100 Subject: [PATCH 388/425] Remove unused import --- youtube_dl/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 7e156d4d4..63437301b 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -56,7 +56,6 @@ from .utils import ( compat_print, DateRange, decodeOption, - determine_ext, get_term_width, DownloadError, get_cachedir, From e302f9ce324c13e24a717027eaa7a918658b4e8a Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 22 Dec 2013 03:57:42 +0100 Subject: [PATCH 389/425] [youtube:user] Speed up --match-title --- youtube_dl/extractor/youtube.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index c860eedda..a68576547 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1717,7 +1717,7 @@ class YoutubeUserIE(InfoExtractor): # page by page until there are no video ids - it means we got # all of them. - video_ids = [] + url_results = [] for pagenum in itertools.count(0): start_index = pagenum * self._GDATA_PAGE_SIZE + 1 @@ -1735,10 +1735,17 @@ class YoutubeUserIE(InfoExtractor): break # Extract video identifiers - ids_in_page = [] - for entry in response['feed']['entry']: - ids_in_page.append(entry['id']['$t'].split('/')[-1]) - video_ids.extend(ids_in_page) + entries = response['feed']['entry'] + for entry in entries: + title = entry['title']['$t'] + video_id = entry['id']['$t'].split('/')[-1] + url_results.append({ + '_type': 'url', + 'url': video_id, + 'ie_key': 'Youtube', + 'id': 'video_id', + 'title': title, + }) # A little optimization - if current page is not # "full", ie. does not contain PAGE_SIZE video ids then @@ -1746,12 +1753,9 @@ class YoutubeUserIE(InfoExtractor): # are no more ids on further pages - no need to query # again. - if len(ids_in_page) < self._GDATA_PAGE_SIZE: + if len(entries) < self._GDATA_PAGE_SIZE: break - url_results = [ - self.url_result(video_id, 'Youtube', video_id=video_id) - for video_id in video_ids] return self.playlist_result(url_results, playlist_title=username) From 1b969041d701e2fb7ff106476b91084dbc67332a Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Sun, 22 Dec 2013 07:43:54 +0100 Subject: [PATCH 390/425] [blinkx] Support mobile URLs (Closes #2022) --- youtube_dl/extractor/blinkx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/blinkx.py b/youtube_dl/extractor/blinkx.py index e118f2e9f..144ce64cc 100644 --- a/youtube_dl/extractor/blinkx.py +++ b/youtube_dl/extractor/blinkx.py @@ -9,7 +9,7 @@ from ..utils import ( class BlinkxIE(InfoExtractor): - _VALID_URL = r'^(?:https?://(?:www\.)blinkx\.com/ce/|blinkx:)(?P<id>[^?]+)' + _VALID_URL = r'^(?:https?://(?:www\.)blinkx\.com/#?ce/|blinkx:)(?P<id>[^?]+)' _IE_NAME = u'blinkx' _TEST = { From 8d0bdeba18962688f1fadfc246c69da5ad17a237 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 23 Dec 2013 03:38:29 +0100 Subject: [PATCH 391/425] [smotri] Make optional attributes optional --- youtube_dl/extractor/smotri.py | 42 +++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py index a589a893b..51aa244e9 100644 --- a/youtube_dl/extractor/smotri.py +++ b/youtube_dl/extractor/smotri.py @@ -152,34 +152,40 @@ class SmotriIE(InfoExtractor): video_description = self._search_meta(u'description', video_page) END_TEXT = u' на сайте Smotri.com' - if video_description.endswith(END_TEXT): + if video_description and video_description.endswith(END_TEXT): video_description = video_description[:-len(END_TEXT)] START_TEXT = u'Смотреть онлайн ролик ' - if video_description.startswith(START_TEXT): + if video_description and video_description.startswith(START_TEXT): video_description = video_description[len(START_TEXT):] video_thumbnail = self._search_meta(u'thumbnail', video_page) upload_date_str = self._search_meta(u'uploadDate', video_page, u'upload date') - upload_date_m = re.search(r'(?P<year>\d{4})\.(?P<month>\d{2})\.(?P<day>\d{2})T', upload_date_str) - video_upload_date = ( - ( - upload_date_m.group('year') + - upload_date_m.group('month') + - upload_date_m.group('day') + if upload_date_str: + upload_date_m = re.search(r'(?P<year>\d{4})\.(?P<month>\d{2})\.(?P<day>\d{2})T', upload_date_str) + video_upload_date = ( + ( + upload_date_m.group('year') + + upload_date_m.group('month') + + upload_date_m.group('day') + ) + if upload_date_m else None ) - if upload_date_m else None - ) + else: + video_upload_date = None duration_str = self._search_meta(u'duration', video_page) - duration_m = re.search(r'T(?P<hours>[0-9]{2})H(?P<minutes>[0-9]{2})M(?P<seconds>[0-9]{2})S', duration_str) - video_duration = ( - ( - (int(duration_m.group('hours')) * 60 * 60) + - (int(duration_m.group('minutes')) * 60) + - int(duration_m.group('seconds')) + if duration_str: + duration_m = re.search(r'T(?P<hours>[0-9]{2})H(?P<minutes>[0-9]{2})M(?P<seconds>[0-9]{2})S', duration_str) + video_duration = ( + ( + (int(duration_m.group('hours')) * 60 * 60) + + (int(duration_m.group('minutes')) * 60) + + int(duration_m.group('seconds')) + ) + if duration_m else None ) - if duration_m else None - ) + else: + video_duration = None video_uploader = self._html_search_regex( u'<div class="DescrUser"><div>Автор.*?onmouseover="popup_user_info[^"]+">(.*?)</a>', From 7dbf5ae5877e6226bd843c4f20675044b9981c00 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 23 Dec 2013 03:44:47 +0100 Subject: [PATCH 392/425] [smotri] Add support for moderated (?) videos (Fixes #2030) --- youtube_dl/extractor/smotri.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py index 51aa244e9..beea58d63 100644 --- a/youtube_dl/extractor/smotri.py +++ b/youtube_dl/extractor/smotri.py @@ -1,5 +1,6 @@ # encoding: utf-8 +import os.path import re import json import hashlib @@ -10,6 +11,7 @@ from ..utils import ( compat_urllib_parse, compat_urllib_request, ExtractorError, + url_basename, ) @@ -132,7 +134,16 @@ class SmotriIE(InfoExtractor): # We will extract some from the video web page instead video_page_url = 'http://' + mobj.group('url') video_page = self._download_webpage(video_page_url, video_id, u'Downloading video page') - + + # Warning if video is unavailable + warning = self._html_search_regex( + r'<div class="videoUnModer">(.*?)</div>', video_page, + u'warning messagef', default=None) + if warning is not None: + self._downloader.report_warning( + u'Video %s may not be available; smotri said: %s ' % + (video_id, warning)) + # Adult content if re.search(u'EroConfirmText">', video_page) is not None: self.report_age_confirmation() @@ -148,7 +159,7 @@ class SmotriIE(InfoExtractor): # Extract the rest of meta data video_title = self._search_meta(u'name', video_page, u'title') if not video_title: - video_title = video_url.rsplit('/', 1)[-1] + video_title = os.path.splitext(url_basename(video_url))[0] video_description = self._search_meta(u'description', video_page) END_TEXT = u' на сайте Smotri.com' From c7b487d96b1307508843321d17949356e03ace6a Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 23 Dec 2013 03:45:02 +0100 Subject: [PATCH 393/425] release 2013.12.23 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index c13af8abd..24855bceb 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.20' +__version__ = '2013.12.23' From 5d681e960db98ec2d9f3d4361bd66f581506a772 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 23 Dec 2013 04:19:20 +0100 Subject: [PATCH 394/425] Use bidiv instead of fribidi if available (Fixes #1912) --- youtube_dl/YoutubeDL.py | 23 ++++++++++++++++------- youtube_dl/__init__.py | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 2a078adfb..e19708e0d 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -183,12 +183,19 @@ class YoutubeDL(object): width_args = [] else: width_args = ['-w', str(width)] - self._fribidi = subprocess.Popen( - ['fribidi', '-c', 'UTF-8'] + width_args, + sp_kwargs = dict( stdin=subprocess.PIPE, stdout=slave, stderr=self._err_file) - self._fribidi_channel = os.fdopen(master, 'rb') + try: + self._output_process = subprocess.Popen( + ['bidiv'] + width_args, **sp_kwargs + ) + except OSError: + print('Falling back to fribidi') + self._output_process = subprocess.Popen( + ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs) + self._output_channel = os.fdopen(master, 'rb') except OSError as ose: if ose.errno == 2: self.report_warning(u'Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.') @@ -243,14 +250,16 @@ class YoutubeDL(object): pp.set_downloader(self) def _bidi_workaround(self, message): - if not hasattr(self, '_fribidi_channel'): + if not hasattr(self, '_output_channel'): + print('WORKAROUND NOT ENABLED') return message + assert hasattr(self, '_output_process') assert type(message) == type(u'') line_count = message.count(u'\n') + 1 - self._fribidi.stdin.write((message + u'\n').encode('utf-8')) - self._fribidi.stdin.flush() - res = u''.join(self._fribidi_channel.readline().decode('utf-8') + self._output_process.stdin.write((message + u'\n').encode('utf-8')) + self._output_process.stdin.flush() + res = u''.join(self._output_channel.readline().decode('utf-8') for _ in range(line_count)) return res[:-len(u'\n')] diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 63437301b..c37d28c59 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -194,7 +194,7 @@ def parseOpts(overrideArguments=None): type=float, default=None, help=optparse.SUPPRESS_HELP) general.add_option( '--bidi-workaround', dest='bidi_workaround', action='store_true', - help=u'Work around terminals that lack bidirectional text support. Requires fribidi executable in PATH') + help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH') selection.add_option( From a94e129a653824f54621f9d603f014e0d7591804 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 23 Dec 2013 04:20:25 +0100 Subject: [PATCH 395/425] release 2013.12.23.1 --- README.md | 3 ++- youtube_dl/version.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index caed94846..91e18e372 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,8 @@ which means you can modify it, redistribute it or use it however you like. /youtube-dl . --no-cache-dir Disable filesystem caching --bidi-workaround Work around terminals that lack bidirectional - text support. Requires fribidi executable in PATH + text support. Requires bidiv or fribidi + executable in PATH ## Video Selection: --playlist-start NUMBER playlist video to start at (default is 1) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 24855bceb..c727dc512 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.23' +__version__ = '2013.12.23.1' From 196938835a77b0987e7743290983ccb03a14ffc1 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 23 Dec 2013 04:30:55 +0100 Subject: [PATCH 396/425] Remove debugging code Introduced by accident in 5d681e960db98ec2d9f3d4361bd66f581506a772 --- youtube_dl/YoutubeDL.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index e19708e0d..766d6e5fe 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -192,7 +192,6 @@ class YoutubeDL(object): ['bidiv'] + width_args, **sp_kwargs ) except OSError: - print('Falling back to fribidi') self._output_process = subprocess.Popen( ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs) self._output_channel = os.fdopen(master, 'rb') @@ -251,7 +250,6 @@ class YoutubeDL(object): def _bidi_workaround(self, message): if not hasattr(self, '_output_channel'): - print('WORKAROUND NOT ENABLED') return message assert hasattr(self, '_output_process') From 466617f539806ec932c8f69ec261d552a20b7b3d Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 23 Dec 2013 04:31:38 +0100 Subject: [PATCH 397/425] [bliptv] Simplify (From #2000) --- youtube_dl/extractor/bliptv.py | 79 +++++++++++++++++----------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py index 5e33a69df..0e63208df 100644 --- a/youtube_dl/extractor/bliptv.py +++ b/youtube_dl/extractor/bliptv.py @@ -70,13 +70,14 @@ class BlipTVIE(InfoExtractor): info = None urlh = self._request_webpage(request, None, False, u'unable to download video info webpage') + if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download basename = url.split('/')[-1] title,ext = os.path.splitext(basename) title = title.decode('UTF-8') ext = ext.replace('.', '') self.report_direct_download(title) - info = { + return { 'id': title, 'url': url, 'uploader': None, @@ -85,49 +86,47 @@ class BlipTVIE(InfoExtractor): 'ext': ext, 'urlhandle': urlh } - if info is None: # Regular URL - try: - json_code_bytes = urlh.read() - json_code = json_code_bytes.decode('utf-8') - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to read video info webpage: %s' % compat_str(err)) - try: - json_data = json.loads(json_code) - if 'Post' in json_data: - data = json_data['Post'] - else: - data = json_data + try: + json_code_bytes = urlh.read() + json_code = json_code_bytes.decode('utf-8') + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + raise ExtractorError(u'Unable to read video info webpage: %s' % compat_str(err)) - upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d') - if 'additionalMedia' in data: - formats = sorted(data['additionalMedia'], key=lambda f: int(f['media_height'])) - best_format = formats[-1] - video_url = best_format['url'] - else: - video_url = data['media']['url'] - umobj = re.match(self._URL_EXT, video_url) - if umobj is None: - raise ValueError('Can not determine filename extension') - ext = umobj.group(1) + try: + json_data = json.loads(json_code) + if 'Post' in json_data: + data = json_data['Post'] + else: + data = json_data - info = { - 'id': compat_str(data['item_id']), - 'url': video_url, - 'uploader': data['display_name'], - 'upload_date': upload_date, - 'title': data['title'], - 'ext': ext, - 'format': data['media']['mimeType'], - 'thumbnail': data['thumbnailUrl'], - 'description': data['description'], - 'player_url': data['embedUrl'], - 'user_agent': 'iTunes/10.6.1', - } - except (ValueError,KeyError) as err: - raise ExtractorError(u'Unable to parse video information: %s' % repr(err)) + upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d') + if 'additionalMedia' in data: + formats = sorted(data['additionalMedia'], key=lambda f: int(f['media_height'])) + best_format = formats[-1] + video_url = best_format['url'] + else: + video_url = data['media']['url'] + umobj = re.match(self._URL_EXT, video_url) + if umobj is None: + raise ValueError('Can not determine filename extension') + ext = umobj.group(1) - return [info] + return { + 'id': compat_str(data['item_id']), + 'url': video_url, + 'uploader': data['display_name'], + 'upload_date': upload_date, + 'title': data['title'], + 'ext': ext, + 'format': data['media']['mimeType'], + 'thumbnail': data['thumbnailUrl'], + 'description': data['description'], + 'player_url': data['embedUrl'], + 'user_agent': 'iTunes/10.6.1', + } + except (ValueError, KeyError) as err: + raise ExtractorError(u'Unable to parse video information: %s' % repr(err)) class BlipTVUserIE(InfoExtractor): From 504c668d3bb4ef03c52a1795511976ceacc52552 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 23 Dec 2013 04:31:45 +0100 Subject: [PATCH 398/425] release 2013.12.23.2 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index c727dc512..80b722608 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.23.1' +__version__ = '2013.12.23.2' From bc2103f3bf2aafdf28897a935a837e817f389626 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 23 Dec 2013 04:39:55 +0100 Subject: [PATCH 399/425] release 2013.12.23.3 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 80b722608..67f427894 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.23.2' +__version__ = '2013.12.23.3' From d80044c235afcbcfa905ab08a82f6f972df25268 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 23 Dec 2013 04:51:42 +0100 Subject: [PATCH 400/425] [youtube] Prefer videos with sound --- youtube_dl/extractor/youtube.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 9fb07b366..55c345e8a 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1432,10 +1432,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): 'height': height, 'format_note': note, }) + def _formats_key(f): - return (f.get('height') if f.get('height') is not None else -1, - f.get('width') if f.get('width') is not None else -1) - formats = sorted(formats, key=_formats_key) + note = f.get('format_note') + if note is None: + note = u'' + is_dash = u'DASH' in note + return ( + 0 if is_dash else 1, + f.get('height') if f.get('height') is not None else -1, + f.get('width') if f.get('width') is not None else -1) + formats.sort(key=_formats_key) return { 'id': video_id, From 8958b6916cdf8b30b347972b452136cb9fdab679 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 23 Dec 2013 05:08:35 +0100 Subject: [PATCH 401/425] release 2013.12.23.4 --- youtube_dl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/version.py b/youtube_dl/version.py index 67f427894..b3d015634 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2013.12.23.3' +__version__ = '2013.12.23.4' From b3d9ef88ec55335db93601f4b2ec9ad30eca93ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Mon, 23 Dec 2013 10:23:13 +0100 Subject: [PATCH 402/425] =?UTF-8?q?YoutubeDL:=20only=20set=20the=20?= =?UTF-8?q?=E2=80=98formats=E2=80=99=20field=20of=20the=20info=5Fdict=20if?= =?UTF-8?q?=20it=20was=20already=20set=20before?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It caused a circular reference error, when trying to dump it to json (for example with the test video for myvideo.de or any other video without formats) --- youtube_dl/YoutubeDL.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 04771c637..adeef23af 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -688,7 +688,12 @@ class YoutubeDL(object): ext_ord) formats = sorted(formats, key=_free_formats_key) - info_dict['formats'] = formats + if formats[0] is not info_dict: + # only set the 'formats' fields if the original info_dict list them + # otherwise we end up with a circular reference, the first (and unique) + # element in the 'formats' field in info_dict is info_dict itself, + # wich can't be exported to json + info_dict['formats'] = formats if self.params.get('listformats', None): self.list_formats(info_dict) return From 933605d7e8c00df065f04024c3206a3352058484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Mon, 23 Dec 2013 10:37:27 +0100 Subject: [PATCH 403/425] YoutubeDL: rename `_fd_progress_hooks` back to `_progress_hooks` In the future it may report more things. --- test/test_download.py | 2 +- youtube_dl/YoutubeDL.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test_download.py b/test/test_download.py index f242b3528..d0be8d27c 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -90,7 +90,7 @@ def generator(test_case): def _hook(status): if status['status'] == 'finished': finished_hook_called.add(status['filename']) - ydl.add_downloader_progress_hook(_hook) + ydl.add_progress_hook(_hook) def get_tc_filename(tc): return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {})) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index adeef23af..ac4627c4e 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -167,7 +167,7 @@ class YoutubeDL(object): self._ies = [] self._ies_instances = {} self._pps = [] - self._fd_progress_hooks = [] + self._progress_hooks = [] self._download_retcode = 0 self._num_downloads = 0 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] @@ -246,9 +246,9 @@ class YoutubeDL(object): self._pps.append(pp) pp.set_downloader(self) - def add_downloader_progress_hook(self, ph): - """Add the progress hook to the file downloader""" - self._fd_progress_hooks.append(ph) + def add_progress_hook(self, ph): + """Add the progress hook (currently only for the file downloader)""" + self._progress_hooks.append(ph) def _bidi_workaround(self, message): if not hasattr(self, '_output_channel'): @@ -888,7 +888,7 @@ class YoutubeDL(object): else: try: fd = get_suitable_downloader(info_dict)(self, self.params) - for ph in self._fd_progress_hooks: + for ph in self._progress_hooks: fd.add_progress_hook(ph) success = fd.download(filename, info_dict) except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: From 9d8d675e0e739504978642f132ec3f68d6690892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= <jaime.marquinez.ferrandiz@gmail.com> Date: Mon, 23 Dec 2013 10:40:28 +0100 Subject: [PATCH 404/425] [subtitles-tests] Fix youtube test It returns now a single info_dict --- test/test_subtitles.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/test_subtitles.py b/test/test_subtitles.py index 23a653124..263b5ac69 100644 --- a/test/test_subtitles.py +++ b/test/test_subtitles.py @@ -36,10 +36,6 @@ class TestYoutubeSubtitles(BaseTestSubtitles): url = 'QRS8MkLhQmM' IE = YoutubeIE - def getSubtitles(self): - info_dict = self.getInfoDict() - return info_dict[0]['subtitles'] - def test_youtube_no_writesubtitles(self): self.DL.params['writesubtitles'] = False subtitles = self.getSubtitles() From 8e3e03229e10cba8fe6ad7aa67d48c939a1c458a Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 23 Dec 2013 13:51:41 +0100 Subject: [PATCH 405/425] [YoutubeDL] fix tests (Closes #2036) --- test/test_YoutubeDL.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index 3100c362a..5203a0273 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -78,12 +78,12 @@ class TestFormatSelection(unittest.TestCase): ydl = YDL({'format_limit': 'good'}) assert ydl.params['format_limit'] == 'good' - ydl.process_ie_result(info_dict) + ydl.process_ie_result(info_dict.copy()) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded[u'format_id'], u'good') ydl = YDL({'format_limit': 'great', 'format': 'all'}) - ydl.process_ie_result(info_dict) + ydl.process_ie_result(info_dict.copy()) self.assertEqual(ydl.downloaded_info_dicts[0][u'format_id'], u'meh') self.assertEqual(ydl.downloaded_info_dicts[1][u'format_id'], u'good') self.assertEqual(ydl.downloaded_info_dicts[2][u'format_id'], u'great') @@ -91,7 +91,7 @@ class TestFormatSelection(unittest.TestCase): ydl = YDL() ydl.params['format_limit'] = 'excellent' - ydl.process_ie_result(info_dict) + ydl.process_ie_result(info_dict.copy()) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded[u'format_id'], u'excellent') @@ -105,27 +105,27 @@ class TestFormatSelection(unittest.TestCase): info_dict = {u'formats': formats, u'extractor': u'test'} ydl = YDL({'format': u'20/47'}) - ydl.process_ie_result(info_dict) + ydl.process_ie_result(info_dict.copy()) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['format_id'], u'47') ydl = YDL({'format': u'20/71/worst'}) - ydl.process_ie_result(info_dict) + ydl.process_ie_result(info_dict.copy()) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['format_id'], u'35') ydl = YDL() - ydl.process_ie_result(info_dict) + ydl.process_ie_result(info_dict.copy()) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['format_id'], u'2') ydl = YDL({'format': u'webm/mp4'}) - ydl.process_ie_result(info_dict) + ydl.process_ie_result(info_dict.copy()) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['format_id'], u'47') ydl = YDL({'format': u'3gp/40/mp4'}) - ydl.process_ie_result(info_dict) + ydl.process_ie_result(info_dict.copy()) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['format_id'], u'35') From 1538eff6d8ee73f2ae4ac1dd832a9ef978c63074 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 23 Dec 2013 15:48:00 +0100 Subject: [PATCH 406/425] [bliptv] Remove support for direct downloads This is now handled by the generic IE --- youtube_dl/YoutubeDL.py | 5 ++--- youtube_dl/downloader/http.py | 2 -- youtube_dl/extractor/bliptv.py | 19 +------------------ youtube_dl/extractor/common.py | 2 -- 4 files changed, 3 insertions(+), 25 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index ac4627c4e..41a9114ad 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -542,7 +542,7 @@ class YoutubeDL(object): def make_result(embedded_info): new_result = ie_result.copy() for f in ('_type', 'url', 'ext', 'player_url', 'formats', - 'entries', 'urlhandle', 'ie_key', 'duration', + 'entries', 'ie_key', 'duration', 'subtitles', 'annotations', 'format', 'thumbnail', 'thumbnails'): if f in new_result: @@ -856,8 +856,7 @@ class YoutubeDL(object): else: self.to_screen(u'[info] Writing video description metadata as JSON to: ' + infofn) try: - json_info_dict = dict((k, v) for k, v in info_dict.items() if not k in ['urlhandle']) - write_json_file(json_info_dict, encodeFilename(infofn)) + write_json_file(info_dict, encodeFilename(infofn)) except (OSError, IOError): self.report_error(u'Cannot write metadata to JSON file ' + infofn) return diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py index 54dc7616b..dbc5af3ca 100644 --- a/youtube_dl/downloader/http.py +++ b/youtube_dl/downloader/http.py @@ -52,8 +52,6 @@ class HttpFD(FileDownloader): while count <= retries: # Establish connection try: - if count == 0 and 'urlhandle' in info_dict: - data = info_dict['urlhandle'] data = compat_urllib_request.urlopen(request) break except (compat_urllib_error.HTTPError, ) as err: diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py index 0e63208df..a32891c17 100644 --- a/youtube_dl/extractor/bliptv.py +++ b/youtube_dl/extractor/bliptv.py @@ -21,7 +21,7 @@ from ..utils import ( class BlipTVIE(InfoExtractor): """Information extractor for blip.tv""" - _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv/((.+/)|(play/)|(api\.swf#))(.+)$' + _VALID_URL = r'^(?:https?://)?(?:www\.)?blip\.tv/((.+/)|(play/)|(api\.swf#))(.+)$' _URL_EXT = r'^.*\.([a-z0-9]+)$' IE_NAME = u'blip.tv' _TEST = { @@ -58,7 +58,6 @@ class BlipTVIE(InfoExtractor): url = 'http://blip.tv/a/a-' + file_id return self._real_extract(url) - if '?' in url: cchar = '&' else: @@ -71,22 +70,6 @@ class BlipTVIE(InfoExtractor): urlh = self._request_webpage(request, None, False, u'unable to download video info webpage') - if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download - basename = url.split('/')[-1] - title,ext = os.path.splitext(basename) - title = title.decode('UTF-8') - ext = ext.replace('.', '') - self.report_direct_download(title) - return { - 'id': title, - 'url': url, - 'uploader': None, - 'upload_date': None, - 'title': title, - 'ext': ext, - 'urlhandle': urlh - } - try: json_code_bytes = urlh.read() json_code = json_code_bytes.decode('utf-8') diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index ba46a7bc7..9d39c632a 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -63,8 +63,6 @@ class InfoExtractor(object): ext: Video filename extension. format: The video format, defaults to ext (used for --get-format) player_url: SWF Player URL (used for rtmpdump). - urlhandle: [internal] The urlHandle to be used to download the file, - like returned by urllib.request.urlopen The following fields are optional: From f45f96f8f83b7004960a269fea8bfab40a14c725 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 23 Dec 2013 15:57:43 +0100 Subject: [PATCH 407/425] [myvideo] Use RTMP instead of RTMPT (Fixes #2032) --- youtube_dl/extractor/common.py | 5 +++++ youtube_dl/extractor/myvideo.py | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 9d39c632a..0dd504444 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -242,6 +242,11 @@ class InfoExtractor(object): xml_string = transform_source(xml_string) return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8')) + def report_warning(self, msg, video_id=None): + idstr = u'' if video_id is None else u'%s: ' % video_id + self._downloader.report_warning( + u'[%s] %s%s' % (self.IE_NAME, idstr, msg)) + def to_screen(self, msg): """Print msg to screen, prefixing it with '[ie_name]'""" self._downloader.to_screen(u'[%s] %s' % (self.IE_NAME, msg)) diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py index 0404e6e43..6d35c7861 100644 --- a/youtube_dl/extractor/myvideo.py +++ b/youtube_dl/extractor/myvideo.py @@ -143,8 +143,10 @@ class MyVideoIE(InfoExtractor): if mobj: video_url = compat_urllib_parse.unquote(mobj.group(1)) if 'myvideo2flash' in video_url: - self._downloader.report_warning(u'forcing RTMPT ...') - video_url = video_url.replace('rtmpe://', 'rtmpt://') + self.report_warning( + u'Rewriting URL to use unencrypted rtmp:// ...', + video_id) + video_url = video_url.replace('rtmpe://', 'rtmp://') if not video_url: # extract non rtmp videos From c25c991809e10dc8fd4bc2ddf37f8675cbd818fb Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 23 Dec 2013 16:00:43 +0100 Subject: [PATCH 408/425] [mplayer] Fix error introduced by downloader separation --- youtube_dl/downloader/mplayer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube_dl/downloader/mplayer.py b/youtube_dl/downloader/mplayer.py index 7dd6e76d0..67e0e4189 100644 --- a/youtube_dl/downloader/mplayer.py +++ b/youtube_dl/downloader/mplayer.py @@ -9,6 +9,7 @@ from ..utils import ( class MplayerFD(FileDownloader): def real_download(self, filename, info_dict): + url = info_dict['url'] self.report_destination(filename) tmpfilename = self.temp_name(filename) From dabc127362ddfe88996e72b7e0d5cd2e4f239c98 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Mon, 23 Dec 2013 16:03:06 +0100 Subject: [PATCH 409/425] Remove dead code --- youtube_dl/downloader/common.py | 10 +++------- youtube_dl/downloader/http.py | 3 --- youtube_dl/extractor/bliptv.py | 2 -- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index e9c9b4473..10143d56a 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -1,7 +1,5 @@ -import math import os import re -import subprocess import sys import time @@ -175,7 +173,7 @@ class FileDownloader(object): return os.rename(encodeFilename(old_filename), encodeFilename(new_filename)) except (IOError, OSError) as err: - self.report_error(u'unable to rename file') + self.report_error(u'unable to rename file: %s' % str(err)) def try_utime(self, filename, last_modified_hdr): """Try to set the last-modified time of the given file.""" @@ -279,8 +277,6 @@ class FileDownloader(object): """Download to a filename using the info from info_dict Return True on success and False otherwise """ - url = info_dict['url'] - # Check file already present if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False): self.report_file_already_downloaded(filename) @@ -290,8 +286,8 @@ class FileDownloader(object): 'total_bytes': os.path.getsize(encodeFilename(filename)), }) return True - else: - return self.real_download(filename, info_dict) + + return self.real_download(filename, info_dict) def real_download(self, filename, info_dict): """Real download process. Redefine in subclasses.""" diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py index dbc5af3ca..14b88efd3 100644 --- a/youtube_dl/downloader/http.py +++ b/youtube_dl/downloader/http.py @@ -1,7 +1,4 @@ import os -import re -import subprocess -import sys import time from .common import FileDownloader diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py index a32891c17..c7e0a53c1 100644 --- a/youtube_dl/extractor/bliptv.py +++ b/youtube_dl/extractor/bliptv.py @@ -1,6 +1,5 @@ import datetime import json -import os import re import socket @@ -66,7 +65,6 @@ class BlipTVIE(InfoExtractor): request = compat_urllib_request.Request(json_url) request.add_header('User-Agent', 'iTunes/10.6.1') self.report_extraction(mobj.group(1)) - info = None urlh = self._request_webpage(request, None, False, u'unable to download video info webpage') From f49d89ee04897b3ec64af74bb9fd94957c42331c Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 24 Dec 2013 11:56:02 +0100 Subject: [PATCH 410/425] Add a resolution field and improve general --list-formats output --- youtube_dl/YoutubeDL.py | 16 +++++++++------- youtube_dl/extractor/common.py | 13 ++++++++++--- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 41a9114ad..d0cb52a51 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1007,13 +1007,15 @@ class YoutubeDL(object): def format_resolution(format, default='unknown'): if format.get('vcodec') == 'none': return 'audio only' - if format.get('_resolution') is not None: - return format['_resolution'] + if format.get('resolution') is not None: + return format['resolution'] if format.get('height') is not None: if format.get('width') is not None: res = u'%sx%s' % (format['width'], format['height']) else: res = u'%sp' % format['height'] + elif format.get('width') is not None: + res = u'?x%d' % format['width'] else: res = default return res @@ -1025,11 +1027,11 @@ class YoutubeDL(object): res += fdict['format_note'] + u' ' if (fdict.get('vcodec') is not None and fdict.get('vcodec') != 'none'): - res += u'%-5s' % fdict['vcodec'] - elif fdict.get('vbr') is not None: - res += u'video' + res += u'%-5s@' % fdict['vcodec'] + elif fdict.get('vbr') is not None and fdict.get('abr') is not None: + res += u'video@' if fdict.get('vbr') is not None: - res += u'@%4dk' % fdict['vbr'] + res += u'%4dk' % fdict['vbr'] if fdict.get('acodec') is not None: if res: res += u', ' @@ -1064,7 +1066,7 @@ class YoutubeDL(object): header_line = line({ 'format_id': u'format code', 'ext': u'extension', - '_resolution': u'resolution', 'format_note': u'note'}, idlen=idlen) + 'resolution': u'resolution', 'format_note': u'note'}, idlen=idlen) self.to_screen(u'[info] Available formats for %s:\n%s\n%s' % (info_dict['id'], header_line, u"\n".join(formats_s))) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 0dd504444..123af43c6 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -37,10 +37,12 @@ class InfoExtractor(object): id: Video identifier. title: Video title, unescaped. - Additionally, it must contain either a formats entry or url and ext: + Additionally, it must contain either a formats entry or a url one: - formats: A list of dictionaries for each format available, it must - be ordered from worst to best quality. Potential fields: + formats: A list of dictionaries for each format available, ordered + from worst to best quality. + + Potential fields: * url Mandatory. The URL of the video file * ext Will be calculated from url if missing * format A human-readable description of the format @@ -53,12 +55,17 @@ class InfoExtractor(object): ("3D" or "DASH video") * width Width of the video, if known * height Height of the video, if known + * resolution Textual description of width and height * abr Average audio bitrate in KBit/s * acodec Name of the audio codec in use * vbr Average video bitrate in KBit/s * vcodec Name of the video codec in use * filesize The number of bytes, if known in advance * player_url SWF Player URL (used for rtmpdump). + * preference Order number of this format. If this field is + present, the formats get sorted by this field. + -1 for default (order by other properties), + -2 or smaller for less than default. url: Final video URL. ext: Video filename extension. format: The video format, defaults to ext (used for --get-format) From 4bcc7bd1f21c479a3ef613e62868a9fe6d9c370f Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 24 Dec 2013 12:25:22 +0100 Subject: [PATCH 411/425] Add temporary _sort_formats helper function --- youtube_dl/YoutubeDL.py | 13 ++--------- youtube_dl/extractor/common.py | 41 +++++++++++++++++++++++++++++++++ youtube_dl/extractor/youtube.py | 11 +-------- 3 files changed, 44 insertions(+), 21 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index d0cb52a51..4910a2912 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -676,17 +676,8 @@ class YoutubeDL(object): formats = list(takewhile_inclusive( lambda f: f['format_id'] != format_limit, formats )) - if self.params.get('prefer_free_formats'): - def _free_formats_key(f): - try: - ext_ord = [u'flv', u'mp4', u'webm'].index(f['ext']) - except ValueError: - ext_ord = -1 - # We only compare the extension if they have the same height and width - return (f.get('height') if f.get('height') is not None else -1, - f.get('width') if f.get('width') is not None else -1, - ext_ord) - formats = sorted(formats, key=_free_formats_key) + + # TODO Central sorting goes here if formats[0] is not info_dict: # only set the 'formats' fields if the original info_dict list them diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 123af43c6..280693d1d 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -436,6 +436,47 @@ class InfoExtractor(object): } return RATING_TABLE.get(rating.lower(), None) + def _sort_formats(self, formats): + def _formats_key(f): + preference = f.get('preference') + if preference is None: + preference = 0 if f.get('url', '').startswith('http') else -0.1 + if f.get('ext') in ['f4f', 'f4m']: # Not yet supported + preference -= 0.5 + + if f.get('vcodec') == 'none': # audio only + if self._downloader.params.get('prefer_free_formats'): + ORDER = [u'aac', u'mp3', u'm4a', u'webm', u'ogg', u'opus'] + else: + ORDER = [u'webm', u'opus', u'ogg', u'mp3', u'aac', u'm4a'] + ext_preference = 0 + try: + audio_ext_preference = ORDER.index(f['ext']) + except ValueError: + audio_ext_preference = -1 + else: + if self._downloader.params.get('prefer_free_formats'): + ORDER = [u'flv', u'mp4', u'webm'] + else: + ORDER = [u'webm', u'flv', u'mp4'] + try: + ext_preference = ORDER.index(f['ext']) + except ValueError: + ext_preference = -1 + audio_ext_preference = 0 + + return ( + preference, + f.get('height') if f.get('height') is not None else -1, + f.get('width') if f.get('width') is not None else -1, + ext_preference, + f.get('vbr') if f.get('vbr') is not None else -1, + f.get('abr') if f.get('abr') is not None else -1, + audio_ext_preference, + f.get('filesize') if f.get('filesize') is not None else -1, + f.get('format_id'), + ) + formats.sort(key=_formats_key) class SearchInfoExtractor(InfoExtractor): diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 55c345e8a..829d84e9b 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1433,16 +1433,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): 'format_note': note, }) - def _formats_key(f): - note = f.get('format_note') - if note is None: - note = u'' - is_dash = u'DASH' in note - return ( - 0 if is_dash else 1, - f.get('height') if f.get('height') is not None else -1, - f.get('width') if f.get('width') is not None else -1) - formats.sort(key=_formats_key) + self._sort_formats(formats) return { 'id': video_id, From 3d4a70b8213f08dbf66a24add4a7b23fcea1ed66 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 24 Dec 2013 12:33:33 +0100 Subject: [PATCH 412/425] Add more tests for format selection --- test/test_YoutubeDL.py | 75 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 66 insertions(+), 9 deletions(-) diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index 5203a0273..01de10e31 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -8,6 +8,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import FakeYDL from youtube_dl import YoutubeDL +from youtube_dl.extractor import YoutubeIE class YDL(FakeYDL): @@ -33,6 +34,8 @@ class TestFormatSelection(unittest.TestCase): {u'ext': u'mp4', u'height': 460}, ] info_dict = {u'formats': formats, u'extractor': u'test'} + yie = YoutubeIE(ydl) + yie._sort_formats(info_dict['formats']) ydl.process_ie_result(info_dict) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded[u'ext'], u'webm') @@ -45,28 +48,46 @@ class TestFormatSelection(unittest.TestCase): {u'ext': u'mp4', u'height': 1080}, ] info_dict[u'formats'] = formats + yie = YoutubeIE(ydl) + yie._sort_formats(info_dict['formats']) ydl.process_ie_result(info_dict) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded[u'ext'], u'mp4') - # No prefer_free_formats => keep original formats order + # No prefer_free_formats => prefer mp4 and flv for greater compatibilty ydl = YDL() ydl.params['prefer_free_formats'] = False formats = [ {u'ext': u'webm', u'height': 720}, + {u'ext': u'mp4', u'height': 720}, {u'ext': u'flv', u'height': 720}, ] info_dict[u'formats'] = formats + yie = YoutubeIE(ydl) + yie._sort_formats(info_dict['formats']) + ydl.process_ie_result(info_dict) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded[u'ext'], u'mp4') + + ydl = YDL() + ydl.params['prefer_free_formats'] = False + formats = [ + {u'ext': u'flv', u'height': 720}, + {u'ext': u'webm', u'height': 720}, + ] + info_dict[u'formats'] = formats + yie = YoutubeIE(ydl) + yie._sort_formats(info_dict['formats']) ydl.process_ie_result(info_dict) downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded[u'ext'], u'flv') def test_format_limit(self): formats = [ - {u'format_id': u'meh', u'url': u'http://example.com/meh'}, - {u'format_id': u'good', u'url': u'http://example.com/good'}, - {u'format_id': u'great', u'url': u'http://example.com/great'}, - {u'format_id': u'excellent', u'url': u'http://example.com/exc'}, + {u'format_id': u'meh', u'url': u'http://example.com/meh', 'preference': 1}, + {u'format_id': u'good', u'url': u'http://example.com/good', 'preference': 2}, + {u'format_id': u'great', u'url': u'http://example.com/great', 'preference': 3}, + {u'format_id': u'excellent', u'url': u'http://example.com/exc', 'preference': 4}, ] info_dict = { u'formats': formats, u'extractor': u'test', 'id': 'testvid'} @@ -97,10 +118,10 @@ class TestFormatSelection(unittest.TestCase): def test_format_selection(self): formats = [ - {u'format_id': u'35', u'ext': u'mp4'}, - {u'format_id': u'45', u'ext': u'webm'}, - {u'format_id': u'47', u'ext': u'webm'}, - {u'format_id': u'2', u'ext': u'flv'}, + {u'format_id': u'35', u'ext': u'mp4', 'preference': 1}, + {u'format_id': u'45', u'ext': u'webm', 'preference': 2}, + {u'format_id': u'47', u'ext': u'webm', 'preference': 3}, + {u'format_id': u'2', u'ext': u'flv', 'preference': 4}, ] info_dict = {u'formats': formats, u'extractor': u'test'} @@ -129,6 +150,42 @@ class TestFormatSelection(unittest.TestCase): downloaded = ydl.downloaded_info_dicts[0] self.assertEqual(downloaded['format_id'], u'35') + def test_youtube_format_selection(self): + order = [ + '38', '37', '46', '22', '45', '35', '44', '18', '34', '43', '6', '5', '36', '17', '13', + # Apple HTTP Live Streaming + '96', '95', '94', '93', '92', '132', '151', + # 3D + '85', '84', '102', '83', '101', '82', '100', + # Dash video + '138', '137', '248', '136', '247', '135', '246', + '245', '244', '134', '243', '133', '242', '160', + # Dash audio + '141', '172', '140', '139', '171', + ] + + for f1id, f2id in zip(order, order[1:]): + f1 = YoutubeIE._formats[f1id].copy() + f1['format_id'] = f1id + f2 = YoutubeIE._formats[f2id].copy() + f2['format_id'] = f2id + + info_dict = {'formats': [f1, f2], 'extractor': 'youtube'} + ydl = YDL() + yie = YoutubeIE(ydl) + yie._sort_formats(info_dict['formats']) + ydl.process_ie_result(info_dict) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], f1id) + + info_dict = {'formats': [f2, f1], 'extractor': 'youtube'} + ydl = YDL() + yie = YoutubeIE(ydl) + yie._sort_formats(info_dict['formats']) + ydl.process_ie_result(info_dict) + downloaded = ydl.downloaded_info_dicts[0] + self.assertEqual(downloaded['format_id'], f1id) + def test_add_extra_info(self): test_dict = { 'extractor': 'Foo', From 2c62dc26c82bfd07a00e6775f6558d1dde7e088a Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 24 Dec 2013 12:34:09 +0100 Subject: [PATCH 413/425] [youtube] Simplify format specification --- youtube_dl/extractor/youtube.py | 220 +++++++++----------------------- 1 file changed, 61 insertions(+), 159 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 829d84e9b..b0e29c2a8 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -150,151 +150,68 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): (?(1).+)? # if we found the ID, everything can follow $""" _NEXT_URL_RE = r'[\?&]next_url=([^&]+)' - # Listed in order of quality - _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '36', '17', '13', - # Apple HTTP Live Streaming - '96', '95', '94', '93', '92', '132', '151', - # 3D - '85', '84', '102', '83', '101', '82', '100', - # Dash video - '138', '137', '248', '136', '247', '135', '246', - '245', '244', '134', '243', '133', '242', '160', - # Dash audio - '141', '172', '140', '171', '139', - ] - _video_extensions = { - '13': '3gp', - '17': '3gp', - '18': 'mp4', - '22': 'mp4', - '36': '3gp', - '37': 'mp4', - '38': 'mp4', - '43': 'webm', - '44': 'webm', - '45': 'webm', - '46': 'webm', + _formats = { + '5': {'ext': 'flv', 'width': 400, 'height': 240}, + '6': {'ext': 'flv', 'width': 450, 'height': 270}, + '13': {'ext': '3gp'}, + '17': {'ext': '3gp', 'width': 176, 'height': 144}, + '18': {'ext': 'mp4', 'width': 640, 'height': 360}, + '22': {'ext': 'mp4', 'width': 1280, 'height': 720}, + '34': {'ext': 'flv', 'width': 640, 'height': 360}, + '35': {'ext': 'flv', 'width': 854, 'height': 480}, + '36': {'ext': '3gp', 'width': 320, 'height': 240}, + '37': {'ext': 'mp4', 'width': 1920, 'height': 1080}, + '38': {'ext': 'mp4', 'width': 4096, 'height': 3072}, + '43': {'ext': 'webm', 'width': 640, 'height': 360}, + '44': {'ext': 'webm', 'width': 854, 'height': 480}, + '45': {'ext': 'webm', 'width': 1280, 'height': 720}, + '46': {'ext': 'webm', 'width': 1920, 'height': 1080}, + # 3d videos - '82': 'mp4', - '83': 'mp4', - '84': 'mp4', - '85': 'mp4', - '100': 'webm', - '101': 'webm', - '102': 'webm', + '82': {'ext': 'mp4', 'height': 360, 'resolution': '360p', 'format_note': '3D', 'preference': -20}, + '83': {'ext': 'mp4', 'height': 480, 'resolution': '480p', 'format_note': '3D', 'preference': -20}, + '84': {'ext': 'mp4', 'height': 720, 'resolution': '720p', 'format_note': '3D', 'preference': -20}, + '85': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': '3D', 'preference': -20}, + '100': {'ext': 'webm', 'height': 360, 'resolution': '360p', 'format_note': '3D', 'preference': -20}, + '101': {'ext': 'webm', 'height': 480, 'resolution': '480p', 'format_note': '3D', 'preference': -20}, + '102': {'ext': 'webm', 'height': 720, 'resolution': '720p', 'format_note': '3D', 'preference': -20}, # Apple HTTP Live Streaming - '92': 'mp4', - '93': 'mp4', - '94': 'mp4', - '95': 'mp4', - '96': 'mp4', - '132': 'mp4', - '151': 'mp4', + '92': {'ext': 'mp4', 'height': 240, 'resolution': '240p', 'format_note': 'HLS', 'preference': -10}, + '93': {'ext': 'mp4', 'height': 360, 'resolution': '360p', 'format_note': 'HLS', 'preference': -10}, + '94': {'ext': 'mp4', 'height': 480, 'resolution': '480p', 'format_note': 'HLS', 'preference': -10}, + '95': {'ext': 'mp4', 'height': 720, 'resolution': '720p', 'format_note': 'HLS', 'preference': -10}, + '96': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': 'HLS', 'preference': -10}, + '132': {'ext': 'mp4', 'height': 240, 'resolution': '240p', 'format_note': 'HLS', 'preference': -10}, + '151': {'ext': 'mp4', 'height': 72, 'resolution': '72p', 'format_note': 'HLS', 'preference': -10}, - # Dash mp4 - '133': 'mp4', - '134': 'mp4', - '135': 'mp4', - '136': 'mp4', - '137': 'mp4', - '138': 'mp4', - '160': 'mp4', + # DASH mp4 video + '133': {'ext': 'mp4', 'height': 240, 'resolution': '240p', 'format_note': 'DASH video', 'preference': -40}, + '134': {'ext': 'mp4', 'height': 360, 'resolution': '360p', 'format_note': 'DASH video', 'preference': -40}, + '135': {'ext': 'mp4', 'height': 480, 'resolution': '480p', 'format_note': 'DASH video', 'preference': -40}, + '136': {'ext': 'mp4', 'height': 720, 'resolution': '720p', 'format_note': 'DASH video', 'preference': -40}, + '137': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': 'DASH video', 'preference': -40}, + '138': {'ext': 'mp4', 'height': 1081, 'resolution': '>1080p', 'format_note': 'DASH video', 'preference': -40}, + '160': {'ext': 'mp4', 'height': 192, 'resolution': '192p', 'format_note': 'DASH video', 'preference': -40}, # Dash mp4 audio - '139': 'm4a', - '140': 'm4a', - '141': 'm4a', + '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 48, 'preference': -50}, + '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 128, 'preference': -50}, + '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 256, 'preference': -50}, # Dash webm - '171': 'webm', - '172': 'webm', - '242': 'webm', - '243': 'webm', - '244': 'webm', - '245': 'webm', - '246': 'webm', - '247': 'webm', - '248': 'webm', - } - _video_dimensions = { - '5': {'width': 400, 'height': 240}, - '6': {}, - '13': {}, - '17': {'width': 176, 'height': 144}, - '18': {'width': 640, 'height': 360}, - '22': {'width': 1280, 'height': 720}, - '34': {'width': 640, 'height': 360}, - '35': {'width': 854, 'height': 480}, - '36': {'width': 320, 'height': 240}, - '37': {'width': 1920, 'height': 1080}, - '38': {'width': 4096, 'height': 3072}, - '43': {'width': 640, 'height': 360}, - '44': {'width': 854, 'height': 480}, - '45': {'width': 1280, 'height': 720}, - '46': {'width': 1920, 'height': 1080}, - '82': {'height': 360, 'display': '360p'}, - '83': {'height': 480, 'display': '480p'}, - '84': {'height': 720, 'display': '720p'}, - '85': {'height': 1080, 'display': '1080p'}, - '92': {'height': 240, 'display': '240p'}, - '93': {'height': 360, 'display': '360p'}, - '94': {'height': 480, 'display': '480p'}, - '95': {'height': 720, 'display': '720p'}, - '96': {'height': 1080, 'display': '1080p'}, - '100': {'height': 360, 'display': '360p'}, - '101': {'height': 480, 'display': '480p'}, - '102': {'height': 720, 'display': '720p'}, - '132': {'height': 240, 'display': '240p'}, - '151': {'height': 72, 'display': '72p'}, - '133': {'height': 240, 'display': '240p'}, - '134': {'height': 360, 'display': '360p'}, - '135': {'height': 480, 'display': '480p'}, - '136': {'height': 720, 'display': '720p'}, - '137': {'height': 1080, 'display': '1080p'}, - '138': {'height': 1081, 'display': '>1080p'}, - '139': {'display': '48k'}, - '140': {'display': '128k'}, - '141': {'display': '256k'}, - '160': {'height': 192, 'display': '192p'}, - '171': {'display': '128k'}, - '172': {'display': '256k'}, - '242': {'height': 240, 'display': '240p'}, - '243': {'height': 360, 'display': '360p'}, - '244': {'height': 480, 'display': '480p'}, - '245': {'height': 480, 'display': '480p'}, - '246': {'height': 480, 'display': '480p'}, - '247': {'height': 720, 'display': '720p'}, - '248': {'height': 1080, 'display': '1080p'}, - } - _special_itags = { - '82': '3D', - '83': '3D', - '84': '3D', - '85': '3D', - '100': '3D', - '101': '3D', - '102': '3D', - '133': 'DASH Video', - '134': 'DASH Video', - '135': 'DASH Video', - '136': 'DASH Video', - '137': 'DASH Video', - '138': 'DASH Video', - '139': 'DASH Audio', - '140': 'DASH Audio', - '141': 'DASH Audio', - '160': 'DASH Video', - '171': 'DASH Audio', - '172': 'DASH Audio', - '242': 'DASH Video', - '243': 'DASH Video', - '244': 'DASH Video', - '245': 'DASH Video', - '246': 'DASH Video', - '247': 'DASH Video', - '248': 'DASH Video', + '242': {'ext': 'webm', 'height': 240, 'resolution': '240p', 'format_note': 'DASH webm', 'preference': -40}, + '243': {'ext': 'webm', 'height': 360, 'resolution': '360p', 'format_note': 'DASH webm', 'preference': -40}, + '244': {'ext': 'webm', 'height': 480, 'resolution': '480p', 'format_note': 'DASH webm', 'preference': -40}, + '245': {'ext': 'webm', 'height': 480, 'resolution': '480p', 'format_note': 'DASH webm', 'preference': -40}, + '246': {'ext': 'webm', 'height': 480, 'resolution': '480p', 'format_note': 'DASH webm', 'preference': -40}, + '247': {'ext': 'webm', 'height': 720, 'resolution': '720p', 'format_note': 'DASH webm', 'preference': -40}, + '248': {'ext': 'webm', 'height': 1080, 'resolution': '1080p', 'format_note': 'DASH webm', 'preference': -40}, + + # Dash webm audio + '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH webm audio', 'abr': 48, 'preference': -50}, + '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH webm audio', 'abr': 256, 'preference': -50}, } IE_NAME = u'youtube' @@ -1148,7 +1065,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): Transform a dictionary in the format {itag:url} to a list of (itag, url) with the requested formats. """ - existing_formats = [x for x in self._available_formats if x in url_map] + existing_formats = [x for x in self._formats if x in url_map] if len(existing_formats) == 0: raise ExtractorError(u'no known formats available for video') video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats @@ -1410,28 +1327,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): formats = [] for itag, video_real_url in video_url_list: - # Extension - video_extension = self._video_extensions.get(itag, 'flv') - resolution = self._video_dimensions.get(itag, {}).get('display') - width = self._video_dimensions.get(itag, {}).get('width') - height = self._video_dimensions.get(itag, {}).get('height') - note = self._special_itags.get(itag) - - video_format = '{0} - {1}{2}'.format(itag if itag else video_extension, - '%dx%d' % (width, height) if width is not None and height is not None else (resolution if resolution is not None else '???'), - ' ('+self._special_itags[itag]+')' if itag in self._special_itags else '') - - formats.append({ - 'url': video_real_url, - 'ext': video_extension, - 'format': video_format, - 'format_id': itag, - 'player_url': player_url, - '_resolution': resolution, - 'width': width, - 'height': height, - 'format_note': note, - }) + dct = { + 'format_id': itag, + 'url': video_real_url, + 'player_url': player_url, + } + dct.update(self._formats[itag]) + formats.append(dct) self._sort_formats(formats) From 55e663a8d73b38922599e7414e24a2620d5763ce Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 24 Dec 2013 12:35:02 +0100 Subject: [PATCH 414/425] [dreisat] Use centralized format sorting --- youtube_dl/extractor/dreisat.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py index cb7226f82..f141c850d 100644 --- a/youtube_dl/extractor/dreisat.py +++ b/youtube_dl/extractor/dreisat.py @@ -52,18 +52,12 @@ class DreiSatIE(InfoExtractor): 'width': int(fe.find('./width').text), 'height': int(fe.find('./height').text), 'url': fe.find('./url').text, - 'ext': determine_ext(fe.find('./url').text), 'filesize': int(fe.find('./filesize').text), 'video_bitrate': int(fe.find('./videoBitrate').text), - '3sat_qualityname': fe.find('./quality').text, } for fe in format_els if not fe.find('./url').text.startswith('http://www.metafilegenerator.de/')] - def _sortkey(format): - qidx = ['low', 'med', 'high', 'veryhigh'].index(format['3sat_qualityname']) - prefer_http = 1 if 'rtmp' in format['url'] else 0 - return (qidx, prefer_http, format['video_bitrate']) - formats.sort(key=_sortkey) + self._sort_formats(formats) return { '_type': 'video', From 719d3927d7072a180682df4dee6fc10fc2170800 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 24 Dec 2013 12:38:08 +0100 Subject: [PATCH 415/425] [mit] Add support for multiple formats --- youtube_dl/extractor/mit.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py index 52be9232f..ab8b7ec3e 100644 --- a/youtube_dl/extractor/mit.py +++ b/youtube_dl/extractor/mit.py @@ -3,6 +3,7 @@ import json from .common import InfoExtractor from ..utils import ( + compat_str, clean_html, get_element_by_id, ) @@ -33,8 +34,18 @@ class TechTVMITIE(InfoExtractor): raw_page, u'base url') formats_json = self._search_regex(r'bitrates: (\[.+?\])', raw_page, u'video formats') - formats = json.loads(formats_json) - formats = sorted(formats, key=lambda f: f['bitrate']) + formats_mit = json.loads(formats_json) + formats = [ + { + 'format_id': f['label'], + 'url': base_url + f['url'].partition(':')[2], + 'ext': f['url'].partition(':')[0], + 'format': f['label'], + 'width': f['width'], + 'vbr': f['bitrate'], + } + for f in formats_mit + ] title = get_element_by_id('edit-title', clean_page) description = clean_html(get_element_by_id('edit-description', clean_page)) @@ -43,8 +54,7 @@ class TechTVMITIE(InfoExtractor): return {'id': video_id, 'title': title, - 'url': base_url + formats[-1]['url'].replace('mp4:', ''), - 'ext': 'mp4', + 'formats': formats, 'description': description, 'thumbnail': thumbnail, } From e6812ac99dfc9117f3dd78425bf4bbe01601ecfd Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 24 Dec 2013 12:40:23 +0100 Subject: [PATCH 416/425] [spiegel] Use centralized sorting --- youtube_dl/extractor/common.py | 5 +++++ youtube_dl/extractor/spiegel.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 280693d1d..0a90382de 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -438,6 +438,11 @@ class InfoExtractor(object): def _sort_formats(self, formats): def _formats_key(f): + # TODO remove the following workaround + from ..utils import determine_ext + if not f.get('ext') and 'url' in f: + f['ext'] = determine_ext(f['url']) + preference = f.get('preference') if preference is None: preference = 0 if f.get('url', '').startswith('http') else -0.1 diff --git a/youtube_dl/extractor/spiegel.py b/youtube_dl/extractor/spiegel.py index 695520524..051a34d5b 100644 --- a/youtube_dl/extractor/spiegel.py +++ b/youtube_dl/extractor/spiegel.py @@ -51,9 +51,10 @@ class SpiegelIE(InfoExtractor): # Blacklist type 6, it's extremely LQ and not available on the same server if n.tag.startswith('type') and n.tag != 'type6' ] - formats.sort(key=lambda f: f['vbr']) duration = float(idoc[0].findall('./duration')[0].text) + self._sort_formats(formats) + info = { 'id': video_id, 'title': video_title, From c7deaa4c745382634ac5a910cc68efa4211f7456 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 24 Dec 2013 23:32:04 +0100 Subject: [PATCH 417/425] [zdf] Use centralized sorting --- youtube_dl/YoutubeDL.py | 2 ++ youtube_dl/extractor/common.py | 10 +++++++++- youtube_dl/extractor/zdf.py | 28 +++++++--------------------- 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 4910a2912..88f4b11f6 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1014,6 +1014,8 @@ class YoutubeDL(object): def list_formats(self, info_dict): def format_note(fdict): res = u'' + if f.get('ext') in ['f4f', 'f4m']: + res += u'(unsupported) ' if fdict.get('format_note') is not None: res += fdict['format_note'] + u' ' if (fdict.get('vcodec') is not None and diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 0a90382de..05f0cb6d5 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -9,6 +9,7 @@ import xml.etree.ElementTree from ..utils import ( compat_http_client, compat_urllib_error, + compat_urllib_parse_urlparse, compat_str, clean_html, @@ -62,6 +63,9 @@ class InfoExtractor(object): * vcodec Name of the video codec in use * filesize The number of bytes, if known in advance * player_url SWF Player URL (used for rtmpdump). + * protocol The protocol that will be used for the actual + download, lower-case. + "http", "https", "rtsp", "rtmp" or so. * preference Order number of this format. If this field is present, the formats get sorted by this field. -1 for default (order by other properties), @@ -445,7 +449,11 @@ class InfoExtractor(object): preference = f.get('preference') if preference is None: - preference = 0 if f.get('url', '').startswith('http') else -0.1 + proto = f.get('protocol') + if proto is None: + proto = compat_urllib_parse_urlparse(f.get('url', '')).scheme + + preference = 0 if proto in ['http', 'https'] else -0.1 if f.get('ext') in ['f4f', 'f4m']: # Not yet supported preference -= 0.5 diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py index 35ece354a..94594c5d6 100644 --- a/youtube_dl/extractor/zdf.py +++ b/youtube_dl/extractor/zdf.py @@ -67,29 +67,13 @@ class ZDFIE(InfoExtractor): ''', format_id) ext = format_m.group('container') - is_supported = ext != 'f4f' - - PROTO_ORDER = ['http', 'rtmp', 'rtsp'] - try: - proto_pref = -PROTO_ORDER.index(format_m.group('proto')) - except ValueError: - proto_pref = -999 + proto = format_m.group('proto') quality = fnode.find('./quality').text - QUALITY_ORDER = ['veryhigh', '300', 'high', 'med', 'low'] - try: - quality_pref = -QUALITY_ORDER.index(quality) - except ValueError: - quality_pref = -999 - abr = int(fnode.find('./audioBitrate').text) // 1000 vbr = int(fnode.find('./videoBitrate').text) // 1000 - pref = (is_available, is_supported, - proto_pref, quality_pref, vbr, abr) format_note = u'' - if not is_supported: - format_note += u'(unsupported)' if not format_note: format_note = None @@ -105,14 +89,16 @@ class ZDFIE(InfoExtractor): 'height': int(fnode.find('./height').text), 'filesize': int(fnode.find('./filesize').text), 'format_note': format_note, - '_pref': pref, + 'protocol': format_m.group('proto').lower(), '_available': is_available, } format_nodes = doc.findall('.//formitaeten/formitaet') - formats = sorted(filter(lambda f: f['_available'], - map(xml_to_format, format_nodes)), - key=operator.itemgetter('_pref')) + formats = list(filter( + lambda f: f['_available'], + map(xml_to_format, format_nodes))) + + self._sort_formats(formats) return { 'id': video_id, From b874fe2da8defdd5fd945e87d746f52ef52a40f2 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Tue, 24 Dec 2013 23:34:11 +0100 Subject: [PATCH 418/425] [mdr] Use centralized format selection --- youtube_dl/extractor/mdr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/mdr.py b/youtube_dl/extractor/mdr.py index 08ce0647f..7aa0080d7 100644 --- a/youtube_dl/extractor/mdr.py +++ b/youtube_dl/extractor/mdr.py @@ -52,10 +52,11 @@ class MDRIE(InfoExtractor): 'format_id': u'%s-%d' % (media_type, vbr), }) formats.append(format) - formats.sort(key=lambda f: (f.get('vbr'), f['abr'])) if not formats: raise ExtractorError(u'Could not find any valid formats') + self._sort_formats(formats) + return { 'id': video_id, 'title': title, From f7e9d77f340250c2ec3cb18fba3ab0f7d8829944 Mon Sep 17 00:00:00 2001 From: dst <dstftw@gmail.com> Date: Wed, 25 Dec 2013 09:02:35 +0700 Subject: [PATCH 419/425] [smotri] Fix typo --- youtube_dl/extractor/smotri.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/smotri.py b/youtube_dl/extractor/smotri.py index beea58d63..99f5b19d2 100644 --- a/youtube_dl/extractor/smotri.py +++ b/youtube_dl/extractor/smotri.py @@ -138,7 +138,7 @@ class SmotriIE(InfoExtractor): # Warning if video is unavailable warning = self._html_search_regex( r'<div class="videoUnModer">(.*?)</div>', video_page, - u'warning messagef', default=None) + u'warning message', default=None) if warning is not None: self._downloader.report_warning( u'Video %s may not be available; smotri said: %s ' % From 7217e148fb03ea27edb12a878c959ebefec4c6c3 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Wed, 25 Dec 2013 15:18:40 +0100 Subject: [PATCH 420/425] [yahoo] Use centralized sorting, and add tbr field --- youtube_dl/YoutubeDL.py | 2 ++ youtube_dl/extractor/common.py | 1 + youtube_dl/extractor/yahoo.py | 12 ++++++------ youtube_dl/utils.py | 4 ++++ 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 88f4b11f6..a9a3639d7 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1018,6 +1018,8 @@ class YoutubeDL(object): res += u'(unsupported) ' if fdict.get('format_note') is not None: res += fdict['format_note'] + u' ' + if fdict.get('tbr') is not None: + res += u'%4dk ' % fdict['tbr'] if (fdict.get('vcodec') is not None and fdict.get('vcodec') != 'none'): res += u'%-5s@' % fdict['vcodec'] diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 05f0cb6d5..6fa60622e 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -57,6 +57,7 @@ class InfoExtractor(object): * width Width of the video, if known * height Height of the video, if known * resolution Textual description of width and height + * tbr Average bitrate of audio and video in KBit/s * abr Average audio bitrate in KBit/s * acodec Name of the audio codec in use * vbr Average video bitrate in KBit/s diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py index 5c9c361b9..e17a39782 100644 --- a/youtube_dl/extractor/yahoo.py +++ b/youtube_dl/extractor/yahoo.py @@ -6,8 +6,8 @@ from .common import InfoExtractor, SearchInfoExtractor from ..utils import ( compat_urllib_parse, compat_urlparse, - determine_ext, clean_html, + int_or_none, ) @@ -68,9 +68,9 @@ class YahooIE(InfoExtractor): formats = [] for s in info['streams']: format_info = { - 'width': s.get('width'), - 'height': s.get('height'), - 'bitrate': s.get('bitrate'), + 'width': int_or_none(s.get('width')), + 'height': int_or_none(s.get('height')), + 'tbr': int_or_none(s.get('bitrate')), } host = s['host'] @@ -84,10 +84,10 @@ class YahooIE(InfoExtractor): else: format_url = compat_urlparse.urljoin(host, path) format_info['url'] = format_url - format_info['ext'] = determine_ext(format_url) formats.append(format_info) - formats = sorted(formats, key=lambda f:(f['height'], f['width'])) + + self._sort_formats(formats) return { 'id': video_id, diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 2e48f187e..4c8bdbb0c 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1098,3 +1098,7 @@ def url_basename(url): class HEADRequest(compat_urllib_request.Request): def get_method(self): return "HEAD" + + +def int_or_none(v): + return v if v is None else int(v) From 539179f45bbeecefecfec4cba40dde7c7f4a813c Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Wed, 25 Dec 2013 15:20:14 +0100 Subject: [PATCH 421/425] [wistia] Use centralized sorting --- youtube_dl/extractor/wistia.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py index e1748c261..584550455 100644 --- a/youtube_dl/extractor/wistia.py +++ b/youtube_dl/extractor/wistia.py @@ -45,7 +45,8 @@ class WistiaIE(InfoExtractor): 'filesize': a['size'], 'ext': a['ext'], }) - formats.sort(key=lambda a: a['filesize']) + + self._sort_formats(formats) return { 'id': video_id, From 7b8af56340bcc7521bb05da14d2dce6d949ce916 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Wed, 25 Dec 2013 15:24:41 +0100 Subject: [PATCH 422/425] [appletrailers] Use centralized format selection --- youtube_dl/extractor/appletrailers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py index ef5644aa5..e7361ae06 100644 --- a/youtube_dl/extractor/appletrailers.py +++ b/youtube_dl/extractor/appletrailers.py @@ -110,7 +110,8 @@ class AppleTrailersIE(InfoExtractor): 'width': format['width'], 'height': int(format['height']), }) - formats = sorted(formats, key=lambda f: (f['height'], f['width'])) + + self._sort_formats(formats) playlist.append({ '_type': 'video', From 61224dbcdd46052b264e422cc45da907fb06fd42 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Wed, 25 Dec 2013 15:33:09 +0100 Subject: [PATCH 423/425] [zdf] Make width extraction more robust --- youtube_dl/extractor/zdf.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py index 94594c5d6..829f002cf 100644 --- a/youtube_dl/extractor/zdf.py +++ b/youtube_dl/extractor/zdf.py @@ -1,10 +1,10 @@ # coding: utf-8 -import operator import re from .common import InfoExtractor from ..utils import ( + int_or_none, unified_strdate, ) @@ -67,7 +67,7 @@ class ZDFIE(InfoExtractor): ''', format_id) ext = format_m.group('container') - proto = format_m.group('proto') + proto = format_m.group('proto').lower() quality = fnode.find('./quality').text abr = int(fnode.find('./audioBitrate').text) // 1000 @@ -85,11 +85,11 @@ class ZDFIE(InfoExtractor): 'vcodec': format_m.group('vcodec'), 'abr': abr, 'vbr': vbr, - 'width': int(fnode.find('./width').text), - 'height': int(fnode.find('./height').text), - 'filesize': int(fnode.find('./filesize').text), + 'width': int_or_none(fnode.find('./width').text), + 'height': int_or_none(fnode.find('./height').text), + 'filesize': int_or_none(fnode.find('./filesize').text), 'format_note': format_note, - 'protocol': format_m.group('proto').lower(), + 'protocol': proto, '_available': is_available, } From 723f83991198c09cf3519abd3306704f64b09af0 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Wed, 25 Dec 2013 15:33:19 +0100 Subject: [PATCH 424/425] Remove unused imports --- youtube_dl/extractor/dreisat.py | 1 - youtube_dl/extractor/mit.py | 1 - 2 files changed, 2 deletions(-) diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py index f141c850d..416e25156 100644 --- a/youtube_dl/extractor/dreisat.py +++ b/youtube_dl/extractor/dreisat.py @@ -4,7 +4,6 @@ import re from .common import InfoExtractor from ..utils import ( - determine_ext, unified_strdate, ) diff --git a/youtube_dl/extractor/mit.py b/youtube_dl/extractor/mit.py index ab8b7ec3e..76b717fe5 100644 --- a/youtube_dl/extractor/mit.py +++ b/youtube_dl/extractor/mit.py @@ -3,7 +3,6 @@ import json from .common import InfoExtractor from ..utils import ( - compat_str, clean_html, get_element_by_id, ) From 608d11f515c9668fa069b422b5850a41e17f52be Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister <phihag@phihag.de> Date: Thu, 26 Dec 2013 13:49:44 +0100 Subject: [PATCH 425/425] [cnn] Add multiple formats, duration, and upload_date --- test/test_utils.py | 8 +++++ youtube_dl/extractor/cnn.py | 71 +++++++++++++++++++++++++++++-------- youtube_dl/utils.py | 16 +++++++++ 3 files changed, 80 insertions(+), 15 deletions(-) diff --git a/test/test_utils.py b/test/test_utils.py index e5778cd83..bee355ee0 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -18,6 +18,7 @@ from youtube_dl.utils import ( find_xpath_attr, get_meta_content, orderedSet, + parse_duration, sanitize_filename, shell_quote, smuggle_url, @@ -192,5 +193,12 @@ class TestUtil(unittest.TestCase): url_basename(u'http://media.w3.org/2010/05/sintel/trailer.mp4'), u'trailer.mp4') + def test_parse_duration(self): + self.assertEqual(parse_duration(None), None) + self.assertEqual(parse_duration('1'), 1) + self.assertEqual(parse_duration('1337:12'), 80232) + self.assertEqual(parse_duration('9:12:43'), 33163) + self.assertEqual(parse_duration('x:y'), None) + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/extractor/cnn.py b/youtube_dl/extractor/cnn.py index a034bb2fb..ecac5e0e9 100644 --- a/youtube_dl/extractor/cnn.py +++ b/youtube_dl/extractor/cnn.py @@ -1,7 +1,10 @@ import re from .common import InfoExtractor -from ..utils import determine_ext +from ..utils import ( + int_or_none, + parse_duration, +) class CNNIE(InfoExtractor): @@ -15,6 +18,8 @@ class CNNIE(InfoExtractor): u'info_dict': { u'title': u'Nadal wins 8th French Open title', u'description': u'World Sport\'s Amanda Davies chats with 2013 French Open champion Rafael Nadal.', + u'duration': 135, + u'upload_date': u'20130609', }, }, { @@ -35,22 +40,58 @@ class CNNIE(InfoExtractor): info = self._download_xml(info_url, page_title) formats = [] + rex = re.compile(r'''(?x) + (?P<width>[0-9]+)x(?P<height>[0-9]+) + (?:_(?P<bitrate>[0-9]+)k)? + ''') for f in info.findall('files/file'): - mf = re.match(r'(\d+)x(\d+)(?:_(.*)k)?',f.attrib['bitrate']) - if mf is not None: - formats.append((int(mf.group(1)), int(mf.group(2)), int(mf.group(3) or 0), f.text)) - formats = sorted(formats) - (_,_,_, video_path) = formats[-1] - video_url = 'http://ht.cdn.turner.com/cnn/big%s' % video_path + video_url = 'http://ht.cdn.turner.com/cnn/big%s' % (f.text.strip()) + fdct = { + 'format_id': f.attrib['bitrate'], + 'url': video_url, + } + + mf = rex.match(f.attrib['bitrate']) + if mf: + fdct['width'] = int(mf.group('width')) + fdct['height'] = int(mf.group('height')) + fdct['tbr'] = int_or_none(mf.group('bitrate')) + else: + mf = rex.search(f.text) + if mf: + fdct['width'] = int(mf.group('width')) + fdct['height'] = int(mf.group('height')) + fdct['tbr'] = int_or_none(mf.group('bitrate')) + else: + mi = re.match(r'ios_(audio|[0-9]+)$', f.attrib['bitrate']) + if mi: + if mi.group(1) == 'audio': + fdct['vcodec'] = 'none' + fdct['ext'] = 'm4a' + else: + fdct['tbr'] = int(mi.group(1)) + + formats.append(fdct) + + self._sort_formats(formats) thumbnails = sorted([((int(t.attrib['height']),int(t.attrib['width'])), t.text) for t in info.findall('images/image')]) thumbs_dict = [{'resolution': res, 'url': t_url} for (res, t_url) in thumbnails] - return {'id': info.attrib['id'], - 'title': info.find('headline').text, - 'url': video_url, - 'ext': determine_ext(video_url), - 'thumbnail': thumbnails[-1][1], - 'thumbnails': thumbs_dict, - 'description': info.find('description').text, - } + metas_el = info.find('metas') + upload_date = ( + metas_el.attrib.get('version') if metas_el is not None else None) + + duration_el = info.find('length') + duration = parse_duration(duration_el.text) + + return { + 'id': info.attrib['id'], + 'title': info.find('headline').text, + 'formats': formats, + 'thumbnail': thumbnails[-1][1], + 'thumbnails': thumbs_dict, + 'description': info.find('description').text, + 'duration': duration, + 'upload_date': upload_date, + } diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 4c8bdbb0c..83a274043 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1102,3 +1102,19 @@ class HEADRequest(compat_urllib_request.Request): def int_or_none(v): return v if v is None else int(v) + + +def parse_duration(s): + if s is None: + return None + + m = re.match( + r'(?:(?:(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)$', s) + if not m: + return None + res = int(m.group('secs')) + if m.group('mins'): + res += int(m.group('mins')) * 60 + if m.group('hours'): + res += int(m.group('hours')) * 60 * 60 + return res