From e2cc9a9c5b960e3ff8d5b2c4bf745a3c8a9c7da4 Mon Sep 17 00:00:00 2001 From: deezerdl Date: Wed, 24 May 2017 13:16:26 +0000 Subject: [PATCH 01/26] Update __init__.py --- youtube_dl/downloader/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py index 2e485df9d..5b0c3f166 100644 --- a/youtube_dl/downloader/__init__.py +++ b/youtube_dl/downloader/__init__.py @@ -7,6 +7,7 @@ from .http import HttpFD from .rtmp import RtmpFD from .dash import DashSegmentsFD from .rtsp import RtspFD +from .deezer import DeezerDownloader from .ism import IsmFD from .external import ( get_external_downloader, @@ -23,6 +24,7 @@ PROTOCOL_MAP = { 'm3u8': FFmpegFD, 'mms': RtspFD, 'rtsp': RtspFD, + 'deezer': DeezerDownloader, 'f4m': F4mFD, 'http_dash_segments': DashSegmentsFD, 'ism': IsmFD, From d9fff7fc39be96c99fe10064c74435a47508381a Mon Sep 17 00:00:00 2001 From: deezerdl Date: Wed, 24 May 2017 13:17:24 +0000 Subject: [PATCH 02/26] Create deezer.py --- youtube_dl/downloader/deezer.py | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 youtube_dl/downloader/deezer.py diff --git a/youtube_dl/downloader/deezer.py b/youtube_dl/downloader/deezer.py new file mode 100644 index 000000000..882e10a1e --- /dev/null +++ b/youtube_dl/downloader/deezer.py @@ -0,0 +1,40 @@ +from Crypto.Cipher import Blowfish +import binascii + +from .common import FileDownloader + +from ..utils import sanitized_Request + +def blowfishDecrypt(data, key): + """ CBC decrypt data with key """ + c = Blowfish.new(key, Blowfish.MODE_CBC, binascii.a2b_hex("0001020304050607")) + return c.decrypt(data) + + +def decryptfile(fh, key, fo): + """ + Decrypt data from file , and write to file . + decrypt using blowfish with . + Only every third 2048 byte block is encrypted. + """ + i = 0 + while True: + data = fh.read(2048) + if not data: + break + + if (i % 3)==0 and len(data)==2048: + data = blowfishDecrypt(data, key) + fo.write(data) + i += 1 + + +class DeezerDownloader(FileDownloader): + def real_download(self, filename, info_dict): + url = info_dict['url'] + request = sanitized_Request(url, None, {}) + data = self.ydl.urlopen(request) + + with open(filename, "wb") as fo: + decryptfile(data, info_dict['key'], fo) + From c926483fc830ad25cb6a678473a7462d95ae3ae3 Mon Sep 17 00:00:00 2001 From: deezerdl Date: Wed, 24 May 2017 13:18:51 +0000 Subject: [PATCH 03/26] Update deezer.py --- youtube_dl/extractor/deezer.py | 108 ++++++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index ec87b94db..05df2235f 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -10,9 +10,53 @@ from ..utils import ( orderedSet, ) +from hashlib import md5 +from Crypto.Cipher import AES, Blowfish +from binascii import b2a_hex + +############ + +def md5hex(data): + """ return hex string of md5 of the given string """ + return md5(data).hexdigest() + + +def hexaescrypt(data, key): + """ returns hex string of aes encrypted data """ + c = AES.new(key, AES.MODE_ECB) + return b2a_hex(c.encrypt(data)) + + +def calcurlkey(songid, md5origin, mediaver=4, fmt=1): + """ Calculate the deezer download url given the songid, origin and media+format """ + data = b'\xa4'.join(_.encode("utf-8") for _ in [md5origin, str(fmt), str(songid), str(mediaver)]) + data = b'\xa4'.join([md5hex(data), data])+b'\xa4' + if len(data)%16: + data += b'\x00' * (16-len(data)%16) + return hexaescrypt(data, "jo6aey6haid2Teih").decode('utf-8') + + +def calcblowfishkey(songid): + """ Calculate the Blowfish decrypt key for a given songid """ + h = md5hex(b"%d" % songid) + key = b"g4el58wc0zvf9na1" + return "".join(chr(ord(h[i]) ^ ord(h[i+16]) ^ ord(key[i])) for i in range(16)) + + +def getformat(song): + """ return format id for a song """ + if song["FILESIZE_MP3_320"]: + return 3 + if song["FILESIZE_MP3_256"]: + return 5 + return 1 + + + +############ class DeezerPlaylistIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?deezer\.com/playlist/(?P[0-9]+)' + _VALID_URL = r'https?://(?:www\.)?deezer\.com/\w+/(?P[0-9]+)' _TEST = { 'url': 'http://www.deezer.com/playlist/176747451', 'info_dict': { @@ -40,6 +84,11 @@ class DeezerPlaylistIE(InfoExtractor): raise ExtractorError( 'Deezer said: %s' % geoblocking_msg, expected=True) + host_stream_cdn = self._search_regex( + r'var HOST_STREAM_CDN = \'(.*?)\'', webpage, 'host stream cdn') + setting_domain_img = self._search_regex( + r'var SETTING_DOMAIN_IMG = \'(.*?)\'', webpage, 'setting domain img') + data_json = self._search_regex( (r'__DZR_APP_STATE__\s*=\s*({.+?})\s*', r'naboo\.display\(\'[^\']+\',\s*(.*?)\);\n'), @@ -47,6 +96,63 @@ class DeezerPlaylistIE(InfoExtractor): data = json.loads(data_json) playlist_title = data.get('DATA', {}).get('TITLE') + playlist_uploader = data.get('DATA', {}).get('PARENT_USERNAME') + #playlist_thumbnail = self._search_regex( r' Date: Wed, 24 May 2017 15:56:49 +0000 Subject: [PATCH 04/26] Update deezer.py --- youtube_dl/extractor/deezer.py | 112 ++++++++++++++++++++++++++++++++- 1 file changed, 109 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index 05df2235f..2dfa2e54a 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -13,13 +13,13 @@ from ..utils import ( from hashlib import md5 from Crypto.Cipher import AES, Blowfish from binascii import b2a_hex +from ..compat import compat_ord, compat_chr ############ def md5hex(data): """ return hex string of md5 of the given string """ - return md5(data).hexdigest() - + return md5(data).hexdigest().encode('utf-8') def hexaescrypt(data, key): """ returns hex string of aes encrypted data """ @@ -40,7 +40,7 @@ def calcblowfishkey(songid): """ Calculate the Blowfish decrypt key for a given songid """ h = md5hex(b"%d" % songid) key = b"g4el58wc0zvf9na1" - return "".join(chr(ord(h[i]) ^ ord(h[i+16]) ^ ord(key[i])) for i in range(16)) + return "".join(compat_chr(compat_ord(h[i]) ^ compat_ord(h[i+16]) ^ compat_ord(key[i])) for i in range(16)) def getformat(song): @@ -53,6 +53,112 @@ def getformat(song): +############ + +class DeezerPlaylistIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?deezer\.com/\w+/(?P[0-9]+)' + _TEST = { + 'url': 'http://www.deezer.com/playlist/176747451', + 'info_dict': { + 'id': '176747451', + 'title': 'Best!', + 'uploader': 'Anonymous', + 'thumbnail': r're:^https?://cdn-images.deezer.com/images/cover/.*\.jpg$', + }, + 'playlist_count': 30, + 'skip': 'Only available in .de', + } + + def _real_extract(self, url): + if 'test' not in self._downloader.params: + self._downloader.report_warning('For now, this extractor only supports the 30 second previews. Patches welcome!') + + mobj = re.match(self._VALID_URL, url) + playlist_id = mobj.group('id') + + webpage = self._download_webpage(url, playlist_id) + geoblocking_msg = self._html_search_regex( + r'

(.*?)

', webpage, 'geoblocking message', + default=None) + if geoblocking_msg is not None: + raise ExtractorError( + 'Deezer said: %s' % geoblocking_msg, expected=True) + + host_stream_cdn = self._search_regex( + r'var HOST_STREAM_CDN = \'(.*?)\'', webpage, 'host stream cdn') + setting_domain_img = self._search_regex( + r'var SETTING_DOMAIN_IMG = \'(.*?)\'', webpage, 'setting domain img') + + data_json = self._search_regex( + (r'__DZR_APP_STATE__\s*=\s*({.+?})\s*', + r'naboo\.display\(\'[^\']+\',\s*(.*?)\);\n'), + webpage, 'data JSON') + data = json.loads(data_json) + + playlist_title = data.get('DATA', {}).get('TITLE') + playlist_uploader = data.get('DATA', {}).get('PARENT_USERNAME') + #playlist_thumbnail = self._search_regex( r' Date: Wed, 24 May 2017 15:59:12 +0000 Subject: [PATCH 05/26] fixing a copy/paste error --- youtube_dl/extractor/deezer.py | 149 --------------------------------- 1 file changed, 149 deletions(-) diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index 2dfa2e54a..d783d29c2 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -152,152 +152,3 @@ class DeezerPlaylistIE(InfoExtractor): #'thumbnail': playlist_thumbnail, 'entries': entries, } - - if song["FILESIZE_MP3_256"]: - return 5 - return 1 - - - -############ - -class DeezerPlaylistIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?deezer\.com/\w+/(?P[0-9]+)' - _TEST = { - 'url': 'http://www.deezer.com/playlist/176747451', - 'info_dict': { - 'id': '176747451', - 'title': 'Best!', - 'uploader': 'Anonymous', - 'thumbnail': r're:^https?://cdn-images.deezer.com/images/cover/.*\.jpg$', - }, - 'playlist_count': 30, - 'skip': 'Only available in .de', - } - - def _real_extract(self, url): - if 'test' not in self._downloader.params: - self._downloader.report_warning('For now, this extractor only supports the 30 second previews. Patches welcome!') - - mobj = re.match(self._VALID_URL, url) - playlist_id = mobj.group('id') - - webpage = self._download_webpage(url, playlist_id) - geoblocking_msg = self._html_search_regex( - r'

(.*?)

', webpage, 'geoblocking message', - default=None) - if geoblocking_msg is not None: - raise ExtractorError( - 'Deezer said: %s' % geoblocking_msg, expected=True) - - host_stream_cdn = self._search_regex( - r'var HOST_STREAM_CDN = \'(.*?)\'', webpage, 'host stream cdn') - setting_domain_img = self._search_regex( - r'var SETTING_DOMAIN_IMG = \'(.*?)\'', webpage, 'setting domain img') - - data_json = self._search_regex( - (r'__DZR_APP_STATE__\s*=\s*({.+?})\s*', - r'naboo\.display\(\'[^\']+\',\s*(.*?)\);\n'), - webpage, 'data JSON') - data = json.loads(data_json) - - playlist_title = data.get('DATA', {}).get('TITLE') - playlist_uploader = data.get('DATA', {}).get('PARENT_USERNAME') - #playlist_thumbnail = self._search_regex( r' Date: Wed, 24 May 2017 16:07:26 +0000 Subject: [PATCH 06/26] made flake8 happy --- youtube_dl/extractor/deezer.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index d783d29c2..fd775196f 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -11,16 +11,16 @@ from ..utils import ( ) from hashlib import md5 -from Crypto.Cipher import AES, Blowfish +from Crypto.Cipher import AES from binascii import b2a_hex from ..compat import compat_ord, compat_chr -############ def md5hex(data): """ return hex string of md5 of the given string """ return md5(data).hexdigest().encode('utf-8') + def hexaescrypt(data, key): """ returns hex string of aes encrypted data """ c = AES.new(key, AES.MODE_ECB) @@ -30,9 +30,9 @@ def hexaescrypt(data, key): def calcurlkey(songid, md5origin, mediaver=4, fmt=1): """ Calculate the deezer download url given the songid, origin and media+format """ data = b'\xa4'.join(_.encode("utf-8") for _ in [md5origin, str(fmt), str(songid), str(mediaver)]) - data = b'\xa4'.join([md5hex(data), data])+b'\xa4' - if len(data)%16: - data += b'\x00' * (16-len(data)%16) + data = b'\xa4'.join([md5hex(data), data]) + b'\xa4' + if len(data) % 16: + data += b'\x00' * (16 - len(data) % 16) return hexaescrypt(data, "jo6aey6haid2Teih").decode('utf-8') @@ -40,7 +40,7 @@ def calcblowfishkey(songid): """ Calculate the Blowfish decrypt key for a given songid """ h = md5hex(b"%d" % songid) key = b"g4el58wc0zvf9na1" - return "".join(compat_chr(compat_ord(h[i]) ^ compat_ord(h[i+16]) ^ compat_ord(key[i])) for i in range(16)) + return "".join(compat_chr(compat_ord(h[i]) ^ compat_ord(h[i + 16]) ^ compat_ord(key[i])) for i in range(16)) def getformat(song): @@ -52,9 +52,6 @@ def getformat(song): return 1 - -############ - class DeezerPlaylistIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?deezer\.com/\w+/(?P[0-9]+)' _TEST = { @@ -85,9 +82,7 @@ class DeezerPlaylistIE(InfoExtractor): 'Deezer said: %s' % geoblocking_msg, expected=True) host_stream_cdn = self._search_regex( - r'var HOST_STREAM_CDN = \'(.*?)\'', webpage, 'host stream cdn') - setting_domain_img = self._search_regex( - r'var SETTING_DOMAIN_IMG = \'(.*?)\'', webpage, 'setting domain img') + r'var HOST_STREAM_CDN = \'(.*?)\'', webpage, 'host stream cdn') data_json = self._search_regex( (r'__DZR_APP_STATE__\s*=\s*({.+?})\s*', @@ -97,11 +92,8 @@ class DeezerPlaylistIE(InfoExtractor): playlist_title = data.get('DATA', {}).get('TITLE') playlist_uploader = data.get('DATA', {}).get('PARENT_USERNAME') - #playlist_thumbnail = self._search_regex( r' Date: Wed, 24 May 2017 16:08:21 +0000 Subject: [PATCH 07/26] made flake8 happy --- youtube_dl/downloader/deezer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/downloader/deezer.py b/youtube_dl/downloader/deezer.py index 882e10a1e..172415f26 100644 --- a/youtube_dl/downloader/deezer.py +++ b/youtube_dl/downloader/deezer.py @@ -5,6 +5,7 @@ from .common import FileDownloader from ..utils import sanitized_Request + def blowfishDecrypt(data, key): """ CBC decrypt data with key """ c = Blowfish.new(key, Blowfish.MODE_CBC, binascii.a2b_hex("0001020304050607")) @@ -23,7 +24,7 @@ def decryptfile(fh, key, fo): if not data: break - if (i % 3)==0 and len(data)==2048: + if (i % 3) == 0 and len(data) == 2048: data = blowfishDecrypt(data, key) fo.write(data) i += 1 @@ -37,4 +38,3 @@ class DeezerDownloader(FileDownloader): with open(filename, "wb") as fo: decryptfile(data, info_dict['key'], fo) - From 3026f25995c8539e8509e8c197a8908c4533d759 Mon Sep 17 00:00:00 2001 From: deezerdl Date: Wed, 24 May 2017 16:20:20 +0000 Subject: [PATCH 08/26] added check for pycrypto --- youtube_dl/extractor/deezer.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index fd775196f..477223484 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -52,6 +52,13 @@ def getformat(song): return 1 +def check_for_blowfish(): + try: + from Crypto.Cipher import Blowfish + except: + raise ExtractorError('DeezerDownloader needs pyCrypto installed.', expected=True) + + class DeezerPlaylistIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?deezer\.com/\w+/(?P[0-9]+)' _TEST = { @@ -67,6 +74,8 @@ class DeezerPlaylistIE(InfoExtractor): } def _real_extract(self, url): + check_for_blowfish() + if 'test' not in self._downloader.params: self._downloader.report_warning('For now, this extractor only supports the 30 second previews. Patches welcome!') From 68a3daff887f1fca1f2d76a8b423f440009aac25 Mon Sep 17 00:00:00 2001 From: deezerdl Date: Wed, 24 May 2017 22:43:57 +0000 Subject: [PATCH 09/26] Create blowfish.py --- youtube_dl/blowfish.py | 311 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 311 insertions(+) create mode 100644 youtube_dl/blowfish.py diff --git a/youtube_dl/blowfish.py b/youtube_dl/blowfish.py new file mode 100644 index 000000000..200015db0 --- /dev/null +++ b/youtube_dl/blowfish.py @@ -0,0 +1,311 @@ +from __future__ import unicode_literals + +import struct + +# Blowfish implementation, translated from original C code by Bruce Schneier. + +N = 16 + +# the initialization data is the binary representation of PI - 3 + +# N+2 words +P0 = [ + 0x243f6a88, 0x85a308d3, 0x13198a2e, 0x03707344, 0xa4093822, + 0x299f31d0, 0x082efa98, 0xec4e6c89, 0x452821e6, 0x38d01377, + 0xbe5466cf, 0x34e90c6c, 0xc0ac29b7, 0xc97c50dd, 0x3f84d5b5, + 0xb5470917, 0x9216d5d9, 0x8979fb1b, +] + + +# 4 * 256 words +S0 = [ + [ + 0xd1310ba6, 0x98dfb5ac, 0x2ffd72db, 0xd01adfb7, 0xb8e1afed, 0x6a267e96, + 0xba7c9045, 0xf12c7f99, 0x24a19947, 0xb3916cf7, 0x0801f2e2, 0x858efc16, + 0x636920d8, 0x71574e69, 0xa458fea3, 0xf4933d7e, 0x0d95748f, 0x728eb658, + 0x718bcd58, 0x82154aee, 0x7b54a41d, 0xc25a59b5, 0x9c30d539, 0x2af26013, + 0xc5d1b023, 0x286085f0, 0xca417918, 0xb8db38ef, 0x8e79dcb0, 0x603a180e, + 0x6c9e0e8b, 0xb01e8a3e, 0xd71577c1, 0xbd314b27, 0x78af2fda, 0x55605c60, + 0xe65525f3, 0xaa55ab94, 0x57489862, 0x63e81440, 0x55ca396a, 0x2aab10b6, + 0xb4cc5c34, 0x1141e8ce, 0xa15486af, 0x7c72e993, 0xb3ee1411, 0x636fbc2a, + 0x2ba9c55d, 0x741831f6, 0xce5c3e16, 0x9b87931e, 0xafd6ba33, 0x6c24cf5c, + 0x7a325381, 0x28958677, 0x3b8f4898, 0x6b4bb9af, 0xc4bfe81b, 0x66282193, + 0x61d809cc, 0xfb21a991, 0x487cac60, 0x5dec8032, 0xef845d5d, 0xe98575b1, + 0xdc262302, 0xeb651b88, 0x23893e81, 0xd396acc5, 0x0f6d6ff3, 0x83f44239, + 0x2e0b4482, 0xa4842004, 0x69c8f04a, 0x9e1f9b5e, 0x21c66842, 0xf6e96c9a, + 0x670c9c61, 0xabd388f0, 0x6a51a0d2, 0xd8542f68, 0x960fa728, 0xab5133a3, + 0x6eef0b6c, 0x137a3be4, 0xba3bf050, 0x7efb2a98, 0xa1f1651d, 0x39af0176, + 0x66ca593e, 0x82430e88, 0x8cee8619, 0x456f9fb4, 0x7d84a5c3, 0x3b8b5ebe, + 0xe06f75d8, 0x85c12073, 0x401a449f, 0x56c16aa6, 0x4ed3aa62, 0x363f7706, + 0x1bfedf72, 0x429b023d, 0x37d0d724, 0xd00a1248, 0xdb0fead3, 0x49f1c09b, + 0x075372c9, 0x80991b7b, 0x25d479d8, 0xf6e8def7, 0xe3fe501a, 0xb6794c3b, + 0x976ce0bd, 0x04c006ba, 0xc1a94fb6, 0x409f60c4, 0x5e5c9ec2, 0x196a2463, + 0x68fb6faf, 0x3e6c53b5, 0x1339b2eb, 0x3b52ec6f, 0x6dfc511f, 0x9b30952c, + 0xcc814544, 0xaf5ebd09, 0xbee3d004, 0xde334afd, 0x660f2807, 0x192e4bb3, + 0xc0cba857, 0x45c8740f, 0xd20b5f39, 0xb9d3fbdb, 0x5579c0bd, 0x1a60320a, + 0xd6a100c6, 0x402c7279, 0x679f25fe, 0xfb1fa3cc, 0x8ea5e9f8, 0xdb3222f8, + 0x3c7516df, 0xfd616b15, 0x2f501ec8, 0xad0552ab, 0x323db5fa, 0xfd238760, + 0x53317b48, 0x3e00df82, 0x9e5c57bb, 0xca6f8ca0, 0x1a87562e, 0xdf1769db, + 0xd542a8f6, 0x287effc3, 0xac6732c6, 0x8c4f5573, 0x695b27b0, 0xbbca58c8, + 0xe1ffa35d, 0xb8f011a0, 0x10fa3d98, 0xfd2183b8, 0x4afcb56c, 0x2dd1d35b, + 0x9a53e479, 0xb6f84565, 0xd28e49bc, 0x4bfb9790, 0xe1ddf2da, 0xa4cb7e33, + 0x62fb1341, 0xcee4c6e8, 0xef20cada, 0x36774c01, 0xd07e9efe, 0x2bf11fb4, + 0x95dbda4d, 0xae909198, 0xeaad8e71, 0x6b93d5a0, 0xd08ed1d0, 0xafc725e0, + 0x8e3c5b2f, 0x8e7594b7, 0x8ff6e2fb, 0xf2122b64, 0x8888b812, 0x900df01c, + 0x4fad5ea0, 0x688fc31c, 0xd1cff191, 0xb3a8c1ad, 0x2f2f2218, 0xbe0e1777, + 0xea752dfe, 0x8b021fa1, 0xe5a0cc0f, 0xb56f74e8, 0x18acf3d6, 0xce89e299, + 0xb4a84fe0, 0xfd13e0b7, 0x7cc43b81, 0xd2ada8d9, 0x165fa266, 0x80957705, + 0x93cc7314, 0x211a1477, 0xe6ad2065, 0x77b5fa86, 0xc75442f5, 0xfb9d35cf, + 0xebcdaf0c, 0x7b3e89a0, 0xd6411bd3, 0xae1e7e49, 0x00250e2d, 0x2071b35e, + 0x226800bb, 0x57b8e0af, 0x2464369b, 0xf009b91e, 0x5563911d, 0x59dfa6aa, + 0x78c14389, 0xd95a537f, 0x207d5ba2, 0x02e5b9c5, 0x83260376, 0x6295cfa9, + 0x11c81968, 0x4e734a41, 0xb3472dca, 0x7b14a94a, 0x1b510052, 0x9a532915, + 0xd60f573f, 0xbc9bc6e4, 0x2b60a476, 0x81e67400, 0x08ba6fb5, 0x571be91f, + 0xf296ec6b, 0x2a0dd915, 0xb6636521, 0xe7b9f9b6, 0xff34052e, 0xc5855664, + 0x53b02d5d, 0xa99f8fa1, 0x08ba4799, 0x6e85076a + ], + [ + 0x4b7a70e9, 0xb5b32944, 0xdb75092e, 0xc4192623, 0xad6ea6b0, 0x49a7df7d, + 0x9cee60b8, 0x8fedb266, 0xecaa8c71, 0x699a17ff, 0x5664526c, 0xc2b19ee1, + 0x193602a5, 0x75094c29, 0xa0591340, 0xe4183a3e, 0x3f54989a, 0x5b429d65, + 0x6b8fe4d6, 0x99f73fd6, 0xa1d29c07, 0xefe830f5, 0x4d2d38e6, 0xf0255dc1, + 0x4cdd2086, 0x8470eb26, 0x6382e9c6, 0x021ecc5e, 0x09686b3f, 0x3ebaefc9, + 0x3c971814, 0x6b6a70a1, 0x687f3584, 0x52a0e286, 0xb79c5305, 0xaa500737, + 0x3e07841c, 0x7fdeae5c, 0x8e7d44ec, 0x5716f2b8, 0xb03ada37, 0xf0500c0d, + 0xf01c1f04, 0x0200b3ff, 0xae0cf51a, 0x3cb574b2, 0x25837a58, 0xdc0921bd, + 0xd19113f9, 0x7ca92ff6, 0x94324773, 0x22f54701, 0x3ae5e581, 0x37c2dadc, + 0xc8b57634, 0x9af3dda7, 0xa9446146, 0x0fd0030e, 0xecc8c73e, 0xa4751e41, + 0xe238cd99, 0x3bea0e2f, 0x3280bba1, 0x183eb331, 0x4e548b38, 0x4f6db908, + 0x6f420d03, 0xf60a04bf, 0x2cb81290, 0x24977c79, 0x5679b072, 0xbcaf89af, + 0xde9a771f, 0xd9930810, 0xb38bae12, 0xdccf3f2e, 0x5512721f, 0x2e6b7124, + 0x501adde6, 0x9f84cd87, 0x7a584718, 0x7408da17, 0xbc9f9abc, 0xe94b7d8c, + 0xec7aec3a, 0xdb851dfa, 0x63094366, 0xc464c3d2, 0xef1c1847, 0x3215d908, + 0xdd433b37, 0x24c2ba16, 0x12a14d43, 0x2a65c451, 0x50940002, 0x133ae4dd, + 0x71dff89e, 0x10314e55, 0x81ac77d6, 0x5f11199b, 0x043556f1, 0xd7a3c76b, + 0x3c11183b, 0x5924a509, 0xf28fe6ed, 0x97f1fbfa, 0x9ebabf2c, 0x1e153c6e, + 0x86e34570, 0xeae96fb1, 0x860e5e0a, 0x5a3e2ab3, 0x771fe71c, 0x4e3d06fa, + 0x2965dcb9, 0x99e71d0f, 0x803e89d6, 0x5266c825, 0x2e4cc978, 0x9c10b36a, + 0xc6150eba, 0x94e2ea78, 0xa5fc3c53, 0x1e0a2df4, 0xf2f74ea7, 0x361d2b3d, + 0x1939260f, 0x19c27960, 0x5223a708, 0xf71312b6, 0xebadfe6e, 0xeac31f66, + 0xe3bc4595, 0xa67bc883, 0xb17f37d1, 0x018cff28, 0xc332ddef, 0xbe6c5aa5, + 0x65582185, 0x68ab9802, 0xeecea50f, 0xdb2f953b, 0x2aef7dad, 0x5b6e2f84, + 0x1521b628, 0x29076170, 0xecdd4775, 0x619f1510, 0x13cca830, 0xeb61bd96, + 0x0334fe1e, 0xaa0363cf, 0xb5735c90, 0x4c70a239, 0xd59e9e0b, 0xcbaade14, + 0xeecc86bc, 0x60622ca7, 0x9cab5cab, 0xb2f3846e, 0x648b1eaf, 0x19bdf0ca, + 0xa02369b9, 0x655abb50, 0x40685a32, 0x3c2ab4b3, 0x319ee9d5, 0xc021b8f7, + 0x9b540b19, 0x875fa099, 0x95f7997e, 0x623d7da8, 0xf837889a, 0x97e32d77, + 0x11ed935f, 0x16681281, 0x0e358829, 0xc7e61fd6, 0x96dedfa1, 0x7858ba99, + 0x57f584a5, 0x1b227263, 0x9b83c3ff, 0x1ac24696, 0xcdb30aeb, 0x532e3054, + 0x8fd948e4, 0x6dbc3128, 0x58ebf2ef, 0x34c6ffea, 0xfe28ed61, 0xee7c3c73, + 0x5d4a14d9, 0xe864b7e3, 0x42105d14, 0x203e13e0, 0x45eee2b6, 0xa3aaabea, + 0xdb6c4f15, 0xfacb4fd0, 0xc742f442, 0xef6abbb5, 0x654f3b1d, 0x41cd2105, + 0xd81e799e, 0x86854dc7, 0xe44b476a, 0x3d816250, 0xcf62a1f2, 0x5b8d2646, + 0xfc8883a0, 0xc1c7b6a3, 0x7f1524c3, 0x69cb7492, 0x47848a0b, 0x5692b285, + 0x095bbf00, 0xad19489d, 0x1462b174, 0x23820e00, 0x58428d2a, 0x0c55f5ea, + 0x1dadf43e, 0x233f7061, 0x3372f092, 0x8d937e41, 0xd65fecf1, 0x6c223bdb, + 0x7cde3759, 0xcbee7460, 0x4085f2a7, 0xce77326e, 0xa6078084, 0x19f8509e, + 0xe8efd855, 0x61d99735, 0xa969a7aa, 0xc50c06c2, 0x5a04abfc, 0x800bcadc, + 0x9e447a2e, 0xc3453484, 0xfdd56705, 0x0e1e9ec9, 0xdb73dbd3, 0x105588cd, + 0x675fda79, 0xe3674340, 0xc5c43465, 0x713e38d8, 0x3d28f89e, 0xf16dff20, + 0x153e21e7, 0x8fb03d4a, 0xe6e39f2b, 0xdb83adf7 + ], + [ + 0xe93d5a68, 0x948140f7, 0xf64c261c, 0x94692934, 0x411520f7, 0x7602d4f7, + 0xbcf46b2e, 0xd4a20068, 0xd4082471, 0x3320f46a, 0x43b7d4b7, 0x500061af, + 0x1e39f62e, 0x97244546, 0x14214f74, 0xbf8b8840, 0x4d95fc1d, 0x96b591af, + 0x70f4ddd3, 0x66a02f45, 0xbfbc09ec, 0x03bd9785, 0x7fac6dd0, 0x31cb8504, + 0x96eb27b3, 0x55fd3941, 0xda2547e6, 0xabca0a9a, 0x28507825, 0x530429f4, + 0x0a2c86da, 0xe9b66dfb, 0x68dc1462, 0xd7486900, 0x680ec0a4, 0x27a18dee, + 0x4f3ffea2, 0xe887ad8c, 0xb58ce006, 0x7af4d6b6, 0xaace1e7c, 0xd3375fec, + 0xce78a399, 0x406b2a42, 0x20fe9e35, 0xd9f385b9, 0xee39d7ab, 0x3b124e8b, + 0x1dc9faf7, 0x4b6d1856, 0x26a36631, 0xeae397b2, 0x3a6efa74, 0xdd5b4332, + 0x6841e7f7, 0xca7820fb, 0xfb0af54e, 0xd8feb397, 0x454056ac, 0xba489527, + 0x55533a3a, 0x20838d87, 0xfe6ba9b7, 0xd096954b, 0x55a867bc, 0xa1159a58, + 0xcca92963, 0x99e1db33, 0xa62a4a56, 0x3f3125f9, 0x5ef47e1c, 0x9029317c, + 0xfdf8e802, 0x04272f70, 0x80bb155c, 0x05282ce3, 0x95c11548, 0xe4c66d22, + 0x48c1133f, 0xc70f86dc, 0x07f9c9ee, 0x41041f0f, 0x404779a4, 0x5d886e17, + 0x325f51eb, 0xd59bc0d1, 0xf2bcc18f, 0x41113564, 0x257b7834, 0x602a9c60, + 0xdff8e8a3, 0x1f636c1b, 0x0e12b4c2, 0x02e1329e, 0xaf664fd1, 0xcad18115, + 0x6b2395e0, 0x333e92e1, 0x3b240b62, 0xeebeb922, 0x85b2a20e, 0xe6ba0d99, + 0xde720c8c, 0x2da2f728, 0xd0127845, 0x95b794fd, 0x647d0862, 0xe7ccf5f0, + 0x5449a36f, 0x877d48fa, 0xc39dfd27, 0xf33e8d1e, 0x0a476341, 0x992eff74, + 0x3a6f6eab, 0xf4f8fd37, 0xa812dc60, 0xa1ebddf8, 0x991be14c, 0xdb6e6b0d, + 0xc67b5510, 0x6d672c37, 0x2765d43b, 0xdcd0e804, 0xf1290dc7, 0xcc00ffa3, + 0xb5390f92, 0x690fed0b, 0x667b9ffb, 0xcedb7d9c, 0xa091cf0b, 0xd9155ea3, + 0xbb132f88, 0x515bad24, 0x7b9479bf, 0x763bd6eb, 0x37392eb3, 0xcc115979, + 0x8026e297, 0xf42e312d, 0x6842ada7, 0xc66a2b3b, 0x12754ccc, 0x782ef11c, + 0x6a124237, 0xb79251e7, 0x06a1bbe6, 0x4bfb6350, 0x1a6b1018, 0x11caedfa, + 0x3d25bdd8, 0xe2e1c3c9, 0x44421659, 0x0a121386, 0xd90cec6e, 0xd5abea2a, + 0x64af674e, 0xda86a85f, 0xbebfe988, 0x64e4c3fe, 0x9dbc8057, 0xf0f7c086, + 0x60787bf8, 0x6003604d, 0xd1fd8346, 0xf6381fb0, 0x7745ae04, 0xd736fccc, + 0x83426b33, 0xf01eab71, 0xb0804187, 0x3c005e5f, 0x77a057be, 0xbde8ae24, + 0x55464299, 0xbf582e61, 0x4e58f48f, 0xf2ddfda2, 0xf474ef38, 0x8789bdc2, + 0x5366f9c3, 0xc8b38e74, 0xb475f255, 0x46fcd9b9, 0x7aeb2661, 0x8b1ddf84, + 0x846a0e79, 0x915f95e2, 0x466e598e, 0x20b45770, 0x8cd55591, 0xc902de4c, + 0xb90bace1, 0xbb8205d0, 0x11a86248, 0x7574a99e, 0xb77f19b6, 0xe0a9dc09, + 0x662d09a1, 0xc4324633, 0xe85a1f02, 0x09f0be8c, 0x4a99a025, 0x1d6efe10, + 0x1ab93d1d, 0x0ba5a4df, 0xa186f20f, 0x2868f169, 0xdcb7da83, 0x573906fe, + 0xa1e2ce9b, 0x4fcd7f52, 0x50115e01, 0xa70683fa, 0xa002b5c4, 0x0de6d027, + 0x9af88c27, 0x773f8641, 0xc3604c06, 0x61a806b5, 0xf0177a28, 0xc0f586e0, + 0x006058aa, 0x30dc7d62, 0x11e69ed7, 0x2338ea63, 0x53c2dd94, 0xc2c21634, + 0xbbcbee56, 0x90bcb6de, 0xebfc7da1, 0xce591d76, 0x6f05e409, 0x4b7c0188, + 0x39720a3d, 0x7c927c24, 0x86e3725f, 0x724d9db9, 0x1ac15bb4, 0xd39eb8fc, + 0xed545578, 0x08fca5b5, 0xd83d7cd3, 0x4dad0fc4, 0x1e50ef5e, 0xb161e6f8, + 0xa28514d9, 0x6c51133c, 0x6fd5c7e7, 0x56e14ec4, 0x362abfce, 0xddc6c837, + 0xd79a3234, 0x92638212, 0x670efa8e, 0x406000e0 + ], + [ + 0x3a39ce37, 0xd3faf5cf, 0xabc27737, 0x5ac52d1b, 0x5cb0679e, 0x4fa33742, + 0xd3822740, 0x99bc9bbe, 0xd5118e9d, 0xbf0f7315, 0xd62d1c7e, 0xc700c47b, + 0xb78c1b6b, 0x21a19045, 0xb26eb1be, 0x6a366eb4, 0x5748ab2f, 0xbc946e79, + 0xc6a376d2, 0x6549c2c8, 0x530ff8ee, 0x468dde7d, 0xd5730a1d, 0x4cd04dc6, + 0x2939bbdb, 0xa9ba4650, 0xac9526e8, 0xbe5ee304, 0xa1fad5f0, 0x6a2d519a, + 0x63ef8ce2, 0x9a86ee22, 0xc089c2b8, 0x43242ef6, 0xa51e03aa, 0x9cf2d0a4, + 0x83c061ba, 0x9be96a4d, 0x8fe51550, 0xba645bd6, 0x2826a2f9, 0xa73a3ae1, + 0x4ba99586, 0xef5562e9, 0xc72fefd3, 0xf752f7da, 0x3f046f69, 0x77fa0a59, + 0x80e4a915, 0x87b08601, 0x9b09e6ad, 0x3b3ee593, 0xe990fd5a, 0x9e34d797, + 0x2cf0b7d9, 0x022b8b51, 0x96d5ac3a, 0x017da67d, 0xd1cf3ed6, 0x7c7d2d28, + 0x1f9f25cf, 0xadf2b89b, 0x5ad6b472, 0x5a88f54c, 0xe029ac71, 0xe019a5e6, + 0x47b0acfd, 0xed93fa9b, 0xe8d3c48d, 0x283b57cc, 0xf8d56629, 0x79132e28, + 0x785f0191, 0xed756055, 0xf7960e44, 0xe3d35e8c, 0x15056dd4, 0x88f46dba, + 0x03a16125, 0x0564f0bd, 0xc3eb9e15, 0x3c9057a2, 0x97271aec, 0xa93a072a, + 0x1b3f6d9b, 0x1e6321f5, 0xf59c66fb, 0x26dcf319, 0x7533d928, 0xb155fdf5, + 0x03563482, 0x8aba3cbb, 0x28517711, 0xc20ad9f8, 0xabcc5167, 0xccad925f, + 0x4de81751, 0x3830dc8e, 0x379d5862, 0x9320f991, 0xea7a90c2, 0xfb3e7bce, + 0x5121ce64, 0x774fbe32, 0xa8b6e37e, 0xc3293d46, 0x48de5369, 0x6413e680, + 0xa2ae0810, 0xdd6db224, 0x69852dfd, 0x09072166, 0xb39a460a, 0x6445c0dd, + 0x586cdecf, 0x1c20c8ae, 0x5bbef7dd, 0x1b588d40, 0xccd2017f, 0x6bb4e3bb, + 0xdda26a7e, 0x3a59ff45, 0x3e350a44, 0xbcb4cdd5, 0x72eacea8, 0xfa6484bb, + 0x8d6612ae, 0xbf3c6f47, 0xd29be463, 0x542f5d9e, 0xaec2771b, 0xf64e6370, + 0x740e0d8d, 0xe75b1357, 0xf8721671, 0xaf537d5d, 0x4040cb08, 0x4eb4e2cc, + 0x34d2466a, 0x0115af84, 0xe1b00428, 0x95983a1d, 0x06b89fb4, 0xce6ea048, + 0x6f3f3b82, 0x3520ab82, 0x011a1d4b, 0x277227f8, 0x611560b1, 0xe7933fdc, + 0xbb3a792b, 0x344525bd, 0xa08839e1, 0x51ce794b, 0x2f32c9b7, 0xa01fbac9, + 0xe01cc87e, 0xbcc7d1f6, 0xcf0111c3, 0xa1e8aac7, 0x1a908749, 0xd44fbd9a, + 0xd0dadecb, 0xd50ada38, 0x0339c32a, 0xc6913667, 0x8df9317c, 0xe0b12b4f, + 0xf79e59b7, 0x43f5bb3a, 0xf2d519ff, 0x27d9459c, 0xbf97222c, 0x15e6fc2a, + 0x0f91fc71, 0x9b941525, 0xfae59361, 0xceb69ceb, 0xc2a86459, 0x12baa8d1, + 0xb6c1075e, 0xe3056a0c, 0x10d25065, 0xcb03a442, 0xe0ec6e0e, 0x1698db3b, + 0x4c98a0be, 0x3278e964, 0x9f1f9532, 0xe0d392df, 0xd3a0342b, 0x8971f21e, + 0x1b0a7441, 0x4ba3348c, 0xc5be7120, 0xc37632d8, 0xdf359f8d, 0x9b992f2e, + 0xe60b6f47, 0x0fe3f11d, 0xe54cda54, 0x1edad891, 0xce6279cf, 0xcd3e7e6f, + 0x1618b166, 0xfd2c1d05, 0x848fd2c5, 0xf6fb2299, 0xf523f357, 0xa6327623, + 0x93a83531, 0x56cccd02, 0xacf08162, 0x5a75ebb5, 0x6e163697, 0x88d273cc, + 0xde966292, 0x81b949d0, 0x4c50901b, 0x71c65614, 0xe6c6c7bd, 0x327a140a, + 0x45e1d006, 0xc3f27b9a, 0xc9aa53fd, 0x62a80f00, 0xbb25bfe2, 0x35bdd2f6, + 0x71126905, 0xb2040222, 0xb6cbcf7c, 0xcd769c2b, 0x53113ec0, 0x1640e3d3, + 0x38abbd60, 0x2547adf0, 0xba38209c, 0xf746ce76, 0x77afa1c5, 0x20756060, + 0x85cbfe4e, 0x8ae88dd8, 0x7aaaf9b0, 0x4cf9aa7e, 0x1948c25c, 0x02fb8a8c, + 0x01c36ae4, 0xd6ebe1f9, 0x90d4f869, 0xa65cdea0, 0x3f09252d, 0xc208e69f, + 0xb74e6132, 0xce77e25b, 0x578fdfe3, 0x3ac372e6 + ], +] + + +class Blowfish: + def __init__(self, key): + j = [0] + + def nextkeybyte(): + data = key[j[0]] + j[0] += 1 + if j[0] >= len(key): + j[0] = 0 + if type(data) != int: + data = ord(data) + return data + + def nextkeyword(): + data = 0 + for _ in range(4): + data <<= 8 + data |= nextkeybyte() + return data + + self.P = [p ^ nextkeyword() for p in P0] + self.S = [[s for s in sblock] for sblock in S0] + + datal, datar = 0, 0 + for i in range(0, N + 2, 2): + datal, datar = self.encipher(datal, datar) + self.P[i] = datal + self.P[i + 1] = datar + + for i in range(4): + for j in range(0, 256, 2): + datal, datar = self.encipher(datal, datar) + self.S[i][j] = datal + self.S[i][j + 1] = datar + + def F(self, x): + d = x & 0xFF + x >>= 8 + c = x & 0xFF + x >>= 8 + b = x & 0xFF + x >>= 8 + a = x & 0xFF + + y = self.S[0][a] + self.S[1][b] + y = y ^ self.S[2][c] + y = y + self.S[3][d] + + return y & 0xFFFFFFFF + + def encipher(self, Xl, Xr): + for i in range(N): + Xl = Xl ^ self.P[i] + Xr = self.F(Xl) ^ Xr + + Xl, Xr = Xr, Xl + + Xl, Xr = Xr, Xl + + Xr = Xr ^ self.P[N] + Xl = Xl ^ self.P[N + 1] + + return Xl, Xr + + def decipher(self, Xl, Xr): + for i in range(N + 1, 1, -1): + Xl = Xl ^ self.P[i] + Xr = self.F(Xl) ^ Xr + + Xl, Xr = Xr, Xl + + Xl, Xr = Xr, Xl + + Xr = Xr ^ self.P[1] + Xl = Xl ^ self.P[0] + + return Xl, Xr + + +def blowfish_cbc_decrypt(data, key, iv): + """ + Decrypt with blowfish in CBC mode + + @param {int[]} data cipher + @param {int[]} key variable length key + @param {int[]} iv 8-Byte IV + @returns {int[]} decrypted data + """ + fmt = ">%dL" % (len(data) // 4) + c = Blowfish(key) + ivl, ivr = struct.unpack(">LL", iv) + out = [] + ind = struct.unpack(fmt, data) + + for i in range(0, len(ind), 2): + l, r = c.decipher(ind[i], ind[i + 1]) + l ^= ivl + r ^= ivr + ivl, ivr = ind[i], ind[i + 1] + + out.append(l) + out.append(r) + + return struct.pack(fmt, *out) + + +__all__ = ['blowfish_cbc_decrypt'] From 0110a157a7021796129f1e734cf769117355eea6 Mon Sep 17 00:00:00 2001 From: deezerdl Date: Wed, 24 May 2017 22:44:37 +0000 Subject: [PATCH 10/26] Create test_blowfish.py --- test/test_blowfish.py | 84 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 test/test_blowfish.py diff --git a/test/test_blowfish.py b/test/test_blowfish.py new file mode 100644 index 000000000..f2a8f4dce --- /dev/null +++ b/test/test_blowfish.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python + +from __future__ import unicode_literals + +from binascii import a2b_hex + +# Allow direct execution + +import os +import sys +import unittest +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +from youtube_dl.blowfish import Blowfish, blowfish_cbc_decrypt + +# ECB Test vector data from http://www.mirrors.wiretapped.net/security/cryptography/algorithms/blowfish/blowfish-TESTVECTORS.txt +testvectors = [ + [[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], (0x00000000, 0x00000000), (0x4ef99745, 0x6198dd78)], + [[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff], (0xffffffff, 0xffffffff), (0x51866fd5, 0xb85ecb8a)], + [[0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff], (0x00000000, 0x00000000), (0xf21e9a77, 0xb71c49bc)], + [[0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11], (0x01234567, 0x89abcdef), (0x7d0cc630, 0xafda1ec7)], + [[0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11], (0x11111111, 0x11111111), (0x2466dd87, 0x8b963c9d)], + [[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], (0xffffffff, 0xffffffff), (0x014933e0, 0xcdaff6e4)], + [[0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01], (0x01234567, 0x89abcdef), (0xfa34ec48, 0x47b268b2)], + [[0x1f, 0x1f, 0x1f, 0x1f, 0x0e, 0x0e, 0x0e, 0x0e], (0x01234567, 0x89abcdef), (0xa7907951, 0x08ea3cae)], + [[0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00], (0x10000000, 0x00000001), (0x7d856f9a, 0x613063f2)], + [[0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef], (0x11111111, 0x11111111), (0x61f9c380, 0x2281b096)], + [[0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10], (0x01234567, 0x89abcdef), (0x0aceab0f, 0xc6a0a28d)], + [[0x7c, 0xa1, 0x10, 0x45, 0x4a, 0x1a, 0x6e, 0x57], (0x01a1d6d0, 0x39776742), (0x59c68245, 0xeb05282b)], + [[0x01, 0x31, 0xd9, 0x61, 0x9d, 0xc1, 0x37, 0x6e], (0x5cd54ca8, 0x3def57da), (0xb1b8cc0b, 0x250f09a0)], + [[0x07, 0xa1, 0x13, 0x3e, 0x4a, 0x0b, 0x26, 0x86], (0x0248d438, 0x06f67172), (0x1730e577, 0x8bea1da4)], + [[0x38, 0x49, 0x67, 0x4c, 0x26, 0x02, 0x31, 0x9e], (0x51454b58, 0x2ddf440a), (0xa25e7856, 0xcf2651eb)], + [[0x04, 0xb9, 0x15, 0xba, 0x43, 0xfe, 0xb5, 0xb6], (0x42fd4430, 0x59577fa2), (0x353882b1, 0x09ce8f1a)], + [[0x01, 0x13, 0xb9, 0x70, 0xfd, 0x34, 0xf2, 0xce], (0x059b5e08, 0x51cf143a), (0x48f4d088, 0x4c379918)], + [[0x01, 0x70, 0xf1, 0x75, 0x46, 0x8f, 0xb5, 0xe6], (0x0756d8e0, 0x774761d2), (0x432193b7, 0x8951fc98)], + [[0x43, 0x29, 0x7f, 0xad, 0x38, 0xe3, 0x73, 0xfe], (0x762514b8, 0x29bf486a), (0x13f04154, 0xd69d1ae5)], + [[0x07, 0xa7, 0x13, 0x70, 0x45, 0xda, 0x2a, 0x16], (0x3bdd1190, 0x49372802), (0x2eedda93, 0xffd39c79)], + [[0x04, 0x68, 0x91, 0x04, 0xc2, 0xfd, 0x3b, 0x2f], (0x26955f68, 0x35af609a), (0xd887e039, 0x3c2da6e3)], + [[0x37, 0xd0, 0x6b, 0xb5, 0x16, 0xcb, 0x75, 0x46], (0x164d5e40, 0x4f275232), (0x5f99d04f, 0x5b163969)], + [[0x1f, 0x08, 0x26, 0x0d, 0x1a, 0xc2, 0x46, 0x5e], (0x6b056e18, 0x759f5cca), (0x4a057a3b, 0x24d3977b)], + [[0x58, 0x40, 0x23, 0x64, 0x1a, 0xba, 0x61, 0x76], (0x004bd6ef, 0x09176062), (0x452031c1, 0xe4fada8e)], + [[0x02, 0x58, 0x16, 0x16, 0x46, 0x29, 0xb0, 0x07], (0x480d3900, 0x6ee762f2), (0x7555ae39, 0xf59b87bd)], + [[0x49, 0x79, 0x3e, 0xbc, 0x79, 0xb3, 0x25, 0x8f], (0x437540c8, 0x698f3cfa), (0x53c55f9c, 0xb49fc019)], + [[0x4f, 0xb0, 0x5e, 0x15, 0x15, 0xab, 0x73, 0xa7], (0x072d43a0, 0x77075292), (0x7a8e7bfa, 0x937e89a3)], + [[0x49, 0xe9, 0x5d, 0x6d, 0x4c, 0xa2, 0x29, 0xbf], (0x02fe5577, 0x8117f12a), (0xcf9c5d7a, 0x4986adb5)], + [[0x01, 0x83, 0x10, 0xdc, 0x40, 0x9b, 0x26, 0xd6], (0x1d9d5c50, 0x18f728c2), (0xd1abb290, 0x658bc778)], + [[0x1c, 0x58, 0x7f, 0x1c, 0x13, 0x92, 0x4f, 0xef], (0x30553228, 0x6d6f295a), (0x55cb3774, 0xd13ef201)], + [[0xe0, 0xfe, 0xe0, 0xfe, 0xf1, 0xfe, 0xf1, 0xfe], (0x01234567, 0x89abcdef), (0xc39e072d, 0x9fac631d)], + [[0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef], (0x00000000, 0x00000000), (0x24594688, 0x5754369a)], + [[0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10], (0xffffffff, 0xffffffff), (0x6b5c5a9c, 0x5d9e0a5a)] +] + + +class TestBlowfish(unittest.TestCase): + def test_koch(self): + # test vector from Paul Koch's blowfish implementation + bf = Blowfish(b"TESTKEY") + l, r = 1, 2 + + l, r = bf.encipher(l, r) + self.assertEqual((l, r), (0xDF333FD2, 0x30A71BB4)) + l, r = bf.decipher(l, r) + + self.assertEqual((l, r), (1, 2)) + + def test_vectors(self): + for key, indata, outdata in testvectors: + bf = Blowfish(key) + l, r = bf.encipher(*indata) + self.assertEqual((l, r), outdata) + + def test_cbc1(self): + key = a2b_hex("0123456789abcdeff0e1d2c3b4a59687") + iv = a2b_hex("fedcba9876543210") + indata = a2b_hex("6b77b4d63006dee605b156e27403979358deb9e7154616d9") + check = a2b_hex("37363534333231204e6f77206973207468652074696d6520") + + outdata = blowfish_cbc_decrypt(indata, key, iv) + self.assertEqual(outdata, check) + + +if __name__ == '__main__': + unittest.main() From f963bfba7964244a26f491f2a55496905fac3803 Mon Sep 17 00:00:00 2001 From: deezerdl Date: Wed, 24 May 2017 22:49:22 +0000 Subject: [PATCH 11/26] added aes_ecb_encrypt --- youtube_dl/aes.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py index c5bb3c4ef..a24e3be48 100644 --- a/youtube_dl/aes.py +++ b/youtube_dl/aes.py @@ -88,6 +88,25 @@ def aes_cbc_encrypt(data, key, iv): return encrypted_data +def aes_ecb_encrypt(data, key): + """ + Encrypt with aes in ECB mode. + + @param {int[]} data cleartext + @param {int[]} key 16/24/32-Byte cipher key + @returns {int[]} encrypted data + """ + expanded_key = key_expansion(key) + block_count = len(data) // BLOCK_SIZE_BYTES + + encrypted_data = [] + for i in range(block_count): + block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES] + encrypted_data += aes_encrypt(block, expanded_key) + + return encrypted_data + + def key_expansion(data): """ Generate key schedule @@ -358,4 +377,4 @@ def inc(data): return data -__all__ = ['aes_encrypt', 'key_expansion', 'aes_ctr_decrypt', 'aes_cbc_decrypt', 'aes_decrypt_text'] +__all__ = ['aes_encrypt', 'key_expansion', 'aes_ctr_decrypt', 'aes_cbc_decrypt', 'aes_decrypt_text', 'aes_ecb_encrypt'] From d00e73c8ea0d86b6c769eee1828f488f7f98879e Mon Sep 17 00:00:00 2001 From: deezerdl Date: Wed, 24 May 2017 22:52:32 +0000 Subject: [PATCH 12/26] unittest for aes_ecb_encrypt --- test/test_aes.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/test_aes.py b/test/test_aes.py index 78a28751b..0c2dfa5dd 100644 --- a/test/test_aes.py +++ b/test/test_aes.py @@ -8,9 +8,10 @@ import sys import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from youtube_dl.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_cbc_encrypt, aes_decrypt_text +from youtube_dl.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_cbc_encrypt, aes_decrypt_text, aes_ecb_encrypt from youtube_dl.utils import bytes_to_intlist, intlist_to_bytes import base64 +from binascii import a2b_hex # the encrypted data can be generate with 'devscripts/generate_aes_testdata.py' @@ -41,6 +42,16 @@ class TestAES(unittest.TestCase): encrypted, b"\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6'\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd") + def test_ecb_encrypt(self): + key = a2b_hex("0123456789abcdeff0e1d2c3b4a59687") + data = a2b_hex("6b77b4d63006dee605b156e27403979358deb9e7154616d90123456789abcdef") + check = a2b_hex("1498ad525d546566da51197e5cd8c5487360057efbf565f10de06b58626aef28") + + data = bytes_to_intlist(data) + key = bytes_to_intlist(key) + encrypted = intlist_to_bytes(aes_ecb_encrypt(data, key)) + self.assertEqual(encrypted, check) + def test_decrypt_text(self): password = intlist_to_bytes(self.key).decode('utf-8') encrypted = base64.b64encode( From 4da45b5e79f8137946ac64643140712f54fe8b13 Mon Sep 17 00:00:00 2001 From: deezerdl Date: Wed, 24 May 2017 22:54:39 +0000 Subject: [PATCH 13/26] now using own implementation of blowfish --- youtube_dl/downloader/deezer.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/youtube_dl/downloader/deezer.py b/youtube_dl/downloader/deezer.py index 172415f26..5b1a9351a 100644 --- a/youtube_dl/downloader/deezer.py +++ b/youtube_dl/downloader/deezer.py @@ -1,17 +1,10 @@ -from Crypto.Cipher import Blowfish -import binascii +from __future__ import unicode_literals +from ..blowfish import blowfish_cbc_decrypt from .common import FileDownloader - from ..utils import sanitized_Request -def blowfishDecrypt(data, key): - """ CBC decrypt data with key """ - c = Blowfish.new(key, Blowfish.MODE_CBC, binascii.a2b_hex("0001020304050607")) - return c.decrypt(data) - - def decryptfile(fh, key, fo): """ Decrypt data from file , and write to file . @@ -25,7 +18,7 @@ def decryptfile(fh, key, fo): break if (i % 3) == 0 and len(data) == 2048: - data = blowfishDecrypt(data, key) + data = blowfish_cbc_decrypt(data, key, b"\x00\x01\x02\x03\x04\x05\x06\x07") fo.write(data) i += 1 From 4dc70bcc5f85ab497639925f758426fadda83348 Mon Sep 17 00:00:00 2001 From: deezerdl Date: Wed, 24 May 2017 22:55:50 +0000 Subject: [PATCH 14/26] no longer depends on pycrypto --- youtube_dl/extractor/deezer.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index 477223484..135e3e12d 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -8,10 +8,13 @@ from ..utils import ( ExtractorError, int_or_none, orderedSet, + bytes_to_intlist, + intlist_to_bytes, ) from hashlib import md5 -from Crypto.Cipher import AES +from ..aes import aes_ecb_encrypt + from binascii import b2a_hex from ..compat import compat_ord, compat_chr @@ -21,19 +24,14 @@ def md5hex(data): return md5(data).hexdigest().encode('utf-8') -def hexaescrypt(data, key): - """ returns hex string of aes encrypted data """ - c = AES.new(key, AES.MODE_ECB) - return b2a_hex(c.encrypt(data)) - - def calcurlkey(songid, md5origin, mediaver=4, fmt=1): """ Calculate the deezer download url given the songid, origin and media+format """ data = b'\xa4'.join(_.encode("utf-8") for _ in [md5origin, str(fmt), str(songid), str(mediaver)]) data = b'\xa4'.join([md5hex(data), data]) + b'\xa4' if len(data) % 16: data += b'\x00' * (16 - len(data) % 16) - return hexaescrypt(data, "jo6aey6haid2Teih").decode('utf-8') + enc = aes_ecb_encrypt(bytes_to_intlist(data), bytes_to_intlist(b"jo6aey6haid2Teih")) + return b2a_hex(intlist_to_bytes(enc)).decode('utf-8') def calcblowfishkey(songid): @@ -52,13 +50,6 @@ def getformat(song): return 1 -def check_for_blowfish(): - try: - from Crypto.Cipher import Blowfish - except: - raise ExtractorError('DeezerDownloader needs pyCrypto installed.', expected=True) - - class DeezerPlaylistIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?deezer\.com/\w+/(?P[0-9]+)' _TEST = { @@ -74,8 +65,6 @@ class DeezerPlaylistIE(InfoExtractor): } def _real_extract(self, url): - check_for_blowfish() - if 'test' not in self._downloader.params: self._downloader.report_warning('For now, this extractor only supports the 30 second previews. Patches welcome!') From 755a3e126c41761ef8726db8d8c277267b1c6bc3 Mon Sep 17 00:00:00 2001 From: deezerdl Date: Thu, 25 May 2017 22:22:43 +0000 Subject: [PATCH 15/26] fixing problem with python 3.2 solves the error message: 'str' does not support the buffer interface --- test/test_aes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_aes.py b/test/test_aes.py index 0c2dfa5dd..21e6e0a55 100644 --- a/test/test_aes.py +++ b/test/test_aes.py @@ -43,9 +43,9 @@ class TestAES(unittest.TestCase): b"\x97\x92+\xe5\x0b\xc3\x18\x91ky9m&\xb3\xb5@\xe6'\xc2\x96.\xc8u\x88\xab9-[\x9e|\xf1\xcd") def test_ecb_encrypt(self): - key = a2b_hex("0123456789abcdeff0e1d2c3b4a59687") - data = a2b_hex("6b77b4d63006dee605b156e27403979358deb9e7154616d90123456789abcdef") - check = a2b_hex("1498ad525d546566da51197e5cd8c5487360057efbf565f10de06b58626aef28") + key = a2b_hex(b"0123456789abcdeff0e1d2c3b4a59687") + data = a2b_hex(b"6b77b4d63006dee605b156e27403979358deb9e7154616d90123456789abcdef") + check = a2b_hex(b"1498ad525d546566da51197e5cd8c5487360057efbf565f10de06b58626aef28") data = bytes_to_intlist(data) key = bytes_to_intlist(key) From f69a9cb60a7f70a0e78a88729fbbcee3eb6481cf Mon Sep 17 00:00:00 2001 From: deezerdl Date: Thu, 25 May 2017 22:28:00 +0000 Subject: [PATCH 16/26] fix for python 3.2 problem --- test/test_blowfish.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_blowfish.py b/test/test_blowfish.py index f2a8f4dce..60853cd16 100644 --- a/test/test_blowfish.py +++ b/test/test_blowfish.py @@ -71,10 +71,10 @@ class TestBlowfish(unittest.TestCase): self.assertEqual((l, r), outdata) def test_cbc1(self): - key = a2b_hex("0123456789abcdeff0e1d2c3b4a59687") - iv = a2b_hex("fedcba9876543210") - indata = a2b_hex("6b77b4d63006dee605b156e27403979358deb9e7154616d9") - check = a2b_hex("37363534333231204e6f77206973207468652074696d6520") + key = a2b_hex(b"0123456789abcdeff0e1d2c3b4a59687") + iv = a2b_hex(b"fedcba9876543210") + indata = a2b_hex(b"6b77b4d63006dee605b156e27403979358deb9e7154616d9") + check = a2b_hex(b"37363534333231204e6f77206973207468652074696d6520") outdata = blowfish_cbc_decrypt(indata, key, iv) self.assertEqual(outdata, check) From 0e41f9aae29c30ab09c8e46280c4146265357415 Mon Sep 17 00:00:00 2001 From: deezerdl Date: Fri, 26 May 2017 15:48:13 +0000 Subject: [PATCH 17/26] solved python 2.6 issue --- youtube_dl/blowfish.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/youtube_dl/blowfish.py b/youtube_dl/blowfish.py index 200015db0..24c0125f0 100644 --- a/youtube_dl/blowfish.py +++ b/youtube_dl/blowfish.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals -import struct +from compat import compat_struct_unpack, compat_struct_pack # Blowfish implementation, translated from original C code by Bruce Schneier. @@ -292,9 +292,9 @@ def blowfish_cbc_decrypt(data, key, iv): """ fmt = ">%dL" % (len(data) // 4) c = Blowfish(key) - ivl, ivr = struct.unpack(">LL", iv) + ivl, ivr = compat_struct_unpack(">LL", iv) out = [] - ind = struct.unpack(fmt, data) + ind = compat_struct_unpack(fmt, data) for i in range(0, len(ind), 2): l, r = c.decipher(ind[i], ind[i + 1]) @@ -305,7 +305,7 @@ def blowfish_cbc_decrypt(data, key, iv): out.append(l) out.append(r) - return struct.pack(fmt, *out) + return compat_struct_pack(fmt, *out) __all__ = ['blowfish_cbc_decrypt'] From c1784d3979900bfde8e4eb2e820917511bedf10e Mon Sep 17 00:00:00 2001 From: deezerdl Date: Fri, 26 May 2017 18:34:10 +0000 Subject: [PATCH 18/26] and making it work in 3.x again as well. --- youtube_dl/blowfish.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/blowfish.py b/youtube_dl/blowfish.py index 24c0125f0..0ef97681a 100644 --- a/youtube_dl/blowfish.py +++ b/youtube_dl/blowfish.py @@ -1,6 +1,6 @@ from __future__ import unicode_literals -from compat import compat_struct_unpack, compat_struct_pack +from .compat import compat_struct_unpack, compat_struct_pack # Blowfish implementation, translated from original C code by Bruce Schneier. From 12ca30db874761af0b07fbcc0737404c81e0ad25 Mon Sep 17 00:00:00 2001 From: deezerdl Date: Sun, 28 May 2017 16:51:03 +0000 Subject: [PATCH 19/26] removed downloader warning about 30 second previews --- youtube_dl/extractor/deezer.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index 135e3e12d..98ae6e4e3 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -65,9 +65,6 @@ class DeezerPlaylistIE(InfoExtractor): } def _real_extract(self, url): - if 'test' not in self._downloader.params: - self._downloader.report_warning('For now, this extractor only supports the 30 second previews. Patches welcome!') - mobj = re.match(self._VALID_URL, url) playlist_id = mobj.group('id') From e6414d00fc7442a1a0618acfd510d25f7db71566 Mon Sep 17 00:00:00 2001 From: deezerdl Date: Mon, 29 May 2017 10:00:16 +0000 Subject: [PATCH 20/26] added progres reporting --- youtube_dl/downloader/deezer.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/youtube_dl/downloader/deezer.py b/youtube_dl/downloader/deezer.py index 5b1a9351a..851098751 100644 --- a/youtube_dl/downloader/deezer.py +++ b/youtube_dl/downloader/deezer.py @@ -3,15 +3,18 @@ from __future__ import unicode_literals from ..blowfish import blowfish_cbc_decrypt from .common import FileDownloader from ..utils import sanitized_Request +import time -def decryptfile(fh, key, fo): +def decryptfile(fh, key, fo, progress, data_len): """ Decrypt data from file , and write to file . decrypt using blowfish with . Only every third 2048 byte block is encrypted. """ i = 0 + byte_counter = 0 + tstart = time.time() while True: data = fh.read(2048) if not data: @@ -22,6 +25,16 @@ def decryptfile(fh, key, fo): fo.write(data) i += 1 + byte_counter += len(data) + + progress._hook_progress({ + 'status': 'downloading', + 'downloaded_bytes': byte_counter, + 'total_bytes': data_len, + 'eta': progress.calc_eta(tstart, time.time(), data_len, byte_counter), + 'speed': progress.calc_speed(tstart, time.time(), byte_counter), + }) + class DeezerDownloader(FileDownloader): def real_download(self, filename, info_dict): @@ -30,4 +43,4 @@ class DeezerDownloader(FileDownloader): data = self.ydl.urlopen(request) with open(filename, "wb") as fo: - decryptfile(data, info_dict['key'], fo) + decryptfile(data, info_dict['key'], fo, self, int(data.info().get('Content-length', 0))) From e3f0c6a2d9e11add73618571389cffd084d5e3e7 Mon Sep 17 00:00:00 2001 From: deezerdl Date: Mon, 29 May 2017 10:04:24 +0000 Subject: [PATCH 21/26] improved thumbs, formats, test result. * now returns all formats. * added support for all types of thumbnails. * downloadtest will now succeed. * updated geolocking check * removed 30 second preview notice. --- youtube_dl/extractor/deezer.py | 108 ++++++++++++++++++++++++--------- 1 file changed, 81 insertions(+), 27 deletions(-) diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index 98ae6e4e3..b274dbf29 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -50,34 +50,78 @@ def getformat(song): return 1 +class Format: + def __init__(self, name, fmtid, ext, bitrate): + self.name = name + self.formatid = fmtid + self.extension = ext + self.bitrate = bitrate + + +_formatinfo = { + 'MP3_MISC': Format('MP3_MISC', 0, 'mp3', 128), + 'MP3_128': Format('MP3_128', 1, 'mp3', 128), + # 2 used to be MP3_192 + 'MP3_320': Format('MP3_320', 3, 'mp3', 320), + # 4 used to be AAC_96 + 'MP3_256': Format('MP3_256', 5, 'mp3', 256), + 'AAC_64': Format('AAC_64', 6, 'aac', 64), # used to be AAC_192 + 'MP3_192': Format('MP3_192', 7, 'mp3', 192), + 'AAC_96': Format('AAC_96', 8, 'aac', 96), + 'FLAC': Format('FLAC', 9, 'flac', 0), + 'MP3_64': Format('MP3_64', 10, 'mp3', 64), + 'MP3_32': Format('MP3_32', 11, 'mp3', 32), +} + + +def convert_to_int(x): + try: + return int(x) + except: + return 0 + + +def songformats(s): + """ + Enumerate all formats for a song + """ + for k, v in s.items(): + if k.startswith("FILESIZE_") and convert_to_int(v): + name = k[9:] + if name in _formatinfo: + yield _formatinfo[name] + + class DeezerPlaylistIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?deezer\.com/\w+/(?P[0-9]+)' + _VALID_URL = r'https?://(?:www\.)?deezer\.com/(?P\w+)/(?P[0-9]+)' _TEST = { 'url': 'http://www.deezer.com/playlist/176747451', 'info_dict': { 'id': '176747451', 'title': 'Best!', - 'uploader': 'Anonymous', - 'thumbnail': r're:^https?://cdn-images.deezer.com/images/cover/.*\.jpg$', + 'uploader': 'anonymous', + 'thumbnail': r're:^https?://\S*cdn-images.deezer.com/images/playlist/.*\.jpg$', }, 'playlist_count': 30, - 'skip': 'Only available in .de', } def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) - playlist_id = mobj.group('id') + list_id = mobj.group('id') + list_type = mobj.group('type') - webpage = self._download_webpage(url, playlist_id) + webpage = self._download_webpage(url, list_id) geoblocking_msg = self._html_search_regex( - r'

(.*?)

', webpage, 'geoblocking message', + r'
', @@ -85,39 +129,48 @@ class DeezerPlaylistIE(InfoExtractor): webpage, 'data JSON') data = json.loads(data_json) - playlist_title = data.get('DATA', {}).get('TITLE') - playlist_uploader = data.get('DATA', {}).get('PARENT_USERNAME') - # playlist_thumbnail = self._search_regex( r' Date: Mon, 29 May 2017 21:34:45 +0000 Subject: [PATCH 22/26] another fix for older python versions. and added several fields to info dict --- youtube_dl/extractor/deezer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/deezer.py b/youtube_dl/extractor/deezer.py index b274dbf29..956df1372 100644 --- a/youtube_dl/extractor/deezer.py +++ b/youtube_dl/extractor/deezer.py @@ -36,7 +36,7 @@ def calcurlkey(songid, md5origin, mediaver=4, fmt=1): def calcblowfishkey(songid): """ Calculate the Blowfish decrypt key for a given songid """ - h = md5hex(b"%d" % songid) + h = md5hex(str(songid).encode('ascii')) key = b"g4el58wc0zvf9na1" return "".join(compat_chr(compat_ord(h[i]) ^ compat_ord(h[i + 16]) ^ compat_ord(key[i])) for i in range(16)) @@ -178,6 +178,10 @@ class DeezerPlaylistIE(InfoExtractor): 'id': s['SNG_ID'], 'duration': int_or_none(s.get('DURATION')), 'title': '%s - %s' % (artists, s['SNG_TITLE']), + 'artist': s['ART_NAME'], + 'track': s['SNG_TITLE'], + 'album': s['ALB_TITLE'], + 'track_number': s['TRACK_NUMBER'], 'uploader': s['ART_NAME'], 'uploader_id': s['ART_ID'], 'age_limit': 16 if convert_to_int(s.get('EXPLICIT_LYRICS')) else 0, From 2418bd784f685c0d0f47bd01c68872d4721d6dee Mon Sep 17 00:00:00 2001 From: deezerdl Date: Mon, 29 May 2017 21:36:49 +0000 Subject: [PATCH 23/26] added id3 tags to mp3 file. --- youtube_dl/downloader/deezer.py | 69 +++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/youtube_dl/downloader/deezer.py b/youtube_dl/downloader/deezer.py index 851098751..9ea60eed8 100644 --- a/youtube_dl/downloader/deezer.py +++ b/youtube_dl/downloader/deezer.py @@ -2,10 +2,66 @@ from __future__ import unicode_literals from ..blowfish import blowfish_cbc_decrypt from .common import FileDownloader -from ..utils import sanitized_Request +from ..utils import sanitized_Request, sanitize_open +from ..compat import compat_struct_pack import time +def writeid3v1(fo, info_dict): + data = compat_struct_pack("3s" "30s" "30s" "30s" "4s" "28s" "BB" "B", + b"TAG", + info_dict["track"].encode('utf-8'), + info_dict["artist"].encode('utf-8'), + info_dict["album"].encode('utf-8'), + b"", # year + b"", # comment + 0, int(info_dict["track_number"] or 0), # tracknum + 255) # genre + fo.write(data) + + +def writeid3v2(fo, info_dict, ydl): + def make28bit(x): + return ((x << 3) & 0x7F000000) | ((x << 2) & 0x7F0000) | ((x << 1) & 0x7F00) | (x & 127) + + def maketag(tag, content): + return compat_struct_pack(">4sLH", tag, len(content), 0) + content + + def makeutf8(txt): + return b"\x03" + (txt.encode('utf-8')) + + def makepic(data): + imgframe = (b"\x00", # text encoding + b"image/jpeg\x00", # mime type + b"\x00", # picture type: 'other' + b"\x00", # description + data) + return b''.join(imgframe) + + id3 = [ + maketag(b"TRCK", makeutf8("%02s" % str(info_dict["track_number"]))), # decimal, no term NUL + maketag(b"TIT2", makeutf8(info_dict["track"])), # tern NUL ? + maketag(b"TPE1", makeutf8(info_dict["artist"])), # tern NUL ? + maketag(b"TALB", makeutf8(info_dict["album"])), # tern NUL ? + ] + try: + fh = ydl.urlopen(url) + id3.append(maketag(b"APIC", makepic(fh.read()))) + except Exception as e: + pass + + id3data = b"".join(id3) + + hdr = compat_struct_pack(">3s" "H" "B" "L", + b"ID3", + 0x400, # version + 0x00, # flags + make28bit(len(id3data))) + + fo.write(hdr) + fo.write(id3data) + + def decryptfile(fh, key, fo, progress, data_len): """ Decrypt data from file , and write to file . @@ -42,5 +98,12 @@ class DeezerDownloader(FileDownloader): request = sanitized_Request(url, None, {}) data = self.ydl.urlopen(request) - with open(filename, "wb") as fo: - decryptfile(data, info_dict['key'], fo, self, int(data.info().get('Content-length', 0))) + stream, realfilename = sanitize_open(filename, "wb") + try: + writeid3v2(stream, info_dict, self.ydl) + decryptfile(data, info_dict['key'], stream, self, int(data.info().get('Content-length', 0))) + writeid3v1(stream, info_dict) + finally: + if realfilename != '-': + stream.close() + From a0827f784ab17c325a6988e79b8feaf8c61bdfdf Mon Sep 17 00:00:00 2001 From: deezerdl Date: Mon, 29 May 2017 21:42:04 +0000 Subject: [PATCH 24/26] fixing albumart download --- youtube_dl/downloader/deezer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/downloader/deezer.py b/youtube_dl/downloader/deezer.py index 9ea60eed8..53e48d93e 100644 --- a/youtube_dl/downloader/deezer.py +++ b/youtube_dl/downloader/deezer.py @@ -45,7 +45,7 @@ def writeid3v2(fo, info_dict, ydl): maketag(b"TALB", makeutf8(info_dict["album"])), # tern NUL ? ] try: - fh = ydl.urlopen(url) + fh = ydl.urlopen(info_dict["thumbnail"]) id3.append(maketag(b"APIC", makepic(fh.read()))) except Exception as e: pass From d431c70c6a83ff80eb91be87c21570951da7178c Mon Sep 17 00:00:00 2001 From: deezerdl Date: Mon, 29 May 2017 21:49:43 +0000 Subject: [PATCH 25/26] removed trailing empty line --- youtube_dl/downloader/deezer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/youtube_dl/downloader/deezer.py b/youtube_dl/downloader/deezer.py index 53e48d93e..073c2d97b 100644 --- a/youtube_dl/downloader/deezer.py +++ b/youtube_dl/downloader/deezer.py @@ -106,4 +106,3 @@ class DeezerDownloader(FileDownloader): finally: if realfilename != '-': stream.close() - From 2fd78e9baf40c1e53f217181ddbbcd5e19e390fd Mon Sep 17 00:00:00 2001 From: deezerdl Date: Tue, 30 May 2017 14:28:16 +0000 Subject: [PATCH 26/26] improved progress reporting --- youtube_dl/downloader/deezer.py | 38 +++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/youtube_dl/downloader/deezer.py b/youtube_dl/downloader/deezer.py index 073c2d97b..6f191d8ab 100644 --- a/youtube_dl/downloader/deezer.py +++ b/youtube_dl/downloader/deezer.py @@ -16,7 +16,7 @@ def writeid3v1(fo, info_dict): b"", # year b"", # comment 0, int(info_dict["track_number"] or 0), # tracknum - 255) # genre + 255) # genre fo.write(data) @@ -62,7 +62,7 @@ def writeid3v2(fo, info_dict, ydl): fo.write(id3data) -def decryptfile(fh, key, fo, progress, data_len): +def decryptfile(fh, key, fo, progressupdate): """ Decrypt data from file , and write to file . decrypt using blowfish with . @@ -70,7 +70,6 @@ def decryptfile(fh, key, fo, progress, data_len): """ i = 0 byte_counter = 0 - tstart = time.time() while True: data = fh.read(2048) if not data: @@ -83,13 +82,8 @@ def decryptfile(fh, key, fo, progress, data_len): byte_counter += len(data) - progress._hook_progress({ - 'status': 'downloading', - 'downloaded_bytes': byte_counter, - 'total_bytes': data_len, - 'eta': progress.calc_eta(tstart, time.time(), data_len, byte_counter), - 'speed': progress.calc_speed(tstart, time.time(), byte_counter), - }) + if (i % 16) == 0: + progressupdate(byte_counter) class DeezerDownloader(FileDownloader): @@ -98,11 +92,33 @@ class DeezerDownloader(FileDownloader): request = sanitized_Request(url, None, {}) data = self.ydl.urlopen(request) + tstart = time.time() + data_len = int(data.info().get('Content-length', 0)) + + def progressupdate(byte_counter): + self._hook_progress({ + 'filename': filename, + 'status': 'downloading', + 'downloaded_bytes': byte_counter, + 'total_bytes': data_len, + 'eta': self.calc_eta(tstart, time.time(), data_len, byte_counter), + 'speed': self.calc_speed(tstart, time.time(), byte_counter), + }) + + def progressfinished(): + self._hook_progress({ + 'filename': filename, + 'status': 'finished', + 'downloaded_bytes': data_len, + 'total_bytes': data_len, + }) + stream, realfilename = sanitize_open(filename, "wb") try: writeid3v2(stream, info_dict, self.ydl) - decryptfile(data, info_dict['key'], stream, self, int(data.info().get('Content-length', 0))) + decryptfile(data, info_dict['key'], stream, progressupdate) writeid3v1(stream, info_dict) + progressfinished() finally: if realfilename != '-': stream.close()