From afa17379a40e0cef9b27a0ca164031920bb98422 Mon Sep 17 00:00:00 2001 From: kabaakaba Date: Fri, 28 Aug 2015 15:40:27 +0300 Subject: [PATCH 1/4] added md5 calculation param --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index cad6b026e..327c0c265 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1524,7 +1524,7 @@ class YoutubeDL(object): fd.add_progress_hook(ph) if self.params.get('verbose'): self.to_stdout('[debug] Invoking downloader on %r' % info.get('url')) - return fd.download(name, info) + return fd.download(name, info, self.params.get("md5")) if info_dict.get('requested_formats') is not None: downloaded = [] From 31b6346cbc2f9b865dc05f4370e0969c837b218e Mon Sep 17 00:00:00 2001 From: kabaakaba Date: Fri, 28 Aug 2015 15:45:18 +0300 Subject: [PATCH 2/4] added md5 calculation param --- youtube_dl/__init__.py | 3 +++ youtube_dl/options.py | 13 +++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 55b22c889..44132ab21 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -189,6 +189,8 @@ def _real_main(argv=None): if opts.allsubtitles and not opts.writeautomaticsub: opts.writesubtitles = True + + outtmpl = ((opts.outtmpl is not None and opts.outtmpl) or (opts.format == '-1' and opts.usetitle and '%(title)s-%(id)s-%(format)s.%(ext)s') or (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s') or @@ -372,6 +374,7 @@ def _real_main(argv=None): 'external_downloader_args': external_downloader_args, 'postprocessor_args': postprocessor_args, 'cn_verification_proxy': opts.cn_verification_proxy, + 'md5': opts.md5 } with YoutubeDL(ydl_opts) as ydl: diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 9016e3498..f39960061 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -162,9 +162,9 @@ def parseOpts(overrideArguments=None): '--ignore-config', action='store_true', help='Do not read configuration files. ' - 'When given in the global configuration file /etc/youtube-dl.conf: ' - 'Do not read the user configuration in ~/.config/youtube-dl/config ' - '(%APPDATA%/youtube-dl/config.txt on Windows)') + 'When given in the global configuration file /etc/youtube-dl.conf: ' + 'Do not read the user configuration in ~/.config/youtube-dl/config ' + '(%APPDATA%/youtube-dl/config.txt on Windows)') general.add_option( '--flat-playlist', action='store_const', dest='extract_flat', const='in_playlist', @@ -175,6 +175,11 @@ def parseOpts(overrideArguments=None): action='store_true', dest='no_color', default=False, help='Do not emit color codes in output') + general.add_option( + '--md5', + action='store_true', dest='md5', + default=False, + help='Calculate md5 of video and save in file') network = optparse.OptionGroup(parser, 'Network Options') network.add_option( @@ -204,7 +209,7 @@ def parseOpts(overrideArguments=None): '--cn-verification-proxy', dest='cn_verification_proxy', default=None, metavar='URL', help='Use this proxy to verify the IP address for some Chinese sites. ' - 'The default proxy specified by --proxy (or none, if the options is not present) is used for the actual downloading. (experimental)' + 'The default proxy specified by --proxy (or none, if the options is not present) is used for the actual downloading. (experimental)' ) selection = optparse.OptionGroup(parser, 'Video Selection') From 076f3a7c9f5179a3a31590a12194685c12825038 Mon Sep 17 00:00:00 2001 From: kabaakaba Date: Fri, 28 Aug 2015 15:45:44 +0300 Subject: [PATCH 3/4] added md5 calculation param --- youtube_dl/downloader/common.py | 6 +++--- youtube_dl/downloader/http.py | 13 ++++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index 97e755d4b..46b5e6841 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -308,7 +308,7 @@ class FileDownloader(object): """Report it was impossible to resume download.""" self.to_screen('[download] Unable to resume') - def download(self, filename, info_dict): + def download(self, filename, info_dict, md5=False): """Download to a filename using the info from info_dict Return True on success and False otherwise """ @@ -339,9 +339,9 @@ class FileDownloader(object): self.to_screen('[download] Sleeping %s seconds...' % sleep_interval) time.sleep(sleep_interval) - return self.real_download(filename, info_dict) + return self.real_download(filename, info_dict, md5) - def real_download(self, filename, info_dict): + def real_download(self, filename, info_dict, *args): """Real download process. Redefine in subclasses.""" raise NotImplementedError('This method must be implemented by subclasses') diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py index a29f5cf31..27133148d 100644 --- a/youtube_dl/downloader/http.py +++ b/youtube_dl/downloader/http.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import errno +import hashlib import os import socket import time @@ -19,7 +20,7 @@ from ..utils import ( class HttpFD(FileDownloader): - def real_download(self, filename, info_dict): + def real_download(self, filename, info_dict, md5=False): url = info_dict['url'] tmpfilename = self.temp_name(filename) stream = None @@ -158,12 +159,17 @@ class HttpFD(FileDownloader): # measure time over whole while-loop, so slow_down() and best_block_size() work together properly now = None # needed for slow_down() in the first loop run before = start # start measuring + if md5: + m = hashlib.md5() + else: + m = None while True: # Download and write data_block = data.read(block_size if not is_test else min(block_size, data_len - byte_counter)) byte_counter += len(data_block) - + if m is not None: + m.update(data_block) # exit loop when download is finished if len(data_block) == 0: break @@ -237,7 +243,8 @@ class HttpFD(FileDownloader): if data_len is not None and byte_counter != data_len: raise ContentTooShortError(byte_counter, int(data_len)) self.try_rename(tmpfilename, filename) - + if m is not None: + open(filename+'.md5', 'w').write(m.hexdigest()) # Update file modification time if self.params.get('updatetime', True): info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None)) From 25b73545c6bf2b99e1ad1f1701fb9c4868c02338 Mon Sep 17 00:00:00 2001 From: kabaakaba Date: Fri, 28 Aug 2015 18:32:45 +0300 Subject: [PATCH 4/4] added md5 calculation param for dash and http --- youtube_dl/downloader/common.py | 2 +- youtube_dl/downloader/dash.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index 46b5e6841..7f2cda086 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -341,7 +341,7 @@ class FileDownloader(object): return self.real_download(filename, info_dict, md5) - def real_download(self, filename, info_dict, *args): + def real_download(self, filename, info_dict): """Real download process. Redefine in subclasses.""" raise NotImplementedError('This method must be implemented by subclasses') diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py index 8b6fa2753..61f6b1f16 100644 --- a/youtube_dl/downloader/dash.py +++ b/youtube_dl/downloader/dash.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals +import hashlib import re @@ -10,7 +11,7 @@ class DashSegmentsFD(FileDownloader): """ Download segments in a DASH manifest """ - def real_download(self, filename, info_dict): + def real_download(self, filename, info_dict, md5=False): self.report_destination(filename) tmpfilename = self.temp_name(filename) base_url = info_dict['url'] @@ -19,18 +20,20 @@ class DashSegmentsFD(FileDownloader): is_test = self.params.get('test', False) remaining_bytes = self._TEST_FILE_SIZE if is_test else None byte_counter = 0 - + if md5: + m = hashlib.md5() + else: + m = None def append_url_to_file(outf, target_url, target_name, remaining_bytes=None): self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name)) req = compat_urllib_request.Request(target_url) if remaining_bytes is not None: req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1)) - data = self.ydl.urlopen(req).read() - if remaining_bytes is not None: data = data[:remaining_bytes] - + if m is not None: + m.update(data) outf.write(data) return len(data) @@ -53,7 +56,8 @@ class DashSegmentsFD(FileDownloader): remaining_bytes -= segment_len if remaining_bytes <= 0: break - + if m is not None: + open(filename, 'w').write(m.hexdigest()) self.try_rename(tmpfilename, filename) self._hook_progress({