From 817e8f523fe6d0d0aeb0b100c4cd10a13546793f Mon Sep 17 00:00:00 2001 From: Nevar Angelo Date: Sat, 6 Nov 2010 22:21:45 +0200 Subject: [PATCH 01/22] Allow comments in batch file. --- youtube-dl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube-dl b/youtube-dl index b099ffd15..24cc454a8 100755 --- a/youtube-dl +++ b/youtube-dl @@ -2212,7 +2212,7 @@ if __name__ == '__main__': batchfd = open(opts.batchfile, 'r') batchurls = batchfd.readlines() batchurls = [x.strip() for x in batchurls] - batchurls = [x for x in batchurls if len(x) > 0] + batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)] except IOError: sys.exit(u'ERROR: batch file could not be read') all_urls = batchurls + args From 1e47d226e10ff985a8a12e0044381bf59450c7f7 Mon Sep 17 00:00:00 2001 From: Nevar Angelo Date: Sat, 6 Nov 2010 22:34:22 +0200 Subject: [PATCH 02/22] Added command line switch -A --auto-number Numbering downloaded URLs was implemented with %(ord)s in the output template. It has been replaced with the %(autonumber)s sequence and is now also available as a command line switch. --- youtube-dl | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/youtube-dl b/youtube-dl index 24cc454a8..2ec980f97 100755 --- a/youtube-dl +++ b/youtube-dl @@ -413,7 +413,7 @@ class FileDownloader(object): try: template_dict = dict(info_dict) template_dict['epoch'] = unicode(long(time.time())) - template_dict['ord'] = unicode('%05d' % self._num_downloads) + template_dict['autonumber'] = unicode('%05d' % self._num_downloads) filename = self.params['outtmpl'] % template_dict except (ValueError, KeyError), err: self.trouble(u'ERROR: invalid system charset or erroneous output template') @@ -2171,6 +2171,8 @@ if __name__ == '__main__': action='store_true', dest='usetitle', help='use title in file name', default=False) filesystem.add_option('-l', '--literal', action='store_true', dest='useliteral', help='use literal title in file name', default=False) + filesystem.add_option('-A', '--auto-number', + action='store_true', dest='autonumber', help='number downloaded URLs starting from 00000', default=False) filesystem.add_option('-o', '--output', dest='outtmpl', metavar='TEMPLATE', help='output filename template') filesystem.add_option('-a', '--batch-file', @@ -2224,8 +2226,8 @@ if __name__ == '__main__': parser.error(u'using .netrc conflicts with giving username/password') if opts.password is not None and opts.username is None: parser.error(u'account username missing') - if opts.outtmpl is not None and (opts.useliteral or opts.usetitle): - parser.error(u'using output template conflicts with using title or literal title') + if opts.outtmpl is not None and (opts.useliteral or opts.usetitle or opts.autonumber): + parser.error(u'using output template conflicts with using title, literal title or auto number') if opts.usetitle and opts.useliteral: parser.error(u'using title conflicts with using literal title') if opts.username is not None and opts.password is None: @@ -2284,8 +2286,11 @@ if __name__ == '__main__': or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s') or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s') or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s') + or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(stitle)s-%(id)s.%(ext)s') + or (opts.useliteral and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s') or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s') or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s') + or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s') or u'%(id)s.%(ext)s'), 'ignoreerrors': opts.ignoreerrors, 'ratelimit': opts.ratelimit, From 5e596cac0adcb7e4321f0c9d84e1a813ab26777e Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Date: Sat, 6 Nov 2010 22:13:59 +0100 Subject: [PATCH 03/22] Minor help text correction --- youtube-dl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube-dl b/youtube-dl index 2ec980f97..dbb505f19 100755 --- a/youtube-dl +++ b/youtube-dl @@ -2172,7 +2172,7 @@ if __name__ == '__main__': filesystem.add_option('-l', '--literal', action='store_true', dest='useliteral', help='use literal title in file name', default=False) filesystem.add_option('-A', '--auto-number', - action='store_true', dest='autonumber', help='number downloaded URLs starting from 00000', default=False) + action='store_true', dest='autonumber', help='number downloaded files starting from 00000', default=False) filesystem.add_option('-o', '--output', dest='outtmpl', metavar='TEMPLATE', help='output filename template') filesystem.add_option('-a', '--batch-file', From b3a27b52171761e23060d6dbd7183359d0bad650 Mon Sep 17 00:00:00 2001 From: Nevar Angelo Date: Wed, 17 Nov 2010 20:55:30 +0200 Subject: [PATCH 04/22] Added 'uploaddate' output sequence for YoutubeIE. --- youtube-dl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/youtube-dl b/youtube-dl index dbb505f19..1a53c2c41 100755 --- a/youtube-dl +++ b/youtube-dl @@ -5,6 +5,7 @@ # Author: Benjamin Johnson # License: Public domain code import cookielib +import datetime import htmlentitydefs import httplib import locale @@ -894,6 +895,18 @@ class YoutubeIE(InfoExtractor): else: # don't panic if we can't find it video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0]) + # upload date + upload_date = u'NA' + mobj = re.search(r'id="eow-date".*?>(.*?)', video_webpage, re.DOTALL) + if mobj is not None: + upload_date = mobj.group(1).split() + format_expressions = ['%d %B %Y', '%B %d, %Y'] + for expression in format_expressions: + try: + upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d') + except: + pass + # description video_description = 'No description available.' if self._downloader.params.get('forcedescription', False): @@ -948,6 +961,7 @@ class YoutubeIE(InfoExtractor): 'id': video_id.decode('utf-8'), 'url': video_real_url.decode('utf-8'), 'uploader': video_uploader.decode('utf-8'), + 'uploaddate': upload_date, 'title': video_title, 'stitle': simple_title, 'ext': video_extension.decode('utf-8'), @@ -1094,6 +1108,7 @@ class MetacafeIE(InfoExtractor): 'id': video_id.decode('utf-8'), 'url': video_url.decode('utf-8'), 'uploader': video_uploader.decode('utf-8'), + 'uploaddate': u'NA', 'title': video_title, 'stitle': simple_title, 'ext': video_extension.decode('utf-8'), @@ -1182,6 +1197,7 @@ class DailymotionIE(InfoExtractor): 'id': video_id.decode('utf-8'), 'url': video_url.decode('utf-8'), 'uploader': video_uploader.decode('utf-8'), + 'uploaddate': u'NA', 'title': video_title, 'stitle': simple_title, 'ext': video_extension.decode('utf-8'), @@ -1291,6 +1307,7 @@ class GoogleIE(InfoExtractor): 'id': video_id.decode('utf-8'), 'url': video_url.decode('utf-8'), 'uploader': u'NA', + 'uploaddate': u'NA', 'title': video_title, 'stitle': simple_title, 'ext': video_extension.decode('utf-8'), @@ -1372,6 +1389,7 @@ class PhotobucketIE(InfoExtractor): 'id': video_id.decode('utf-8'), 'url': video_url.decode('utf-8'), 'uploader': video_uploader, + 'uploaddate': u'NA', 'title': video_title, 'stitle': simple_title, 'ext': video_extension.decode('utf-8'), @@ -1526,6 +1544,7 @@ class YahooIE(InfoExtractor): 'id': video_id.decode('utf-8'), 'url': video_url, 'uploader': video_uploader, + 'uploaddate': u'NA', 'title': video_title, 'stitle': simple_title, 'ext': video_extension.decode('utf-8'), @@ -1628,6 +1647,7 @@ class GenericIE(InfoExtractor): 'id': video_id.decode('utf-8'), 'url': video_url.decode('utf-8'), 'uploader': video_uploader, + 'uploaddate': u'NA', 'title': video_title, 'stitle': simple_title, 'ext': video_extension.decode('utf-8'), From b620a5f811d94a0296a9809391e3a121ad8d68e9 Mon Sep 17 00:00:00 2001 From: jamiejones Date: Tue, 16 Nov 2010 13:52:23 -0800 Subject: [PATCH 05/22] Correctly parse the player URL in RTMP downloads (closes #11) Fixed several problems courtesy of jamiejones: The parsing for the SWF url was wrong (the "//" are now escaped and the initial .*match needs to be 'ungreedy'), so the -W setting to rtmpdump was not set, causing the decryption of the video to be wrong. Finally, add "&has_verified=1" to the fetch of the HMTL page to allow fetching of age-restricted videos. --- youtube-dl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) mode change 100755 => 100644 youtube-dl diff --git a/youtube-dl b/youtube-dl old mode 100755 new mode 100644 index dbb505f19..6a888cc3f --- a/youtube-dl +++ b/youtube-dl @@ -831,7 +831,7 @@ class YoutubeIE(InfoExtractor): # Get video webpage self.report_video_webpage_download(video_id) - request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en' % video_id, None, std_headers) + request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id, None, std_headers) try: video_webpage = urllib2.urlopen(request).read() except (urllib2.URLError, httplib.HTTPException, socket.error), err: @@ -839,9 +839,9 @@ class YoutubeIE(InfoExtractor): return # Attempt to extract SWF player URL - mobj = re.search(r'swfConfig.*"(http://.*?watch.*?-.*?\.swf)"', video_webpage) + mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage) if mobj is not None: - player_url = mobj.group(1) + player_url = re.sub(r'\\(.)', r'\1', mobj.group(1)) else: player_url = None From 05df0c1d4acbc7077187579f16c6661260e62014 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Date: Fri, 19 Nov 2010 18:37:07 +0100 Subject: [PATCH 06/22] Restore file permissions --- youtube-dl | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 youtube-dl diff --git a/youtube-dl b/youtube-dl old mode 100644 new mode 100755 From 138b11f36ee5e8018c29621d39c324d98d8291cc Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Date: Fri, 19 Nov 2010 19:31:26 +0100 Subject: [PATCH 07/22] Rework upload date mechanism after detecting problems in several tests --- youtube-dl | 49 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/youtube-dl b/youtube-dl index e164d5c8c..3d20a9d6d 100755 --- a/youtube-dl +++ b/youtube-dl @@ -5,7 +5,6 @@ # Author: Benjamin Johnson # License: Public domain code import cookielib -import datetime import htmlentitydefs import httplib import locale @@ -37,6 +36,21 @@ std_headers = { simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii') +month_name_to_number = { + 'January': '01', + 'February': '02', + 'March': '03', + 'April': '04', + 'May': '05', + 'June': '06', + 'July': '07', + 'August': '08', + 'September': '09', + 'October': '10', + 'November': '11', + 'December': '12', +} + def preferredencoding(): """Get preferred encoding. @@ -899,13 +913,18 @@ class YoutubeIE(InfoExtractor): upload_date = u'NA' mobj = re.search(r'id="eow-date".*?>(.*?)', video_webpage, re.DOTALL) if mobj is not None: - upload_date = mobj.group(1).split() - format_expressions = ['%d %B %Y', '%B %d, %Y'] - for expression in format_expressions: - try: - upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d') - except: - pass + try: + if ',' in mobj.group(1): + # Month Day, Year + m, d, y = mobj.group(1).replace(',', '').split() + else: + # Day Month Year, we'll suppose + d, m, y = mobj.group(1).split() + m = month_name_to_number[m] + d = '%02d' % (long(d)) + upload_date = '%s%s%s' % (y, m, d) + except: + upload_date = u'NA' # description video_description = 'No description available.' @@ -961,7 +980,7 @@ class YoutubeIE(InfoExtractor): 'id': video_id.decode('utf-8'), 'url': video_real_url.decode('utf-8'), 'uploader': video_uploader.decode('utf-8'), - 'uploaddate': upload_date, + 'upload_date': upload_date, 'title': video_title, 'stitle': simple_title, 'ext': video_extension.decode('utf-8'), @@ -1108,7 +1127,7 @@ class MetacafeIE(InfoExtractor): 'id': video_id.decode('utf-8'), 'url': video_url.decode('utf-8'), 'uploader': video_uploader.decode('utf-8'), - 'uploaddate': u'NA', + 'upload_date': u'NA', 'title': video_title, 'stitle': simple_title, 'ext': video_extension.decode('utf-8'), @@ -1197,7 +1216,7 @@ class DailymotionIE(InfoExtractor): 'id': video_id.decode('utf-8'), 'url': video_url.decode('utf-8'), 'uploader': video_uploader.decode('utf-8'), - 'uploaddate': u'NA', + 'upload_date': u'NA', 'title': video_title, 'stitle': simple_title, 'ext': video_extension.decode('utf-8'), @@ -1307,7 +1326,7 @@ class GoogleIE(InfoExtractor): 'id': video_id.decode('utf-8'), 'url': video_url.decode('utf-8'), 'uploader': u'NA', - 'uploaddate': u'NA', + 'upload_date': u'NA', 'title': video_title, 'stitle': simple_title, 'ext': video_extension.decode('utf-8'), @@ -1389,7 +1408,7 @@ class PhotobucketIE(InfoExtractor): 'id': video_id.decode('utf-8'), 'url': video_url.decode('utf-8'), 'uploader': video_uploader, - 'uploaddate': u'NA', + 'upload_date': u'NA', 'title': video_title, 'stitle': simple_title, 'ext': video_extension.decode('utf-8'), @@ -1544,7 +1563,7 @@ class YahooIE(InfoExtractor): 'id': video_id.decode('utf-8'), 'url': video_url, 'uploader': video_uploader, - 'uploaddate': u'NA', + 'upload_date': u'NA', 'title': video_title, 'stitle': simple_title, 'ext': video_extension.decode('utf-8'), @@ -1647,7 +1666,7 @@ class GenericIE(InfoExtractor): 'id': video_id.decode('utf-8'), 'url': video_url.decode('utf-8'), 'uploader': video_uploader, - 'uploaddate': u'NA', + 'upload_date': u'NA', 'title': video_title, 'stitle': simple_title, 'ext': video_extension.decode('utf-8'), From 010ebaf7832f41a5842c2a9ac316943b0efc5c2b Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Date: Fri, 19 Nov 2010 19:40:18 +0100 Subject: [PATCH 08/22] Update User-Agent string --- youtube-dl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube-dl b/youtube-dl index 3d20a9d6d..94e04d4cd 100755 --- a/youtube-dl +++ b/youtube-dl @@ -28,7 +28,7 @@ except ImportError: from cgi import parse_qs std_headers = { - 'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.11) Gecko/20101019 Firefox/3.6.11', + 'User-Agent': 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.12) Gecko/20101028 Firefox/3.6.12', 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-us,en;q=0.5', From f8dc4414305f111a17d31f53d4a04650d80d34f5 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Date: Fri, 19 Nov 2010 19:41:09 +0100 Subject: [PATCH 09/22] Bump version number --- LATEST_VERSION | 2 +- youtube-dl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LATEST_VERSION b/LATEST_VERSION index 8418210b7..4f8405c40 100644 --- a/LATEST_VERSION +++ b/LATEST_VERSION @@ -1 +1 @@ -2010.10.24 +2010.11.19 diff --git a/youtube-dl b/youtube-dl index 94e04d4cd..24722d292 100755 --- a/youtube-dl +++ b/youtube-dl @@ -2145,7 +2145,7 @@ if __name__ == '__main__': # Parse command line parser = optparse.OptionParser( usage='Usage: %prog [options] url...', - version='2010.10.24', + version='2010.11.19', conflict_handler='resolve', ) From a1f03c7b06d0a811d18499cb2ef09405c52a5db9 Mon Sep 17 00:00:00 2001 From: Nevar Angelo Date: Tue, 30 Nov 2010 18:51:00 +0200 Subject: [PATCH 10/22] Reworked 'upload_date' code for output sequence in YoutubeIE. Reverted to previous version of 'upload_date' and fixed a mistake that prevented the code from working properly. --- youtube-dl | 35 ++++++++--------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/youtube-dl b/youtube-dl index 24722d292..4cea2e85e 100755 --- a/youtube-dl +++ b/youtube-dl @@ -5,6 +5,7 @@ # Author: Benjamin Johnson # License: Public domain code import cookielib +import datetime import htmlentitydefs import httplib import locale @@ -36,21 +37,6 @@ std_headers = { simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii') -month_name_to_number = { - 'January': '01', - 'February': '02', - 'March': '03', - 'April': '04', - 'May': '05', - 'June': '06', - 'July': '07', - 'August': '08', - 'September': '09', - 'October': '10', - 'November': '11', - 'December': '12', -} - def preferredencoding(): """Get preferred encoding. @@ -913,18 +899,13 @@ class YoutubeIE(InfoExtractor): upload_date = u'NA' mobj = re.search(r'id="eow-date".*?>(.*?)', video_webpage, re.DOTALL) if mobj is not None: - try: - if ',' in mobj.group(1): - # Month Day, Year - m, d, y = mobj.group(1).replace(',', '').split() - else: - # Day Month Year, we'll suppose - d, m, y = mobj.group(1).split() - m = month_name_to_number[m] - d = '%02d' % (long(d)) - upload_date = '%s%s%s' % (y, m, d) - except: - upload_date = u'NA' + upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split()) + format_expressions = ['%d %B %Y', '%B %d %Y'] + for expression in format_expressions: + try: + upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d') + except: + pass # description video_description = 'No description available.' From 62cf7aaf9a3b9b74ba90d1c17303eb5cabd3f52c Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Date: Sat, 4 Dec 2010 10:38:53 +0100 Subject: [PATCH 11/22] Use a temporary filename to download files --- youtube-dl | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/youtube-dl b/youtube-dl index 4cea2e85e..22dd230ee 100755 --- a/youtube-dl +++ b/youtube-dl @@ -235,6 +235,11 @@ class FileDownloader(object): if not os.path.exists(dir): os.mkdir(dir) + @staticmethod + def temp_name(filename): + """Returns a temporary filename for the given filename.""" + return filename + '.part' + @staticmethod def format_bytes(bytes): if bytes is None: @@ -353,6 +358,12 @@ class FileDownloader(object): speed = float(byte_counter) / elapsed if speed > rate_limit: time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) + + def try_rename(self, old_filename, new_filename): + try: + os.rename(old_filename, new_filename) + except (IOError, OSError), err: + self.trouble(u'ERROR: unable to rename file') def report_destination(self, filename): """Report destination filename.""" @@ -484,6 +495,7 @@ class FileDownloader(object): def _download_with_rtmpdump(self, filename, url, player_url): self.report_destination(filename) + tmpfilename = self.temp_name(filename) # Check for rtmpdump first try: @@ -495,36 +507,43 @@ class FileDownloader(object): # Download using rtmpdump. rtmpdump returns exit code 2 when # the connection was interrumpted and resuming appears to be # possible. This is part of rtmpdump's normal usage, AFAIK. - basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', filename] + basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename] retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]) while retval == 2 or retval == 1: - prevsize = os.path.getsize(filename) + prevsize = os.path.getsize(tmpfilename) self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True) time.sleep(5.0) # This seems to be needed retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) - cursize = os.path.getsize(filename) + cursize = os.path.getsize(tmpfilename) if prevsize == cursize and retval == 1: break if retval == 0: - self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(filename)) + self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(tmpfilename)) + self.try_rename(tmpfilename, filename) return True else: self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval) return False def _do_download(self, filename, url, player_url): + # Check file already present + if self.params.get('continuedl', False) and os.path.isfile(filename): + self.report_file_already_downloaded(filename) + return True + # Attempt to download using rtmpdump if url.startswith('rtmp'): return self._download_with_rtmpdump(filename, url, player_url) + tmpfilename = self.temp_name(filename) stream = None open_mode = 'wb' basic_request = urllib2.Request(url, None, std_headers) request = urllib2.Request(url, None, std_headers) # Establish possible resume length - if os.path.isfile(filename): - resume_len = os.path.getsize(filename) + if os.path.isfile(tmpfilename): + resume_len = os.path.getsize(tmpfilename) else: resume_len = 0 @@ -566,6 +585,7 @@ class FileDownloader(object): # completely downloaded if the file size differs less than 100 bytes from # the one in the hard drive. self.report_file_already_downloaded(filename) + self.try_rename(tmpfilename, filename) return True else: # The length does not match, we start the download over @@ -599,7 +619,7 @@ class FileDownloader(object): # Open file just in time if stream is None: try: - (stream, filename) = sanitize_open(filename, open_mode) + (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode) self.report_destination(filename) except (OSError, IOError), err: self.trouble(u'ERROR: unable to open for writing: %s' % str(err)) @@ -623,6 +643,7 @@ class FileDownloader(object): self.report_finish() if data_len is not None and str(byte_counter) != data_len: raise ContentTooShortError(byte_counter, long(data_len)) + self.try_rename(tmpfilename, filename) return True class InfoExtractor(object): From a9806fd83d7efc4cc1dcd11174c839f637d0c105 Mon Sep 17 00:00:00 2001 From: Vasyl' Vavrychuk Date: Sun, 5 Dec 2010 20:48:22 +0200 Subject: [PATCH 12/22] report_extraction was never called for GenericIE --- youtube-dl | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube-dl b/youtube-dl index 22dd230ee..d0584a312 100755 --- a/youtube-dl +++ b/youtube-dl @@ -1619,6 +1619,7 @@ class GenericIE(InfoExtractor): self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) return + self.report_extraction(video_id) # Start with something easy: JW Player in SWFObject mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage) if mobj is None: From 6f0ff3bab9f213790817e3344bb5025a7fbea1e4 Mon Sep 17 00:00:00 2001 From: Vasyl' Vavrychuk Date: Sun, 5 Dec 2010 20:57:46 +0200 Subject: [PATCH 13/22] Fixed failure of os.rename after receiving file finished due to file not being closed. Following error happen while at the end of _do_download called try_rename WindowsError: [Error 32] The process cannot access the file because it is being used by another process --- youtube-dl | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube-dl b/youtube-dl index d0584a312..6b5f5693d 100755 --- a/youtube-dl +++ b/youtube-dl @@ -640,6 +640,7 @@ class FileDownloader(object): # Apply rate limit self.slow_down(start, byte_counter) + stream.close() self.report_finish() if data_len is not None and str(byte_counter) != data_len: raise ContentTooShortError(byte_counter, long(data_len)) From 27179cfdba64c40607ec089f53ed754efceff64a Mon Sep 17 00:00:00 2001 From: Vasyl' Vavrychuk Date: Sun, 5 Dec 2010 21:09:14 +0200 Subject: [PATCH 14/22] Implemented depositfiles.com support --- youtube-dl | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/youtube-dl b/youtube-dl index 6b5f5693d..0e9699807 100755 --- a/youtube-dl +++ b/youtube-dl @@ -110,7 +110,6 @@ def sanitize_open(filename, open_mode): stream = open(filename, open_mode) return (stream, filename) - class DownloadError(Exception): """Download Error exception. @@ -2077,6 +2076,85 @@ class YoutubeUserIE(InfoExtractor): self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id) return +class DepositFilesIE(InfoExtractor): + """Information extractor for depositfiles.com""" + + _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles.com/(?:../(?#locale))?files/(.+)' + + def __init__(self, downloader=None): + InfoExtractor.__init__(self, downloader) + + @staticmethod + def suitable(url): + return (re.match(DepositFilesIE._VALID_URL, url) is not None) + + def report_download_webpage(self, file_id): + """Report webpage download.""" + self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id) + + def report_extraction(self, file_id): + """Report information extraction.""" + self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id) + + def _real_initialize(self): + return + + def _real_extract(self, url): + # At this point we have a new file + self._downloader.increment_downloads() + + file_id = url.split('/')[-1] + # Rebuild url in english locale + url = 'http://depositfiles.com/en/files/' + file_id + + # Retrieve file webpage with 'Free download' button pressed + free_download_indication = { 'gateway_result' : '1' } + request = urllib2.Request(url, urllib.urlencode(free_download_indication), std_headers) + try: + self.report_download_webpage(file_id) + webpage = urllib2.urlopen(request).read() + except (urllib2.URLError, httplib.HTTPException, socket.error), err: + self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % str(err)) + return + + # Search for the real file URL + mobj = re.search(r'
(Attention.*?)', webpage, re.DOTALL) + if (mobj is not None) and (mobj.group(1) is not None): + restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip() + self._downloader.trouble(u'ERROR: %s' % restriction_message) + else: + self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url) + return + + file_url = mobj.group(1) + file_extension = os.path.splitext(file_url)[1][1:] + + # Search for file title + mobj = re.search(r'', webpage) + if mobj is None: + self._downloader.trouble(u'ERROR: unable to extract title') + return + file_title = mobj.group(1).decode('utf-8') + + try: + # Process file information + self._downloader.process_info({ + 'id': file_id.decode('utf-8'), + 'url': file_url.decode('utf-8'), + 'uploader': u'NA', + 'upload_date': u'NA', + 'title': file_title, + 'stitle': file_title, + 'ext': file_extension.decode('utf-8'), + 'format': u'NA', + 'player_url': None, + }) + except UnavailableVideoError, err: + self._downloader.trouble(u'ERROR: unable to download file') + class PostProcessor(object): """Post Processor class. @@ -2310,6 +2388,7 @@ if __name__ == '__main__': photobucket_ie = PhotobucketIE() yahoo_ie = YahooIE() yahoo_search_ie = YahooSearchIE(yahoo_ie) + deposit_files_ie = DepositFilesIE() generic_ie = GenericIE() # File downloader @@ -2356,6 +2435,7 @@ if __name__ == '__main__': fd.add_info_extractor(photobucket_ie) fd.add_info_extractor(yahoo_ie) fd.add_info_extractor(yahoo_search_ie) + fd.add_info_extractor(deposit_files_ie) # This must come last since it's the # fallback if none of the others work From e567ef93d8833a566f063bab9f9ee2310a131ab1 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Date: Wed, 8 Dec 2010 11:04:22 +0100 Subject: [PATCH 15/22] Add Vasyl' Vavrychuk to the list of authors --- youtube-dl | 1 + 1 file changed, 1 insertion(+) diff --git a/youtube-dl b/youtube-dl index 0e9699807..d60fa60c9 100755 --- a/youtube-dl +++ b/youtube-dl @@ -3,6 +3,7 @@ # Author: Ricardo Garcia Gonzalez # Author: Danny Colligan # Author: Benjamin Johnson +# Author: Vasyl' Vavrychuk # License: Public domain code import cookielib import datetime From d157d2597a5fe99db60304fe1e89523de78b7981 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Date: Thu, 9 Dec 2010 19:22:32 +0100 Subject: [PATCH 16/22] Fix YoutubeIE after recent YouTube changes (closes #34) --- youtube-dl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube-dl b/youtube-dl index d60fa60c9..415cf7073 100755 --- a/youtube-dl +++ b/youtube-dl @@ -955,9 +955,9 @@ class YoutubeIE(InfoExtractor): self._downloader.trouble(u'ERROR: no known formats available for video') return if requested_format is None: - video_url_list = [(existing_formats[0], get_video_template % existing_formats[0])] # Best quality + video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality elif requested_format == '-1': - video_url_list = [(f, get_video_template % f) for f in existing_formats] # All formats + video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats else: video_url_list = [(requested_format, get_video_template % requested_format)] # Specific format From 7d950ca1d6543d51d066eaceb9ea1d5ac96d92f4 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Date: Thu, 9 Dec 2010 19:33:04 +0100 Subject: [PATCH 17/22] Improve temporary filename handling of special cases --- youtube-dl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/youtube-dl b/youtube-dl index 415cf7073..64c590fbf 100755 --- a/youtube-dl +++ b/youtube-dl @@ -238,7 +238,9 @@ class FileDownloader(object): @staticmethod def temp_name(filename): """Returns a temporary filename for the given filename.""" - return filename + '.part' + if filename == u'-' or (os.path.exists(filename) and not os.path.isfile(filename)): + return filename + return filename + u'.part' @staticmethod def format_bytes(bytes): @@ -361,6 +363,8 @@ class FileDownloader(object): def try_rename(self, old_filename, new_filename): try: + if old_filename == new_filename: + return os.rename(old_filename, new_filename) except (IOError, OSError), err: self.trouble(u'ERROR: unable to rename file') From f148ea4473697a1e6666e789b0bfae8f0b4804e4 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Date: Thu, 9 Dec 2010 19:37:07 +0100 Subject: [PATCH 18/22] Bump version number --- LATEST_VERSION | 2 +- youtube-dl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LATEST_VERSION b/LATEST_VERSION index 4f8405c40..a1c4173c8 100644 --- a/LATEST_VERSION +++ b/LATEST_VERSION @@ -1 +1 @@ -2010.11.19 +2010.12.09 diff --git a/youtube-dl b/youtube-dl index 64c590fbf..e6177e363 100755 --- a/youtube-dl +++ b/youtube-dl @@ -2232,7 +2232,7 @@ if __name__ == '__main__': # Parse command line parser = optparse.OptionParser( usage='Usage: %prog [options] url...', - version='2010.11.19', + version='2010.12.09', conflict_handler='resolve', ) From f83ae7816b64ff23fbef602d7ade3fd00b57e5aa Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Date: Thu, 9 Dec 2010 19:57:39 +0100 Subject: [PATCH 19/22] Fix problem when requesting an existing format explicitly with -f --- youtube-dl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/youtube-dl b/youtube-dl index e6177e363..a8e3bd36c 100755 --- a/youtube-dl +++ b/youtube-dl @@ -944,7 +944,7 @@ class YoutubeIE(InfoExtractor): video_token = urllib.unquote_plus(video_info['token'][0]) # Decide which formats to download - requested_format = self._downloader.params.get('format', None) + req_format = self._downloader.params.get('format', None) get_video_template = 'http://www.youtube.com/get_video?video_id=%s&t=%s&eurl=&el=&ps=&asv=&fmt=%%s' % (video_id, video_token) if 'fmt_url_map' in video_info: @@ -958,12 +958,15 @@ class YoutubeIE(InfoExtractor): if len(existing_formats) == 0: self._downloader.trouble(u'ERROR: no known formats available for video') return - if requested_format is None: + if req_format is None: video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality - elif requested_format == '-1': + elif req_format == '-1': video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats else: - video_url_list = [(requested_format, get_video_template % requested_format)] # Specific format + if req_format in url_map: + video_url_list = [(req_format, url_map[req_format])] # Specific format + else: + video_url_list = [(req_format, get_video_template % req_format)] # Specific format elif 'conn' in video_info and video_info['conn'][0].startswith('rtmp'): self.report_rtmp_download() From 106d091e80f609802d3f6dcc3512525458900042 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Date: Sat, 11 Dec 2010 11:32:13 +0100 Subject: [PATCH 20/22] Do not use 0% as the starting point in resumed downloads (closes #40) --- youtube-dl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/youtube-dl b/youtube-dl index a8e3bd36c..22b9eba17 100755 --- a/youtube-dl +++ b/youtube-dl @@ -606,8 +606,10 @@ class FileDownloader(object): return False data_len = data.info().get('Content-length', None) + if data_len is not None: + data_len = long(data_len) + resume_len data_len_str = self.format_bytes(data_len) - byte_counter = 0 + byte_counter = 0 + resume_len block_size = 1024 start = time.time() while True: From 5c1327931a91fbc2f621b7f107255463bdf698b9 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Date: Sat, 11 Dec 2010 11:34:10 +0100 Subject: [PATCH 21/22] Stop attempting to use get_video and detect missing formats ourselves --- youtube-dl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/youtube-dl b/youtube-dl index 22b9eba17..8f895a5cb 100755 --- a/youtube-dl +++ b/youtube-dl @@ -947,7 +947,6 @@ class YoutubeIE(InfoExtractor): # Decide which formats to download req_format = self._downloader.params.get('format', None) - get_video_template = 'http://www.youtube.com/get_video?video_id=%s&t=%s&eurl=&el=&ps=&asv=&fmt=%%s' % (video_id, video_token) if 'fmt_url_map' in video_info: url_map = dict(tuple(pair.split('|')) for pair in video_info['fmt_url_map'][0].split(',')) @@ -965,10 +964,11 @@ class YoutubeIE(InfoExtractor): elif req_format == '-1': video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats else: - if req_format in url_map: - video_url_list = [(req_format, url_map[req_format])] # Specific format - else: - video_url_list = [(req_format, get_video_template % req_format)] # Specific format + # Specific format + if req_format not in url_map: + self._downloader.trouble(u'ERROR: requested format not available') + return + video_url_list = [(req_format, url_map[req_format])] # Specific format elif 'conn' in video_info and video_info['conn'][0].startswith('rtmp'): self.report_rtmp_download() @@ -1002,7 +1002,7 @@ class YoutubeIE(InfoExtractor): 'player_url': player_url, }) except UnavailableVideoError, err: - self._downloader.trouble(u'ERROR: unable to download video (format may not be available)') + self._downloader.trouble(u'ERROR: unable to download video') class MetacafeIE(InfoExtractor): From ef4f4544a24633655a9347176ed475322e6dacf0 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Date: Sat, 11 Dec 2010 11:35:28 +0100 Subject: [PATCH 22/22] Remove deprecated -b option and nonworking -m option (closes #39) --- youtube-dl | 6 ------ 1 file changed, 6 deletions(-) diff --git a/youtube-dl b/youtube-dl index 8f895a5cb..4cb84b22a 100755 --- a/youtube-dl +++ b/youtube-dl @@ -2270,14 +2270,10 @@ if __name__ == '__main__': video_format = optparse.OptionGroup(parser, 'Video Format Options') video_format.add_option('-f', '--format', action='store', dest='format', metavar='FORMAT', help='video format code') - video_format.add_option('-m', '--mobile-version', - action='store_const', dest='format', help='alias for -f 17', const='17') video_format.add_option('--all-formats', action='store_const', dest='format', help='download all available video formats', const='-1') video_format.add_option('--max-quality', action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download') - video_format.add_option('-b', '--best-quality', - action='store_true', dest='bestquality', help='download the best video quality (DEPRECATED)') parser.add_option_group(video_format) verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options') @@ -2351,8 +2347,6 @@ if __name__ == '__main__': all_urls = batchurls + args # Conflicting, missing and erroneous options - if opts.bestquality: - print >>sys.stderr, u'\nWARNING: -b/--best-quality IS DEPRECATED AS IT IS THE DEFAULT BEHAVIOR NOW\n' if opts.usenetrc and (opts.username is not None or opts.password is not None): parser.error(u'using .netrc conflicts with giving username/password') if opts.password is not None and opts.username is None: