From ebd92e71f25c6bcdf53b26af3e7ce8eaac1d98d0 Mon Sep 17 00:00:00 2001 From: Henrik Hank Date: Mon, 14 May 2018 22:27:23 +0200 Subject: [PATCH 01/10] Allow creation of internet shortcut files with new --write-link switch and similar ones. --- README.md | 28 ++++++++++++-- test/parameters.json | 5 +++ test/test_compat.py | 35 ++++++++++++++++-- test/test_utils.py | 27 ++++++++++++++ youtube_dl/YoutubeDL.py | 67 ++++++++++++++++++++++++++++++++- youtube_dl/__init__.py | 4 ++ youtube_dl/compat.py | 26 ++++++++++++- youtube_dl/options.py | 25 +++++++++++-- youtube_dl/utils.py | 82 +++++++++++++++++++++++++++++++++++++++++ 9 files changed, 284 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index d9fe2350a..e4a13e6b8 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,11 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo age --download-archive FILE Download only videos not listed in the archive file. Record the IDs of all - downloaded videos in it. + downloaded videos in it. When the switches + --write-link (or similar) and + --skip-download are used additionally, the + IDs will also be recorded, even though + nothing was actually downloaded. --include-ads Download advertisements as well (experimental) @@ -268,12 +272,24 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo --no-cache-dir Disable filesystem caching --rm-cache-dir Delete all filesystem cache files -## Thumbnail images: +## Thumbnail Images: --write-thumbnail Write thumbnail image to disk --write-all-thumbnails Write all thumbnail image formats to disk --list-thumbnails Simulate and list all available thumbnail formats +## Internet Shortcut Options: + --write-link Write an internet shortcut file, depending + on the current platform (.url/.webloc/ + .desktop). The URL may be cached by the OS. + --write-url-link Write a Windows internet shortcut file + (.url). Note that the OS caches the URL + based on the file path. + --write-webloc-link Write a macOS internet shortcut file + (.webloc) + --write-desktop-link Write a Linux internet shortcut file + (.desktop) + ## Verbosity / Simulation Options: -q, --quiet Activate quiet mode --no-warnings Ignore warnings @@ -385,7 +401,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo --ap-list-mso List all supported multiple-system operators -## Post-processing Options: +## Post-Processing Options: -x, --extract-audio Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe) @@ -497,7 +513,11 @@ The `-o` option allows users to indicate a template for the output file names. **tl;dr:** [navigate me to examples](#output-template-examples). -The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "https://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [python string formatting operations](https://docs.python.org/2/library/stdtypes.html#string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by a formatting operations. Allowed names along with sequence type are: +The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "https://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [Python string formatting operations](https://docs.python.org/2/library/stdtypes.html#string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by a formatting operation. + +You can, e.g., limit the number of characters of the title to prevent errors with too long filenames or file paths: `%(title).100s`. You may want to check the length of the rest of your filename or path and adapt the number accordingly. + +Allowed names along with sequence type are: - `id` (string): Video identifier - `title` (string): Video title diff --git a/test/parameters.json b/test/parameters.json index 7bf59c25f..2e186e2d5 100644 --- a/test/parameters.json +++ b/test/parameters.json @@ -35,6 +35,11 @@ "verbose": true, "writedescription": false, "writeinfojson": true, + "writeannotations": false, + "writelink": false, + "writeurllink": false, + "writewebloclink": false, + "writedesktoplink": false, "writesubtitles": false, "allsubtitles": false, "listssubtitles": false, diff --git a/test/test_compat.py b/test/test_compat.py index d6c54e135..63ae8e799 100644 --- a/test/test_compat.py +++ b/test/test_compat.py @@ -18,6 +18,8 @@ from youtube_dl.compat import ( compat_shlex_split, compat_str, compat_struct_unpack, + compat_urllib_parse_quote, + compat_urllib_parse_quote_plus, compat_urllib_parse_unquote, compat_urllib_parse_unquote_plus, compat_urllib_parse_urlencode, @@ -52,6 +54,29 @@ class TestCompat(unittest.TestCase): dir(youtube_dl.compat))) - set(['unicode_literals']) self.assertEqual(all_names, sorted(present_names)) + def test_compat_urllib_parse_quote(self): + self.assertEqual(compat_urllib_parse_quote('abc def'), 'abc%20def') + self.assertEqual(compat_urllib_parse_quote('/~user/abc+def'), '/%7Euser/abc%2Bdef') + self.assertEqual(compat_urllib_parse_quote('/~user/abc+def', safe='/~+'), '/~user/abc+def') + self.assertEqual(compat_urllib_parse_quote(''), '') + self.assertEqual(compat_urllib_parse_quote('%'), '%25') + self.assertEqual(compat_urllib_parse_quote('%', safe='%'), '%') + self.assertEqual(compat_urllib_parse_quote('津波'), '%E6%B4%A5%E6%B3%A2') + self.assertEqual( + compat_urllib_parse_quote( + ''' + %%a''', safe='<>=":%/ \r\n'), + ''' + %%a''') + self.assertEqual( + compat_urllib_parse_quote( + '''(^◣_◢^)っ︻デ═一 ⇀ ⇀ ⇀ ⇀ ⇀ ↶%I%Break%25Things%''', safe='% '), + '''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%86%B6%I%Break%25Things%''') + + def test_compat_urllib_parse_quote_plus(self): + self.assertEqual(compat_urllib_parse_quote_plus('abc def'), 'abc+def') + self.assertEqual(compat_urllib_parse_quote_plus('~/abc def'), '%7E%2Fabc+def') + def test_compat_urllib_parse_unquote(self): self.assertEqual(compat_urllib_parse_unquote('abc%20def'), 'abc def') self.assertEqual(compat_urllib_parse_unquote('%7e/abc+def'), '~/abc+def') @@ -63,12 +88,14 @@ class TestCompat(unittest.TestCase): self.assertEqual(compat_urllib_parse_unquote('%2f'), '/') self.assertEqual(compat_urllib_parse_unquote('%E6%B4%A5%E6%B3%A2'), '津波') self.assertEqual( - compat_urllib_parse_unquote(''' -%%a'''), + compat_urllib_parse_unquote( + ''' + %%a'''), ''' -%%a''') + %%a''') self.assertEqual( - compat_urllib_parse_unquote('''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%86%B6%I%Break%25Things%'''), + compat_urllib_parse_unquote( + '''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%86%B6%I%Break%25Things%'''), '''(^◣_◢^)っ︻デ═一 ⇀ ⇀ ⇀ ⇀ ⇀ ↶%I%Break%Things%''') def test_compat_urllib_parse_unquote_plus(self): diff --git a/test/test_utils.py b/test/test_utils.py index 14503ab53..f881f12a4 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -96,6 +96,7 @@ from youtube_dl.utils import ( cli_valueless_option, cli_bool_option, parse_codecs, + iri_to_uri, ) from youtube_dl.compat import ( compat_chr, @@ -1333,6 +1334,32 @@ Line 1 self.assertEqual(get_elements_by_attribute('class', 'foo bar', html), ['nice', 'also nice']) self.assertEqual(get_elements_by_attribute('class', 'foo', html), []) self.assertEqual(get_elements_by_attribute('class', 'no-such-foo', html), []) + + def test_iri_to_uri(self): + self.assertEqual( + iri_to_uri('https://www.google.com/search?q=foo&ie=utf-8&oe=utf-8&client=firefox-b'), + 'https://www.google.com/search?q=foo&ie=utf-8&oe=utf-8&client=firefox-b') # Same + self.assertEqual( + iri_to_uri('https://www.google.com/search?q=Käsesoßenrührlöffel'), # German for cheese sauce stirring spoon + 'https://www.google.com/search?q=K%C3%A4seso%C3%9Fenr%C3%BChrl%C3%B6ffel') + self.assertEqual( + iri_to_uri('https://www.google.com/search?q=lt<+gt>+eq%3D+amp%26+percent%25+hash%23+colon%3A+tilde~#trash=?&garbage=#'), + 'https://www.google.com/search?q=lt%3C+gt%3E+eq%3D+amp%26+percent%25+hash%23+colon%3A+tilde~#trash=?&garbage=#') + self.assertEqual( + iri_to_uri('http://правозащита38.рф/category/news/'), + 'http://xn--38-6kcaak9aj5chl4a3g.xn--p1ai/category/news/') + self.assertEqual( + iri_to_uri('http://www.правозащита38.рф/category/news/'), + 'http://www.xn--38-6kcaak9aj5chl4a3g.xn--p1ai/category/news/') + self.assertEqual( + iri_to_uri('https://i❤.ws/emojidomain/👍👏🤝💪'), + 'https://xn--i-7iq.ws/emojidomain/%F0%9F%91%8D%F0%9F%91%8F%F0%9F%A4%9D%F0%9F%92%AA') + self.assertEqual( + iri_to_uri('http://日本語.jp/'), + 'http://xn--wgv71a119e.jp/') + self.assertEqual( + iri_to_uri('http://导航.中国/'), + 'http://xn--fet810g.xn--fiqs8s/') if __name__ == '__main__': diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 046e03247..20fd53b69 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -49,6 +49,7 @@ from .utils import ( date_from_str, DateRange, DEFAULT_OUTTMPL, + DESKTOP_LINK_TEMPLATE, determine_ext, determine_protocol, DownloadError, @@ -61,6 +62,7 @@ from .utils import ( formatSeconds, GeoRestrictedError, int_or_none, + iri_to_uri, ISO3166Utils, locked_file, make_HTTPS_handler, @@ -83,9 +85,12 @@ from .utils import ( sanitized_Request, std_headers, subtitles_filename, + to_high_limit_path, UnavailableVideoError, url_basename, + URL_LINK_TEMPLATE, version_tuple, + WEBLOC_LINK_TEMPLATE, write_json_file, write_string, YoutubeDLCookieProcessor, @@ -178,6 +183,11 @@ class YoutubeDL(object): writeannotations: Write the video annotations to a .annotations.xml file writethumbnail: Write the thumbnail image to a file write_all_thumbnails: Write all thumbnail formats to files + writelink: Write an internet shortcut file, depending on the + current platform (.url/.webloc/.desktop) + writeurllink: Write a Windows internet shortcut file (.url) + writewebloclink: Write a macOS internet shortcut file (.webloc) + writedesktoplink: Write a Linux internet shortcut file (.desktop) writesubtitles: Write the video subtitles to a file writeautomaticsub: Write the automatically generated subtitles to a file allsubtitles: Downloads all the subtitles of the video @@ -204,7 +214,9 @@ class YoutubeDL(object): downloaded. None for no limit. download_archive: File name of a file where all downloads are recorded. Videos already present in the file are not downloaded - again. + again. When 'writelink' (or similar) and + 'skip_download' are also present, the videos will be + recorded, too. cookiefile: File name where cookies should be read from and dumped to. nocheckcertificate:Do not verify SSL certificates prefer_insecure: Use HTTP instead of HTTPS to retrieve information. @@ -1407,6 +1419,8 @@ class YoutubeDL(object): raise ExtractorError('Missing "id" field in extractor result') if 'title' not in info_dict: raise ExtractorError('Missing "title" field in extractor result') + if 'webpage_url' not in info_dict: + raise ExtractorError('Missing "webpage_url" field in extractor result. Should have been augmented with it.') def report_force_conversion(field, field_not, conversion): self.report_warning( @@ -1836,7 +1850,56 @@ class YoutubeDL(object): self._write_thumbnails(info_dict, filename) - if not self.params.get('skip_download', False): + # Write internet shortcut files + url_link = webloc_link = desktop_link = False + if self.params.get('writelink', False): + if sys.platform == "darwin": # macOS. + webloc_link = True + elif sys.platform.startswith("linux"): + desktop_link = True + else: # if sys.platform in ['win32', 'cygwin']: + url_link = True + if self.params.get('writeurllink', False): + url_link = True + if self.params.get('writewebloclink', False): + webloc_link = True + if self.params.get('writedesktoplink', False): + desktop_link = True + + if url_link or webloc_link or desktop_link: + ascii_url = iri_to_uri(info_dict['webpage_url']) + + def _write_link_file(extension, template, newline, embed_filename): + linkfn = replace_extension(filename, extension, info_dict.get('ext')) + if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(linkfn)): + self.to_screen('[info] Internet shortcut is already present') + else: + try: + self.to_screen('[info] Writing internet shortcut to: ' + linkfn) + with io.open(encodeFilename(to_high_limit_path(linkfn)), 'w', encoding='utf-8', newline=newline) as linkfile: + template_vars = { 'url': ascii_url } + if embed_filename: + template_vars['filename'] = linkfn[:-len(extension) - 1] + linkfile.write(template % template_vars) + except (OSError, IOError): + self.report_error('Cannot write internet shortcut ' + linkfn) + return False + return True + + if url_link: + if not _write_link_file('url', URL_LINK_TEMPLATE, '\r\n', embed_filename=False): return + if webloc_link: + if not _write_link_file('webloc', WEBLOC_LINK_TEMPLATE, '\n', embed_filename=False): return + if desktop_link: + if not _write_link_file('desktop', DESKTOP_LINK_TEMPLATE, '\n', embed_filename=True ): return + + if self.params.get('skip_download', False): + # Regarding the download archive, consider internet shortcut creation in conjunction with the `--skip-download` switch as everything the user wants. (See also help for the`--download-archive` switch.) + if url_link or webloc_link or desktop_link: + self.record_download_archive(info_dict) + + # Download + else: # No `--skip-download` try: def dl(name, info): fd = get_suitable_downloader(info, self.params)(self, self.params) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index ba435ea42..b05494c39 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -369,6 +369,10 @@ def _real_main(argv=None): 'writeinfojson': opts.writeinfojson, 'writethumbnail': opts.writethumbnail, 'write_all_thumbnails': opts.write_all_thumbnails, + 'writelink': opts.writelink, + 'writeurllink': opts.writeurllink, + 'writewebloclink': opts.writewebloclink, + 'writedesktoplink': opts.writedesktoplink, 'writesubtitles': opts.writesubtitles, 'writeautomaticsub': opts.writeautomaticsub, 'allsubtitles': opts.allsubtitles, diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 4a611f183..1268e7379 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -37,15 +37,20 @@ try: except ImportError: # Python 2 import urllib as compat_urllib_parse +try: + import urllib.parse as compat_urlparse +except ImportError: # Python 2 + import urlparse as compat_urlparse + try: from urllib.parse import urlparse as compat_urllib_parse_urlparse except ImportError: # Python 2 from urlparse import urlparse as compat_urllib_parse_urlparse try: - import urllib.parse as compat_urlparse + from urllib.parse import urlunparse as compat_urllib_parse_urlunparse except ImportError: # Python 2 - import urlparse as compat_urlparse + from urlparse import urlunparse as compat_urllib_parse_urlunparse try: import urllib.response as compat_urllib_response @@ -2354,6 +2359,20 @@ try: except NameError: compat_str = str +try: + from urllib.parse import quote as compat_urllib_parse_quote + from urllib.parse import quote_plus as compat_urllib_parse_quote_plus +except ImportError: # Python 2 + def compat_urllib_parse_quote(string, safe='/'): + return compat_urllib_parse.quote( + string.encode('utf-8'), + str(safe)) + + def compat_urllib_parse_quote_plus(string, safe=''): + return compat_urllib_parse.quote_plus( + string.encode('utf-8'), + str(safe)) + try: from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes from urllib.parse import unquote as compat_urllib_parse_unquote @@ -2992,11 +3011,14 @@ __all__ = [ 'compat_tokenize_tokenize', 'compat_urllib_error', 'compat_urllib_parse', + 'compat_urllib_parse_quote', + 'compat_urllib_parse_quote_plus', 'compat_urllib_parse_unquote', 'compat_urllib_parse_unquote_plus', 'compat_urllib_parse_unquote_to_bytes', 'compat_urllib_parse_urlencode', 'compat_urllib_parse_urlparse', + 'compat_urllib_parse_urlunparse', 'compat_urllib_request', 'compat_urllib_request_DataHandler', 'compat_urllib_response', diff --git a/youtube_dl/options.py b/youtube_dl/options.py index b692c6b3b..54d44af36 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -343,7 +343,7 @@ def parseOpts(overrideArguments=None): selection.add_option( '--download-archive', metavar='FILE', dest='download_archive', - help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.') + help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it. When the switches --write-link (or similar) and --skip-download are used additionally, the IDs will also be recorded, even though nothing was actually downloaded.') selection.add_option( '--include-ads', dest='include_ads', action='store_true', @@ -764,7 +764,7 @@ def parseOpts(overrideArguments=None): action='store_true', dest='rm_cachedir', help='Delete all filesystem cache files') - thumbnail = optparse.OptionGroup(parser, 'Thumbnail images') + thumbnail = optparse.OptionGroup(parser, 'Thumbnail Images') thumbnail.add_option( '--write-thumbnail', action='store_true', dest='writethumbnail', default=False, @@ -778,7 +778,25 @@ def parseOpts(overrideArguments=None): action='store_true', dest='list_thumbnails', default=False, help='Simulate and list all available thumbnail formats') - postproc = optparse.OptionGroup(parser, 'Post-processing Options') + link = optparse.OptionGroup(parser, 'Internet Shortcut Options') + link.add_option( + '--write-link', + action='store_true', dest='writelink', default=False, + help='Write an internet shortcut file, depending on the current platform (.url/.webloc/.desktop). The URL may be cached by the OS.') + link.add_option( + '--write-url-link', + action='store_true', dest='writeurllink', default=False, + help='Write a Windows internet shortcut file (.url). Note that the OS caches the URL based on the file path.') + link.add_option( + '--write-webloc-link', + action='store_true', dest='writewebloclink', default=False, + help='Write a macOS internet shortcut file (.webloc)') + link.add_option( + '--write-desktop-link', + action='store_true', dest='writedesktoplink', default=False, + help='Write a Linux internet shortcut file (.desktop)') + + postproc = optparse.OptionGroup(parser, 'Post-Processing Options') postproc.add_option( '-x', '--extract-audio', action='store_true', dest='extractaudio', default=False, @@ -866,6 +884,7 @@ def parseOpts(overrideArguments=None): parser.add_option_group(downloader) parser.add_option_group(filesystem) parser.add_option_group(thumbnail) + parser.add_option_group(link) parser.add_option_group(verbosity) parser.add_option_group(workarounds) parser.add_option_group(video_format) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index f9ca63c58..f1aa620fc 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -57,6 +57,9 @@ from .compat import ( compat_urllib_parse, compat_urllib_parse_urlencode, compat_urllib_parse_urlparse, + compat_urllib_parse_urlunparse, + compat_urllib_parse_quote, + compat_urllib_parse_quote_plus, compat_urllib_parse_unquote_plus, compat_urllib_request, compat_urlparse, @@ -3902,3 +3905,82 @@ def random_birthday(year_field, month_field, day_field): month_field: str(random.randint(1, 12)), day_field: str(random.randint(1, 31)), } + + +# Templates for internet shortcut files, which are plain text files. +URL_LINK_TEMPLATE = ''' +[InternetShortcut] +URL=%(url)s +'''.lstrip() + +WEBLOC_LINK_TEMPLATE = ''' + + + + +\tURL +\t%(url)s + + +'''.lstrip() + +DESKTOP_LINK_TEMPLATE = ''' +[Desktop Entry] +Encoding=UTF-8 +Name=Link to %(filename)s +Type=Link +URL=%(url)s +Icon=text-html +'''.lstrip() + + +def iri_to_uri(iri): + """ + Converts an IRI (Internationalized Resource Identifier, allowing Unicode characters) to a URI (Uniform Resource Identifier, ASCII-only). + + The function doesn't add an additional layer of escaping; e.g., it doesn't escape `%3C` as `%253C`. Instead, it percent-escapes characters with an underlying UTF-8 encoding *besides* those already escaped, leaving the URI intact. + """ + + iri_parts = compat_urllib_parse_urlparse(iri) + + if '[' in iri_parts.netloc: + raise ValueError('IPv6 URIs are not, yet, supported.') + # Querying `.netloc`, when there's only one bracket, also raises a ValueError. + + # The `safe` argument values, that the following code uses, contain the characters that should not be percent-encoded. Everything else but letters, digits and '_.-' will be percent-encoded with an underlying UTF-8 encoding. Everything already percent-encoded will be left as is. + + net_location = '' + if iri_parts.username: + net_location += compat_urllib_parse_quote(iri_parts.username, safe=r"!$%&'()*+,~") + if iri_parts.password is not None: + net_location += ':' + compat_urllib_parse_quote(iri_parts.password, safe=r"!$%&'()*+,~") + net_location += '@' + + net_location += iri_parts.hostname.encode('idna').decode('utf-8') # Punycode for Unicode hostnames. + # The 'idna' encoding produces ASCII text. + if iri_parts.port is not None and iri_parts.port != 80: + net_location += ':' + str(iri_parts.port) + + return compat_urllib_parse_urlunparse( ( + iri_parts.scheme, + net_location, + + compat_urllib_parse_quote_plus(iri_parts.path, safe=r"!$%&'()*+,/:;=@|~"), + + # Unsure about the `safe` argument , since this is a legacy way of handling parameters. + compat_urllib_parse_quote_plus(iri_parts.params, safe=r"!$%&'()*+,/:;=@|~"), + + # Not totally sure about the `safe` argument, since the source does not explicitly mention the query URI component. + compat_urllib_parse_quote_plus(iri_parts.query, safe=r"!$%&'()*+,/:;=?@{|}~"), + + compat_urllib_parse_quote_plus(iri_parts.fragment, safe=r"!#$%&'()*+,/:;=?@{|}~")) ) + + # Source for `safe` arguments: https://url.spec.whatwg.org/#percent-encoded-bytes. + + +def to_high_limit_path(path): + if sys.platform in ['win32', 'cygwin']: + # Work around MAX_PATH limitation on Windows. The maximum allowed length for the individual path segments may still be quite limited. + return r'\\?\ '.rstrip() + os.path.abspath(path) + + return path From e662c030701bfd1f23494ff112e64c2cb99178b4 Mon Sep 17 00:00:00 2001 From: Henrik Hank Date: Mon, 14 May 2018 23:31:22 +0200 Subject: [PATCH 02/10] More flake8 compatibility. --- test/test_compat.py | 22 +++++++++------------- test/test_utils.py | 2 +- youtube_dl/YoutubeDL.py | 6 +++--- youtube_dl/compat.py | 4 ++-- youtube_dl/utils.py | 26 +++++++++++++------------- 5 files changed, 28 insertions(+), 32 deletions(-) diff --git a/test/test_compat.py b/test/test_compat.py index 63ae8e799..b25bcb44c 100644 --- a/test/test_compat.py +++ b/test/test_compat.py @@ -63,16 +63,14 @@ class TestCompat(unittest.TestCase): self.assertEqual(compat_urllib_parse_quote('%', safe='%'), '%') self.assertEqual(compat_urllib_parse_quote('津波'), '%E6%B4%A5%E6%B3%A2') self.assertEqual( - compat_urllib_parse_quote( - ''' - %%a''', safe='<>=":%/ \r\n'), + compat_urllib_parse_quote(''' +%%a''', safe='<>=":%/ \r\n'), ''' - %%a''') +%%a''') self.assertEqual( - compat_urllib_parse_quote( - '''(^◣_◢^)っ︻デ═一 ⇀ ⇀ ⇀ ⇀ ⇀ ↶%I%Break%25Things%''', safe='% '), + compat_urllib_parse_quote('''(^◣_◢^)っ︻デ═一 ⇀ ⇀ ⇀ ⇀ ⇀ ↶%I%Break%25Things%''', safe='% '), '''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%86%B6%I%Break%25Things%''') - + def test_compat_urllib_parse_quote_plus(self): self.assertEqual(compat_urllib_parse_quote_plus('abc def'), 'abc+def') self.assertEqual(compat_urllib_parse_quote_plus('~/abc def'), '%7E%2Fabc+def') @@ -88,14 +86,12 @@ class TestCompat(unittest.TestCase): self.assertEqual(compat_urllib_parse_unquote('%2f'), '/') self.assertEqual(compat_urllib_parse_unquote('%E6%B4%A5%E6%B3%A2'), '津波') self.assertEqual( - compat_urllib_parse_unquote( - ''' - %%a'''), + compat_urllib_parse_unquote(''' +%%a'''), ''' - %%a''') +%%a''') self.assertEqual( - compat_urllib_parse_unquote( - '''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%86%B6%I%Break%25Things%'''), + compat_urllib_parse_unquote('''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%86%B6%I%Break%25Things%'''), '''(^◣_◢^)っ︻デ═一 ⇀ ⇀ ⇀ ⇀ ⇀ ↶%I%Break%Things%''') def test_compat_urllib_parse_unquote_plus(self): diff --git a/test/test_utils.py b/test/test_utils.py index f881f12a4..f377e6405 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1334,7 +1334,7 @@ Line 1 self.assertEqual(get_elements_by_attribute('class', 'foo bar', html), ['nice', 'also nice']) self.assertEqual(get_elements_by_attribute('class', 'foo', html), []) self.assertEqual(get_elements_by_attribute('class', 'no-such-foo', html), []) - + def test_iri_to_uri(self): self.assertEqual( iri_to_uri('https://www.google.com/search?q=foo&ie=utf-8&oe=utf-8&client=firefox-b'), diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 20fd53b69..98d4e58ee 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1877,7 +1877,7 @@ class YoutubeDL(object): try: self.to_screen('[info] Writing internet shortcut to: ' + linkfn) with io.open(encodeFilename(to_high_limit_path(linkfn)), 'w', encoding='utf-8', newline=newline) as linkfile: - template_vars = { 'url': ascii_url } + template_vars = {'url': ascii_url} if embed_filename: template_vars['filename'] = linkfn[:-len(extension) - 1] linkfile.write(template % template_vars) @@ -1892,12 +1892,12 @@ class YoutubeDL(object): if not _write_link_file('webloc', WEBLOC_LINK_TEMPLATE, '\n', embed_filename=False): return if desktop_link: if not _write_link_file('desktop', DESKTOP_LINK_TEMPLATE, '\n', embed_filename=True ): return - + if self.params.get('skip_download', False): # Regarding the download archive, consider internet shortcut creation in conjunction with the `--skip-download` switch as everything the user wants. (See also help for the`--download-archive` switch.) if url_link or webloc_link or desktop_link: self.record_download_archive(info_dict) - + # Download else: # No `--skip-download` try: diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 1268e7379..f4d23dc5a 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -50,7 +50,7 @@ except ImportError: # Python 2 try: from urllib.parse import urlunparse as compat_urllib_parse_urlunparse except ImportError: # Python 2 - from urlparse import urlunparse as compat_urllib_parse_urlunparse + from urlparse import urlunparse as compat_urllib_parse_urlunparse try: import urllib.response as compat_urllib_response @@ -2360,7 +2360,7 @@ except NameError: compat_str = str try: - from urllib.parse import quote as compat_urllib_parse_quote + from urllib.parse import quote as compat_urllib_parse_quote from urllib.parse import quote_plus as compat_urllib_parse_quote_plus except ImportError: # Python 2 def compat_urllib_parse_quote(string, safe='/'): diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index f1aa620fc..46a12d342 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -3951,29 +3951,29 @@ def iri_to_uri(iri): net_location = '' if iri_parts.username: - net_location += compat_urllib_parse_quote(iri_parts.username, safe=r"!$%&'()*+,~") + net_location += compat_urllib_parse_quote(iri_parts.username, safe=r"!$%&'()*+,~") if iri_parts.password is not None: net_location += ':' + compat_urllib_parse_quote(iri_parts.password, safe=r"!$%&'()*+,~") net_location += '@' - + net_location += iri_parts.hostname.encode('idna').decode('utf-8') # Punycode for Unicode hostnames. # The 'idna' encoding produces ASCII text. if iri_parts.port is not None and iri_parts.port != 80: net_location += ':' + str(iri_parts.port) - - return compat_urllib_parse_urlunparse( ( - iri_parts.scheme, - net_location, - compat_urllib_parse_quote_plus(iri_parts.path, safe=r"!$%&'()*+,/:;=@|~"), + return compat_urllib_parse_urlunparse( + (iri_parts.scheme, + net_location, - # Unsure about the `safe` argument , since this is a legacy way of handling parameters. - compat_urllib_parse_quote_plus(iri_parts.params, safe=r"!$%&'()*+,/:;=@|~"), + compat_urllib_parse_quote_plus(iri_parts.path, safe=r"!$%&'()*+,/:;=@|~"), - # Not totally sure about the `safe` argument, since the source does not explicitly mention the query URI component. - compat_urllib_parse_quote_plus(iri_parts.query, safe=r"!$%&'()*+,/:;=?@{|}~"), + # Unsure about the `safe` argument , since this is a legacy way of handling parameters. + compat_urllib_parse_quote_plus(iri_parts.params, safe=r"!$%&'()*+,/:;=@|~"), - compat_urllib_parse_quote_plus(iri_parts.fragment, safe=r"!#$%&'()*+,/:;=?@{|}~")) ) + # Not totally sure about the `safe` argument, since the source does not explicitly mention the query URI component. + compat_urllib_parse_quote_plus(iri_parts.query, safe=r"!$%&'()*+,/:;=?@{|}~"), + + compat_urllib_parse_quote_plus(iri_parts.fragment, safe=r"!#$%&'()*+,/:;=?@{|}~"))) # Source for `safe` arguments: https://url.spec.whatwg.org/#percent-encoded-bytes. @@ -3982,5 +3982,5 @@ def to_high_limit_path(path): if sys.platform in ['win32', 'cygwin']: # Work around MAX_PATH limitation on Windows. The maximum allowed length for the individual path segments may still be quite limited. return r'\\?\ '.rstrip() + os.path.abspath(path) - + return path From ecfc5538439893d042911937841b21dae92b0362 Mon Sep 17 00:00:00 2001 From: Henrik Hank Date: Tue, 15 May 2018 01:04:40 +0200 Subject: [PATCH 03/10] Satisfied tests in `test/test_YoutubeDL.py`. --- test/test_YoutubeDL.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index f0f5a8470..6733b8811 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -42,6 +42,7 @@ def _make_result(formats, **kwargs): 'title': 'testttitle', 'extractor': 'testex', 'extractor_key': 'TestEx', + 'webpage_url': 'http://example.com/watch?v=shenanigans', } res.update(**kwargs) return res @@ -497,6 +498,7 @@ class TestYoutubeDL(unittest.TestCase): 'subtitles': subtitles, 'automatic_captions': auto_captions, 'extractor': 'TEST', + 'webpage_url': 'http://example.com/watch?v=shenanigans', } def get_info(params={}): @@ -660,6 +662,7 @@ class TestYoutubeDL(unittest.TestCase): 'playlist_id': '42', 'uploader': "變態妍字幕版 太妍 тест", 'creator': "тест ' 123 ' тест--", + 'webpage_url': 'http://example.com/watch?v=shenanigans', } second = { 'id': '2', @@ -671,6 +674,7 @@ class TestYoutubeDL(unittest.TestCase): 'filesize': 5 * 1024, 'playlist_id': '43', 'uploader': "тест 123", + 'webpage_url': 'http://example.com/watch?v=SHENANIGANS', } videos = [first, second] From c5b79181cca1a6f7407cfea07f8305f266d48afd Mon Sep 17 00:00:00 2001 From: Henrik Hank Date: Tue, 15 May 2018 03:53:14 +0200 Subject: [PATCH 04/10] Changed internet shortcut file template for Linux. A test under Ubuntu showed, that "Link to " is actually displayed contrary to what a previously consulted source seemed to imply. --- youtube_dl/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 46a12d342..84cc54042 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -3927,7 +3927,7 @@ WEBLOC_LINK_TEMPLATE = ''' DESKTOP_LINK_TEMPLATE = ''' [Desktop Entry] Encoding=UTF-8 -Name=Link to %(filename)s +Name=%(filename)s Type=Link URL=%(url)s Icon=text-html From 57d10df3d6fffe34f25e254704faf23b9f310d77 Mon Sep 17 00:00:00 2001 From: Henrik Hank Date: Tue, 15 May 2018 05:02:38 +0200 Subject: [PATCH 05/10] Superfluous space. --- youtube_dl/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 84cc54042..049ad7704 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -3967,7 +3967,7 @@ def iri_to_uri(iri): compat_urllib_parse_quote_plus(iri_parts.path, safe=r"!$%&'()*+,/:;=@|~"), - # Unsure about the `safe` argument , since this is a legacy way of handling parameters. + # Unsure about the `safe` argument, since this is a legacy way of handling parameters. compat_urllib_parse_quote_plus(iri_parts.params, safe=r"!$%&'()*+,/:;=@|~"), # Not totally sure about the `safe` argument, since the source does not explicitly mention the query URI component. From 19daf26c088c9259f13f2c54e563534b62ef4945 Mon Sep 17 00:00:00 2001 From: Henrik Hank Date: Wed, 16 May 2018 02:58:57 +0200 Subject: [PATCH 06/10] `flake8` compatibility. --- test/test_utils.py | 16 ++++++++-------- youtube_dl/YoutubeDL.py | 9 ++++++--- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/test/test_utils.py b/test/test_utils.py index f377e6405..efe354477 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1338,28 +1338,28 @@ Line 1 def test_iri_to_uri(self): self.assertEqual( iri_to_uri('https://www.google.com/search?q=foo&ie=utf-8&oe=utf-8&client=firefox-b'), - 'https://www.google.com/search?q=foo&ie=utf-8&oe=utf-8&client=firefox-b') # Same + 'https://www.google.com/search?q=foo&ie=utf-8&oe=utf-8&client=firefox-b') # Same self.assertEqual( iri_to_uri('https://www.google.com/search?q=Käsesoßenrührlöffel'), # German for cheese sauce stirring spoon - 'https://www.google.com/search?q=K%C3%A4seso%C3%9Fenr%C3%BChrl%C3%B6ffel') + 'https://www.google.com/search?q=K%C3%A4seso%C3%9Fenr%C3%BChrl%C3%B6ffel') self.assertEqual( iri_to_uri('https://www.google.com/search?q=lt<+gt>+eq%3D+amp%26+percent%25+hash%23+colon%3A+tilde~#trash=?&garbage=#'), - 'https://www.google.com/search?q=lt%3C+gt%3E+eq%3D+amp%26+percent%25+hash%23+colon%3A+tilde~#trash=?&garbage=#') + 'https://www.google.com/search?q=lt%3C+gt%3E+eq%3D+amp%26+percent%25+hash%23+colon%3A+tilde~#trash=?&garbage=#') self.assertEqual( iri_to_uri('http://правозащита38.рф/category/news/'), - 'http://xn--38-6kcaak9aj5chl4a3g.xn--p1ai/category/news/') + 'http://xn--38-6kcaak9aj5chl4a3g.xn--p1ai/category/news/') self.assertEqual( iri_to_uri('http://www.правозащита38.рф/category/news/'), - 'http://www.xn--38-6kcaak9aj5chl4a3g.xn--p1ai/category/news/') + 'http://www.xn--38-6kcaak9aj5chl4a3g.xn--p1ai/category/news/') self.assertEqual( iri_to_uri('https://i❤.ws/emojidomain/👍👏🤝💪'), - 'https://xn--i-7iq.ws/emojidomain/%F0%9F%91%8D%F0%9F%91%8F%F0%9F%A4%9D%F0%9F%92%AA') + 'https://xn--i-7iq.ws/emojidomain/%F0%9F%91%8D%F0%9F%91%8F%F0%9F%A4%9D%F0%9F%92%AA') self.assertEqual( iri_to_uri('http://日本語.jp/'), - 'http://xn--wgv71a119e.jp/') + 'http://xn--wgv71a119e.jp/') self.assertEqual( iri_to_uri('http://导航.中国/'), - 'http://xn--fet810g.xn--fiqs8s/') + 'http://xn--fet810g.xn--fiqs8s/') if __name__ == '__main__': diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 98d4e58ee..24b660f93 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1887,11 +1887,14 @@ class YoutubeDL(object): return True if url_link: - if not _write_link_file('url', URL_LINK_TEMPLATE, '\r\n', embed_filename=False): return + if not _write_link_file('url', URL_LINK_TEMPLATE, '\r\n', embed_filename=False): + return if webloc_link: - if not _write_link_file('webloc', WEBLOC_LINK_TEMPLATE, '\n', embed_filename=False): return + if not _write_link_file('webloc', WEBLOC_LINK_TEMPLATE, '\n', embed_filename=False): + return if desktop_link: - if not _write_link_file('desktop', DESKTOP_LINK_TEMPLATE, '\n', embed_filename=True ): return + if not _write_link_file('desktop', DESKTOP_LINK_TEMPLATE, '\n', embed_filename=True): + return if self.params.get('skip_download', False): # Regarding the download archive, consider internet shortcut creation in conjunction with the `--skip-download` switch as everything the user wants. (See also help for the`--download-archive` switch.) From 56688abb4d2455d207c4c481636a4ce16b23e280 Mon Sep 17 00:00:00 2001 From: Henrik Hank Date: Wed, 16 May 2018 03:00:42 +0200 Subject: [PATCH 07/10] Reverted `README.md` as far as the introduced information is available elsewhere. --- README.md | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index e4a13e6b8..de542488f 100644 --- a/README.md +++ b/README.md @@ -176,11 +176,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo age --download-archive FILE Download only videos not listed in the archive file. Record the IDs of all - downloaded videos in it. When the switches - --write-link (or similar) and - --skip-download are used additionally, the - IDs will also be recorded, even though - nothing was actually downloaded. + downloaded videos in it. --include-ads Download advertisements as well (experimental) @@ -272,24 +268,12 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo --no-cache-dir Disable filesystem caching --rm-cache-dir Delete all filesystem cache files -## Thumbnail Images: +## Thumbnail images: --write-thumbnail Write thumbnail image to disk --write-all-thumbnails Write all thumbnail image formats to disk --list-thumbnails Simulate and list all available thumbnail formats -## Internet Shortcut Options: - --write-link Write an internet shortcut file, depending - on the current platform (.url/.webloc/ - .desktop). The URL may be cached by the OS. - --write-url-link Write a Windows internet shortcut file - (.url). Note that the OS caches the URL - based on the file path. - --write-webloc-link Write a macOS internet shortcut file - (.webloc) - --write-desktop-link Write a Linux internet shortcut file - (.desktop) - ## Verbosity / Simulation Options: -q, --quiet Activate quiet mode --no-warnings Ignore warnings @@ -401,7 +385,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo --ap-list-mso List all supported multiple-system operators -## Post-Processing Options: +## Post-processing Options: -x, --extract-audio Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe) From 6cbe83852f2c462ffb2a5662869cf9a145dbd716 Mon Sep 17 00:00:00 2001 From: Henrik Hank Date: Wed, 16 May 2018 07:03:52 +0200 Subject: [PATCH 08/10] New switch `--force-write-download-archive` instead of modifying the behavior of `--download-archive`. --- test/parameters.json | 1 + youtube_dl/YoutubeDL.py | 29 ++++++++++++++++------------- youtube_dl/__init__.py | 1 + youtube_dl/options.py | 9 ++++++--- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/test/parameters.json b/test/parameters.json index 2e186e2d5..1b7aa2ccd 100644 --- a/test/parameters.json +++ b/test/parameters.json @@ -7,6 +7,7 @@ "forcethumbnail": false, "forcetitle": false, "forceurl": false, + "force_write_download_archive": false, "format": "best", "ignoreerrors": false, "listformats": null, diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 24b660f93..8f2615b8f 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -162,6 +162,8 @@ class YoutubeDL(object): forcejson: Force printing info_dict as JSON. dump_single_json: Force printing the info_dict of the whole playlist (or video) as a single JSON line. + force_write_download_archive: Force writing download archive regardless of + 'skip_download' or 'simulate'. simulate: Do not download the video files. format: Video format code. See options.py for more information. outtmpl: Template for output names. @@ -214,9 +216,7 @@ class YoutubeDL(object): downloaded. None for no limit. download_archive: File name of a file where all downloads are recorded. Videos already present in the file are not downloaded - again. When 'writelink' (or similar) and - 'skip_download' are also present, the videos will be - recorded, too. + again. cookiefile: File name where cookies should be read from and dumped to. nocheckcertificate:Do not verify SSL certificates prefer_insecure: Use HTTP instead of HTTPS to retrieve information. @@ -1419,8 +1419,6 @@ class YoutubeDL(object): raise ExtractorError('Missing "id" field in extractor result') if 'title' not in info_dict: raise ExtractorError('Missing "title" field in extractor result') - if 'webpage_url' not in info_dict: - raise ExtractorError('Missing "webpage_url" field in extractor result. Should have been augmented with it.') def report_force_conversion(field, field_not, conversion): self.report_warning( @@ -1751,8 +1749,11 @@ class YoutubeDL(object): if self.params.get('forcejson', False): self.to_stdout(json.dumps(info_dict)) - # Do nothing else if in simulate mode if self.params.get('simulate', False): + if self.params.get('force_write_download_archive', False): + self.record_download_archive(info_dict) + + # Do nothing else if in simulate mode return if filename is None: @@ -1867,6 +1868,9 @@ class YoutubeDL(object): desktop_link = True if url_link or webloc_link or desktop_link: + if 'webpage_url' not in info_dict: + self.report_error('Cannot write internet shortcut file because the "webpage_url" field is missing in the media information') + return ascii_url = iri_to_uri(info_dict['webpage_url']) def _write_link_file(extension, template, newline, embed_filename): @@ -1896,13 +1900,9 @@ class YoutubeDL(object): if not _write_link_file('desktop', DESKTOP_LINK_TEMPLATE, '\n', embed_filename=True): return - if self.params.get('skip_download', False): - # Regarding the download archive, consider internet shortcut creation in conjunction with the `--skip-download` switch as everything the user wants. (See also help for the`--download-archive` switch.) - if url_link or webloc_link or desktop_link: - self.record_download_archive(info_dict) - # Download - else: # No `--skip-download` + must_record_download_archive = False + if not self.params.get('skip_download', False): try: def dl(name, info): fd = get_suitable_downloader(info, self.params)(self, self.params) @@ -2049,7 +2049,10 @@ class YoutubeDL(object): except (PostProcessingError) as err: self.report_error('postprocessing: %s' % str(err)) return - self.record_download_archive(info_dict) + must_record_download_archive = True + + if must_record_download_archive or self.params.get('force_write_download_archive', False): + self.record_download_archive(info_dict) def download(self, url_list): """Download a given list of URLs.""" diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index b05494c39..792ff2d59 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -333,6 +333,7 @@ def _real_main(argv=None): 'forceformat': opts.getformat, 'forcejson': opts.dumpjson or opts.print_json, 'dump_single_json': opts.dump_single_json, + 'force_write_download_archive': opts.force_write_download_archive, 'simulate': opts.simulate or any_getting, 'skip_download': opts.skip_download, 'format': opts.format, diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 54d44af36..a17ee6322 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -343,7 +343,7 @@ def parseOpts(overrideArguments=None): selection.add_option( '--download-archive', metavar='FILE', dest='download_archive', - help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it. When the switches --write-link (or similar) and --skip-download are used additionally, the IDs will also be recorded, even though nothing was actually downloaded.') + help='Download only videos not listed in the archive file. Record the IDs of all downloaded videos in it.') selection.add_option( '--include-ads', dest='include_ads', action='store_true', @@ -633,8 +633,11 @@ def parseOpts(overrideArguments=None): verbosity.add_option( '--print-json', action='store_true', dest='print_json', default=False, - help='Be quiet and print the video information as JSON (video is still being downloaded).', - ) + help='Be quiet and print the video information as JSON (video is still being downloaded).') + verbosity.add_option( + '--force-write-download-archive', + action='store_true', dest='force_write_download_archive', default=False, + help='Force download archive entries to be written as far as no errors occur, even though --skip-download or any simulation switch is used.') verbosity.add_option( '--newline', action='store_true', dest='progress_with_newline', default=False, From a4642440fd37b85f6ef80d7d664141490b4b64db Mon Sep 17 00:00:00 2001 From: Henrik Hank Date: Sat, 26 May 2018 03:42:39 +0200 Subject: [PATCH 09/10] `DOT_` in front of link template constants --- youtube_dl/YoutubeDL.py | 12 ++++++------ youtube_dl/utils.py | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 8f2615b8f..23af35a66 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -49,9 +49,11 @@ from .utils import ( date_from_str, DateRange, DEFAULT_OUTTMPL, - DESKTOP_LINK_TEMPLATE, determine_ext, determine_protocol, + DOT_DESKTOP_LINK_TEMPLATE, + DOT_URL_LINK_TEMPLATE, + DOT_WEBLOC_LINK_TEMPLATE, DownloadError, encode_compat_str, encodeFilename, @@ -88,9 +90,7 @@ from .utils import ( to_high_limit_path, UnavailableVideoError, url_basename, - URL_LINK_TEMPLATE, version_tuple, - WEBLOC_LINK_TEMPLATE, write_json_file, write_string, YoutubeDLCookieProcessor, @@ -1891,13 +1891,13 @@ class YoutubeDL(object): return True if url_link: - if not _write_link_file('url', URL_LINK_TEMPLATE, '\r\n', embed_filename=False): + if not _write_link_file('url', DOT_URL_LINK_TEMPLATE, '\r\n', embed_filename=False): return if webloc_link: - if not _write_link_file('webloc', WEBLOC_LINK_TEMPLATE, '\n', embed_filename=False): + if not _write_link_file('webloc', DOT_WEBLOC_LINK_TEMPLATE, '\n', embed_filename=False): return if desktop_link: - if not _write_link_file('desktop', DESKTOP_LINK_TEMPLATE, '\n', embed_filename=True): + if not _write_link_file('desktop', DOT_DESKTOP_LINK_TEMPLATE, '\n', embed_filename=True): return # Download diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 049ad7704..2de062e8f 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -3908,12 +3908,12 @@ def random_birthday(year_field, month_field, day_field): # Templates for internet shortcut files, which are plain text files. -URL_LINK_TEMPLATE = ''' +DOT_URL_LINK_TEMPLATE = ''' [InternetShortcut] URL=%(url)s '''.lstrip() -WEBLOC_LINK_TEMPLATE = ''' +DOT_WEBLOC_LINK_TEMPLATE = ''' @@ -3924,7 +3924,7 @@ WEBLOC_LINK_TEMPLATE = ''' '''.lstrip() -DESKTOP_LINK_TEMPLATE = ''' +DOT_DESKTOP_LINK_TEMPLATE = ''' [Desktop Entry] Encoding=UTF-8 Name=%(filename)s From a949b3a4069947faf2528fbc34474590a1379f7c Mon Sep 17 00:00:00 2001 From: Henrik Hank Date: Sat, 26 May 2018 03:50:40 +0200 Subject: [PATCH 10/10] Slightly more intuitive way of stripping off extension --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 23af35a66..1ccec7852 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1883,7 +1883,7 @@ class YoutubeDL(object): with io.open(encodeFilename(to_high_limit_path(linkfn)), 'w', encoding='utf-8', newline=newline) as linkfile: template_vars = {'url': ascii_url} if embed_filename: - template_vars['filename'] = linkfn[:-len(extension) - 1] + template_vars['filename'] = linkfn[:-(len(extension) + 1)] linkfile.write(template % template_vars) except (OSError, IOError): self.report_error('Cannot write internet shortcut ' + linkfn)