From 071275116555a005fdf9e4b0ce2dcf5acb2416b9 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Mon, 30 Nov 2015 00:57:00 +0800 Subject: [PATCH 01/53] Introduce GNU gettext infrastructure --- .gitignore | 4 +++- Makefile | 20 +++++++++++++++++++- youtube_dl/utils.py | 9 +++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 0422adf44..358ea3bd7 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,6 @@ test/testdata .tox youtube-dl.zsh .idea -.idea/* \ No newline at end of file +.idea/* +*.mo +*.po.old diff --git a/Makefile b/Makefile index f826c1685..13378fbf9 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ offlinetest: codetest tar: youtube-dl.tar.gz -.PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion ot offlinetest codetest supportedsites +.PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion ot offlinetest codetest supportedsites update-po update-gmo pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 youtube-dl.fish @@ -109,3 +109,21 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash- Makefile MANIFEST.in youtube-dl.1 youtube-dl.bash-completion \ youtube-dl.zsh youtube-dl.fish setup.py \ youtube-dl + +update-po: po/youtube_dl.pot + for file in po/*.po ; do \ + lang=$$(echo $$file | sed -e 's#.*/\([^/]\+\).po#\1#') ; \ + mv $$file $$file.old ; \ + msgmerge -N $$file.old po/youtube_dl.pot > $$file ; \ + done + +po/youtube_dl.pot: youtube_dl/*.py youtube_dl/*/*.py + touch po/youtube_dl.pot && \ + xgettext -d youtube_dl -j -k -kg --from-code=utf-8 -o $@ youtube_dl/*.py youtube_dl/*/*.py + +update-gmo: + for file in po/*.po ; do \ + lang=$$(echo $$file | sed -e 's#.*/\([^/]\+\).po#\1#') ; \ + install -d share/locale/$$lang/LC_MESSAGES; \ + msgfmt -o share/locale/$$lang/LC_MESSAGES/youtube_dl.mo $$file; \ + done diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index d0606b4bc..ef034eb9b 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -12,6 +12,7 @@ import datetime import email.utils import errno import functools +import gettext import gzip import itertools import io @@ -2512,6 +2513,14 @@ class ISO3166Utils(object): return cls._country_map.get(code.upper()) +def g(s): + if gettext.textdomain() != 'youtube_dl': + gettext.textdomain('youtube_dl') + gettext.bindtextdomain('youtube_dl', localedir='share/locale/') + + return gettext.gettext(s) + + class PerRequestProxyHandler(compat_urllib_request.ProxyHandler): def __init__(self, proxies=None): # Set default handlers From e88797e4636a30754dc8a1ccbfaefb2efbd0fd8f Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Mon, 30 Nov 2015 00:57:33 +0800 Subject: [PATCH 02/53] Use gettext for some strings --- youtube_dl/YoutubeDL.py | 9 +++++---- youtube_dl/downloader/common.py | 5 +++-- youtube_dl/extractor/common.py | 3 ++- youtube_dl/extractor/testurl.py | 7 +++++-- youtube_dl/extractor/youtube.py | 9 +++++---- youtube_dl/postprocessor/ffmpeg.py | 3 ++- 6 files changed, 22 insertions(+), 14 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index c642a1fbf..04fb4ba08 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -51,6 +51,7 @@ from .utils import ( ExtractorError, format_bytes, formatSeconds, + g, locked_file, make_HTTPS_handler, MaxDownloadsReached, @@ -1545,7 +1546,7 @@ class YoutubeDL(object): for ph in self._progress_hooks: fd.add_progress_hook(ph) if self.params.get('verbose'): - self.to_stdout('[debug] Invoking downloader on %r' % info.get('url')) + self.to_stdout(g('[debug] Invoking downloader on %r') % info.get('url')) return fd.download(name, info) if info_dict.get('requested_formats') is not None: @@ -1589,8 +1590,8 @@ class YoutubeDL(object): filename = '%s.%s' % (filename_wo_ext, info_dict['ext']) if os.path.exists(encodeFilename(filename)): self.to_screen( - '[download] %s has already been downloaded and ' - 'merged' % filename) + g('[download] %s has already been downloaded and ' + 'merged') % filename) else: for f in requested_formats: new_info = dict(info_dict) @@ -1723,7 +1724,7 @@ class YoutubeDL(object): self.report_error(e.msg) if files_to_delete and not self.params.get('keepvideo', False): for old_filename in files_to_delete: - self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename) + self.to_screen(g('Deleting original file %s (pass -k to keep)') % old_filename) try: os.remove(encodeFilename(old_filename)) except (IOError, OSError): diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index b8bf8daf8..358790aaa 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -10,6 +10,7 @@ from ..utils import ( encodeFilename, decodeArgument, format_bytes, + g, timeconvert, ) @@ -211,10 +212,10 @@ class FileDownloader(object): def report_destination(self, filename): """Report destination filename.""" - self.to_screen('[download] Destination: ' + filename) + self.to_screen(g('[download] Destination: %s') % filename) def _report_progress_status(self, msg, is_last_line=False): - fullmsg = '[download] ' + msg + fullmsg = g('[download] ') + msg if self.params.get('progress_with_newline', False): self.to_screen(fullmsg) else: diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 6ab2d68d6..92458bffe 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -33,6 +33,7 @@ from ..utils import ( ExtractorError, fix_xml_ampersands, float_or_none, + g, int_or_none, RegexNotFoundError, sanitize_filename, @@ -503,7 +504,7 @@ class InfoExtractor(object): def report_download_webpage(self, video_id): """Report webpage download.""" - self.to_screen('%s: Downloading webpage' % video_id) + self.to_screen(g('%s: Downloading webpage') % video_id) def report_age_confirmation(self): """Report attempt to confirm age.""" diff --git a/youtube_dl/extractor/testurl.py b/youtube_dl/extractor/testurl.py index c7d559315..f2c920951 100644 --- a/youtube_dl/extractor/testurl.py +++ b/youtube_dl/extractor/testurl.py @@ -3,7 +3,10 @@ from __future__ import unicode_literals import re from .common import InfoExtractor -from ..utils import ExtractorError +from ..utils import ( + ExtractorError, + g, +) class TestURLIE(InfoExtractor): @@ -59,7 +62,7 @@ class TestURLIE(InfoExtractor): (num, len(testcases))), expected=True) - self.to_screen('Test URL: %s' % tc['url']) + self.to_screen(g('Test URL: %s') % tc['url']) return { '_type': 'url', diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 9b39505ba..73435e6ec 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -42,6 +42,7 @@ from ..utils import ( unsmuggle_url, uppercase_escape, ISO3166Utils, + g, ) @@ -743,11 +744,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor): def report_video_info_webpage_download(self, video_id): """Report attempt to download video info webpage.""" - self.to_screen('%s: Downloading video info webpage' % video_id) + self.to_screen(g('%s: Downloading video info webpage') % video_id) def report_information_extraction(self, video_id): """Report attempt to extract video information.""" - self.to_screen('%s: Extracting video information' % video_id) + self.to_screen(g('%s: Extracting video information') % video_id) def report_unavailable_format(self, video_id, format): """Report extracted video URL.""" @@ -1024,8 +1025,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): dash_manifest_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, dash_manifest_url) dash_doc = self._download_xml( dash_manifest_url, video_id, - note='Downloading DASH manifest', - errnote='Could not download DASH manifest', + note=g('Downloading DASH manifest'), + errnote=g('Could not download DASH manifest'), fatal=fatal) if dash_doc is False: diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index daca5d814..af6e39b87 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -22,6 +22,7 @@ from ..utils import ( subtitles_filename, dfxp2srt, ISO639Utils, + g, ) @@ -403,7 +404,7 @@ class FFmpegMergerPP(FFmpegPostProcessor): filename = info['filepath'] temp_filename = prepend_extension(filename, 'temp') args = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0'] - self._downloader.to_screen('[ffmpeg] Merging formats into "%s"' % filename) + self._downloader.to_screen(g('[ffmpeg] Merging formats into "%s"') % filename) self.run_ffmpeg_multiple_files(info['__files_to_merge'], temp_filename, args) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) return info['__files_to_merge'], info From ddd2737be5bc8c3e1765e96de19883a9b978a984 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Mon, 30 Nov 2015 00:57:53 +0800 Subject: [PATCH 03/53] Add po template and and zh_TW translation --- po/youtube_dl.pot | 76 +++++++++++++++++++++++++++++++++++++++++++++++ po/zh_TW.po | 76 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 po/youtube_dl.pot create mode 100644 po/zh_TW.po diff --git a/po/youtube_dl.pot b/po/youtube_dl.pot new file mode 100644 index 000000000..38795e114 --- /dev/null +++ b/po/youtube_dl.pot @@ -0,0 +1,76 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2015-11-30 00:03+0800\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=CHARSET\n" +"Content-Transfer-Encoding: 8bit\n" + +#: youtube_dl/YoutubeDL.py:1543 youtube_dl/YoutubeDL.py:1544 +#, python-format +msgid "[debug] Invoking downloader on %r" +msgstr "" + +#: youtube_dl/YoutubeDL.py:1587 youtube_dl/YoutubeDL.py:1588 +#, python-format +msgid "[download] %s has already been downloaded and merged" +msgstr "" + +#: youtube_dl/YoutubeDL.py:1721 youtube_dl/YoutubeDL.py:1722 +#, python-format +msgid "Deleting original file %s (pass -k to keep)" +msgstr "" + +#: youtube_dl/downloader/common.py:215 youtube_dl/downloader/common.py:216 +#, python-format +msgid "[download] Destination: %s" +msgstr "" + +#: youtube_dl/downloader/common.py:218 youtube_dl/downloader/common.py:219 +msgid "[download] " +msgstr "" + +#: youtube_dl/extractor/common.py:507 youtube_dl/extractor/common.py:508 +#, python-format +msgid "%s: Downloading webpage" +msgstr "" + +#: youtube_dl/extractor/testurl.py:64 youtube_dl/extractor/testurl.py:65 +#, python-format +msgid "Test URL: %s" +msgstr "" + +#: youtube_dl/extractor/youtube.py:742 youtube_dl/extractor/youtube.py:743 +#, python-format +msgid "%s: Downloading video info webpage" +msgstr "" + +#: youtube_dl/extractor/youtube.py:746 youtube_dl/extractor/youtube.py:747 +#, python-format +msgid "%s: Extracting video information" +msgstr "" + +#: youtube_dl/extractor/youtube.py:1023 youtube_dl/extractor/youtube.py:1024 +msgid "Downloading DASH manifest" +msgstr "" + +#: youtube_dl/extractor/youtube.py:1024 youtube_dl/extractor/youtube.py:1025 +msgid "Could not download DASH manifest" +msgstr "" + +#: youtube_dl/postprocessor/ffmpeg.py:405 +#: youtube_dl/postprocessor/ffmpeg.py:406 +#, python-format +msgid "[ffmpeg] Merging formats into \"%s\"" +msgstr "" diff --git a/po/zh_TW.po b/po/zh_TW.po new file mode 100644 index 000000000..fe43363da --- /dev/null +++ b/po/zh_TW.po @@ -0,0 +1,76 @@ +# Chinese translations for PACKAGE package +# PACKAGE 套件的傳統字漢語翻譯. +# Copyright (C) 2015 THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# , 2015. +# +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2015-11-30 00:03+0800\n" +"PO-Revision-Date: 2015-11-29 21:27+0800\n" +"Last-Translator: \n" +"Language-Team: Chinese (traditional)\n" +"Language: zh_TW\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#: youtube_dl/YoutubeDL.py:1543 youtube_dl/YoutubeDL.py:1544 +#, python-format +msgid "[debug] Invoking downloader on %r" +msgstr "[debug] 開始下載：%r" + +#: youtube_dl/YoutubeDL.py:1587 youtube_dl/YoutubeDL.py:1588 +#, python-format +msgid "[download] %s has already been downloaded and merged" +msgstr "[下載] %s 已經下載並合併完成了" + +#: youtube_dl/YoutubeDL.py:1721 youtube_dl/YoutubeDL.py:1722 +#, python-format +msgid "Deleting original file %s (pass -k to keep)" +msgstr "刪除原始檔案%s（傳入-k以保留）" + +#: youtube_dl/downloader/common.py:215 youtube_dl/downloader/common.py:216 +#, python-format +msgid "[download] Destination: %s" +msgstr "[下載] 目標檔案：%s" + +#: youtube_dl/downloader/common.py:218 youtube_dl/downloader/common.py:219 +msgid "[download] " +msgstr "[下載] " + +#: youtube_dl/extractor/common.py:507 youtube_dl/extractor/common.py:508 +#, python-format +msgid "%s: Downloading webpage" +msgstr "%s: 下載網頁" + +#: youtube_dl/extractor/testurl.py:64 youtube_dl/extractor/testurl.py:65 +#, python-format +msgid "Test URL: %s" +msgstr "測試網址：%s" + +#: youtube_dl/extractor/youtube.py:742 youtube_dl/extractor/youtube.py:743 +#, python-format +msgid "%s: Downloading video info webpage" +msgstr "%s：下載影片訊息頁" + +#: youtube_dl/extractor/youtube.py:746 youtube_dl/extractor/youtube.py:747 +#, python-format +msgid "%s: Extracting video information" +msgstr "%s：取得影片資訊" + +#: youtube_dl/extractor/youtube.py:1023 youtube_dl/extractor/youtube.py:1024 +msgid "Downloading DASH manifest" +msgstr "下載DASH索引" + +#: youtube_dl/extractor/youtube.py:1024 youtube_dl/extractor/youtube.py:1025 +msgid "Could not download DASH manifest" +msgstr "無法下載DASH索引" + +#: youtube_dl/postprocessor/ffmpeg.py:405 +#: youtube_dl/postprocessor/ffmpeg.py:406 +#, python-format +msgid "[ffmpeg] Merging formats into \"%s\"" +msgstr "[ffmpeg] 合併所有格式為：\"%s\"" From 22be0be224c093a6a68feb4d5a36d9d570cfe4b7 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Wed, 2 Dec 2015 10:54:00 +0800 Subject: [PATCH 04/53] Use Python scripts for better portability --- Makefile | 15 ++------- devscripts/i18n.py | 79 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 12 deletions(-) create mode 100644 devscripts/i18n.py diff --git a/Makefile b/Makefile index 13378fbf9..97819568f 100644 --- a/Makefile +++ b/Makefile @@ -111,19 +111,10 @@ youtube-dl.tar.gz: youtube-dl README.md README.txt youtube-dl.1 youtube-dl.bash- youtube-dl update-po: po/youtube_dl.pot - for file in po/*.po ; do \ - lang=$$(echo $$file | sed -e 's#.*/\([^/]\+\).po#\1#') ; \ - mv $$file $$file.old ; \ - msgmerge -N $$file.old po/youtube_dl.pot > $$file ; \ - done + $(PYTHON) devscripts/i18n.py update-po po/youtube_dl.pot: youtube_dl/*.py youtube_dl/*/*.py - touch po/youtube_dl.pot && \ - xgettext -d youtube_dl -j -k -kg --from-code=utf-8 -o $@ youtube_dl/*.py youtube_dl/*/*.py + $(PYTHON) devscripts/i18n.py update-pot update-gmo: - for file in po/*.po ; do \ - lang=$$(echo $$file | sed -e 's#.*/\([^/]\+\).po#\1#') ; \ - install -d share/locale/$$lang/LC_MESSAGES; \ - msgfmt -o share/locale/$$lang/LC_MESSAGES/youtube_dl.mo $$file; \ - done + $(PYTHON) devscripts/i18n.py update-gmo diff --git a/devscripts/i18n.py b/devscripts/i18n.py new file mode 100644 index 000000000..0b2e967da --- /dev/null +++ b/devscripts/i18n.py @@ -0,0 +1,79 @@ +import errno +import glob +import os +import subprocess +import sys + + +def mkdir_p(path): + try: + os.makedirs(path) + except OSError as e: + if e.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + + +class I18N_Utils(object): + GETTEXT_DOMAIN = 'youtube_dl' + + def get_po_root(self): + return 'po/' + + def get_pot_filename(self): + return os.path.join(self.get_po_root(), '%s.pot' % self.GETTEXT_DOMAIN) + + @staticmethod + def _run_subprocess(cmds): + print(' '.join(cmds)) + subprocess.check_call(cmds) + + def update_gmo_internal(self, lang, po_file): + locale_dir = 'share/locale/%s/LC_MESSAGES' % lang + mkdir_p(locale_dir) + self._run_subprocess([ + 'msgfmt', '-o', + os.path.join(locale_dir, 'youtube_dl.mo'), + po_file]) + + def update_po_internal(self, lang, po_file): + old_po_file = po_file + '.old' + os.rename(po_file, old_po_file) + self._run_subprocess([ + 'msgmerge', '-N', old_po_file, '-o', po_file, self.get_pot_filename()]) + + def for_all_po(self, callback): + for f in os.listdir(self.get_po_root()): + name, ext = os.path.splitext(f) + if ext != '.po': + continue + + callback(name, os.path.join(self.get_po_root(), f)) + + def update_gmo(self): + self.for_all_po(self.update_gmo_internal) + + def update_po(self): + self.for_all_po(self.update_po_internal) + + def update_pot(self): + self._run_subprocess + cmds = [ + 'xgettext', '-d', self.GETTEXT_DOMAIN, '-j', '-k', '-kg', '--from-code=utf-8', '-o', + self.get_pot_filename()] + cmds.extend(glob.glob('youtube_dl/*.py') + glob.glob('youtube_dl/*/*.py')) + self._run_subprocess(cmds) + + +def main(argv): + instance = I18N_Utils() + if argv[1] == 'update-po': + instance.update_po() + elif argv[1] == 'update-gmo': + instance.update_gmo() + elif argv[1] == 'update-pot': + instance.update_pot() + +if __name__ == '__main__': + main(sys.argv) From 3719a34a22eead5b164cfa7f1b6752aa67a676dd Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Wed, 2 Dec 2015 14:55:24 +0800 Subject: [PATCH 05/53] [utils] Fix gettext in Python2 In Python 2, gettext() returns bytes --- youtube_dl/utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index ef034eb9b..7dc32e85e 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -2518,7 +2518,12 @@ def g(s): gettext.textdomain('youtube_dl') gettext.bindtextdomain('youtube_dl', localedir='share/locale/') - return gettext.gettext(s) + ret = gettext.gettext(s) + + if isinstance(ret, bytes): + ret = ret.decode('utf-8') + + return ret class PerRequestProxyHandler(compat_urllib_request.ProxyHandler): From bf8c5886ed3d4f6b15a901faa40e723e71cae85c Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Wed, 2 Dec 2015 15:08:17 +0800 Subject: [PATCH 06/53] [utils] Detect language on Windows --- youtube_dl/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 7dc32e85e..3bc85ded0 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -2518,11 +2518,15 @@ def g(s): gettext.textdomain('youtube_dl') gettext.bindtextdomain('youtube_dl', localedir='share/locale/') - ret = gettext.gettext(s) + lang, _ = locale.getdefaultlocale() + try: + t = gettext.translation('youtube_dl', 'share/locale/', [lang]) + except (OSError, IOError): # OSError for 3.3+ and IOError otherwise + return s + ret = t.gettext(s) if isinstance(ret, bytes): ret = ret.decode('utf-8') - return ret From ef95dc78161ae9d703089f0efdc7226f38b319d8 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Wed, 2 Dec 2015 15:24:14 +0800 Subject: [PATCH 07/53] [devscripts/i18n] Fix rename for Windows --- devscripts/i18n.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/devscripts/i18n.py b/devscripts/i18n.py index 0b2e967da..63bfe5da8 100644 --- a/devscripts/i18n.py +++ b/devscripts/i18n.py @@ -15,6 +15,15 @@ def mkdir_p(path): raise +def rename_file(old_path, new_path): + if sys.platform == 'win32': + try: + os.unlink(new_path) + except OSError: + pass + os.rename(old_path, new_path) + + class I18N_Utils(object): GETTEXT_DOMAIN = 'youtube_dl' @@ -39,7 +48,7 @@ class I18N_Utils(object): def update_po_internal(self, lang, po_file): old_po_file = po_file + '.old' - os.rename(po_file, old_po_file) + rename_file(po_file, old_po_file) self._run_subprocess([ 'msgmerge', '-N', old_po_file, '-o', po_file, self.get_pot_filename()]) From 7309cee4a05f4af529c7a82c959af274f56ecb90 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Wed, 2 Dec 2015 16:48:49 +0800 Subject: [PATCH 08/53] [utils] Remove unused codes --- youtube_dl/utils.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 3bc85ded0..3d508498d 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -2514,10 +2514,6 @@ class ISO3166Utils(object): def g(s): - if gettext.textdomain() != 'youtube_dl': - gettext.textdomain('youtube_dl') - gettext.bindtextdomain('youtube_dl', localedir='share/locale/') - lang, _ = locale.getdefaultlocale() try: t = gettext.translation('youtube_dl', 'share/locale/', [lang]) From f344e796f8b4e4608ca5816b23072d06be5ebfaa Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 5 Dec 2015 02:51:50 +0800 Subject: [PATCH 09/53] Bundle .mo files into a Windows EXE resource --- setup.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/setup.py b/setup.py index bfe931f5b..97b551386 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,8 @@ from __future__ import print_function import os.path import warnings import sys +import io +import zipfile try: from setuptools import setup @@ -23,6 +25,16 @@ except ImportError: print("Cannot import py2exe", file=sys.stderr) exit(1) + +def zipped_folder(topdir): + f = io.BytesIO() + zipf = zipfile.ZipFile(f, mode='w') + for dirpath, dirnames, filenames in os.walk(topdir): + for filename in filenames: + zipf.write(os.path.join(dirpath, filename)) + zipf.close() + return f.getvalue() + py2exe_options = { "bundle_files": 1, "compressed": 1, @@ -34,6 +46,7 @@ py2exe_options = { py2exe_console = [{ "script": "./youtube_dl/__main__.py", "dest_base": "youtube-dl", + "other_resources": [(u'LOCALE_DATA', u'locale_data.zip', zipped_folder('share'))], }] py2exe_params = { From bbfacce13c19d4e741b366d99ff198801b353e09 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 5 Dec 2015 02:52:14 +0800 Subject: [PATCH 10/53] [utils] Load .mo in easy_install installation and Windows EXE --- youtube_dl/utils.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 3d508498d..6710cae54 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -32,6 +32,7 @@ import sys import tempfile import traceback import xml.etree.ElementTree +import zipfile import zlib from .compat import ( @@ -2513,11 +2514,36 @@ class ISO3166Utils(object): return cls._country_map.get(code.upper()) +def _load_exe_resource(res_type, res_name): + kernel32 = ctypes.windll.kernel32 + + exe_handle = 0 # NULL: Current process + + res_info = kernel32.FindResourceW(exe_handle, res_name, res_type) + res_handle = kernel32.LoadResource(exe_handle, res_info) + res_data = kernel32.LockResource(res_handle) + res_len = kernel32.SizeofResource(exe_handle, res_info) + res_arr = ctypes.cast(res_data, ctypes.POINTER(ctypes.c_char))[:res_len] + return res_arr + + def g(s): + DOMAIN = 'youtube_dl' lang, _ = locale.getdefaultlocale() try: - t = gettext.translation('youtube_dl', 'share/locale/', [lang]) + t = gettext.translation(DOMAIN, os.path.join(get_root_dir(), '../share/locale/'), [lang]) except (OSError, IOError): # OSError for 3.3+ and IOError otherwise + t = None + + if t is None and sys.platform == 'win32' and hasattr(sys, 'frozen'): + locale_data_zip = _load_exe_resource('LOCALE_DATA', 'LOCALE_DATA.ZIP') + f = io.BytesIO(locale_data_zip) + zipf = zipfile.ZipFile(f) + with zipf.open('share/locale/%s/LC_MESSAGES/%s.mo' % (lang, DOMAIN)) as mo_file: + t = gettext.GNUTranslations(mo_file) + zipf.close() + + if t is None: return s ret = t.gettext(s) @@ -2526,6 +2552,10 @@ def g(s): return ret +def get_root_dir(): + return os.path.dirname(os.path.abspath(__file__)) + + class PerRequestProxyHandler(compat_urllib_request.ProxyHandler): def __init__(self, proxies=None): # Set default handlers From 840f6d04b652612d22ae50618c6d9d88e981094d Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 5 Dec 2015 04:36:34 +0800 Subject: [PATCH 11/53] [Makefile] Shortcuts for handling mo files --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 97819568f..3a390f0ce 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ all: youtube-dl README.md CONTRIBUTING.md README.txt youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish supportedsites clean: - rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe + rm -rf youtube-dl.1.temp.md youtube-dl.1 youtube-dl.bash-completion README.txt MANIFEST build/ dist/ .coverage cover/ youtube-dl.tar.gz youtube-dl.zsh youtube-dl.fish *.dump *.part *.info.json *.mp4 *.flv *.mp3 *.avi CONTRIBUTING.md.tmp youtube-dl youtube-dl.exe share/locale/*/LC_MESSAGES/* find . -name "*.pyc" -delete PREFIX ?= /usr/local @@ -50,7 +50,7 @@ tar: youtube-dl.tar.gz .PHONY: all clean install test tar bash-completion pypi-files zsh-completion fish-completion ot offlinetest codetest supportedsites update-po update-gmo -pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 youtube-dl.fish +pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 youtube-dl.fish update-gmo youtube-dl: youtube_dl/*.py youtube_dl/*/*.py zip --quiet youtube-dl youtube_dl/*.py youtube_dl/*/*.py From 8b92b7fb8dc8dc8b5711fd232df2717cd72dadee Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 5 Dec 2015 04:37:24 +0800 Subject: [PATCH 12/53] Include mo files in pip installations --- MANIFEST.in | 1 + setup.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 5743f605a..1f0d6acd3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,3 +5,4 @@ include youtube-dl.bash-completion include youtube-dl.fish include youtube-dl.1 recursive-include docs Makefile conf.py *.rst +include share/locale/*/LC_MESSAGES/youtube_dl.mo diff --git a/setup.py b/setup.py index 97b551386..e8cc4de8c 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,7 @@ from __future__ import print_function import os.path import warnings import sys +import glob import io import zipfile @@ -64,6 +65,9 @@ else: ('share/doc/youtube_dl', ['README.txt']), ('share/man/man1', ['youtube-dl.1']) ] + for mo_file in glob.glob('share/locale/*/LC_MESSAGES/youtube_dl.mo'): + files_spec.append((os.path.dirname(mo_file), [mo_file])) + root = os.path.dirname(os.path.abspath(__file__)) data_files = [] for dirname, files in files_spec: From 89442a0e26e21c8695420818c525b2edbf54a9eb Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 5 Dec 2015 04:37:47 +0800 Subject: [PATCH 13/53] [utils] Get root dir from root module's __file__ --- youtube_dl/utils.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 6710cae54..04aa917d8 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -2531,7 +2531,7 @@ def g(s): DOMAIN = 'youtube_dl' lang, _ = locale.getdefaultlocale() try: - t = gettext.translation(DOMAIN, os.path.join(get_root_dir(), '../share/locale/'), [lang]) + t = gettext.translation(DOMAIN, find_file_in_root('share/locale/'), [lang]) except (OSError, IOError): # OSError for 3.3+ and IOError otherwise t = None @@ -2552,8 +2552,25 @@ def g(s): return ret -def get_root_dir(): - return os.path.dirname(os.path.abspath(__file__)) +def get_root_dirs(): + ret = [] + + import __main__ + if hasattr(__main__, '__file__'): # __main__ has no __file__ if youtube_dl is imported from stdin + # ../../ of bin/youtube-dl + ret.append(os.path.dirname(os.path.dirname(os.path.abspath(__main__.__file__)))) + + # ../../ of youtube_dl/utils.py + ret.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + return ret + + +def find_file_in_root(file_path): + for root in get_root_dirs(): + full_path = os.path.join(root, file_path) + if os.path.exists(full_path): + return full_path class PerRequestProxyHandler(compat_urllib_request.ProxyHandler): From 995f5036ca0eee575f5a8e15c62766f690ce10e2 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 13 Dec 2015 00:22:37 +0800 Subject: [PATCH 14/53] Drop Windows EXE support for gettext --- setup.py | 12 ------------ youtube_dl/utils.py | 25 ------------------------- 2 files changed, 37 deletions(-) diff --git a/setup.py b/setup.py index e8cc4de8c..0355afd0b 100644 --- a/setup.py +++ b/setup.py @@ -7,8 +7,6 @@ import os.path import warnings import sys import glob -import io -import zipfile try: from setuptools import setup @@ -27,15 +25,6 @@ except ImportError: exit(1) -def zipped_folder(topdir): - f = io.BytesIO() - zipf = zipfile.ZipFile(f, mode='w') - for dirpath, dirnames, filenames in os.walk(topdir): - for filename in filenames: - zipf.write(os.path.join(dirpath, filename)) - zipf.close() - return f.getvalue() - py2exe_options = { "bundle_files": 1, "compressed": 1, @@ -47,7 +36,6 @@ py2exe_options = { py2exe_console = [{ "script": "./youtube_dl/__main__.py", "dest_base": "youtube-dl", - "other_resources": [(u'LOCALE_DATA', u'locale_data.zip', zipped_folder('share'))], }] py2exe_params = { diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 04aa917d8..b7d2043a7 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -32,7 +32,6 @@ import sys import tempfile import traceback import xml.etree.ElementTree -import zipfile import zlib from .compat import ( @@ -2514,36 +2513,12 @@ class ISO3166Utils(object): return cls._country_map.get(code.upper()) -def _load_exe_resource(res_type, res_name): - kernel32 = ctypes.windll.kernel32 - - exe_handle = 0 # NULL: Current process - - res_info = kernel32.FindResourceW(exe_handle, res_name, res_type) - res_handle = kernel32.LoadResource(exe_handle, res_info) - res_data = kernel32.LockResource(res_handle) - res_len = kernel32.SizeofResource(exe_handle, res_info) - res_arr = ctypes.cast(res_data, ctypes.POINTER(ctypes.c_char))[:res_len] - return res_arr - - def g(s): DOMAIN = 'youtube_dl' lang, _ = locale.getdefaultlocale() try: t = gettext.translation(DOMAIN, find_file_in_root('share/locale/'), [lang]) except (OSError, IOError): # OSError for 3.3+ and IOError otherwise - t = None - - if t is None and sys.platform == 'win32' and hasattr(sys, 'frozen'): - locale_data_zip = _load_exe_resource('LOCALE_DATA', 'LOCALE_DATA.ZIP') - f = io.BytesIO(locale_data_zip) - zipf = zipfile.ZipFile(f) - with zipf.open('share/locale/%s/LC_MESSAGES/%s.mo' % (lang, DOMAIN)) as mo_file: - t = gettext.GNUTranslations(mo_file) - zipf.close() - - if t is None: return s ret = t.gettext(s) From 1574e2ba647ed69a05bcc818bb68721e05447f64 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 13 Dec 2015 00:42:53 +0800 Subject: [PATCH 15/53] [devscripts/i18n] Sorted POT file --- devscripts/i18n.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devscripts/i18n.py b/devscripts/i18n.py index 63bfe5da8..d28f40fd8 100644 --- a/devscripts/i18n.py +++ b/devscripts/i18n.py @@ -69,7 +69,7 @@ class I18N_Utils(object): def update_pot(self): self._run_subprocess cmds = [ - 'xgettext', '-d', self.GETTEXT_DOMAIN, '-j', '-k', '-kg', '--from-code=utf-8', '-o', + 'xgettext', '-d', self.GETTEXT_DOMAIN, '-j', '-k', '-kg', '--from-code=utf-8', '-F', '-o', self.get_pot_filename()] cmds.extend(glob.glob('youtube_dl/*.py') + glob.glob('youtube_dl/*/*.py')) self._run_subprocess(cmds) From 6730d3da386a4a9debb0eac893ad7f23d072b488 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 13 Dec 2015 00:45:10 +0800 Subject: [PATCH 16/53] [po] Add descriptive comments to youtube_dl.pot --- po/youtube_dl.pot | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/po/youtube_dl.pot b/po/youtube_dl.pot index 38795e114..63425252a 100644 --- a/po/youtube_dl.pot +++ b/po/youtube_dl.pot @@ -1,14 +1,14 @@ -# SOME DESCRIPTIVE TITLE. -# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER -# This file is distributed under the same license as the PACKAGE package. -# FIRST AUTHOR , YEAR. +# Translation template for youtube-dl +# UNLICENSE 2008-2015 +# This file is distributed under the same license as the youtube-dl package. +# Ricardo García , 2008. # #, fuzzy msgid "" msgstr "" -"Project-Id-Version: PACKAGE VERSION\n" +"Project-Id-Version: 1.0\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2015-11-30 00:03+0800\n" +"POT-Creation-Date: 2015-12-13 00:33+0800\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -17,60 +17,59 @@ msgstr "" "Content-Type: text/plain; charset=CHARSET\n" "Content-Transfer-Encoding: 8bit\n" -#: youtube_dl/YoutubeDL.py:1543 youtube_dl/YoutubeDL.py:1544 +#: youtube_dl/YoutubeDL.py:1549 #, python-format msgid "[debug] Invoking downloader on %r" msgstr "" -#: youtube_dl/YoutubeDL.py:1587 youtube_dl/YoutubeDL.py:1588 +#: youtube_dl/YoutubeDL.py:1593 #, python-format msgid "[download] %s has already been downloaded and merged" msgstr "" -#: youtube_dl/YoutubeDL.py:1721 youtube_dl/YoutubeDL.py:1722 +#: youtube_dl/YoutubeDL.py:1727 #, python-format msgid "Deleting original file %s (pass -k to keep)" msgstr "" -#: youtube_dl/downloader/common.py:215 youtube_dl/downloader/common.py:216 +#: youtube_dl/downloader/common.py:215 #, python-format msgid "[download] Destination: %s" msgstr "" -#: youtube_dl/downloader/common.py:218 youtube_dl/downloader/common.py:219 +#: youtube_dl/downloader/common.py:218 msgid "[download] " msgstr "" -#: youtube_dl/extractor/common.py:507 youtube_dl/extractor/common.py:508 +#: youtube_dl/extractor/common.py:507 #, python-format msgid "%s: Downloading webpage" msgstr "" -#: youtube_dl/extractor/testurl.py:64 youtube_dl/extractor/testurl.py:65 +#: youtube_dl/extractor/testurl.py:65 #, python-format msgid "Test URL: %s" msgstr "" -#: youtube_dl/extractor/youtube.py:742 youtube_dl/extractor/youtube.py:743 +#: youtube_dl/extractor/youtube.py:747 #, python-format msgid "%s: Downloading video info webpage" msgstr "" -#: youtube_dl/extractor/youtube.py:746 youtube_dl/extractor/youtube.py:747 +#: youtube_dl/extractor/youtube.py:751 #, python-format msgid "%s: Extracting video information" msgstr "" -#: youtube_dl/extractor/youtube.py:1023 youtube_dl/extractor/youtube.py:1024 +#: youtube_dl/extractor/youtube.py:1028 msgid "Downloading DASH manifest" msgstr "" -#: youtube_dl/extractor/youtube.py:1024 youtube_dl/extractor/youtube.py:1025 +#: youtube_dl/extractor/youtube.py:1029 msgid "Could not download DASH manifest" msgstr "" -#: youtube_dl/postprocessor/ffmpeg.py:405 -#: youtube_dl/postprocessor/ffmpeg.py:406 +#: youtube_dl/postprocessor/ffmpeg.py:407 #, python-format msgid "[ffmpeg] Merging formats into \"%s\"" msgstr "" From 37ee86e29a2b6097d66f03564df079ba93b2caf3 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 13 Dec 2015 00:45:55 +0800 Subject: [PATCH 17/53] [devscripts/i18n] Remove an unused line --- devscripts/i18n.py | 1 - 1 file changed, 1 deletion(-) diff --git a/devscripts/i18n.py b/devscripts/i18n.py index d28f40fd8..67550ae84 100644 --- a/devscripts/i18n.py +++ b/devscripts/i18n.py @@ -67,7 +67,6 @@ class I18N_Utils(object): self.for_all_po(self.update_po_internal) def update_pot(self): - self._run_subprocess cmds = [ 'xgettext', '-d', self.GETTEXT_DOMAIN, '-j', '-k', '-kg', '--from-code=utf-8', '-F', '-o', self.get_pot_filename()] From efd28c9cd59fb592a7b15c41c1172e1e0e2160d8 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 13 Dec 2015 00:58:55 +0800 Subject: [PATCH 18/53] [devscripts/i18n] Keep youtube_dl.pot metadata in update-pot --- devscripts/i18n.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/devscripts/i18n.py b/devscripts/i18n.py index 67550ae84..4344dfffc 100644 --- a/devscripts/i18n.py +++ b/devscripts/i18n.py @@ -1,6 +1,7 @@ import errno import glob import os +import shutil import subprocess import sys @@ -38,6 +39,9 @@ class I18N_Utils(object): print(' '.join(cmds)) subprocess.check_call(cmds) + def merge_messages(self, old_file, new_file): + self._run_subprocess(['msgmerge', '-N', old_file, '-o', new_file, self.get_pot_filename()]) + def update_gmo_internal(self, lang, po_file): locale_dir = 'share/locale/%s/LC_MESSAGES' % lang mkdir_p(locale_dir) @@ -49,8 +53,7 @@ class I18N_Utils(object): def update_po_internal(self, lang, po_file): old_po_file = po_file + '.old' rename_file(po_file, old_po_file) - self._run_subprocess([ - 'msgmerge', '-N', old_po_file, '-o', po_file, self.get_pot_filename()]) + self.merge_messages(old_po_file, po_file) def for_all_po(self, callback): for f in os.listdir(self.get_po_root()): @@ -67,11 +70,15 @@ class I18N_Utils(object): self.for_all_po(self.update_po_internal) def update_pot(self): + pot_file = self.get_pot_filename() + old_pot_file = self.get_pot_filename() + '.old' + shutil.copy2(pot_file, old_pot_file) cmds = [ 'xgettext', '-d', self.GETTEXT_DOMAIN, '-j', '-k', '-kg', '--from-code=utf-8', '-F', '-o', - self.get_pot_filename()] + pot_file] cmds.extend(glob.glob('youtube_dl/*.py') + glob.glob('youtube_dl/*/*.py')) self._run_subprocess(cmds) + self.merge_messages(old_pot_file, pot_file) def main(argv): From 3f0b6207f877348e15be5a7048b9975d104f702d Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 13 Dec 2015 01:09:33 +0800 Subject: [PATCH 19/53] [po/zh_TW] Use halfwidth colons --- po/zh_TW.po | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/po/zh_TW.po b/po/zh_TW.po index fe43363da..bbeca2e9d 100644 --- a/po/zh_TW.po +++ b/po/zh_TW.po @@ -6,16 +6,17 @@ # msgid "" msgstr "" -"Project-Id-Version: PACKAGE VERSION\n" +"Project-Id-Version: \n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2015-11-30 00:03+0800\n" -"PO-Revision-Date: 2015-11-29 21:27+0800\n" -"Last-Translator: \n" +"PO-Revision-Date: 2015-12-13 01:04+0800\n" +"Last-Translator: yan12125@gmail.com\n" "Language-Team: Chinese (traditional)\n" "Language: zh_TW\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" +"X-Generator: Poedit 1.8.6\n" #: youtube_dl/YoutubeDL.py:1543 youtube_dl/YoutubeDL.py:1544 #, python-format @@ -54,12 +55,12 @@ msgstr "測試網址：%s" #: youtube_dl/extractor/youtube.py:742 youtube_dl/extractor/youtube.py:743 #, python-format msgid "%s: Downloading video info webpage" -msgstr "%s：下載影片訊息頁" +msgstr "%s: 下載影片訊息頁" #: youtube_dl/extractor/youtube.py:746 youtube_dl/extractor/youtube.py:747 #, python-format msgid "%s: Extracting video information" -msgstr "%s：取得影片資訊" +msgstr "%s: 取得影片資訊" #: youtube_dl/extractor/youtube.py:1023 youtube_dl/extractor/youtube.py:1024 msgid "Downloading DASH manifest" From b8feb749b08e5d87dcf442c7f80871359321164a Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 13 Dec 2015 01:12:40 +0800 Subject: [PATCH 20/53] Ignore .pot.old files as well --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 358ea3bd7..2a33b5f1c 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ youtube-dl.zsh .idea/* *.mo *.po.old +*.pot.old From d13823ec7ca523f813533b3c992ca0d6f1c44d11 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 13 Dec 2015 01:16:00 +0800 Subject: [PATCH 21/53] [po/zh_TW] Update comments for zh_TW translation --- po/zh_TW.po | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/po/zh_TW.po b/po/zh_TW.po index bbeca2e9d..567e77d01 100644 --- a/po/zh_TW.po +++ b/po/zh_TW.po @@ -1,14 +1,14 @@ -# Chinese translations for PACKAGE package -# PACKAGE 套件的傳統字漢語翻譯. -# Copyright (C) 2015 THE PACKAGE'S COPYRIGHT HOLDER -# This file is distributed under the same license as the PACKAGE package. -# , 2015. +# Chinese translations for youtube-dl package +# youtube-dl 套件的傳統字漢語翻譯. +# UNLICENSE 2008-2015 +# This file is distributed under the same license as the youtube-dl package. +# Yen Chi Hsuan , 2015. # msgid "" msgstr "" "Project-Id-Version: \n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2015-11-30 00:03+0800\n" +"POT-Creation-Date: 2015-12-13 00:33+0800\n" "PO-Revision-Date: 2015-12-13 01:04+0800\n" "Last-Translator: yan12125@gmail.com\n" "Language-Team: Chinese (traditional)\n" @@ -18,60 +18,59 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "X-Generator: Poedit 1.8.6\n" -#: youtube_dl/YoutubeDL.py:1543 youtube_dl/YoutubeDL.py:1544 +#: youtube_dl/YoutubeDL.py:1549 #, python-format msgid "[debug] Invoking downloader on %r" msgstr "[debug] 開始下載：%r" -#: youtube_dl/YoutubeDL.py:1587 youtube_dl/YoutubeDL.py:1588 +#: youtube_dl/YoutubeDL.py:1593 #, python-format msgid "[download] %s has already been downloaded and merged" msgstr "[下載] %s 已經下載並合併完成了" -#: youtube_dl/YoutubeDL.py:1721 youtube_dl/YoutubeDL.py:1722 +#: youtube_dl/YoutubeDL.py:1727 #, python-format msgid "Deleting original file %s (pass -k to keep)" msgstr "刪除原始檔案%s（傳入-k以保留）" -#: youtube_dl/downloader/common.py:215 youtube_dl/downloader/common.py:216 +#: youtube_dl/downloader/common.py:215 #, python-format msgid "[download] Destination: %s" msgstr "[下載] 目標檔案：%s" -#: youtube_dl/downloader/common.py:218 youtube_dl/downloader/common.py:219 +#: youtube_dl/downloader/common.py:218 msgid "[download] " msgstr "[下載] " -#: youtube_dl/extractor/common.py:507 youtube_dl/extractor/common.py:508 +#: youtube_dl/extractor/common.py:507 #, python-format msgid "%s: Downloading webpage" msgstr "%s: 下載網頁" -#: youtube_dl/extractor/testurl.py:64 youtube_dl/extractor/testurl.py:65 +#: youtube_dl/extractor/testurl.py:65 #, python-format msgid "Test URL: %s" msgstr "測試網址：%s" -#: youtube_dl/extractor/youtube.py:742 youtube_dl/extractor/youtube.py:743 +#: youtube_dl/extractor/youtube.py:747 #, python-format msgid "%s: Downloading video info webpage" msgstr "%s: 下載影片訊息頁" -#: youtube_dl/extractor/youtube.py:746 youtube_dl/extractor/youtube.py:747 +#: youtube_dl/extractor/youtube.py:751 #, python-format msgid "%s: Extracting video information" msgstr "%s: 取得影片資訊" -#: youtube_dl/extractor/youtube.py:1023 youtube_dl/extractor/youtube.py:1024 +#: youtube_dl/extractor/youtube.py:1028 msgid "Downloading DASH manifest" msgstr "下載DASH索引" -#: youtube_dl/extractor/youtube.py:1024 youtube_dl/extractor/youtube.py:1025 +#: youtube_dl/extractor/youtube.py:1029 msgid "Could not download DASH manifest" msgstr "無法下載DASH索引" -#: youtube_dl/postprocessor/ffmpeg.py:405 -#: youtube_dl/postprocessor/ffmpeg.py:406 +#: youtube_dl/postprocessor/ffmpeg.py:407 #, python-format msgid "[ffmpeg] Merging formats into \"%s\"" msgstr "[ffmpeg] 合併所有格式為：\"%s\"" From 4fe08dc18d4dc9318f1dda7caebbc5275e467f5c Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 13 Dec 2015 01:18:21 +0800 Subject: [PATCH 22/53] Remove an extra blank line --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 0355afd0b..4403f162b 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,6 @@ except ImportError: print("Cannot import py2exe", file=sys.stderr) exit(1) - py2exe_options = { "bundle_files": 1, "compressed": 1, From d8257c57d880c25c7c5a9e3bc8a6c036770f51d9 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 13 Dec 2015 03:33:40 +0800 Subject: [PATCH 23/53] [i18n] Use tr() instead of g() --- devscripts/i18n.py | 2 +- po/youtube_dl.pot | 2 +- youtube_dl/YoutubeDL.py | 10 +++++----- youtube_dl/downloader/common.py | 6 +++--- youtube_dl/extractor/common.py | 4 ++-- youtube_dl/extractor/testurl.py | 4 ++-- youtube_dl/extractor/youtube.py | 10 +++++----- youtube_dl/postprocessor/ffmpeg.py | 4 ++-- youtube_dl/utils.py | 2 +- 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/devscripts/i18n.py b/devscripts/i18n.py index 4344dfffc..229cbd8b9 100644 --- a/devscripts/i18n.py +++ b/devscripts/i18n.py @@ -74,7 +74,7 @@ class I18N_Utils(object): old_pot_file = self.get_pot_filename() + '.old' shutil.copy2(pot_file, old_pot_file) cmds = [ - 'xgettext', '-d', self.GETTEXT_DOMAIN, '-j', '-k', '-kg', '--from-code=utf-8', '-F', '-o', + 'xgettext', '-d', self.GETTEXT_DOMAIN, '-j', '-k', '-ktr', '--from-code=utf-8', '-F', '-o', pot_file] cmds.extend(glob.glob('youtube_dl/*.py') + glob.glob('youtube_dl/*/*.py')) self._run_subprocess(cmds) diff --git a/po/youtube_dl.pot b/po/youtube_dl.pot index 63425252a..b29b329c5 100644 --- a/po/youtube_dl.pot +++ b/po/youtube_dl.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: 1.0\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2015-12-13 00:33+0800\n" +"POT-Creation-Date: 2015-12-13 03:32+0800\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 04fb4ba08..0cb218174 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -51,7 +51,6 @@ from .utils import ( ExtractorError, format_bytes, formatSeconds, - g, locked_file, make_HTTPS_handler, MaxDownloadsReached, @@ -68,6 +67,7 @@ from .utils import ( sanitized_Request, std_headers, subtitles_filename, + tr, UnavailableVideoError, url_basename, version_tuple, @@ -1546,7 +1546,7 @@ class YoutubeDL(object): for ph in self._progress_hooks: fd.add_progress_hook(ph) if self.params.get('verbose'): - self.to_stdout(g('[debug] Invoking downloader on %r') % info.get('url')) + self.to_stdout(tr('[debug] Invoking downloader on %r') % info.get('url')) return fd.download(name, info) if info_dict.get('requested_formats') is not None: @@ -1590,8 +1590,8 @@ class YoutubeDL(object): filename = '%s.%s' % (filename_wo_ext, info_dict['ext']) if os.path.exists(encodeFilename(filename)): self.to_screen( - g('[download] %s has already been downloaded and ' - 'merged') % filename) + tr('[download] %s has already been downloaded and ' + 'merged') % filename) else: for f in requested_formats: new_info = dict(info_dict) @@ -1724,7 +1724,7 @@ class YoutubeDL(object): self.report_error(e.msg) if files_to_delete and not self.params.get('keepvideo', False): for old_filename in files_to_delete: - self.to_screen(g('Deleting original file %s (pass -k to keep)') % old_filename) + self.to_screen(tr('Deleting original file %s (pass -k to keep)') % old_filename) try: os.remove(encodeFilename(old_filename)) except (IOError, OSError): diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index 358790aaa..b15cb1b08 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -10,8 +10,8 @@ from ..utils import ( encodeFilename, decodeArgument, format_bytes, - g, timeconvert, + tr, ) @@ -212,10 +212,10 @@ class FileDownloader(object): def report_destination(self, filename): """Report destination filename.""" - self.to_screen(g('[download] Destination: %s') % filename) + self.to_screen(tr('[download] Destination: %s') % filename) def _report_progress_status(self, msg, is_last_line=False): - fullmsg = g('[download] ') + msg + fullmsg = tr('[download] ') + msg if self.params.get('progress_with_newline', False): self.to_screen(fullmsg) else: diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 92458bffe..4e8fce607 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -33,11 +33,11 @@ from ..utils import ( ExtractorError, fix_xml_ampersands, float_or_none, - g, int_or_none, RegexNotFoundError, sanitize_filename, sanitized_Request, + tr, unescapeHTML, unified_strdate, url_basename, @@ -504,7 +504,7 @@ class InfoExtractor(object): def report_download_webpage(self, video_id): """Report webpage download.""" - self.to_screen(g('%s: Downloading webpage') % video_id) + self.to_screen(tr('%s: Downloading webpage') % video_id) def report_age_confirmation(self): """Report attempt to confirm age.""" diff --git a/youtube_dl/extractor/testurl.py b/youtube_dl/extractor/testurl.py index f2c920951..1a4ee1220 100644 --- a/youtube_dl/extractor/testurl.py +++ b/youtube_dl/extractor/testurl.py @@ -5,7 +5,7 @@ import re from .common import InfoExtractor from ..utils import ( ExtractorError, - g, + tr, ) @@ -62,7 +62,7 @@ class TestURLIE(InfoExtractor): (num, len(testcases))), expected=True) - self.to_screen(g('Test URL: %s') % tc['url']) + self.to_screen(tr('Test URL: %s') % tc['url']) return { '_type': 'url', diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 73435e6ec..2d5dcce95 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -42,7 +42,7 @@ from ..utils import ( unsmuggle_url, uppercase_escape, ISO3166Utils, - g, + tr, ) @@ -744,11 +744,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor): def report_video_info_webpage_download(self, video_id): """Report attempt to download video info webpage.""" - self.to_screen(g('%s: Downloading video info webpage') % video_id) + self.to_screen(tr('%s: Downloading video info webpage') % video_id) def report_information_extraction(self, video_id): """Report attempt to extract video information.""" - self.to_screen(g('%s: Extracting video information') % video_id) + self.to_screen(tr('%s: Extracting video information') % video_id) def report_unavailable_format(self, video_id, format): """Report extracted video URL.""" @@ -1025,8 +1025,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): dash_manifest_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, dash_manifest_url) dash_doc = self._download_xml( dash_manifest_url, video_id, - note=g('Downloading DASH manifest'), - errnote=g('Could not download DASH manifest'), + note=tr('Downloading DASH manifest'), + errnote=tr('Could not download DASH manifest'), fatal=fatal) if dash_doc is False: diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index af6e39b87..2be3d36a5 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -22,7 +22,7 @@ from ..utils import ( subtitles_filename, dfxp2srt, ISO639Utils, - g, + tr, ) @@ -404,7 +404,7 @@ class FFmpegMergerPP(FFmpegPostProcessor): filename = info['filepath'] temp_filename = prepend_extension(filename, 'temp') args = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0'] - self._downloader.to_screen(g('[ffmpeg] Merging formats into "%s"') % filename) + self._downloader.to_screen(tr('[ffmpeg] Merging formats into "%s"') % filename) self.run_ffmpeg_multiple_files(info['__files_to_merge'], temp_filename, args) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) return info['__files_to_merge'], info diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index b7d2043a7..e3c3c3696 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -2513,7 +2513,7 @@ class ISO3166Utils(object): return cls._country_map.get(code.upper()) -def g(s): +def tr(s): DOMAIN = 'youtube_dl' lang, _ = locale.getdefaultlocale() try: From 638a4d868931adcf4ee2606d31509d05180b4d72 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 13 Dec 2015 03:40:09 +0800 Subject: [PATCH 24/53] Revert "Drop Windows EXE support for gettext" This reverts commit 995f5036ca0eee575f5a8e15c62766f690ce10e2. --- setup.py | 13 +++++++++++++ youtube_dl/utils.py | 25 +++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/setup.py b/setup.py index 4403f162b..e8cc4de8c 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,8 @@ import os.path import warnings import sys import glob +import io +import zipfile try: from setuptools import setup @@ -24,6 +26,16 @@ except ImportError: print("Cannot import py2exe", file=sys.stderr) exit(1) + +def zipped_folder(topdir): + f = io.BytesIO() + zipf = zipfile.ZipFile(f, mode='w') + for dirpath, dirnames, filenames in os.walk(topdir): + for filename in filenames: + zipf.write(os.path.join(dirpath, filename)) + zipf.close() + return f.getvalue() + py2exe_options = { "bundle_files": 1, "compressed": 1, @@ -35,6 +47,7 @@ py2exe_options = { py2exe_console = [{ "script": "./youtube_dl/__main__.py", "dest_base": "youtube-dl", + "other_resources": [(u'LOCALE_DATA', u'locale_data.zip', zipped_folder('share'))], }] py2exe_params = { diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index e3c3c3696..e6a770b01 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -32,6 +32,7 @@ import sys import tempfile import traceback import xml.etree.ElementTree +import zipfile import zlib from .compat import ( @@ -2513,12 +2514,36 @@ class ISO3166Utils(object): return cls._country_map.get(code.upper()) +def _load_exe_resource(res_type, res_name): + kernel32 = ctypes.windll.kernel32 + + exe_handle = 0 # NULL: Current process + + res_info = kernel32.FindResourceW(exe_handle, res_name, res_type) + res_handle = kernel32.LoadResource(exe_handle, res_info) + res_data = kernel32.LockResource(res_handle) + res_len = kernel32.SizeofResource(exe_handle, res_info) + res_arr = ctypes.cast(res_data, ctypes.POINTER(ctypes.c_char))[:res_len] + return res_arr + + def tr(s): DOMAIN = 'youtube_dl' lang, _ = locale.getdefaultlocale() try: t = gettext.translation(DOMAIN, find_file_in_root('share/locale/'), [lang]) except (OSError, IOError): # OSError for 3.3+ and IOError otherwise + t = None + + if t is None and sys.platform == 'win32' and hasattr(sys, 'frozen'): + locale_data_zip = _load_exe_resource('LOCALE_DATA', 'LOCALE_DATA.ZIP') + f = io.BytesIO(locale_data_zip) + zipf = zipfile.ZipFile(f) + with zipf.open('share/locale/%s/LC_MESSAGES/%s.mo' % (lang, DOMAIN)) as mo_file: + t = gettext.GNUTranslations(mo_file) + zipf.close() + + if t is None: return s ret = t.gettext(s) From 7969009f00b50f04818864f1e8b30ea1833b7168 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 13 Dec 2015 03:40:55 +0800 Subject: [PATCH 25/53] [setup.py] Windows resource names are always capitalized --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e8cc4de8c..230ff33ee 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ py2exe_options = { py2exe_console = [{ "script": "./youtube_dl/__main__.py", "dest_base": "youtube-dl", - "other_resources": [(u'LOCALE_DATA', u'locale_data.zip', zipped_folder('share'))], + "other_resources": [(u'LOCALE_DATA', u'LOCALE_DATA.ZIP', zipped_folder('share'))], }] py2exe_params = { From 7da2b9e3bc3849bb94277a917ad3039fb797f273 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 13 Dec 2015 05:00:26 +0800 Subject: [PATCH 26/53] [options/utils] Add --default-language --- youtube_dl/__init__.py | 4 ++++ youtube_dl/options.py | 4 ++++ youtube_dl/utils.py | 54 ++++++++++++++++++++++++++---------------- 3 files changed, 42 insertions(+), 20 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 9f131f5db..e13efe511 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -27,6 +27,7 @@ from .utils import ( decodeOption, DEFAULT_OUTTMPL, DownloadError, + i18n_service, match_filter_func, MaxDownloadsReached, preferredencoding, @@ -56,6 +57,9 @@ def _real_main(argv=None): parser, opts, args = parseOpts(argv) + if opts.default_language is not None: + i18n_service.set_default_language(opts.default_language) + # Set user agent if opts.user_agent is not None: std_headers['User-Agent'] = opts.user_agent diff --git a/youtube_dl/options.py b/youtube_dl/options.py index c46e136bf..156977bdc 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -175,6 +175,10 @@ def parseOpts(overrideArguments=None): action='store_true', dest='no_color', default=False, help='Do not emit color codes in output') + general.add_option( + '--default-language', + dest='default_language', metavar='LANG_CODE', + help='Specify the default user interface language. Language codes should correspond to RFC 1766 standard, such as en_US or zh_TW.UTF-8. Default is en_US.') network = optparse.OptionGroup(parser, 'Network Options') network.add_option( diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index e6a770b01..4e4435e16 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -2527,29 +2527,43 @@ def _load_exe_resource(res_type, res_name): return res_arr -def tr(s): - DOMAIN = 'youtube_dl' - lang, _ = locale.getdefaultlocale() - try: - t = gettext.translation(DOMAIN, find_file_in_root('share/locale/'), [lang]) - except (OSError, IOError): # OSError for 3.3+ and IOError otherwise - t = None +class I18N(object): + def __init__(self): + self.default_lang = None + self.domain = 'youtube_dl' - if t is None and sys.platform == 'win32' and hasattr(sys, 'frozen'): - locale_data_zip = _load_exe_resource('LOCALE_DATA', 'LOCALE_DATA.ZIP') - f = io.BytesIO(locale_data_zip) - zipf = zipfile.ZipFile(f) - with zipf.open('share/locale/%s/LC_MESSAGES/%s.mo' % (lang, DOMAIN)) as mo_file: - t = gettext.GNUTranslations(mo_file) - zipf.close() + def translate(self, s): + if self.default_lang is not None: + lang = self.default_lang + else: + lang, _ = locale.getdefaultlocale() - if t is None: - return s + try: + t = gettext.translation(self.domain, find_file_in_root('share/locale/'), [lang]) + except (OSError, IOError): # OSError for 3.3+ and IOError otherwise + t = None - ret = t.gettext(s) - if isinstance(ret, bytes): - ret = ret.decode('utf-8') - return ret + if t is None and sys.platform == 'win32' and hasattr(sys, 'frozen'): + locale_data_zip = _load_exe_resource('LOCALE_DATA', 'LOCALE_DATA.ZIP') + f = io.BytesIO(locale_data_zip) + zipf = zipfile.ZipFile(f) + with zipf.open('share/locale/%s/LC_MESSAGES/%s.mo' % (lang, self.domain)) as mo_file: + t = gettext.GNUTranslations(mo_file) + zipf.close() + + if t is None: + return s + + ret = t.gettext(s) + if isinstance(ret, bytes): + ret = ret.decode('utf-8') + return ret + + def set_default_language(self, default_lang): + self.default_lang = default_lang + +i18n_service = I18N() +tr = i18n_service.translate def get_root_dirs(): From c0450e2014a88812dde21340f838613ac9eb0734 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 13 Dec 2015 05:09:41 +0800 Subject: [PATCH 27/53] [utils] Capture the error that mo files not found in LOCALE_DATA.ZIP --- youtube_dl/utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 4e4435e16..7e94bddeb 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -2547,8 +2547,13 @@ class I18N(object): locale_data_zip = _load_exe_resource('LOCALE_DATA', 'LOCALE_DATA.ZIP') f = io.BytesIO(locale_data_zip) zipf = zipfile.ZipFile(f) - with zipf.open('share/locale/%s/LC_MESSAGES/%s.mo' % (lang, self.domain)) as mo_file: - t = gettext.GNUTranslations(mo_file) + try: + zinfo = zipf.getinfo('share/locale/%s/LC_MESSAGES/%s.mo' % (lang, self.domain)) + except KeyError: + zinfo = None + if zinfo is not None: + with zipf.open(zinfo) as mo_file: + t = gettext.GNUTranslations(mo_file) zipf.close() if t is None: From e556aa643d738e51313d469fe819ad625eeae908 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 20 Dec 2015 18:57:14 +0800 Subject: [PATCH 28/53] [utils] Preliminary fix for Python2 with non-ASCII paths --- youtube_dl/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 7e94bddeb..235c1318b 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -2582,7 +2582,7 @@ def get_root_dirs(): # ../../ of youtube_dl/utils.py ret.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - return ret + return map(decodeFilename, ret) def find_file_in_root(file_path): From 163f0a511c772b721d8dbfd3165f197bd67b22ed Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 26 Dec 2015 05:00:54 +0800 Subject: [PATCH 29/53] [devscripts/i18n] unicode_literals --- devscripts/i18n.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/devscripts/i18n.py b/devscripts/i18n.py index 229cbd8b9..3f72baeb7 100644 --- a/devscripts/i18n.py +++ b/devscripts/i18n.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import errno import glob import os From a5f07800ddc566801cdf97216743c2407fdc20a6 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 26 Dec 2015 05:05:19 +0800 Subject: [PATCH 30/53] [utils] Check .mo files instead of directories in root dirs For example, nosetests imports yuotube_dl, get_root_dir() returns /usr and the current directory. /usr/share/locale is picked in the original implementation. --- youtube_dl/utils.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 235c1318b..02f0c839d 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -2538,10 +2538,13 @@ class I18N(object): else: lang, _ = locale.getdefaultlocale() - try: - t = gettext.translation(self.domain, find_file_in_root('share/locale/'), [lang]) - except (OSError, IOError): # OSError for 3.3+ and IOError otherwise - t = None + for root in get_root_dirs(): + try: + t = gettext.translation(self.domain, os.path.join(root, 'share', 'locale'), [lang]) + if t is not None: + break + except (OSError, IOError): # OSError for 3.3+ and IOError otherwise + t = None if t is None and sys.platform == 'win32' and hasattr(sys, 'frozen'): locale_data_zip = _load_exe_resource('LOCALE_DATA', 'LOCALE_DATA.ZIP') @@ -2585,13 +2588,6 @@ def get_root_dirs(): return map(decodeFilename, ret) -def find_file_in_root(file_path): - for root in get_root_dirs(): - full_path = os.path.join(root, file_path) - if os.path.exists(full_path): - return full_path - - class PerRequestProxyHandler(compat_urllib_request.ProxyHandler): def __init__(self, proxies=None): # Set default handlers From af9c89ba07195aae59d8c5ffcea0b17afb6a17d0 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 26 Dec 2015 05:13:50 +0800 Subject: [PATCH 31/53] [test_utils] Add a basic test for i18n --- .travis.yml | 4 ++++ Makefile | 4 ++-- test/test_utils.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index cc21fae8f..76434202e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,3 +17,7 @@ notifications: # channels: # - "irc.freenode.org#youtube-dl" # skip_join: true +addons: + apt: + packages: + - gettext diff --git a/Makefile b/Makefile index 3a390f0ce..728844ca0 100644 --- a/Makefile +++ b/Makefile @@ -36,14 +36,14 @@ install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtu codetest: flake8 . -test: +test: update-gmo #nosetests --with-coverage --cover-package=youtube_dl --cover-html --verbose --processes 4 test nosetests --verbose test $(MAKE) codetest ot: offlinetest -offlinetest: codetest +offlinetest: codetest update-gmo nosetests --verbose test --exclude test_download.py --exclude test_age_restriction.py --exclude test_subtitles.py --exclude test_write_annotations.py --exclude test_youtube_lists.py tar: youtube-dl.tar.gz diff --git a/test/test_utils.py b/test/test_utils.py index 501355c74..ae3cc66f2 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -30,6 +30,7 @@ from youtube_dl.utils import ( fix_xml_ampersands, InAdvancePagedList, intlist_to_bytes, + I18N, is_html, js_to_json, limit_length, @@ -746,6 +747,35 @@ The first line {'nocheckcertificate': False}, '--check-certificate', 'nocheckcertificate', 'false', 'true', '='), ['--check-certificate=true']) + def test_i18n(self): + old_locale = os.environ.get('LC_ALL') + if old_locale is not None: + del os.environ['LC_ALL'] + + instance = I18N() + instance.set_default_language('en_US') + self.assertEqual(instance.translate('Test URL: %s'), 'Test URL: %s') + + instance = I18N() + instance.set_default_language('zh_TW') + self.assertEqual(instance.translate('Test URL: %s'), '測試網址：%s') + + # A fake language code + instance = I18N() + instance.set_default_language('pp_QQ') + self.assertEqual(instance.translate('Test URL: %s'), 'Test URL: %s') + + # setting locale via environ is not applicable on Windows + if not sys.platform.startswith('win'): + os.environ['LC_ALL'] = 'zh_TW' + instance = I18N() + self.assertEqual(instance.translate('Test URL: %s'), '測試網址：%s') + + if old_locale is not None: + os.environ['LC_ALL'] = old_locale + else: + del os.environ['LC_ALL'] + if __name__ == '__main__': unittest.main() From 312bfeca77ab98ca7c64db29d2188f7e58ec4b5d Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 26 Dec 2015 05:29:52 +0800 Subject: [PATCH 32/53] [utils] Also search .mo files in sys.prefix --- youtube_dl/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 02f0c839d..506c3b9b1 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -2585,6 +2585,9 @@ def get_root_dirs(): # ../../ of youtube_dl/utils.py ret.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + if sys.prefix not in ret: + ret.append(sys.prefix) # fallback + return map(decodeFilename, ret) From cedf7b1510511835e2b6fc3c221aa2bae01b9dfc Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 26 Dec 2015 05:47:51 +0800 Subject: [PATCH 33/53] [utils] Add a cache for translations --- youtube_dl/utils.py | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 506c3b9b1..a17db849b 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -2531,20 +2531,22 @@ class I18N(object): def __init__(self): self.default_lang = None self.domain = 'youtube_dl' + self._translation_cache = {} - def translate(self, s): - if self.default_lang is not None: - lang = self.default_lang - else: - lang, _ = locale.getdefaultlocale() + def _load_translation(self, lang): + t = self._translation_cache.get(lang) - for root in get_root_dirs(): - try: - t = gettext.translation(self.domain, os.path.join(root, 'share', 'locale'), [lang]) - if t is not None: - break - except (OSError, IOError): # OSError for 3.3+ and IOError otherwise - t = None + if t is not None: + return t + + if t is None: + for root in get_root_dirs(): + try: + t = gettext.translation(self.domain, os.path.join(root, 'share', 'locale'), [lang]) + if t is not None: + break + except (OSError, IOError): # OSError for 3.3+ and IOError otherwise + t = None if t is None and sys.platform == 'win32' and hasattr(sys, 'frozen'): locale_data_zip = _load_exe_resource('LOCALE_DATA', 'LOCALE_DATA.ZIP') @@ -2559,6 +2561,18 @@ class I18N(object): t = gettext.GNUTranslations(mo_file) zipf.close() + if t is not None: + self._translation_cache[lang] = t + + return t + + def translate(self, s): + if self.default_lang is not None: + lang = self.default_lang + else: + lang, _ = locale.getdefaultlocale() + + t = self._load_translation(lang) if t is None: return s From 8a54fa704f65d16a80ab219de1aab827aa06f624 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 26 Dec 2015 15:28:29 +0800 Subject: [PATCH 34/53] Generate .mo files before running test --- .travis.yml | 2 +- devscripts/run_travis_test.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 devscripts/run_travis_test.sh diff --git a/.travis.yml b/.travis.yml index 76434202e..1cbbcf7bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ python: - "3.4" - "3.5" sudo: false -script: nosetests test --verbose +script: bash ./devscripts/run_travis_test.sh notifications: email: - filippo.valsorda@gmail.com diff --git a/devscripts/run_travis_test.sh b/devscripts/run_travis_test.sh new file mode 100644 index 000000000..20d1f50a1 --- /dev/null +++ b/devscripts/run_travis_test.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +python devscripts/i18n.py update-gmo +nosetests test --verbose From 7bdc9cc044a1b57a47fcda7f4d548c01ad0680b7 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 27 Dec 2015 21:56:57 +0800 Subject: [PATCH 35/53] [utils] Remove a redundant check in I18N --- youtube_dl/utils.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index a17db849b..333de7a64 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -2539,14 +2539,13 @@ class I18N(object): if t is not None: return t - if t is None: - for root in get_root_dirs(): - try: - t = gettext.translation(self.domain, os.path.join(root, 'share', 'locale'), [lang]) - if t is not None: - break - except (OSError, IOError): # OSError for 3.3+ and IOError otherwise - t = None + for root in get_root_dirs(): + try: + t = gettext.translation(self.domain, os.path.join(root, 'share', 'locale'), [lang]) + if t is not None: + break + except (OSError, IOError): # OSError for 3.3+ and IOError otherwise + t = None if t is None and sys.platform == 'win32' and hasattr(sys, 'frozen'): locale_data_zip = _load_exe_resource('LOCALE_DATA', 'LOCALE_DATA.ZIP') From bf3a04bc00d26900a45544a5019d2f278568e40d Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Tue, 29 Dec 2015 03:29:23 +0800 Subject: [PATCH 36/53] [__init__] Disable localization if --verbose given --verbose is used for debugging, and we don't want to receive non-English bug reports. --- youtube_dl/__init__.py | 3 +++ youtube_dl/options.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index e13efe511..3978555e4 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -60,6 +60,9 @@ def _real_main(argv=None): if opts.default_language is not None: i18n_service.set_default_language(opts.default_language) + if opts.verbose: + i18n_service.set_default_language('en_US') + # Set user agent if opts.user_agent is not None: std_headers['User-Agent'] = opts.user_agent diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 156977bdc..8853e4b75 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -542,7 +542,7 @@ def parseOpts(overrideArguments=None): verbosity.add_option( '-v', '--verbose', action='store_true', dest='verbose', default=False, - help='Print various debugging information') + help='Print various debugging information. Localization is disabled with this option.') verbosity.add_option( '--dump-pages', '--dump-intermediate-pages', action='store_true', dest='dump_intermediate_pages', default=False, From ca282231116fecb9fe1afd9d65e57ca41d59a50b Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Tue, 29 Dec 2015 03:35:55 +0800 Subject: [PATCH 37/53] [devscripts/i18n] Remove -j when regenerating youtube_dl.pot This option leave unnecessary comments in youtube_dl.pot --- devscripts/i18n.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devscripts/i18n.py b/devscripts/i18n.py index 3f72baeb7..94653dbeb 100644 --- a/devscripts/i18n.py +++ b/devscripts/i18n.py @@ -76,7 +76,7 @@ class I18N_Utils(object): old_pot_file = self.get_pot_filename() + '.old' shutil.copy2(pot_file, old_pot_file) cmds = [ - 'xgettext', '-d', self.GETTEXT_DOMAIN, '-j', '-k', '-ktr', '--from-code=utf-8', '-F', '-o', + 'xgettext', '-d', self.GETTEXT_DOMAIN, '-k', '-ktr', '--from-code=utf-8', '-F', '-o', pot_file] cmds.extend(glob.glob('youtube_dl/*.py') + glob.glob('youtube_dl/*/*.py')) self._run_subprocess(cmds) From 3242f7edd289faf6f03a48b6dbe238d40889170b Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sat, 2 Jan 2016 01:13:26 +0800 Subject: [PATCH 38/53] [options] Rename --default-language to --lang --- youtube_dl/__init__.py | 4 ++-- youtube_dl/options.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 3978555e4..be28a9c54 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -57,8 +57,8 @@ def _real_main(argv=None): parser, opts, args = parseOpts(argv) - if opts.default_language is not None: - i18n_service.set_default_language(opts.default_language) + if opts.i18n_language is not None: + i18n_service.set_default_language(opts.i18n_language) if opts.verbose: i18n_service.set_default_language('en_US') diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 8853e4b75..197db3332 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -176,8 +176,8 @@ def parseOpts(overrideArguments=None): default=False, help='Do not emit color codes in output') general.add_option( - '--default-language', - dest='default_language', metavar='LANG_CODE', + '--lang', + dest='i18n_language', metavar='LANG_CODE', help='Specify the default user interface language. Language codes should correspond to RFC 1766 standard, such as en_US or zh_TW.UTF-8. Default is en_US.') network = optparse.OptionGroup(parser, 'Network Options') From a2aae628a0a1a6c95664ace01960b3656cdce13c Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 17 Jan 2016 16:33:26 +0800 Subject: [PATCH 39/53] [i18n] New extractor for I18N testing --- po/youtube_dl.pot | 6 +++++- po/zh_TW.po | 6 +++++- youtube_dl/extractor/__init__.py | 1 + youtube_dl/extractor/i18ntest.py | 13 +++++++++++++ 4 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 youtube_dl/extractor/i18ntest.py diff --git a/po/youtube_dl.pot b/po/youtube_dl.pot index b29b329c5..199819e02 100644 --- a/po/youtube_dl.pot +++ b/po/youtube_dl.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: 1.0\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2015-12-13 03:32+0800\n" +"POT-Creation-Date: 2016-01-17 16:08+0800\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -46,6 +46,10 @@ msgstr "" msgid "%s: Downloading webpage" msgstr "" +#: youtube_dl/extractor/i18ntest.py:13 +msgid "I18N test message" +msgstr "" + #: youtube_dl/extractor/testurl.py:65 #, python-format msgid "Test URL: %s" diff --git a/po/zh_TW.po b/po/zh_TW.po index 567e77d01..99484cec3 100644 --- a/po/zh_TW.po +++ b/po/zh_TW.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: \n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2015-12-13 00:33+0800\n" +"POT-Creation-Date: 2016-01-17 16:08+0800\n" "PO-Revision-Date: 2015-12-13 01:04+0800\n" "Last-Translator: yan12125@gmail.com\n" "Language-Team: Chinese (traditional)\n" @@ -47,6 +47,10 @@ msgstr "[下載] " msgid "%s: Downloading webpage" msgstr "%s: 下載網頁" +#: youtube_dl/extractor/i18ntest.py:13 +msgid "I18N test message" +msgstr "I18N測試訊息" + #: youtube_dl/extractor/testurl.py:65 #, python-format msgid "Test URL: %s" diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 3cd95ba01..541b329c4 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -251,6 +251,7 @@ from .howcast import HowcastIE from .howstuffworks import HowStuffWorksIE from .huffpost import HuffPostIE from .hypem import HypemIE +from .i18ntest import I18NTestIE from .iconosquare import IconosquareIE from .ign import IGNIE, OneUPIE from .imdb import ( diff --git a/youtube_dl/extractor/i18ntest.py b/youtube_dl/extractor/i18ntest.py new file mode 100644 index 000000000..858db723b --- /dev/null +++ b/youtube_dl/extractor/i18ntest.py @@ -0,0 +1,13 @@ +# coding: utf-8 + +from __future__ import unicode_literals + +from .common import InfoExtractor +from ..utils import tr + + +class I18NTestIE(InfoExtractor): + _VALID_URL = 'i18n:test' + + def _real_extract(self, url): + self._downloader.to_screen(tr('I18N test message')) From 8843ac8c54773c6a146e91a0b4f36483b1024c32 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 17 Jan 2016 16:33:47 +0800 Subject: [PATCH 40/53] [test_i18n] Add a test case for I18N --- test/test_i18n.py | 111 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 test/test_i18n.py diff --git a/test/test_i18n.py b/test/test_i18n.py new file mode 100644 index 000000000..a5b5bd988 --- /dev/null +++ b/test/test_i18n.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# coding: utf-8 + +from __future__ import unicode_literals + +import contextlib +import os +import shutil +import subprocess +import sys + +try: + import unittest + unittest.TestCase.setUpClass +except AttributeError: + import unittest2 as unittest + +# Allow direct execution +rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, rootDir) + +from youtube_dl.version import __version__ +from youtube_dl.compat import subprocess_check_output +from youtube_dl.utils import decodeFilename + + +@contextlib.contextmanager +def chdir_to(path): + oldpwd = os.getcwd() + os.chdir(os.path.join(decodeFilename(rootDir), path)) + yield + os.chdir(oldpwd) + + +class I18NTestCase(object): + @classmethod + def setUpClass(cls): + with chdir_to('.'): + cls.make('update-gmo') + + cls.install() + + # avoid false positive + with chdir_to('.'): + shutil.rmtree(os.path.join('share', 'locale')) + + @classmethod + def tearDownClass(cls): + cls.uninstall() + + @classmethod + def make(cls, target): + with chdir_to('.'): + subprocess.check_call([ + 'make', target], env=dict(os.environ, PYTHON=sys.executable)) + + def test_lang_opt(self): + def output_with_lang_opt(lang): + return subprocess_check_output([ + sys.executable, self.PROGRAM, 'i18n:test', '--lang', lang + ]).decode('utf-8').strip() + + self.assertEqual(output_with_lang_opt('en_US.UTF-8'), 'I18N test message') + self.assertEqual(output_with_lang_opt('zh_TW.UTF-8'), 'I18N測試訊息') + + def test_lc_all(self): + def output_with_lc_all(lang): + return subprocess_check_output([ + sys.executable, self.PROGRAM, 'i18n:test' + ], env=dict(os.environ, LC_ALL=lang)).decode('utf-8').strip() + + self.assertEqual(output_with_lc_all('en_US.UTF-8'), 'I18N test message') + self.assertEqual(output_with_lc_all('zh_TW.UTF-8'), 'I18N測試訊息') + + +@unittest.skipUnless(os.environ.get('VIRTUAL_ENV'), 'Requires virtualenv because of call to pip install') +class TestPipInstall(I18NTestCase, unittest.TestCase): + PROGRAM = os.path.join(os.environ['VIRTUAL_ENV'], 'bin', 'youtube-dl') + + @classmethod + def install(cls): + with chdir_to('.'): + subprocess.check_call([sys.executable, 'setup.py', '--quiet', 'sdist']) + + with chdir_to('test'): + subprocess.check_call([ + 'pip', 'install', '--quiet', + os.path.join('..', 'dist', 'youtube_dl-%s.tar.gz' % __version__)]) + + @classmethod + def uninstall(cls): + with chdir_to('test'): + subprocess.check_call(['pip', 'uninstall', '--yes', 'youtube_dl']) + + +class TestDirectInstall(I18NTestCase, unittest.TestCase): + PROGRAM = os.path.join(os.environ['VIRTUAL_ENV'], 'bin', 'youtube-dl') + + @classmethod + def install(cls): + with chdir_to('.'): + subprocess.check_call([sys.executable, 'setup.py', '--quiet', 'install']) + + @classmethod + def uninstall(cls): + with chdir_to('test'): + subprocess.check_call(['pip', 'uninstall', '--yes', 'youtube_dl']) + + +if __name__ == '__main__': + unittest.main() From 216c21f5329b3b60cdea04a305278e35f37a39e7 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 17 Jan 2016 16:51:05 +0800 Subject: [PATCH 41/53] [Makefile] Create zipped app with *.mo file --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 728844ca0..c38a3e16f 100644 --- a/Makefile +++ b/Makefile @@ -52,8 +52,8 @@ tar: youtube-dl.tar.gz pypi-files: youtube-dl.bash-completion README.txt youtube-dl.1 youtube-dl.fish update-gmo -youtube-dl: youtube_dl/*.py youtube_dl/*/*.py - zip --quiet youtube-dl youtube_dl/*.py youtube_dl/*/*.py +youtube-dl: youtube_dl/*.py youtube_dl/*/*.py update-gmo + zip --quiet youtube-dl youtube_dl/*.py youtube_dl/*/*.py share/locale/*/LC_MESSAGES/*.mo zip --quiet --junk-paths youtube-dl youtube_dl/__main__.py echo '#!$(PYTHON)' > youtube-dl cat youtube-dl.zip >> youtube-dl From fecfb0554caa42150a986729c26d45506f4a869b Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 17 Jan 2016 16:53:21 +0800 Subject: [PATCH 42/53] [utils] Load translations from zipped app --- youtube_dl/utils.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 333de7a64..9628a8339 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -33,6 +33,7 @@ import tempfile import traceback import xml.etree.ElementTree import zipfile +import zipimport import zlib from .compat import ( @@ -1789,9 +1790,8 @@ def is_outdated_version(version, limit, assume_new=True): def ytdl_is_updateable(): """ Returns if youtube-dl can be updated with -U """ - from zipimport import zipimporter - return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen') + return isinstance(globals().get('__loader__'), zipimport.zipimporter) or hasattr(sys, 'frozen') def args_to_str(args): @@ -2533,6 +2533,19 @@ class I18N(object): self.domain = 'youtube_dl' self._translation_cache = {} + def _load_translation_from_zipfile(self, f, lang): + t = None + zipf = zipfile.ZipFile(f) + try: + zinfo = zipf.getinfo('share/locale/%s/LC_MESSAGES/%s.mo' % (lang, self.domain)) + except KeyError: + zinfo = None + if zinfo is not None: + with zipf.open(zinfo) as mo_file: + t = gettext.GNUTranslations(mo_file) + zipf.close() + return t + def _load_translation(self, lang): t = self._translation_cache.get(lang) @@ -2547,18 +2560,15 @@ class I18N(object): except (OSError, IOError): # OSError for 3.3+ and IOError otherwise t = None + if t is None and isinstance(globals().get('__loader__'), zipimport.zipimporter): + myself = globals()['__loader__'].archive + with open(myself, 'rb') as f: + t = self._load_translation_from_zipfile(f, lang) + if t is None and sys.platform == 'win32' and hasattr(sys, 'frozen'): locale_data_zip = _load_exe_resource('LOCALE_DATA', 'LOCALE_DATA.ZIP') f = io.BytesIO(locale_data_zip) - zipf = zipfile.ZipFile(f) - try: - zinfo = zipf.getinfo('share/locale/%s/LC_MESSAGES/%s.mo' % (lang, self.domain)) - except KeyError: - zinfo = None - if zinfo is not None: - with zipf.open(zinfo) as mo_file: - t = gettext.GNUTranslations(mo_file) - zipf.close() + t = self._load_translation_from_zipfile(f, lang) if t is not None: self._translation_cache[lang] = t @@ -2581,7 +2591,8 @@ class I18N(object): return ret def set_default_language(self, default_lang): - self.default_lang = default_lang + # split to save locale only, for example zh_TW.UTF-8 => zh_TW + self.default_lang = default_lang.split('.')[0] i18n_service = I18N() tr = i18n_service.translate From 5ebbe780fed8525ac15e7da74ea0569cae528997 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 17 Jan 2016 16:53:44 +0800 Subject: [PATCH 43/53] [test_i18n] Add a test case for zipped app --- test/test_i18n.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test/test_i18n.py b/test/test_i18n.py index a5b5bd988..e837bcf43 100644 --- a/test/test_i18n.py +++ b/test/test_i18n.py @@ -23,11 +23,13 @@ from youtube_dl.version import __version__ from youtube_dl.compat import subprocess_check_output from youtube_dl.utils import decodeFilename +rootDir_u = decodeFilename(rootDir) + @contextlib.contextmanager def chdir_to(path): oldpwd = os.getcwd() - os.chdir(os.path.join(decodeFilename(rootDir), path)) + os.chdir(os.path.join(rootDir_u, path)) yield os.chdir(oldpwd) @@ -107,5 +109,18 @@ class TestDirectInstall(I18NTestCase, unittest.TestCase): subprocess.check_call(['pip', 'uninstall', '--yes', 'youtube_dl']) +class TestZippedApp(I18NTestCase, unittest.TestCase): + PROGRAM = os.path.join(rootDir_u, 'youtube-dl') + + @classmethod + def install(cls): + with chdir_to('.'): + cls.make('youtube-dl') + + @classmethod + def uninstall(cls): + with chdir_to('.'): + os.unlink('youtube-dl') + if __name__ == '__main__': unittest.main() From 1aee79da3e40f7d48211c484c818e37c736ef9ee Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 17 Jan 2016 16:58:38 +0800 Subject: [PATCH 44/53] [test_i18n] Correctly skip tests in non-virtualenv --- test/test_i18n.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/test_i18n.py b/test/test_i18n.py index e837bcf43..a0a488db5 100644 --- a/test/test_i18n.py +++ b/test/test_i18n.py @@ -77,10 +77,10 @@ class I18NTestCase(object): @unittest.skipUnless(os.environ.get('VIRTUAL_ENV'), 'Requires virtualenv because of call to pip install') class TestPipInstall(I18NTestCase, unittest.TestCase): - PROGRAM = os.path.join(os.environ['VIRTUAL_ENV'], 'bin', 'youtube-dl') - @classmethod def install(cls): + cls.PROGRAM = os.path.join(os.environ['VIRTUAL_ENV'], 'bin', 'youtube-dl') + with chdir_to('.'): subprocess.check_call([sys.executable, 'setup.py', '--quiet', 'sdist']) @@ -95,11 +95,12 @@ class TestPipInstall(I18NTestCase, unittest.TestCase): subprocess.check_call(['pip', 'uninstall', '--yes', 'youtube_dl']) +@unittest.skipUnless(os.environ.get('VIRTUAL_ENV'), 'Requires virtualenv because of call to setup.py install') class TestDirectInstall(I18NTestCase, unittest.TestCase): - PROGRAM = os.path.join(os.environ['VIRTUAL_ENV'], 'bin', 'youtube-dl') - @classmethod def install(cls): + cls.PROGRAM = os.path.join(os.environ['VIRTUAL_ENV'], 'bin', 'youtube-dl') + with chdir_to('.'): subprocess.check_call([sys.executable, 'setup.py', '--quiet', 'install']) From b42444d38b7f801c883c209772d4af82422ec380 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 17 Jan 2016 17:09:39 +0800 Subject: [PATCH 45/53] [utils] I18N: fix for Python 2.6 Using zipfile directly as a context manager is not supported until Python 2.7 --- youtube_dl/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 9628a8339..8f643d52f 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -2541,7 +2541,7 @@ class I18N(object): except KeyError: zinfo = None if zinfo is not None: - with zipf.open(zinfo) as mo_file: + with contextlib.closing(zipf.open(zinfo)) as mo_file: t = gettext.GNUTranslations(mo_file) zipf.close() return t From 04546db7b07298a7c2597dbeebbbf617342ebff5 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 17 Jan 2016 17:39:09 +0800 Subject: [PATCH 46/53] [setup.py] Fix for Python 3.2 --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 230ff33ee..bae063eee 100644 --- a/setup.py +++ b/setup.py @@ -44,10 +44,14 @@ py2exe_options = { "dll_excludes": ['w9xpopen.exe', 'crypt32.dll'], } + +def u(s): + return s.decode('utf-8') + py2exe_console = [{ "script": "./youtube_dl/__main__.py", "dest_base": "youtube-dl", - "other_resources": [(u'LOCALE_DATA', u'LOCALE_DATA.ZIP', zipped_folder('share'))], + "other_resources": [(u(b'LOCALE_DATA'), u(b'LOCALE_DATA.ZIP'), zipped_folder('share'))], }] py2exe_params = { From 0c9c77b654075b2418c6f7ec574e09b5c4bebe18 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 17 Jan 2016 18:00:46 +0800 Subject: [PATCH 47/53] [test_i18n] Skip LC_ALL test on Windows --- test/test_i18n.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_i18n.py b/test/test_i18n.py index a0a488db5..28c98e76c 100644 --- a/test/test_i18n.py +++ b/test/test_i18n.py @@ -65,6 +65,7 @@ class I18NTestCase(object): self.assertEqual(output_with_lang_opt('en_US.UTF-8'), 'I18N test message') self.assertEqual(output_with_lang_opt('zh_TW.UTF-8'), 'I18N測試訊息') + @unittest.skipIf(sys.platform.startswith('win'), 'LC_ALL not applicable on Windows') def test_lc_all(self): def output_with_lc_all(lang): return subprocess_check_output([ From 8347b46331a74e4b64037069080117cfd6011362 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 17 Jan 2016 18:29:22 +0800 Subject: [PATCH 48/53] [test_i18n] Fix for Windows 1. Theoretically using zipped youtube-dl on Windows is possible but building it needs too many dependencies, so I skip it 2. I18NTestCase.PROGRAM is now a list so that .exe can be tested --- test/test_i18n.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/test/test_i18n.py b/test/test_i18n.py index 28c98e76c..bf257b2d1 100644 --- a/test/test_i18n.py +++ b/test/test_i18n.py @@ -21,7 +21,10 @@ sys.path.insert(0, rootDir) from youtube_dl.version import __version__ from youtube_dl.compat import subprocess_check_output -from youtube_dl.utils import decodeFilename +from youtube_dl.utils import ( + decodeFilename, + get_subprocess_encoding, +) rootDir_u = decodeFilename(rootDir) @@ -34,6 +37,13 @@ def chdir_to(path): os.chdir(oldpwd) +def ydl_path(venv_root): + if sys.platform.startswith('win'): + return [os.path.join(venv_root, 'Scripts', 'youtube-dl.exe')] + + return [sys.executable, os.path.join(venv_root, 'bin', 'youtube-dl')] + + class I18NTestCase(object): @classmethod def setUpClass(cls): @@ -58,9 +68,9 @@ class I18NTestCase(object): def test_lang_opt(self): def output_with_lang_opt(lang): - return subprocess_check_output([ - sys.executable, self.PROGRAM, 'i18n:test', '--lang', lang - ]).decode('utf-8').strip() + return subprocess_check_output(self.PROGRAM + [ + 'i18n:test', '--lang', lang + ]).decode(get_subprocess_encoding()).strip() self.assertEqual(output_with_lang_opt('en_US.UTF-8'), 'I18N test message') self.assertEqual(output_with_lang_opt('zh_TW.UTF-8'), 'I18N測試訊息') @@ -68,9 +78,9 @@ class I18NTestCase(object): @unittest.skipIf(sys.platform.startswith('win'), 'LC_ALL not applicable on Windows') def test_lc_all(self): def output_with_lc_all(lang): - return subprocess_check_output([ - sys.executable, self.PROGRAM, 'i18n:test' - ], env=dict(os.environ, LC_ALL=lang)).decode('utf-8').strip() + return subprocess_check_output(self.PROGRAM + [ + 'i18n:test' + ], env=dict(os.environ, LC_ALL=lang)).decode(get_subprocess_encoding()).strip() self.assertEqual(output_with_lc_all('en_US.UTF-8'), 'I18N test message') self.assertEqual(output_with_lc_all('zh_TW.UTF-8'), 'I18N測試訊息') @@ -80,7 +90,7 @@ class I18NTestCase(object): class TestPipInstall(I18NTestCase, unittest.TestCase): @classmethod def install(cls): - cls.PROGRAM = os.path.join(os.environ['VIRTUAL_ENV'], 'bin', 'youtube-dl') + cls.PROGRAM = ydl_path(os.environ['VIRTUAL_ENV']) with chdir_to('.'): subprocess.check_call([sys.executable, 'setup.py', '--quiet', 'sdist']) @@ -100,7 +110,7 @@ class TestPipInstall(I18NTestCase, unittest.TestCase): class TestDirectInstall(I18NTestCase, unittest.TestCase): @classmethod def install(cls): - cls.PROGRAM = os.path.join(os.environ['VIRTUAL_ENV'], 'bin', 'youtube-dl') + cls.PROGRAM = ydl_path(os.environ['VIRTUAL_ENV']) with chdir_to('.'): subprocess.check_call([sys.executable, 'setup.py', '--quiet', 'install']) @@ -111,8 +121,9 @@ class TestDirectInstall(I18NTestCase, unittest.TestCase): subprocess.check_call(['pip', 'uninstall', '--yes', 'youtube_dl']) +@unittest.skipIf(sys.platform.startswith('win'), 'Zipped youtube-dl is not designed for Windows') class TestZippedApp(I18NTestCase, unittest.TestCase): - PROGRAM = os.path.join(rootDir_u, 'youtube-dl') + PROGRAM = [sys.executable, os.path.join(rootDir_u, 'youtube-dl')] @classmethod def install(cls): From fb1da6be36caf504f0c182f5174d2d6176a39591 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 17 Jan 2016 19:31:49 +0800 Subject: [PATCH 49/53] [test_i18n] Handle normalized version by pip --- test/test_i18n.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/test_i18n.py b/test/test_i18n.py index bf257b2d1..8d00eb4e5 100644 --- a/test/test_i18n.py +++ b/test/test_i18n.py @@ -20,7 +20,10 @@ rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, rootDir) from youtube_dl.version import __version__ -from youtube_dl.compat import subprocess_check_output +from youtube_dl.compat import ( + subprocess_check_output, + compat_str, +) from youtube_dl.utils import ( decodeFilename, get_subprocess_encoding, @@ -44,6 +47,15 @@ def ydl_path(venv_root): return [sys.executable, os.path.join(venv_root, 'bin', 'youtube-dl')] +def normalized_version(ver): + try: + from pip._vendor import packaging + except ImportError: + return version + + return compat_str(packaging.version.Version(ver)) + + class I18NTestCase(object): @classmethod def setUpClass(cls): @@ -98,7 +110,7 @@ class TestPipInstall(I18NTestCase, unittest.TestCase): with chdir_to('test'): subprocess.check_call([ 'pip', 'install', '--quiet', - os.path.join('..', 'dist', 'youtube_dl-%s.tar.gz' % __version__)]) + os.path.join('..', 'dist', 'youtube_dl-%s.tar.gz' % normalized_version(__version__))]) @classmethod def uninstall(cls): From 1d5db005dd4aa0f659eaeb2b5454ddcef2acc708 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 17 Jan 2016 19:35:32 +0800 Subject: [PATCH 50/53] Install unittest2 in Python 2.6 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..46fbfa468 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +unittest2 ; python_version < '2.7' From d391152f4d080254a67e05dd2fcc66df687da2de Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Sun, 17 Jan 2016 22:10:50 +0800 Subject: [PATCH 51/53] [test_i18n] Correct a variable name --- test/test_i18n.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_i18n.py b/test/test_i18n.py index 8d00eb4e5..d3a52d704 100644 --- a/test/test_i18n.py +++ b/test/test_i18n.py @@ -51,7 +51,7 @@ def normalized_version(ver): try: from pip._vendor import packaging except ImportError: - return version + return ver return compat_str(packaging.version.Version(ver)) From 9998fbd6e87b411898f97034dfac8239a42c596b Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Mon, 18 Jan 2016 03:33:44 +0800 Subject: [PATCH 52/53] [travis] Install missing locales --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 1cbbcf7bb..6222d33ab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,3 +21,4 @@ addons: apt: packages: - gettext + - language-pack-zh-hant From 8859763182f1966a882edc03180d556952fd0792 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Fri, 4 Mar 2016 01:36:08 +0800 Subject: [PATCH 53/53] [appveyor] Add appveyor.yml --- appveyor.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 000000000..63df6f291 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,15 @@ +# Python 3.2 is not installed on Appveyor +environment: + matrix: + - PYTHON: "C:\\Python26\\python.exe" + - PYTHON: "C:\\Python27\\python.exe" + - PYTHON: "C:\\Python33\\python.exe" + - PYTHON: "C:\\Python34\\python.exe" + - PYTHON: "C:\\Python35\\python.exe" + +install: + - "%PYTHON% -m pip install --verbose -r requirements.txt" + +test_script: + - "%PYTHON% devscripts/i18n.py" + - "%PYTHON% -m nose --verbose test"