Refactor some parts of the option parsing code

devscripts/bash-completion.py

@@ -6,7 +6,7 @@ from os.path import dirname as dirn
 import sys
 
 sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
-import youtube_dl
+from youtube_dl.options import build_option_parser
 
 BASH_COMPLETION_FILE = "youtube-dl.bash-completion"
 BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in"
@@ -25,5 +25,5 @@ def build_completion(opt_parser):
         filled_template = template.replace("{{flags}}", " ".join(opts_flag))
         f.write(filled_template)
 
-parser = youtube_dl.parseOpts()[0]
+parser = build_option_parser()
 build_completion(parser)

devscripts/fish-completion.py

@@ -7,7 +7,7 @@ from os.path import dirname as dirn
 import sys
 
 sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
-import youtube_dl
+from youtube_dl.options import build_option_parser
 from youtube_dl.utils import shell_quote
 
 FISH_COMPLETION_FILE = 'youtube-dl.fish'
@@ -44,5 +44,5 @@ def build_completion(opt_parser):
     with open(FISH_COMPLETION_FILE, 'w') as f:
         f.write(filled_template)
 
-parser = youtube_dl.parseOpts()[0]
+parser = build_option_parser()
 build_completion(parser)

devscripts/zsh-completion.py

@@ -6,7 +6,7 @@ from os.path import dirname as dirn
 import sys
 
 sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
-import youtube_dl
+from youtube_dl.options import build_option_parser
 
 ZSH_COMPLETION_FILE = "youtube-dl.zsh"
 ZSH_COMPLETION_TEMPLATE = "devscripts/zsh-completion.in"
@@ -44,5 +44,5 @@ def build_completion(opt_parser):
     with open(ZSH_COMPLETION_FILE, "w") as f:
         f.write(template)
 
-parser = youtube_dl.parseOpts()[0]
+parser = build_option_parser()
 build_completion(parser)

youtube_dl/__init__.py

@@ -44,80 +44,7 @@ from .extractor import gen_extractors, list_extractors
 from .YoutubeDL import YoutubeDL
 
 
-def _real_main(argv=None):
+def _build_ydl_opts(opts, parser):
-    # Compatibility fixes for Windows
-    if sys.platform == 'win32':
-        # https://github.com/rg3/youtube-dl/issues/820
-        codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
-
-    workaround_optparse_bug9161()
-
-    setproctitle('youtube-dl')
-
-    parser, opts, args = parseOpts(argv)
-
-    # Set user agent
-    if opts.user_agent is not None:
-        std_headers['User-Agent'] = opts.user_agent
-
-    # Set referer
-    if opts.referer is not None:
-        std_headers['Referer'] = opts.referer
-
-    # Custom HTTP headers
-    if opts.headers is not None:
-        for h in opts.headers:
-            if h.find(':', 1) < 0:
-                parser.error('wrong header formatting, it should be key:value, not "%s"' % h)
-            key, value = h.split(':', 2)
-            if opts.verbose:
-                write_string('[debug] Adding header from command line option %s:%s\n' % (key, value))
-            std_headers[key] = value
-
-    # Dump user agent
-    if opts.dump_user_agent:
-        compat_print(std_headers['User-Agent'])
-        sys.exit(0)
-
-    # Batch file verification
-    batch_urls = []
-    if opts.batchfile is not None:
-        try:
-            if opts.batchfile == '-':
-                batchfd = sys.stdin
-            else:
-                batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
-            batch_urls = read_batch_urls(batchfd)
-            if opts.verbose:
-                write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n')
-        except IOError:
-            sys.exit('ERROR: batch file could not be read')
-    all_urls = batch_urls + args
-    all_urls = [url.strip() for url in all_urls]
-    _enc = preferredencoding()
-    all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
-
-    if opts.list_extractors:
-        for ie in list_extractors(opts.age_limit):
-            compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
-            matchedUrls = [url for url in all_urls if ie.suitable(url)]
-            for mu in matchedUrls:
-                compat_print('  ' + mu)
-        sys.exit(0)
-    if opts.list_extractor_descriptions:
-        for ie in list_extractors(opts.age_limit):
-            if not ie._WORKING:
-                continue
-            desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
-            if desc is False:
-                continue
-            if hasattr(ie, 'SEARCH_KEY'):
-                _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny', 'burping cow')
-                _COUNTS = ('', '5', '10', 'all')
-                desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
-            compat_print(desc)
-        sys.exit(0)
-
     # Conflicting, missing and erroneous options
     if opts.usenetrc and (opts.username is not None or opts.password is not None):
         parser.error('using .netrc conflicts with giving username/password')
@@ -276,7 +203,7 @@ def _real_main(argv=None):
         None if opts.match_filter is None
         else match_filter_func(opts.match_filter))
 
-    ydl_opts = {
+    return {
         'usenetrc': opts.usenetrc,
         'username': opts.username,
         'password': opts.password,
@@ -383,6 +310,83 @@ def _real_main(argv=None):
         'cn_verification_proxy': opts.cn_verification_proxy,
     }
 
+
+def _real_main(argv=None):
+    # Compatibility fixes for Windows
+    if sys.platform == 'win32':
+        # https://github.com/rg3/youtube-dl/issues/820
+        codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
+
+    workaround_optparse_bug9161()
+
+    setproctitle('youtube-dl')
+
+    parser, opts, args = parseOpts(argv)
+
+    # Set user agent
+    if opts.user_agent is not None:
+        std_headers['User-Agent'] = opts.user_agent
+
+    # Set referer
+    if opts.referer is not None:
+        std_headers['Referer'] = opts.referer
+
+    # Custom HTTP headers
+    if opts.headers is not None:
+        for h in opts.headers:
+            if h.find(':', 1) < 0:
+                parser.error('wrong header formatting, it should be key:value, not "%s"' % h)
+            key, value = h.split(':', 2)
+            if opts.verbose:
+                write_string('[debug] Adding header from command line option %s:%s\n' % (key, value))
+            std_headers[key] = value
+
+    # Dump user agent
+    if opts.dump_user_agent:
+        compat_print(std_headers['User-Agent'])
+        sys.exit(0)
+
+    # Batch file verification
+    batch_urls = []
+    if opts.batchfile is not None:
+        try:
+            if opts.batchfile == '-':
+                batchfd = sys.stdin
+            else:
+                batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore')
+            batch_urls = read_batch_urls(batchfd)
+            if opts.verbose:
+                write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n')
+        except IOError:
+            sys.exit('ERROR: batch file could not be read')
+    all_urls = batch_urls + args
+    all_urls = [url.strip() for url in all_urls]
+    _enc = preferredencoding()
+    all_urls = [url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls]
+
+    if opts.list_extractors:
+        for ie in list_extractors(opts.age_limit):
+            compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
+            matchedUrls = [url for url in all_urls if ie.suitable(url)]
+            for mu in matchedUrls:
+                compat_print('  ' + mu)
+        sys.exit(0)
+    if opts.list_extractor_descriptions:
+        for ie in list_extractors(opts.age_limit):
+            if not ie._WORKING:
+                continue
+            desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
+            if desc is False:
+                continue
+            if hasattr(ie, 'SEARCH_KEY'):
+                _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny', 'burping cow')
+                _COUNTS = ('', '5', '10', 'all')
+                desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES))
+            compat_print(desc)
+        sys.exit(0)
+
+    ydl_opts = _build_ydl_opts(opts, parser)
+
     with YoutubeDL(ydl_opts) as ydl:
         # Update version
         if opts.update_self:

youtube_dl/options.py

@@ -19,57 +19,7 @@ from .utils import (
 from .version import __version__
 
 
-def parseOpts(overrideArguments=None):
+def build_option_parser():
-    def _readOptions(filename_bytes, default=[]):
-        try:
-            optionf = open(filename_bytes)
-        except IOError:
-            return default  # silently skip if file is not present
-        try:
-            res = []
-            for l in optionf:
-                res += compat_shlex_split(l, comments=True)
-        finally:
-            optionf.close()
-        return res
-
-    def _readUserConf():
-        xdg_config_home = compat_getenv('XDG_CONFIG_HOME')
-        if xdg_config_home:
-            userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
-            if not os.path.isfile(userConfFile):
-                userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
-        else:
-            userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl', 'config')
-            if not os.path.isfile(userConfFile):
-                userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl.conf')
-        userConf = _readOptions(userConfFile, None)
-
-        if userConf is None:
-            appdata_dir = compat_getenv('appdata')
-            if appdata_dir:
-                userConf = _readOptions(
-                    os.path.join(appdata_dir, 'youtube-dl', 'config'),
-                    default=None)
-                if userConf is None:
-                    userConf = _readOptions(
-                        os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
-                        default=None)
-
-        if userConf is None:
-            userConf = _readOptions(
-                os.path.join(compat_expanduser('~'), 'youtube-dl.conf'),
-                default=None)
-        if userConf is None:
-            userConf = _readOptions(
-                os.path.join(compat_expanduser('~'), 'youtube-dl.conf.txt'),
-                default=None)
-
-        if userConf is None:
-            userConf = []
-
-        return userConf
-
     def _format_option_string(option):
         ''' ('-o', '--option') -> -o, --format METAVAR'''
 
@@ -90,16 +40,6 @@ def parseOpts(overrideArguments=None):
     def _comma_separated_values_options_callback(option, opt_str, value, parser):
         setattr(parser.values, option.dest, value.split(','))
 
-    def _hide_login_info(opts):
-        opts = list(opts)
-        for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
-            try:
-                i = opts.index(private_opt)
-                opts[i + 1] = 'PRIVATE'
-            except ValueError:
-                pass
-        return opts
-
     # No need to wrap help messages if we're on a wide console
     columns = compat_get_terminal_size().columns
     max_width = columns if columns else 80
@@ -786,6 +726,72 @@ def parseOpts(overrideArguments=None):
     parser.add_option_group(authentication)
     parser.add_option_group(postproc)
 
+    return parser
+
+
+def parseOpts(overrideArguments=None):
+    def _readOptions(filename_bytes, default=[]):
+        try:
+            optionf = open(filename_bytes)
+        except IOError:
+            return default  # silently skip if file is not present
+        try:
+            res = []
+            for l in optionf:
+                res += compat_shlex_split(l, comments=True)
+        finally:
+            optionf.close()
+        return res
+
+    def _readUserConf():
+        xdg_config_home = compat_getenv('XDG_CONFIG_HOME')
+        if xdg_config_home:
+            userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
+            if not os.path.isfile(userConfFile):
+                userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
+        else:
+            userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl', 'config')
+            if not os.path.isfile(userConfFile):
+                userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl.conf')
+        userConf = _readOptions(userConfFile, None)
+
+        if userConf is None:
+            appdata_dir = compat_getenv('appdata')
+            if appdata_dir:
+                userConf = _readOptions(
+                    os.path.join(appdata_dir, 'youtube-dl', 'config'),
+                    default=None)
+                if userConf is None:
+                    userConf = _readOptions(
+                        os.path.join(appdata_dir, 'youtube-dl', 'config.txt'),
+                        default=None)
+
+        if userConf is None:
+            userConf = _readOptions(
+                os.path.join(compat_expanduser('~'), 'youtube-dl.conf'),
+                default=None)
+        if userConf is None:
+            userConf = _readOptions(
+                os.path.join(compat_expanduser('~'), 'youtube-dl.conf.txt'),
+                default=None)
+
+        if userConf is None:
+            userConf = []
+
+        return userConf
+
+    def _hide_login_info(opts):
+        opts = list(opts)
+        for private_opt in ['-p', '--password', '-u', '--username', '--video-password']:
+            try:
+                i = opts.index(private_opt)
+                opts[i + 1] = 'PRIVATE'
+            except ValueError:
+                pass
+        return opts
+
+    parser = build_option_parser()
+
     if overrideArguments is not None:
         opts, args = parser.parse_args(overrideArguments)
         if opts.verbose: