Load additional extractor module .py files placed in ~/.config/youtube-dl/modules (or equivalent), or path set by --module-dir option

Modules can simply be placed in the directory without editing an __init__.py

Modules are written as normal extractor modules and require no changes,
(ie from .common import InfoExtractor, etc) as they're loaded into
the youtube_dl.extractor. namespace

youtube_dl/__init__.py

@@ -40,7 +40,7 @@ from .update import update_self
 from .downloader import (
     FileDownloader,
 )
-from .extractor import gen_extractors, list_extractors
+from .extractor import gen_extractors, list_extractors, add_extractors
 from .YoutubeDL import YoutubeDL
 
 
@@ -56,6 +56,10 @@ def _real_main(argv=None):
 
     parser, opts, args = parseOpts(argv)
 
+
+    from .autoload import load_dynamic_extractors
+    add_extractors( load_dynamic_extractors(opts.module_dir) )
+
     # Set user agent
     if opts.user_agent is not None:
         std_headers['User-Agent'] = opts.user_agent

youtube_dl/autoload.py

@@ -0,0 +1,42 @@
+import os
+import glob
+import imp
+from .compat import (
+    compat_expanduser,
+    compat_getenv,
+)
+
+
+mdirs = [ os.path.join(os.path.dirname(__file__),'/auto') ]
+
+def confdirs():
+    cfg_home = compat_getenv('XDG_CONFIG_HOME') or compat_getenv('appdata') or os.path.join(compat_expanduser('~'), '.config')
+
+    if cfg_home:
+       cfg_dir = os.path.join(cfg_home, 'youtube-dl','modules')
+       if os.path.isdir(cfg_dir):
+           return [cfg_dir]
+    return []
+
+def load_dynamic_extractors(module_dir=None):
+    mdirs.extend(confdirs())
+    if module_dir != None:
+        if not os.path.isdir(module_dir):
+            raise OSError('No such directory: '+module_dir)
+        mdirs.append(module_dir)
+
+    ret = {}
+    for mdir in mdirs:
+        files = glob.glob(os.path.join(mdir,"*.py"))
+        for f in [ os.path.basename(f)[:-3] for f in files]:
+           # force extractor namespace upon /any/path.py
+           fh, filename, desc = imp.find_module(f, [mdir])
+           module = imp.load_module('youtube_dl.extractor.'+f, fh, filename, desc)
+
+           for name in dir(module):
+              if name.endswith('IE') and name != 'GenericIE':
+                  ci = getattr(module,name)
+                  #globals()[name] = ci
+                  ret[name] = ci
+                  print('[autoload]: '+mdir+' '+f+': '+name)
+    return ret

youtube_dl/extractor/__init__.py

@@ -814,3 +814,9 @@ def list_extractors(age_limit):
 def get_info_extractor(ie_name):
     """Returns the info extractor class with the given ie_name"""
     return globals()[ie_name + 'IE']
+
+
+def add_extractors(extractors):
+    for k, c in extractors.items():
+       globals()[k] = c
+       _ALL_CLASSES[:0] = [c]

youtube_dl/options.py

@@ -142,6 +142,10 @@ def parseOpts(overrideArguments=None):
         '--dump-user-agent',
         action='store_true', dest='dump_user_agent', default=False,
         help='Display the current browser identification')
+    general.add_option(
+        '--module-dir',
+        dest='module_dir', default=None, metavar='DIR',
+        help='Load additional extractor modules from DIR. Modules are also loaded from $XDG_CACHE_HOME/youtube-dl/modules or ~/.config/youtube-dl/modules.')
     general.add_option(
         '--list-extractors',
         action='store_true', dest='list_extractors', default=False,
@@ -716,6 +720,16 @@ def parseOpts(overrideArguments=None):
         '--add-metadata',
         action='store_true', dest='addmetadata', default=False,
         help='Write metadata to the video file')
+    postproc.add_option(
+        '--custom-meta',
+        action='append', dest='custommeta', default=[], metavar='TAG=FORMAT',
+        help='Write specific information to a metadata tag.'
+             'Syntax: "tagname=string to add with %(format)s" '
+             'The formatting syntax is the same as output. '
+             'Example: --custom-meta "comment=%(webpage_url)s\\n%(description)s" will '
+             'add a line with the url, then the description in the "comment" tag. '
+             'Tags are format-specific, common ones include: artist, comment, title, copyright, uploader. '
+             'This can be invoked multiple times for different tags.')
     postproc.add_option(
         '--metadata-from-title',
         metavar='FORMAT', dest='metafromtitle',