From bce3fcd786aa3ebc3793301fdfcea92fe360771f Mon Sep 17 00:00:00 2001
From: fnord <fnord@fnord.mobi>
Date: Mon, 13 Jul 2015 09:21:33 -0500
Subject: [PATCH] Load additional extractor module .py files placed in
 ~/.config/youtube-dl/modules (or equivalent), or path set by --module-dir
 option

Modules can simply be placed in the directory without editing an __init__.py

Modules are written as normal extractor modules and require no changes,
(ie from .common import InfoExtractor, etc) as they're loaded into
the youtube_dl.extractor. namespace
---
 youtube_dl/__init__.py           |  6 ++++-
 youtube_dl/autoload.py           | 42 ++++++++++++++++++++++++++++++++
 youtube_dl/extractor/__init__.py |  6 +++++
 youtube_dl/options.py            | 14 +++++++++++
 4 files changed, 67 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/autoload.py

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 55b22c889..1e01ae4f1 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -40,7 +40,7 @@ from .update import update_self
 from .downloader import (
     FileDownloader,
 )
-from .extractor import gen_extractors, list_extractors
+from .extractor import gen_extractors, list_extractors, add_extractors
 from .YoutubeDL import YoutubeDL
 
 
@@ -56,6 +56,10 @@ def _real_main(argv=None):
 
     parser, opts, args = parseOpts(argv)
 
+
+    from .autoload import load_dynamic_extractors
+    add_extractors( load_dynamic_extractors(opts.module_dir) )
+
     # Set user agent
     if opts.user_agent is not None:
         std_headers['User-Agent'] = opts.user_agent
diff --git a/youtube_dl/autoload.py b/youtube_dl/autoload.py
new file mode 100644
index 000000000..fb6b0578d
--- /dev/null
+++ b/youtube_dl/autoload.py
@@ -0,0 +1,42 @@
+import os
+import glob
+import imp
+from .compat import (
+    compat_expanduser,
+    compat_getenv,
+)
+
+
+mdirs = [ os.path.join(os.path.dirname(__file__),'/auto') ]
+
+def confdirs():
+    cfg_home = compat_getenv('XDG_CONFIG_HOME') or compat_getenv('appdata') or os.path.join(compat_expanduser('~'), '.config')
+
+    if cfg_home:
+       cfg_dir = os.path.join(cfg_home, 'youtube-dl','modules')
+       if os.path.isdir(cfg_dir):
+           return [cfg_dir]
+    return []
+
+def load_dynamic_extractors(module_dir=None):
+    mdirs.extend(confdirs())
+    if module_dir != None:
+        if not os.path.isdir(module_dir):
+            raise OSError('No such directory: '+module_dir)
+        mdirs.append(module_dir)
+
+    ret = {}
+    for mdir in mdirs:
+        files = glob.glob(os.path.join(mdir,"*.py"))
+        for f in [ os.path.basename(f)[:-3] for f in files]:
+           # force extractor namespace upon /any/path.py
+           fh, filename, desc = imp.find_module(f, [mdir])
+           module = imp.load_module('youtube_dl.extractor.'+f, fh, filename, desc)
+
+           for name in dir(module):
+              if name.endswith('IE') and name != 'GenericIE':
+                  ci = getattr(module,name)
+                  #globals()[name] = ci
+                  ret[name] = ci
+                  print('[autoload]: '+mdir+' '+f+': '+name)
+    return ret
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index cbaa07391..3216a580e 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -814,3 +814,9 @@ def list_extractors(age_limit):
 def get_info_extractor(ie_name):
     """Returns the info extractor class with the given ie_name"""
     return globals()[ie_name + 'IE']
+
+
+def add_extractors(extractors):
+    for k, c in extractors.items():
+       globals()[k] = c
+       _ALL_CLASSES[:0] = [c]
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 85365d769..2322a0615 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -142,6 +142,10 @@ def parseOpts(overrideArguments=None):
         '--dump-user-agent',
         action='store_true', dest='dump_user_agent', default=False,
         help='Display the current browser identification')
+    general.add_option(
+        '--module-dir',
+        dest='module_dir', default=None, metavar='DIR',
+        help='Load additional extractor modules from DIR. Modules are also loaded from $XDG_CACHE_HOME/youtube-dl/modules or ~/.config/youtube-dl/modules.')
     general.add_option(
         '--list-extractors',
         action='store_true', dest='list_extractors', default=False,
@@ -716,6 +720,16 @@ def parseOpts(overrideArguments=None):
         '--add-metadata',
         action='store_true', dest='addmetadata', default=False,
         help='Write metadata to the video file')
+    postproc.add_option(
+        '--custom-meta',
+        action='append', dest='custommeta', default=[], metavar='TAG=FORMAT',
+        help='Write specific information to a metadata tag.'
+             'Syntax: "tagname=string to add with %(format)s" '
+             'The formatting syntax is the same as output. '
+             'Example: --custom-meta "comment=%(webpage_url)s\\n%(description)s" will '
+             'add a line with the url, then the description in the "comment" tag. '
+             'Tags are format-specific, common ones include: artist, comment, title, copyright, uploader. '
+             'This can be invoked multiple times for different tags.')
     postproc.add_option(
         '--metadata-from-title',
         metavar='FORMAT', dest='metafromtitle',