Load additional extractor module .py files placed in ~/.config/youtube-dl/modules, $XDG_CACHE_HOME/youtube-dl/modules, or a directory specified with --module-dir

Modules can simply be placed in the directory without editing an __init__.py
Modules are written as normal extractor modules and require no changes,
(ie from .common import InfoExtractor, etc) as they're loaded into
the youtube_dl.extractor. namespace

youtube_dl/__init__.py

@@ -40,7 +40,7 @@ from .update import update_self
 from .downloader import (
     FileDownloader,
 )
-from .extractor import gen_extractors, list_extractors
+from .extractor import gen_extractors, list_extractors, add_extractors
 from .YoutubeDL import YoutubeDL
 
 
@@ -56,6 +56,10 @@ def _real_main(argv=None):
 
     parser, opts, args = parseOpts(argv)
 
+
+    from .autoload import load_dynamic_extractors
+    add_extractors( load_dynamic_extractors(opts.module_dir) )
+
     # Set user agent
     if opts.user_agent is not None:
         std_headers['User-Agent'] = opts.user_agent
@@ -372,6 +376,7 @@ def _real_main(argv=None):
         'external_downloader_args': external_downloader_args,
         'postprocessor_args': postprocessor_args,
         'cn_verification_proxy': opts.cn_verification_proxy,
+        'custommeta': opts.custommeta,
     }
 
     with YoutubeDL(ydl_opts) as ydl:

youtube_dl/autoload.py

@@ -0,0 +1,42 @@
+import os
+import glob
+import imp
+from .compat import (
+    compat_expanduser,
+    compat_getenv,
+)
+
+
+mdirs = [ os.path.dirname(__file__)+'/auto' ]
+
+def confdirs():
+    cfg_home = compat_getenv('XDG_CONFIG_HOME') or compat_getenv('appdata') or os.path.join(compat_expanduser('~'), '.config')
+
+    if cfg_home:
+       cfg_dir = os.path.join(cfg_home, 'youtube-dl','modules')
+       if os.path.isdir(cfg_dir):
+           return [cfg_dir]
+    return []
+
+def load_dynamic_extractors(module_dir=None):
+    mdirs.extend(confdirs())
+    if module_dir != None:
+        if not os.path.isdir(module_dir):
+            raise OSError('No such directory: '+module_dir)
+        mdirs.append(module_dir)
+
+    ret = {}
+    for mdir in mdirs:
+        files = glob.glob(mdir+"/*.py")
+        for f in [ os.path.basename(f)[:-3] for f in files]:
+           # force extractor namespace upon /any/path.py
+           fh, filename, desc = imp.find_module(f, [mdir])
+           module = imp.load_module('youtube_dl.extractor.'+f, fh, filename, desc)
+
+           for name in dir(module):
+              if name.endswith('IE') and name != 'GenericIE':
+                  ci = getattr(module,name)
+                  #globals()[name] = ci
+                  ret[name] = ci
+                  print('[autoload]: '+mdir+' '+f+': '+name)
+    return ret

youtube_dl/extractor/__init__.py

@@ -38,7 +38,7 @@ from .azubu import AzubuIE
 from .baidu import BaiduVideoIE
 from .bambuser import BambuserIE, BambuserChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE
-from .bbccouk import BBCCoUkIE
+from .bbc import BBCCoUkIE, BBCNewsIE
 from .beeg import BeegIE
 from .behindkink import BehindKinkIE
 from .beatportpro import BeatportProIE
@@ -112,6 +112,7 @@ from .daum import DaumIE
 from .dbtv import DBTVIE
 from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
+from .democracynow import DemocracynowIE
 from .dfb import DFBIE
 from .dhm import DHMIE
 from .dotsub import DotsubIE
@@ -256,6 +257,12 @@ from .karrierevideos import KarriereVideosIE
 from .keezmovies import KeezMoviesIE
 from .khanacademy import KhanAcademyIE
 from .kickstarter import KickStarterIE
+from .kissanime import (
+    KissAnimeIE,
+    KissCartoonIE,
+    KissAnimePlaylistIE,
+    KissCartoonPlaylistIE,
+)
 from .keek import KeekIE
 from .kontrtube import KontrTubeIE
 from .krasview import KrasViewIE
@@ -299,6 +306,7 @@ from .malemotion import MalemotionIE
 from .mdr import MDRIE
 from .megavideoz import MegaVideozIE
 from .metacafe import MetacafeIE
+from .memri import MemriIE
 from .metacritic import MetacriticIE
 from .mgoon import MgoonIE
 from .minhateca import MinhatecaIE
@@ -757,6 +765,7 @@ from .yandexmusic import (
 from .yesjapan import YesJapanIE
 from .yinyuetai import YinYueTaiIE
 from .ynet import YnetIE
+from .yospace import YospaceIE, ReutersIE
 from .youjizz import YouJizzIE
 from .youku import YoukuIE
 from .youporn import YouPornIE
@@ -814,3 +823,9 @@ def list_extractors(age_limit):
 def get_info_extractor(ie_name):
     """Returns the info extractor class with the given ie_name"""
     return globals()[ie_name + 'IE']
+
+
+def add_extractors(extractors):
+    for k, c in extractors.items():
+       globals()[k] = c
+       _ALL_CLASSES[:0] = [c]

youtube_dl/options.py

@@ -142,6 +142,10 @@ def parseOpts(overrideArguments=None):
         '--dump-user-agent',
         action='store_true', dest='dump_user_agent', default=False,
         help='Display the current browser identification')
+    general.add_option(
+        '--module-dir',
+        dest='module_dir', default=None, metavar='DIR',
+        help='Load additional extractor modules from DIR. Modules are also loaded from $XDG_CACHE_HOME/youtube-dl/modules or ~/.config/youtube-dl/modules.')
     general.add_option(
         '--list-extractors',
         action='store_true', dest='list_extractors', default=False,
@@ -716,6 +720,16 @@ def parseOpts(overrideArguments=None):
         '--add-metadata',
         action='store_true', dest='addmetadata', default=False,
         help='Write metadata to the video file')
+    postproc.add_option(
+        '--custom-meta',
+        action='append', dest='custommeta', default=[], metavar='TAG=FORMAT',
+        help='Write specific information to a metadata tag.'
+             'Syntax: "tagname=string to add with %(format)s" '
+             'The formatting syntax is the same as output. '
+             'Example: --custom-meta "comment=%(webpage_url)s\\n%(description)s" will '
+             'add a line with the url, then the description in the "comment" tag. '
+             'Tags are format-specific, common ones include: artist, comment, title, copyright, uploader. '
+             'This can be invoked multiple times for different tags.')
     postproc.add_option(
         '--metadata-from-title',
         metavar='FORMAT', dest='metafromtitle',