From ad02f392ffc1ee97b5c364d71be2a61c990c352b Mon Sep 17 00:00:00 2001 From: fnord Date: Mon, 13 Jul 2015 07:58:49 -0500 Subject: [PATCH 1/2] Load additional extractor module .py files placed in ~/.config/youtube-dl/modules, $XDG_CACHE_HOME/youtube-dl/modules, or a directory specified with --module-dir Modules can simply be placed in the directory without editing an __init__.py Modules are written as normal extractor modules and require no changes, (ie from .common import InfoExtractor, etc) as they're loaded into the youtube_dl.extractor. namespace --- youtube_dl/__init__.py | 7 +++++- youtube_dl/autoload.py | 42 ++++++++++++++++++++++++++++++++ youtube_dl/extractor/__init__.py | 17 ++++++++++++- youtube_dl/options.py | 14 +++++++++++ 4 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 youtube_dl/autoload.py diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 55b22c889..cd83ae9ae 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -40,7 +40,7 @@ from .update import update_self from .downloader import ( FileDownloader, ) -from .extractor import gen_extractors, list_extractors +from .extractor import gen_extractors, list_extractors, add_extractors from .YoutubeDL import YoutubeDL @@ -56,6 +56,10 @@ def _real_main(argv=None): parser, opts, args = parseOpts(argv) + + from .autoload import load_dynamic_extractors + add_extractors( load_dynamic_extractors(opts.module_dir) ) + # Set user agent if opts.user_agent is not None: std_headers['User-Agent'] = opts.user_agent @@ -372,6 +376,7 @@ def _real_main(argv=None): 'external_downloader_args': external_downloader_args, 'postprocessor_args': postprocessor_args, 'cn_verification_proxy': opts.cn_verification_proxy, + 'custommeta': opts.custommeta, } with YoutubeDL(ydl_opts) as ydl: diff --git a/youtube_dl/autoload.py b/youtube_dl/autoload.py new file mode 100644 index 000000000..6683a5b0c --- /dev/null +++ b/youtube_dl/autoload.py @@ -0,0 +1,42 @@ +import os +import glob +import imp +from .compat import ( + compat_expanduser, + compat_getenv, +) + + +mdirs = [ os.path.dirname(__file__)+'/auto' ] + +def confdirs(): + cfg_home = compat_getenv('XDG_CONFIG_HOME') or compat_getenv('appdata') or os.path.join(compat_expanduser('~'), '.config') + + if cfg_home: + cfg_dir = os.path.join(cfg_home, 'youtube-dl','modules') + if os.path.isdir(cfg_dir): + return [cfg_dir] + return [] + +def load_dynamic_extractors(module_dir=None): + mdirs.extend(confdirs()) + if module_dir != None: + if not os.path.isdir(module_dir): + raise OSError('No such directory: '+module_dir) + mdirs.append(module_dir) + + ret = {} + for mdir in mdirs: + files = glob.glob(mdir+"/*.py") + for f in [ os.path.basename(f)[:-3] for f in files]: + # force extractor namespace upon /any/path.py + fh, filename, desc = imp.find_module(f, [mdir]) + module = imp.load_module('youtube_dl.extractor.'+f, fh, filename, desc) + + for name in dir(module): + if name.endswith('IE') and name != 'GenericIE': + ci = getattr(module,name) + #globals()[name] = ci + ret[name] = ci + print('[autoload]: '+mdir+' '+f+': '+name) + return ret diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index cbaa07391..f45c16cb6 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -38,7 +38,7 @@ from .azubu import AzubuIE from .baidu import BaiduVideoIE from .bambuser import BambuserIE, BambuserChannelIE from .bandcamp import BandcampIE, BandcampAlbumIE -from .bbccouk import BBCCoUkIE +from .bbc import BBCCoUkIE, BBCNewsIE from .beeg import BeegIE from .behindkink import BehindKinkIE from .beatportpro import BeatportProIE @@ -112,6 +112,7 @@ from .daum import DaumIE from .dbtv import DBTVIE from .dctp import DctpTvIE from .deezer import DeezerPlaylistIE +from .democracynow import DemocracynowIE from .dfb import DFBIE from .dhm import DHMIE from .dotsub import DotsubIE @@ -256,6 +257,12 @@ from .karrierevideos import KarriereVideosIE from .keezmovies import KeezMoviesIE from .khanacademy import KhanAcademyIE from .kickstarter import KickStarterIE +from .kissanime import ( + KissAnimeIE, + KissCartoonIE, + KissAnimePlaylistIE, + KissCartoonPlaylistIE, +) from .keek import KeekIE from .kontrtube import KontrTubeIE from .krasview import KrasViewIE @@ -299,6 +306,7 @@ from .malemotion import MalemotionIE from .mdr import MDRIE from .megavideoz import MegaVideozIE from .metacafe import MetacafeIE +from .memri import MemriIE from .metacritic import MetacriticIE from .mgoon import MgoonIE from .minhateca import MinhatecaIE @@ -757,6 +765,7 @@ from .yandexmusic import ( from .yesjapan import YesJapanIE from .yinyuetai import YinYueTaiIE from .ynet import YnetIE +from .yospace import YospaceIE, ReutersIE from .youjizz import YouJizzIE from .youku import YoukuIE from .youporn import YouPornIE @@ -814,3 +823,9 @@ def list_extractors(age_limit): def get_info_extractor(ie_name): """Returns the info extractor class with the given ie_name""" return globals()[ie_name + 'IE'] + + +def add_extractors(extractors): + for k, c in extractors.items(): + globals()[k] = c + _ALL_CLASSES[:0] = [c] diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 85365d769..2322a0615 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -142,6 +142,10 @@ def parseOpts(overrideArguments=None): '--dump-user-agent', action='store_true', dest='dump_user_agent', default=False, help='Display the current browser identification') + general.add_option( + '--module-dir', + dest='module_dir', default=None, metavar='DIR', + help='Load additional extractor modules from DIR. Modules are also loaded from $XDG_CACHE_HOME/youtube-dl/modules or ~/.config/youtube-dl/modules.') general.add_option( '--list-extractors', action='store_true', dest='list_extractors', default=False, @@ -716,6 +720,16 @@ def parseOpts(overrideArguments=None): '--add-metadata', action='store_true', dest='addmetadata', default=False, help='Write metadata to the video file') + postproc.add_option( + '--custom-meta', + action='append', dest='custommeta', default=[], metavar='TAG=FORMAT', + help='Write specific information to a metadata tag.' + 'Syntax: "tagname=string to add with %(format)s" ' + 'The formatting syntax is the same as output. ' + 'Example: --custom-meta "comment=%(webpage_url)s\\n%(description)s" will ' + 'add a line with the url, then the description in the "comment" tag. ' + 'Tags are format-specific, common ones include: artist, comment, title, copyright, uploader. ' + 'This can be invoked multiple times for different tags.') postproc.add_option( '--metadata-from-title', metavar='FORMAT', dest='metafromtitle', From c8f17aa8ade94d2b98d167e9f12185caf857edb9 Mon Sep 17 00:00:00 2001 From: fnord Date: Mon, 13 Jul 2015 09:05:30 -0500 Subject: [PATCH 2/2] use os.path.join --- youtube_dl/autoload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youtube_dl/autoload.py b/youtube_dl/autoload.py index 6683a5b0c..fb6b0578d 100644 --- a/youtube_dl/autoload.py +++ b/youtube_dl/autoload.py @@ -7,7 +7,7 @@ from .compat import ( ) -mdirs = [ os.path.dirname(__file__)+'/auto' ] +mdirs = [ os.path.join(os.path.dirname(__file__),'/auto') ] def confdirs(): cfg_home = compat_getenv('XDG_CONFIG_HOME') or compat_getenv('appdata') or os.path.join(compat_expanduser('~'), '.config') @@ -27,7 +27,7 @@ def load_dynamic_extractors(module_dir=None): ret = {} for mdir in mdirs: - files = glob.glob(mdir+"/*.py") + files = glob.glob(os.path.join(mdir,"*.py")) for f in [ os.path.basename(f)[:-3] for f in files]: # force extractor namespace upon /any/path.py fh, filename, desc = imp.find_module(f, [mdir])