495b5dba4 · cryptotaxi247/netdata-1

@@ -31,8 +31,8 @@ import os

31 import pprint

32 import re

33 import sys

34 -import time

34 import threading

35 +import time

36 import types

37

38 try:

@@ -50,6 +50,7 @@ else:

50 ENV_NETDATA_USER_CONFIG_DIR = 'NETDATA_USER_CONFIG_DIR'

51 ENV_NETDATA_STOCK_CONFIG_DIR = 'NETDATA_STOCK_CONFIG_DIR'

52 ENV_NETDATA_PLUGINS_DIR = 'NETDATA_PLUGINS_DIR'

53 +ENV_NETDATA_USER_PLUGINS_DIRS = 'NETDATA_USER_PLUGINS_DIRS'

54 ENV_NETDATA_LIB_DIR = 'NETDATA_LIB_DIR'

55 ENV_NETDATA_UPDATE_EVERY = 'NETDATA_UPDATE_EVERY'

56 ENV_NETDATA_LOCK_DIR = 'NETDATA_LOCK_DIR'

@@ -99,6 +100,9 @@ def dirs():

100 modules_user_config = os.path.join(plugin_user_config, 'python.d')

101 modules_stock_config = os.path.join(plugin_stock_config, 'python.d')

102 modules = os.path.abspath(pluginsd + '/../python.d')

103 + user_modules = [os.path.join(p, 'python.d') for p in

104 + os.getenv(ENV_NETDATA_USER_PLUGINS_DIRS, "").split(" ") if

105 + p]

106

107 Dirs = collections.namedtuple(

108 'Dirs',

@@ -108,6 +112,7 @@ def dirs():

112 'modules_user_config',

113 'modules_stock_config',

114 'modules',

115 + 'user_modules',

116 'var_lib',

117 'locks',

118 ]

@@ -118,6 +123,7 @@ def dirs():

123 modules_user_config,

124 modules_stock_config,

125 modules,

126 + user_modules,

127 var_lib,

128 locks,

129 )

@@ -130,6 +136,28 @@ IS_ATTY = sys.stdout.isatty()

136 MODULE_SUFFIX = '.chart.py'

137

138

139 +def find_available_modules(*directories):

140 + AvailableModule = collections.namedtuple(

141 + 'AvailableModule',

142 + [

143 + 'filepath',

144 + 'name',

145 + ]

146 + )

147 + available = list()

148 + for d in directories:

149 + try:

150 + if not os.path.isdir(d):

151 + continue

152 + files = sorted(os.listdir(d))

153 + except OSError:

154 + continue

155 + modules = [m for m in files if m.endswith(MODULE_SUFFIX)]

156 + available.extend([AvailableModule(os.path.join(d, m), m[:-len(MODULE_SUFFIX)]) for m in modules])

157 +

158 + return available

159 +

160 +

161 def available_modules():

162 obsolete = (

163 'apache_cache', # replaced by web_log

@@ -143,10 +171,17 @@ def available_modules():

171 'unbound', # rewritten in Go

172 )

173

146 - files = sorted(os.listdir(DIRS.modules))

147 - modules = [m[:-len(MODULE_SUFFIX)] for m in files if m.endswith(MODULE_SUFFIX)]

148 - avail = [m for m in modules if m not in obsolete]

149 - return tuple(avail)

174 + stock = [m for m in find_available_modules(DIRS.modules) if m.name not in obsolete]

175 + user = find_available_modules(*DIRS.user_modules)

176 +

177 + available, seen = list(), set()

178 + for m in user + stock:

179 + if m.name in seen:

180 + continue

181 + seen.add(m.name)

182 + available.append(m)

183 +

184 + return available

185

186

187 AVAILABLE_MODULES = available_modules()

@@ -176,9 +211,8 @@ def multi_path_find(name, *paths):

211 return str()

212

213

179 -def load_module(name):

180 - abs_path = os.path.join(DIRS.modules, '{0}{1}'.format(name, MODULE_SUFFIX))

181 - module = SourceFileLoader('pythond_' + name, abs_path)

214 +def load_module(name, filepath):

215 + module = SourceFileLoader('pythond_' + name, filepath)

216 if isinstance(module, types.ModuleType):

217 return module

218 return module.load_module()

@@ -331,12 +365,13 @@ class Job(threading.Thread):

365

366

367 class ModuleSrc:

334 - def __init__(self, name):

335 - self.name = name

368 + def __init__(self, m):

369 + self.name = m.name

370 + self.filepath = m.filepath

371 self.src = None

372

373 def load(self):

339 - self.src = load_module(self.name)

374 + self.src = load_module(self.name, self.filepath)

375

376 def get(self, key):

377 return getattr(self.src, key, None)

@@ -553,37 +588,38 @@ class Plugin:

588 builder.min_update_every = self.min_update_every

589

590 jobs = list()

556 - for mod_name in self.modules_to_run:

557 - if not self.config.is_module_enabled(mod_name):

558 - self.log.info("[{0}] is disabled in the configuration file, skipping it".format(mod_name))

591 + for m in self.modules_to_run:

592 + if not self.config.is_module_enabled(m.name):

593 + self.log.info("[{0}] is disabled in the configuration file, skipping it".format(m.name))

594 continue

595

561 - src = ModuleSrc(mod_name)

596 + src = ModuleSrc(m)

597 try:

598 src.load()

599 except Exception as error:

565 - self.log.warning("[{0}] error on loading source : {1}, skipping it".format(mod_name, repr(error)))

600 + self.log.warning("[{0}] error on loading source : {1}, skipping it".format(m.name, repr(error)))

601 continue

602 + self.log.debug("[{0}] loaded module source : '{1}'".format(m.name, m.filepath))

603

604 if not (src.service() and callable(src.service())):

569 - self.log.warning("[{0}] has no callable Service object, skipping it".format(mod_name))

605 + self.log.warning("[{0}] has no callable Service object, skipping it".format(m.name))

606 continue

607

572 - if src.is_disabled_by_default() and not self.config.is_module_explicitly_enabled(mod_name):

573 - self.log.info("[{0}] is disabled by default, skipping it".format(mod_name))

608 + if src.is_disabled_by_default() and not self.config.is_module_explicitly_enabled(m.name):

609 + self.log.info("[{0}] is disabled by default, skipping it".format(m.name))

610 continue

611

612 builder.module_defaults = src.defaults()

577 - configs = builder.build(mod_name)

613 + configs = builder.build(m.name)

614 if not configs:

579 - self.log.info("[{0}] has no job configs, skipping it".format(mod_name))

615 + self.log.info("[{0}] has no job configs, skipping it".format(m.name))

616 continue

617

618 for config in configs:

619 config['job_name'] = re.sub(r'\s+', '_', config['job_name'])

620 config['override_name'] = re.sub(r'\s+', '_', config.pop('name'))

621

586 - job = Job(src.service(), mod_name, config)

622 + job = Job(src.service(), m.name, config)

623

624 was_previously_active = job_statuses and job_statuses.has(job.module_name, job.real_name)

625 if was_previously_active and job.autodetection_retry == 0:

@@ -811,6 +847,20 @@ def disable():

847 exit(0)

848

849

850 +def get_modules_to_run(cmd):

851 + if not cmd.modules_to_run:

852 + return AVAILABLE_MODULES

853 +

854 + modules_to_run, seen = list(), set()

855 + for m in AVAILABLE_MODULES:

856 + if m.name not in cmd.modules_to_run or m.name in seen:

857 + continue

858 + seen.add(m.name)

859 + modules_to_run.append(m)

860 +

861 + return modules_to_run

862 +

863 +

864 def main():

865 cmd = parse_command_line()

866 log = PythonDLogger()

@@ -822,21 +872,22 @@ def main():

872

873 log.info('using python v{0}'.format(PY_VERSION[0]))

874

825 - unknown = set(cmd.modules_to_run) - set(AVAILABLE_MODULES)

875 + if DIRS.locks and not cmd.nolock:

876 + registry = FileLockRegistry(DIRS.locks)

877 + else:

878 + registry = DummyRegistry()

879 +

880 + unique_avail_module_names = set([m.name for m in AVAILABLE_MODULES])

881 + unknown = set(cmd.modules_to_run) - unique_avail_module_names

882 if unknown:

883 log.error('unknown modules : {0}'.format(sorted(list(unknown))))

828 - guessed = guess_module(AVAILABLE_MODULES, *cmd.modules_to_run)

884 + guessed = guess_module(unique_avail_module_names, *cmd.modules_to_run)

885 if guessed:

886 log.info('probably you meant : \n{0}'.format(pprint.pformat(guessed, width=1)))

887 return

888

833 - if DIRS.locks and not cmd.nolock:

834 - registry = FileLockRegistry(DIRS.locks)

835 - else:

836 - registry = DummyRegistry()

837 -

889 p = Plugin(

839 - cmd.modules_to_run or AVAILABLE_MODULES,

890 + get_modules_to_run(cmd),

891 cmd.update_every,

892 registry,

893 )

feat(python.d): load modules from user plugin directories (NETDATA_USER_PLUGINS_DIRS) (#13214)