@cryptotaxi247 / netdata-1 / commits / 082f4c72e

different intervals for different modules; better handling of crashing modules; code formatting

paulfantom committed Jun 11, 2016 at 12:16 UTC 082f4c72eecc64823ac410ec2a7940b4b9699610
2 files changed +144 -81
plugins.d/python.d.plugin
+142 -81
@@ -6,187 +6,237 @@ import time
6 import importlib.machinery
7 import configparser
8
9 +
10 class PythonCharts(object):
10 - def __init__(self,interval=1,modules=[],modules_path='../python.d/',modules_configs='../conf.d/'):
11 +
12 + def __init__(self,
13 + interval=1,
14 + modules=[],
15 + modules_path='../python.d/',
16 + modules_configs='../conf.d/'):
17 self.interval = interval
18 # check if plugin directory exists
19 if not os.path.isdir(modules_path):
14 - debug("cannot find charts directory ",modules_path)
20 + debug("cannot find charts directory ", modules_path)
21 sys.exit(1)
16 - self.configs=modules_configs
17 -
22 + self.configs = modules_configs
23 +
24 self.modules = []
25 if len(modules) > 0:
26 for m in modules:
21 - self.modules.append(self.import_plugin(modules_path+m+".chart.py"))
27 + self.modules.append(
28 + self.import_plugin(
29 + modules_path + m + ".chart.py"))
30 else:
31 self.load_modules(modules_path)
32 if len(self.modules) == 0:
25 - debug("cannot find charts directory",modules_path)
26 -# sys.stdout.write("DISABLE")
33 + debug("cannot find modules directory", modules_path)
34 + # sys.stdout.write("DISABLE\n")
35 sys.exit(1)
36
37 self.load_configs()
38
31 - # set last execution dict
39 + # set last execution and execution frequency dict
40 self.timetable = {}
33 - t = time.time()
41 + #last_exec = time.time()
42 + last_exec = 0
43 for m in self.modules:
35 - self.timetable[m.__name__] = t
44 + try:
45 + interval = m.update_every
46 + except AttributeError:
47 + interval = self.interval
48 + self.timetable[m.__name__] = [last_exec, int(interval)]
49
37 - def import_plugin(self,path,name=None):
50 + def import_plugin(self, path, name=None):
51 if name is None:
52 name = path.split('/')[-1]
53 if name[-9:] != ".chart.py":
41 - return None
42 - name=name[:-9]
54 + return None
55 + name = name[:-9]
56 try:
44 - return importlib.machinery.SourceFileLoader(name,path).load_module()
45 - except FileNotFoundError:
57 + return importlib.machinery.SourceFileLoader(name, path).load_module()
58 + except Exception as e:
59 + debug(str(e))
60 return None
61
48 - def load_modules(self,path):
62 + def load_modules(self, path):
63 names = os.listdir(path)
64 for mod in names:
51 - m = self.import_plugin(path+mod)
65 + m = self.import_plugin(path + mod)
66 if m is not None:
53 - debug("loading chart: '"+path+mod+"'")
67 + debug("loading chart: '" + path + mod + "'")
68 self.modules.append(m)
69
70 def load_configs(self):
71 for m in self.modules:
58 - if os.path.isfile(self.configs+m.__name__+".conf"):
59 - debug("loading chart options: '"+self.configs+m.__name__+".conf'")
60 - for k,v in read_config(self.configs+m.__name__+".conf").items():
61 - setattr(m,k,v)
72 + configfile = self.configs + m.__name__ + ".conf"
73 + if os.path.isfile(configfile):
74 + debug("loading chart options: '" + configfile + "'")
75 + for k, v in read_config(configfile).items():
76 + setattr(m, k, v)
77 else:
63 - debug(m.__name__+": configuration file '"+self.configs+m.__name__+".conf' not found. Using defaults.")
78 + debug(
79 + m.__name__ +
80 + ": configuration file '" +
81 + configfile +
82 + "' not found. Using defaults.")
83
65 - def disable_module(self,mod,reason=None):
84 + def disable_module(self, mod, reason=None):
85 self.modules.remove(mod)
86 del self.timetable[mod.__name__]
87 if reason is None:
88 return
89 elif reason[:3] == "no ":
71 - debug("chart '"+ mod.__name__,"' does not seem to have "+reason[3:]+"() function. Disabling it.")
90 + debug(
91 + "chart '" +
92 + mod.__name__,
93 + "' does not seem to have " +
94 + reason[3:] +
95 + "() function. Disabling it.")
96 elif reason[:7] == "failed ":
73 - debug("chart '"+mod.__name__+reason[3:]+"() function. reports failure.")
97 + debug(
98 + "chart '" +
99 + mod.__name__ +
100 + reason[3:] +
101 + "() function. reports failure.")
102 elif reason[:13] == "configuration":
75 - debug(mod.__name__,"configuration file '"+self.configs+mod.__name__+".conf' not found. Using defaults.")
103 + debug(
104 + mod.__name__,
105 + "configuration file '" +
106 + self.configs +
107 + mod.__name__ +
108 + ".conf' not found. Using defaults.")
109 elif reason[:11] == "misbehaving":
77 - debug(mod.__name__,"is misbeaving. Disabling it")
78 - #sys.stdout.write('DISABLE')
79 -
110 + debug(mod.__name__, "is misbeaving. Disabling it")
111 + # sys.stdout.write('DISABLE')
112
113 def check(self):
114 for mod in self.modules:
115 try:
116 if not mod.check():
85 - self.disable_module(mod,"failed check")
117 + self.disable_module(mod, "failed check")
118 except AttributeError:
87 - self.disable_module(mod,"no check")
119 + self.disable_module(mod, "no check")
120 except UnboundLocalError:
89 - self.disable_module(mod,"misbehaving")
121 + self.disable_module(mod, "misbehaving")
122
123 def create(self):
124 for mod in self.modules:
125 try:
126 if not mod.create():
95 - self.disable_module(mod,"failed create")
127 + self.disable_module(mod, "failed create")
128 else:
97 - chart=mod.__name__
98 - sys.stdout.write("CHART netdata.plugin_pythond_"+chart+" '' 'Execution time for "+chart+" plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 "+str(self.interval)+'\n')
99 - sys.stdout.write("DIMENSION run_time 'run time' absolute 1 1\n\n")
129 + chart = mod.__name__
130 + # sys.stdout.write("CHART netdata.plugin_pythond_"+chart+" '' 'Execution time for "+chart+" plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 "+str(self.interval)+'\n')
131 + sys.stdout.write(
132 + "CHART netdata.plugin_pythond_" +
133 + chart +
134 + " '' 'Execution time for " +
135 + chart +
136 + " plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 " +
137 + str(self.timetable[mod.__name__][1]) +
138 + '\n')
139 + sys.stdout.write(
140 + "DIMENSION run_time 'run time' absolute 1 1\n\n")
141 sys.stdout.flush()
142 except AttributeError:
102 - self.disable_module(mod,"no create")
143 + self.disable_module(mod, "no create")
144 except UnboundLocalError:
104 - self.disable_module(mod,"misbehaving")
145 + self.disable_module(mod, "misbehaving")
146
106 - def update_module(self,mod):
147 + def update_module(self, mod):
148 t1 = time.time()
149 + # check if it is time to execute module update() function
150 + if (t1 - self.timetable[mod.__name__][0]) < self.timetable[mod.__name__][1]:
151 + return
152 try:
109 -# t=int((t1- self.timetable[mod.__name__]) * 1000)
110 -# if t == 0: t=""
111 - t = "" #FIXME
153 + # t=int((t1- self.timetable[mod.__name__][0]) * 1000)
154 + # if t == 0: t=""
155 + t = "" # FIXME
156 if not mod.update(t):
113 -# if not mod.update(int((t1- self.timetable[mod.__name__]) * 1000)):
157 + # if not mod.update(int((t1- self.timetable[mod.__name__][0]) *
158 + # 1000)):
159 self.disable_module(mod, "update failed")
160 return
161 except AttributeError:
162 self.disable_module(mod, "no update")
163 return
164 except UnboundLocalError:
120 - self.disable_module(mod,"misbehaving")
165 + self.disable_module(mod, "misbehaving")
166 return
167 t2 = time.time()
123 -# dt = int((t2 - self.timetable[mod.__name__]) * 1000)
168 +# dt = int((t2 - self.timetable[mod.__name__][0]) * 1000)
169 # if dt == 0: dt=""
125 - dt="" #FIXME
126 - sys.stdout.write("BEGIN netdata.plugin_pythond_"+mod.__name__+" "+str(dt)+'\n')
127 - sys.stdout.write("SET run_time = "+str(int((t2 - t1)*1000))+'\n')
170 + # dt="" #FIXME
171 + #sys.stdout.write("BEGIN netdata.plugin_pythond_"+mod.__name__+" "+str(dt)+'\n')
172 + sys.stdout.write("BEGIN netdata.plugin_pythond_" + mod.__name__ + "\n")
173 + sys.stdout.write("SET run_time = " + str(int((t2 - t1) * 1000)) + '\n')
174 sys.stdout.write("END\n")
175 sys.stdout.flush()
130 - self.timetable[mod.__name__] = t1
176 + self.timetable[mod.__name__][0] = t1
177
178 def update(self):
179 while True:
180 t_begin = time.time()
181 for mod in self.modules:
182 self.update_module(mod)
137 - time.sleep(self.interval - (time.time()-t_begin))
138 -
183 + time.sleep((self.interval - (time.time() - t_begin)) / 2)
184 +
185
186 def read_config(path):
141 - #TODO make it bulletproof
187 + # TODO make it bulletproof
188 config = configparser.ConfigParser()
189 config_str = ""
190 try:
191 with open(path, 'r', encoding="utf_8") as f:
192 config_str = '[config]\n' + f.read()
193 except IsADirectoryError:
148 - debug(str(path),"is a directory")
194 + debug(str(path), "is a directory")
195 return
196 config.read_string(config_str)
197 return dict(config.items('config'))
198
199 +
200 def debug(*args):
201 if not DEBUG_FLAG:
202 return
156 - sys.stderr.write(PROGRAM+":")
203 + sys.stderr.write(PROGRAM + ":")
204 for i in args:
158 - sys.stderr.write(" "+str(i))
205 + sys.stderr.write(" " + str(i))
206 sys.stderr.write("\n")
207 sys.stderr.flush()
208
162 -def parse_cmdline(directory,*commands):
209 +
210 +def parse_cmdline(directory, *commands):
211 global DEBUG_FLAG, PROGRAM
164 - DEBUG_FLAG=False
212 + DEBUG_FLAG = False
213
214 mods = []
215 for cmd in commands[1:]:
216 if cmd == "check":
217 pass
218 elif cmd == "debug" or cmd == "all":
171 - DEBUG_FLAG=True
219 + DEBUG_FLAG = True
220 # redirect stderr to stdout?
173 - elif os.path.isfile(directory+cmd+".chart.py") or os.path.isfile(directory+cmd):
174 - DEBUG_FLAG=True
175 - mods.append(cmd.replace(".chart.py",""))
221 + elif os.path.isfile(directory + cmd + ".chart.py") or os.path.isfile(directory + cmd):
222 + DEBUG_FLAG = True
223 + mods.append(cmd.replace(".chart.py", ""))
224
225 PROGRAM = commands[0].split('/')[-1].split('.plugin')[0]
178 - debug("started from",commands[0],"with options:",*commands[1:])
226 + debug("started from", commands[0], "with options:", *commands[1:])
227
228 return mods
229
182 -#if __name__ == '__main__':
230 +
231 +# if __name__ == '__main__':
232 def run():
233 # parse env variables
234 # https://github.com/firehol/netdata/wiki/External-Plugins#environment-variables
186 - main_dir = os.getenv('NETDATA_PLUGINS_DIR',os.path.abspath(__file__).strip("python.d.plugin.py"))
187 - config_dir = os.getenv('NETDATA_CONFIG_DIR',"/etc/netdata/")
235 + main_dir = os.getenv('NETDATA_PLUGINS_DIR',
236 + os.path.abspath(__file__).strip("python.d.plugin.py"))
237 + config_dir = os.getenv('NETDATA_CONFIG_DIR', "/etc/netdata/")
238 # logs_dir = os.getenv('NETDATA_LOGS_DIR',None) #TODO logging?
189 - interval = int(os.getenv('NETDATA_UPDATE_EVERY',1))
239 + interval = int(os.getenv('NETDATA_UPDATE_EVERY', 1))
240
241 # read configuration file
242 if config_dir[-1] != '/':
@@ -200,29 +250,40 @@ def run():
250 if str(conf['enable']) == "no":
251 debug("disabled in configuration file")
252 sys.exit(1)
203 - except (KeyError,TypeError): pass
204 - try: modules_conf = conf['plugins_config_dir']
205 - except (KeyError,TypeError): modules_conf = config_dir # default configuration directory
206 - try: modules_dir = conf['plugins_dir']
207 - except (KeyError,TypeError): modules_dir = main_dir.replace("plugins.d","python.d")
208 - try: interval = int(conf['interval'])
209 - except (KeyError,TypeError): pass # default interval
253 + except (KeyError, TypeError):
254 + pass
255 + try:
256 + modules_conf = conf['plugins_config_dir']
257 + except (KeyError, TypeError):
258 + modules_conf = config_dir # default configuration directory
259 + try:
260 + modules_dir = conf['plugins_dir']
261 + except (KeyError, TypeError):
262 + modules_dir = main_dir.replace("plugins.d", "python.d")
263 + try:
264 + interval = int(conf['interval'])
265 + except (KeyError, TypeError):
266 + pass # default interval
267 global DEBUG_FLAG
211 - try: DEBUG_FLAG = bool(conf['debug'])
212 - except (KeyError,TypeError): pass
268 + try:
269 + DEBUG_FLAG = bool(conf['debug'])
270 + except (KeyError, TypeError):
271 + pass
272 except FileNotFoundError:
273 modules_conf = config_dir
215 - modules_dir = main_dir.replace("plugins.d","python.d")
274 + modules_dir = main_dir.replace("plugins.d", "python.d")
275
276 # directories should end with '/'
218 - if modules_dir[-1] != '/': modules_dir+="/"
219 - if modules_conf[-1] != '/': modules_conf+="/"
277 + if modules_dir[-1] != '/':
278 + modules_dir += "/"
279 + if modules_conf[-1] != '/':
280 + modules_conf += "/"
281
282 # parse passed command line arguments
222 - modules = parse_cmdline(modules_dir,*sys.argv)
283 + modules = parse_cmdline(modules_dir, *sys.argv)
284
285 # run plugins
225 - charts = PythonCharts(interval,modules,modules_dir,modules_conf)
286 + charts = PythonCharts(interval, modules, modules_dir, modules_conf)
287 charts.check()
288 charts.create()
289 charts.update()
python.d/example.chart.py
+2
@@ -1,5 +1,7 @@
1 import random
2
3 +update_every=10
4 +
5 def check():
6 return True
7