job creator in python.d.plugin
paulfantom committed
Jun 17, 2016 at 21:04 UTC
dfef7f8ce209325dcf41db7045f1809877a1d872
3 files changed
+116
-129
plugins.d/python.d.plugin
+89
-109
@@ -1,11 +1,11 @@
1
-#!/usr/bin/python3 -u
1
+#!/usr/bin/env python3
2
3
import os
4
import sys
5
import time
6
+import json
7
try:
8
assert sys.version_info >= (3,1)
8
- import configparser
9
import importlib.machinery
10
except AssertionError:
11
sys.stderr.write('Not supported python version. Needed python >= 3.1\n')
@@ -22,7 +22,6 @@ class PythonCharts(object):
22
modules_configs='../conf.d/',
23
modules_disabled=[]):
24
self.first_run = True
25
- self.default_priority = 90000
25
# set configuration directory
26
self.configs = modules_configs
27
@@ -35,37 +34,16 @@ class PythonCharts(object):
34
# load configuration files
35
self._load_configs()
36
37
+ # good economy and prosperity:
38
self.jobs = self._create_jobs(self.modules)
39
-
40
- # set timetable dict (last execution, next execution and frequency)
41
- # set priorities
42
- self.timetable = {}
43
- freq = 1
44
- for job in self.jobs:
45
- try:
46
- job.priority = int(job.priority)
47
- except (AttributeError,ValueError):
48
- job.priority = self.default_priority
49
-
50
- if interval is None:
51
- try:
52
- freq = int(job.update_every)
53
- except (AttributeError, ValueError):
54
- freq = 1
55
-
56
- now = time.time()
57
- self.timetable[job.__name__] = {'last' : now,
58
- 'next' : now - (now % freq) + freq,
59
- 'freq' : freq}
39
40
41
def _create_jobs(self,modules):
63
- # modules are a list of modules to load
64
- # module store a definition of Service class
65
- # module store configuration in module.config
66
- # configs are list of dicts or a dict
67
- # one dict is one service
68
- # iterate over list of modules and inside one loop iterate over configs
42
+ # module store a definition of Service class
43
+ # module store configuration in module.config
44
+ # configs are list of dicts or a dict
45
+ # one dict is one service
46
+ # iterate over list of modules and inside one loop iterate over configs
47
jobs = []
48
for module in modules:
49
if type(module.config) is dict:
@@ -78,34 +56,41 @@ class PythonCharts(object):
56
57
58
def _job(self,module,conf):
81
- update = 0
82
- prio = 0
83
- retries = 0
84
- try:
85
- update = int(conf['update_every'])
86
- except (KeyError,ValueError):
87
- update = module.update_every
88
- try:
89
- priority = int(conf['priority'])
90
- except (KeyError,ValueError):
91
- priority = module.priority
92
- try:
93
- retries = int(conf['retries'])
94
- except (KeyError,ValueError):
95
- retries = module.retries
59
+ # register a new job
60
+ # default job settings:
61
+ settings = {'update_every' : 11,
62
+ 'priority' : 12345,
63
+ 'retries' : 0}
64
+ # ensure job has basic variables
65
+ # first check in job config
66
+ # next check in module
67
+ # at last assign defaults
68
+ for key in settings:
69
+ try:
70
+ settings[key] = int(conf[key])
71
+ except (KeyError,ValueError):
72
+ try:
73
+ settings[key] = int(getattr(module, key))
74
+ except (AttributeError,ValueError):
75
+ pass
76
77
try:
98
- return module.Service(configuration=conf,
99
- update_every=update,
100
- priority=prio,
101
- retries=retries)
78
+ job = module.Service(configuration=conf)
79
except Exception as e:
103
- print(e)
104
- #TODO print some error
80
+ sys.stderr.write(module.__name__ +
81
+ ": Couldn't start one of jobs " +
82
+ str(e))
83
return None
84
+ else:
85
+ # set execution_name (needed to plot run time graphs)
86
+ try:
87
+ job.execution_name = module.__name__ + \
88
+ "_" + \
89
+ str(conf['name'])
90
+ except KeyError:
91
+ job.execution_name = module.__name__
92
107
-
108
- def _import_plugin(self, path, name=None):
93
+ def _import_module(self, path, name=None):
94
# try to import module using only its path
95
if name is None:
96
name = path.split('/')[-1]
@@ -129,14 +114,14 @@ class PythonCharts(object):
114
loaded = []
115
if len(modules) > 0:
116
for m in modules:
132
- mod = self._import_plugin(path + m + ".chart.py")
117
+ mod = self._import_module(path + m + ".chart.py")
118
if mod is not None:
119
loaded.append(mod)
120
else:
121
# scan directory specified in path and load all modules from there
122
names = os.listdir(path)
123
for mod in names:
139
- m = self._import_plugin(path + mod)
124
+ m = self._import_module(path + mod)
125
if m is not None:
126
debug("loading chart: '" + path + mod + "'")
127
loaded.append(m)
@@ -144,7 +129,6 @@ class PythonCharts(object):
129
130
def _load_configs(self):
131
# function modifies every loaded module in self.modules
147
- # TODO ensure module has update_every, priority and retries variables
132
for m in self.modules:
133
configfile = self.configs + m.__name__ + ".conf"
134
if os.path.isfile(configfile):
@@ -160,98 +144,97 @@ class PythonCharts(object):
144
configfile +
145
"' not found. Using defaults.")
146
163
- def _disable_module(self, mod, reason=None):
164
- # modifies self.modules
165
- self.modules.remove(mod)
166
- del self.timetable[mod.__name__]
147
+ def _stop(self, job, reason=None): #FIXME test if Service has __name__
148
+ # modifies self.jobs
149
+ self.jobs.remove(job)
150
if reason is None:
151
return
152
elif reason[:3] == "no ":
153
debug("chart '" +
171
- mod.__name__,
154
+ job.__name__,
155
"' does not seem to have " +
156
reason[3:] +
157
"() function. Disabling it.")
158
elif reason[:7] == "failed ":
159
debug("chart '" +
177
- mod.__name__ + "' " +
160
+ job.__name__ + "' " +
161
reason[7:] +
162
"() function reports failure.")
163
elif reason[:13] == "configuration":
181
- debug(mod.__name__,
164
+ debug(job.__name__,
165
"configuration file '" +
166
self.configs +
184
- mod.__name__ +
167
+ job.__name__ +
168
".conf' not found. Using defaults.")
169
elif reason[:11] == "misbehaving":
187
- debug(mod.__name__, "is "+reason)
170
+ debug(job.__name__, "is "+reason)
171
172
def check(self):
190
- # try to execute check() on every loaded module
191
- for mod in self.modules:
173
+ # try to execute check() on every job
174
+ for job in self.jobs:
175
try:
193
- if not mod.check():
194
- self._disable_module(mod, "failed check")
176
+ if not job.check():
177
+ self._stop(job, "failed check")
178
except AttributeError:
196
- self._disable_module(mod, "no check")
179
+ self._stop(job, "no check")
180
except (UnboundLocalError, Exception) as e:
198
- self._disable_module(mod, "misbehaving. Reason: " + str(e))
181
+ self._stop(job, "misbehaving. Reason: " + str(e))
182
183
def create(self):
201
- # try to execute create() on every loaded module
202
- for mod in self.modules:
184
+ # try to execute create() on every job
185
+ for job in self.jobs:
186
try:
204
- if not mod.create():
205
- self._disable_module(mod, "failed create")
187
+ if not job.create():
188
+ self._stop(job, "failed create")
189
else:
207
- chart = mod.__name__
190
+ chart = job.execution_name
191
sys.stdout.write(
192
"CHART netdata.plugin_pythond_" +
193
chart +
194
" '' 'Execution time for " +
195
chart +
196
" plugin' 'milliseconds / run' python.d netdata.plugin_python area 145000 " +
214
- str(self.timetable[mod.__name__]['freq']) +
197
+ str(job.timetable['freq']) +
198
'\n')
199
sys.stdout.write("DIMENSION run_time 'run time' absolute 1 1\n\n")
200
sys.stdout.flush()
201
except AttributeError:
219
- self._disable_module(mod, "no create")
202
+ self._stop(job, "no create")
203
except (UnboundLocalError, Exception) as e:
221
- self._disable_module(mod, "misbehaving. Reason: " + str(e))
204
+ self._stop(job, "misbehaving. Reason: " + str(e))
205
223
- def _update_module(self, mod):
224
- # try to execute update() on every module and draw run time graph
206
+ def _update_job(self, job):
207
+ # try to execute update() on every job and draw run time graph
208
t_start = time.time()
226
- # check if it is time to execute module update() function
227
- if self.timetable[mod.__name__]['next'] > t_start:
209
+ # check if it is time to execute job update() function
210
+ if job.timetable['next'] > t_start:
211
return
212
try:
213
if self.first_run:
214
since_last = 0
215
else:
233
- since_last = int((t_start - self.timetable[mod.__name__]['last']) * 1000000)
234
- if not mod.update(since_last):
235
- self._disable_module(mod, "update failed")
216
+ since_last = int((t_start - job.timetable['last']) * 1000000)
217
+ if not job.update(since_last):
218
+ self._stop(job, "update failed")
219
return
220
except AttributeError:
238
- self._disable_module(mod, "no update")
221
+ self._stop(job, "no update")
222
return
223
except (UnboundLocalError, Exception) as e:
241
- self._disable_module(mod, "misbehaving. Reason: " + str(e))
224
+ self._stop(job, "misbehaving. Reason: " + str(e))
225
return
226
t_end = time.time()
244
- self.timetable[mod.__name__]['next'] = t_end - (t_end % self.timetable[mod.__name__]['freq']) + self.timetable[mod.__name__]['freq']
227
+ job.timetable['next'] = t_end - (t_end % job.timetable['freq']) + job.timetable['freq']
228
# draw performance graph
229
if self.first_run:
230
dt = 0
231
else:
249
- dt = int((t_end - self.timetable[mod.__name__]['last']) * 1000000)
250
- sys.stdout.write("BEGIN netdata.plugin_pythond_"+mod.__name__+" "+str(since_last)+'\n')
232
+ dt = int((t_end - job.timetable['last']) * 1000000)
233
+ sys.stdout.write("BEGIN netdata.plugin_pythond_"+job.execution_name+" "+str(since_last)+'\n')
234
sys.stdout.write("SET run_time = " + str(int((t_end - t_start) * 1000)) + '\n')
235
sys.stdout.write("END\n")
236
sys.stdout.flush()
254
- self.timetable[mod.__name__]['last'] = t_start
237
+ job.timetable['last'] = t_start
238
self.first_run = False
239
240
def update(self):
@@ -260,10 +243,10 @@ class PythonCharts(object):
243
while True:
244
t_begin = time.time()
245
next_runs = []
263
- for mod in self.modules:
264
- self._update_module(mod)
246
+ for job in self.jobs:
247
+ self._update_job(job)
248
try:
266
- next_runs.append(self.timetable[mod.__name__]['next'])
249
+ next_runs.append(job.timetable['next'])
250
except KeyError:
251
pass
252
if len(next_runs) == 0:
@@ -274,20 +257,17 @@ class PythonCharts(object):
257
258
259
def read_config(path):
277
- config = configparser.ConfigParser()
278
- config_str = ""
279
- try:
280
- with open(path, 'r', encoding="utf_8") as f:
281
- config_str = '[config]\n' + f.read()
282
- except IsADirectoryError:
283
- debug(str(path), "is a directory")
284
- return
285
- try:
286
- config.read_string(config_str)
287
- except configparser.ParsingError as e:
288
- debug("Malformed configuration file: "+str(e))
289
- return
290
- return dict(config.items('config'))
260
+ # FIXME move to JSON configurations
261
+# try:
262
+# with open(path, 'r') as f:
263
+# config_str = '[config]\n' + f.read()
264
+# except IsADirectoryError:
265
+# debug(str(path), "is a directory")
266
+# return
267
+# try:
268
+# debug("Malformed configuration file: "+str(e))
269
+# return
270
+ return None
271
272
273
def debug(*args):
python.d/mysql.chart.py
+7
-2
@@ -34,6 +34,11 @@ config = [
34
}
35
]
36
37
+# default module values (can be overridden per job in `config`)
38
+update_every = 3
39
+priority = 90000
40
+retries = 7
41
+
42
# query executed on MySQL server
43
QUERY = "SHOW GLOBAL STATUS"
44
@@ -329,8 +334,8 @@ CHARTS = {
334
335
336
class Service(BaseService):
332
- def __init__(self,configuration=None,update_every=3,priority=90000,retries=2):
333
- super().__init__(configuration,update_every,priority,retries)
337
+ def __init__(self,configuration=None):
338
+ super().__init__(configuration)
339
self.configuration = self._parse_config(configuration)
340
self.connection = None
341
self.defs = {}
python.d/python_modules/base.py
+20
-18
@@ -1,31 +1,33 @@
1
# Description: base for netdata python.d plugins
2
# Author: Pawel Krupa (paulfantom)
3
4
+from time import time
5
+
6
+
7
class BaseService(object):
5
- def __init__(self,configuration=None,update_every=None,priority=None,retries=None):
6
- if None in (configuration,update_every,priority,retries):
8
+ def __init__(self,configuration=None):
9
+ if configuration is None:
10
# use defaults
11
self.error("BaseService: no configuration parameters supplied. Cannot create Service.")
12
raise RuntimeError
13
else:
11
- self._parse_base_config(configuration,update_every,priority,retries)
12
-
13
- def _parse_base_config(self,config,update_every,priority,retries):
14
- # parse configuration options to run this Service
15
- try:
16
- self.update_every = int(config['update_every'])
17
- except (KeyError, ValueError):
18
- self.update_every = update_every
19
- try:
20
- self.priority = int(config['priority'])
21
- except (KeyError, ValueError):
22
- self.priority = priority
23
- try:
24
- self.retries = int(config['retries'])
25
- except (KeyError, ValueError):
26
- self.retries = retries
14
+ self._extract_base_config(configuration)
15
+ self._create_timetable()
16
+ self.execution_name = ""
17
+
18
+ def _extract_base_config(self,config):
19
+ self.update_every = int(config['update_every'])
20
+ self.priority = int(config['priority'])
21
+ self.retries = int(config['retries'])
22
self.retries_left = self.retries
23
24
+ def _create_timetable(self):
25
+ now = time()
26
+ self.timetable = {'last' : now,
27
+ 'next' : now - (now % self.update_every) + self.update_every,
28
+ 'freq' : self.update_every}
29
+
30
+
31
def error(self, msg, exception=""):
32
if exception != "":
33
exception = " " + str(exception).replace("\n"," ")