rewrite `apache`, `nginx` and `phpfpm` plugins to use wrappers on chart creation
paulfantom committed
Jun 24, 2016 at 11:22 UTC
e26438c9a7cb79d6c32f9755e3a1358a10e864d6
4 files changed
+81
-123
python.d/apache.chart.py
+21
-32
@@ -22,58 +22,47 @@ ORDER = ['requests', 'connections', 'conns_async', 'net', 'workers', 'reqpersec'
22
23
CHARTS = {
24
'bytesperreq': {
25
- 'options': "'' 'apache Lifetime Avg. Response Size' 'bytes/request' statistics apache.bytesperreq area",
25
+ 'options': [None, 'apache Lifetime Avg. Response Size', 'bytes/request', 'statistics', 'apache.bytesperreq', 'area'],
26
'lines': [
27
- {"name": "size_req",
28
- "options": "'' absolute 1 1"}
27
+ ["size_req"]
28
]},
29
'workers': {
31
- 'options': "'' 'apache Workers' 'workers' workers apache.workers stacked",
30
+ 'options': [None, 'apache Workers', 'workers', 'workers', 'apache.workers', 'stacked'],
31
'lines': [
33
- {"name": "idle",
34
- "options": "'' absolute 1 1"},
35
- {"name": "busy",
36
- "options": "'' absolute 1 1"}
32
+ ["idle"],
33
+ ["busy"]
34
]},
35
'reqpersec': {
39
- 'options': "'' 'apache Lifetime Avg. Requests/s' 'requests/s' statistics apache.reqpersec area",
36
+ 'options': [None, 'apache Lifetime Avg. Requests/s', 'requests/s', 'statistics', 'apache.reqpersec', 'area'],
37
'lines': [
41
- {"name": "requests_sec",
42
- "options": "'' absolute 1 1"}
38
+ ["requests_sec"]
39
]},
40
'bytespersec': {
45
- 'options': "'' 'apache Lifetime Avg. Bandwidth/s' 'kilobytes/s' statistics apache.bytesperreq area",
41
+ 'options': [None, 'apache Lifetime Avg. Bandwidth/s', 'kilobytes/s', 'statistics', 'apache.bytesperreq', 'area'],
42
'lines': [
47
- {"name": "size_sec",
48
- "options": "'' absolute 1 1000"}
43
+ ["size_sec", None, 'absolute', 1, 1000]
44
]},
45
'requests': {
51
- 'options': "'' 'apache Requests' 'requests/s' requests apache.requests line",
46
+ 'options': [None, 'apache Requests', 'requests/s', 'requests', 'apache.requests', 'line'],
47
'lines': [
53
- {"name": "requests",
54
- "options": "'' incremental 1 1"}
48
+ ["requests", None, 'incremental']
49
]},
50
'net': {
57
- 'options': "'' 'apache Bandwidth' 'kilobytes/s' bandwidth apache.net area",
51
+ 'options': [None, 'apache Bandwidth', 'kilobytes/s', 'bandwidth', 'apache.net', 'area'],
52
'lines': [
59
- {"name": "sent",
60
- "options": "'' incremental 1 1"}
53
+ ["sent", None, 'incremental']
54
]},
55
'connections': {
63
- 'options': "'' 'apache Connections' 'connections' connections apache.connections line",
56
+ 'options': [None, 'apache Connections', 'connections', 'connections', 'apache.connections', 'line'],
57
'lines': [
65
- {"name": "connections",
66
- "options": "'' absolute 1 1"}
58
+ ["connections"]
59
]},
60
'conns_async': {
69
- 'options': "'' 'apache Async Connections' 'connections' connections apache.conns_async stacked",
61
+ 'options': [None, 'apache Async Connections', 'connections', 'connections', 'apache.conns_async', 'stacked'],
62
'lines': [
71
- {"name": "keepalive",
72
- "options": "'' absolute 1 1"},
73
- {"name": "closing",
74
- "options": "'' absolute 1 1"},
75
- {"name": "writing",
76
- "options": "'' absolute 1 1"}
63
+ ["keepalive"],
64
+ ["closing"],
65
+ ["writing"]
66
]}
67
}
68
@@ -82,9 +71,9 @@ class Service(UrlService):
71
def __init__(self, configuration=None, name=None):
72
UrlService.__init__(self, configuration=configuration, name=name)
73
if len(self.url) == 0:
85
- self.url = "http://localhost/server-status?auto"
74
+ self.url = "http://www.apache.org/server-status?auto"
75
self.order = ORDER
87
- self.charts = CHARTS
76
+ self.definitions = CHARTS
77
self.assignment = {"BytesPerReq": 'size_req',
78
"IdleWorkers": 'idle',
79
"BusyWorkers": 'busy',
python.d/nginx.chart.py
+12
-19
@@ -22,34 +22,27 @@ ORDER = ['connections', 'requests', 'connection_status', 'connect_rate']
22
23
CHARTS = {
24
'connections': {
25
- 'options': "'' 'nginx Active Connections' 'connections' nginx nginx.connections line",
25
+ 'options': [None, 'nginx Active Connections', 'connections', 'nginx', 'nginx.connections', 'line'],
26
'lines': [
27
- {"name": "active",
28
- "options": "'' absolute 1 1"}
27
+ ["active"]
28
]},
29
'requests': {
31
- 'options': "'' 'nginx Requests' 'requests/s' nginx nginx.requests line",
30
+ 'options': [None, 'nginx Requests', 'requests/s', 'nginx', 'nginx.requests', 'line'],
31
'lines': [
33
- {"name": "requests",
34
- "options": "'' incremental 1 1"}
32
+ ["requests", None, 'incremental']
33
]},
34
'connection_status': {
37
- 'options': "'' 'nginx Active Connections by Status' 'connections' nginx nginx.connection.status line",
35
+ 'options': [None, 'nginx Active Connections by Status', 'connections', 'nginx', 'nginx.connection.status', 'line'],
36
'lines': [
39
- {"name": "reading",
40
- "options": "'' absolute 1 1"},
41
- {"name": "writing",
42
- "options": "'' absolute 1 1"},
43
- {"name": "waiting",
44
- "options": "idle absolute 1 1"}
37
+ ["reading"],
38
+ ["writing"],
39
+ ["waiting", "idle"]
40
]},
41
'connect_rate': {
47
- 'options': "'' 'nginx Connections Rate' 'connections/s' nginx nginx.performance line",
42
+ 'options': [None, 'nginx Connections Rate', 'connections/s', 'nginx', 'nginx.performance', 'line'],
43
'lines': [
49
- {"name": "accepts",
50
- "options": "accepted incremental 1 1"},
51
- {"name": "handled",
52
- "options": "'' incremental 1 1"}
44
+ ["accepts", "accepted", "incremental"],
45
+ ["handled", None, "incremental"]
46
]}
47
}
48
@@ -60,7 +53,7 @@ class Service(UrlService):
53
if len(self.url) == 0:
54
self.url = "http://localhost/stub_status"
55
self.order = ORDER
63
- self.charts = CHARTS
56
+ self.definitions = CHARTS
57
58
def _formatted_data(self):
59
"""
python.d/phpfpm.chart.py
+10
-17
@@ -22,29 +22,22 @@ ORDER = ['connections', 'requests', 'performance']
22
23
CHARTS = {
24
'connections': {
25
- 'options': "'' 'PHP-FPM Active Connections' 'connections' phpfpm phpfpm.connections line",
25
+ 'options': [None, 'PHP-FPM Active Connections', 'connections', 'phpfpm', 'phpfpm.connections', 'line'],
26
'lines': [
27
- {"name": "active",
28
- "options": "'' absolute 1 1"},
29
- {"name": "maxActive",
30
- "options": "'max active' absolute 1 1"},
31
- {"name": "idle",
32
- "options": "'' absolute 1 1"}
27
+ ["active"],
28
+ ["maxActive", 'max active'],
29
+ ["idle"]
30
]},
31
'requests': {
35
- 'options': "'' 'PHP-FPM Requests' 'requests/s' phpfpm phpfpm.requests line",
32
+ 'options': [None, 'PHP-FPM Requests', 'requests/s', 'phpfpm', 'phpfpm.requests', 'line'],
33
'lines': [
37
- {"name": "requests",
38
- "options": "'' incremental 1 1"}
34
+ ["requests", None, "incremental"]
35
]},
36
'performance': {
41
-
42
- 'options': "'' 'PHP-FPM Performance' 'status' phpfpm phpfpm.performance line",
37
+ 'options': [None, 'PHP-FPM Performance', 'status', 'phpfpm', 'phpfpm.performance', 'line'],
38
'lines': [
44
- {"name": "reached",
45
- "options": "'max children reached' absolute 1 1"},
46
- {"name": "slow",
47
- "options": "'slow requests' absolute 1 1"}
39
+ ["reached", 'max children reached'],
40
+ ["slow", 'slow requests']
41
]}
42
}
43
@@ -55,7 +48,7 @@ class Service(UrlService):
48
if len(self.url) == 0:
49
self.url = "http://localhost/status"
50
self.order = ORDER
58
- self.charts = CHARTS
51
+ self.definitions = CHARTS
52
self.assignment = {"active processes": 'active',
53
"max active processes": 'maxActive',
54
"idle processes": 'idle',
python.d/python_modules/base.py
+38
-55
@@ -25,7 +25,7 @@ class BaseService(threading.Thread):
25
:param name: str
26
"""
27
threading.Thread.__init__(self)
28
- self.data_stream = ""
28
+ self._data_stream = ""
29
self.daemon = True
30
self.retries = 0
31
self.retries_left = 0
@@ -34,8 +34,8 @@ class BaseService(threading.Thread):
34
self.name = name
35
self.override_name = None
36
self.chart_name = ""
37
- self.dimensions = []
38
- self.charts = []
37
+ self._dimensions = []
38
+ self._charts = []
39
if configuration is None:
40
self.error("BaseService: no configuration parameters supplied. Cannot create Service.")
41
raise RuntimeError
@@ -142,15 +142,18 @@ class BaseService(threading.Thread):
142
Converts *params to string and joins them with one space between every one.
143
:param params: str/int/float
144
"""
145
- self.data_stream += instruction
145
+ self._data_stream += instruction
146
for p in params:
147
- p = str(p)
147
+ if p is None:
148
+ p = ""
149
+ else:
150
+ p = str(p)
151
if len(p) == 0:
152
p = "''"
153
if ' ' in p:
154
p = "'" + p + "'"
152
- self.data_stream += " " + p
153
- self.data_stream += "\n"
155
+ self._data_stream += " " + p
156
+ self._data_stream += "\n"
157
158
def chart(self, type_id, name="", title="", units="", family="",
159
category="", charttype="line", priority="", update_every=""):
@@ -166,8 +169,7 @@ class BaseService(threading.Thread):
169
:param priority: int/str
170
:param update_every: int/str
171
"""
169
- self.charts.append(type_id)
170
-
172
+ self._charts.append(type_id)
173
self._line("CHART", type_id, name, title, units, family, category, charttype, priority, update_every)
174
175
def dimension(self, id, name=None, algorithm="absolute", multiplier=1, divisor=1, hidden=False):
@@ -196,7 +198,7 @@ class BaseService(threading.Thread):
198
if algorithm not in ("absolute", "incremental", "percentage-of-absolute-row", "percentage-of-incremental-row"):
199
algorithm = "absolute"
200
199
- self.dimensions.append(id)
201
+ self._dimensions.append(id)
202
if hidden:
203
self._line("DIMENSION", id, name, algorithm, multiplier, divisor, "hidden")
204
else:
@@ -209,7 +211,7 @@ class BaseService(threading.Thread):
211
:param microseconds: int
212
:return: boolean
213
"""
212
- if type_id not in self.charts:
214
+ if type_id not in self._charts:
215
self.error("wrong chart type_id:", type_id)
216
return False
217
try:
@@ -228,7 +230,7 @@ class BaseService(threading.Thread):
230
:param value: int/float
231
:return: boolean
232
"""
231
- if id not in self.dimensions:
233
+ if id not in self._dimensions:
234
self.error("wrong dimension id:", id)
235
return False
236
try:
@@ -246,8 +248,8 @@ class BaseService(threading.Thread):
248
"""
249
Upload new data to netdata
250
"""
249
- print(self.data_stream)
250
- self.data_stream = ""
251
+ print(self._data_stream)
252
+ self._data_stream = ""
253
254
def error(self, *params):
255
"""
@@ -364,40 +366,21 @@ class UrlService(BaseService):
366
Create charts
367
:return: boolean
368
"""
367
- for name in self.order:
368
- if name not in self.charts:
369
- continue
370
- self.definitions[name] = []
371
- for line in self.charts[name]['lines']:
372
- self.definitions[name].append(line['name'])
373
-
374
- idx = 0
369
data = self._formatted_data()
370
if data is None:
371
return False
378
- data_stream = ""
372
+
373
+ idx = 0
374
for name in self.order:
380
- header = "CHART " + \
381
- self.__module__ + "_" + \
382
- self.name + "." + \
383
- name + " " + \
384
- self.charts[name]['options'] + " " + \
385
- str(self.priority + idx) + " " + \
386
- str(self.update_every)
387
- content = ""
375
+ options = self.definitions[name]['options'] + [self.priority + idx, self.update_every]
376
+ self.chart(self.__module__ + "_" + self.name + "." + name, *options)
377
# check if server has this datapoint
389
- for line in self.charts[name]['lines']:
390
- if line['name'] in data:
391
- content += "\nDIMENSION " + line['name'] + " " + line['options']
392
-
393
- if len(content) > 0:
394
- data_stream += header + content + "\n"
395
- idx += 1
396
-
397
- print(data_stream)
378
+ for line in self.definitions[name]['lines']:
379
+ if line[0] in data:
380
+ self.dimension(*line)
381
+ idx += 1
382
399
- if idx == 0:
400
- return False
383
+ self.commit()
384
return True
385
386
def update(self, interval):
@@ -410,17 +393,17 @@ class UrlService(BaseService):
393
if data is None:
394
return False
395
413
- data_stream = ""
414
- for chart, dimensions in self.definitions.items():
415
- header = "BEGIN " + self.__module__ + "_" + str(self.name) + "." + chart + " " + str(interval)
416
- c = ""
417
- for dim in dimensions:
418
- try:
419
- c += "\nSET " + dim + " = " + str(data[dim])
420
- except KeyError:
421
- pass
422
- if len(c) != 0:
423
- data_stream += header + c + "\nEND\n"
424
- print(data_stream)
396
+ updated = False
397
+ for chart in self.order:
398
+ if self.begin(self.__module__ + "_" + str(self.name) + "." + chart, interval):
399
+ updated = True
400
+ for dim in self.definitions[chart]['lines']:
401
+ try:
402
+ self.set(dim[0], data[dim[0]])
403
+ except KeyError:
404
+ pass
405
+ self.end()
406
426
- return True
407
+ self.commit()
408
+
409
+ return updated