@cryptotaxi247 / netdata-1 / commits / fd2d00358

nginx_plus module initial version added

lgz committed Jan 19, 2018 at 15:43 UTC fd2d00358c00abd4dd453e08b67a71fd2ad6a553
1 file changed +490
python.d/nginx_plus.chart.py new
+490
@@ -0,0 +1,490 @@
1 +# -*- coding: utf-8 -*-
2 +# Description: apache netdata python.d module
3 +# Author: Ilya Mashchenko (l2isbad)
4 +
5 +import re
6 +
7 +from collections import defaultdict
8 +from json import loads
9 +
10 +try:
11 + from collections import OrderedDict
12 +except ImportError:
13 + from third_party.ordereddict import OrderedDict
14 +
15 +from bases.FrameworkServices.UrlService import UrlService
16 +
17 +# default module values (can be overridden per job in `config`)
18 +update_every = 1
19 +priority = 60000
20 +retries = 60
21 +
22 +# charts order (can be overridden if you want less charts, or different order)
23 +ORDER = ['requests_total', 'requests_current',
24 + 'connections_statistics', 'connections_workers',
25 + 'ssl_handshakes', 'ssl_session_reuses', 'ssl_memory_usage',
26 + 'processes']
27 +
28 +CHARTS = {
29 + 'requests_total': {
30 + 'options': [None, 'Requests Total', 'requests/s',
31 + 'requests', 'nginx_plus.requests_total', 'line'],
32 + 'lines': [
33 + ['requests_total', 'total', 'incremental']
34 + ]},
35 + 'requests_current': {
36 + 'options': [None, 'Requests Current', 'requests',
37 + 'requests', 'nginx_plus.requests_current', 'line'],
38 + 'lines': [
39 + ['requests_current', 'current']
40 + ]},
41 + 'connections_statistics': {
42 + 'options': [None, 'Connections Statistics', 'connections/s',
43 + 'connections', 'nginx_plus.connections_statistics', 'stacked'],
44 + 'lines': [
45 + ['connections_accepted', 'accepted', 'incremental'],
46 + ['connections_dropped', 'dropped', 'incremental']
47 + ]},
48 + 'connections_workers': {
49 + 'options': [None, 'Workers Statistics', 'workers',
50 + 'connections', 'nginx_plus.connections_workers', 'stacked'],
51 + 'lines': [
52 + ['connections_idle', 'idle'],
53 + ['connections_active', 'active']
54 + ]},
55 + 'ssl_handshakes': {
56 + 'options': [None, 'SSL Handshakes', 'handshakes/s',
57 + 'ssl', 'nginx_plus.ssl_handshakes', 'stacked'],
58 + 'lines': [
59 + ['ssl_handshakes', 'successful', 'incremental'],
60 + ['ssl_handshakes_failed', 'failed', 'incremental']
61 + ]},
62 + 'ssl_session_reuses': {
63 + 'options': [None, 'Session Reuses', 'sessions/s',
64 + 'ssl', 'nginx_plus.ssl_session_reuses', 'line'],
65 + 'lines': [
66 + ['ssl_session_reuses', 'reused', 'incremental']
67 + ]},
68 + 'ssl_memory_usage': {
69 + 'options': [None, 'Memory Usage', '%',
70 + 'ssl', 'nginx_plus.ssl_memory_usage', 'area'],
71 + 'lines': [
72 + ['ssl_memory_usage', 'usage', 'absolute', 1, 100]
73 + ]},
74 + 'processes': {
75 + 'options': [None, 'Processes', 'processes',
76 + 'processes', 'nginx_plus.processes', 'line'],
77 + 'lines': [
78 + ['processes_respawned', 'respawned']
79 + ]}
80 +}
81 +
82 +
83 +def cache_charts(cache):
84 + family = 'cache {0}'.format(cache.real_name)
85 + charts = OrderedDict()
86 +
87 + charts['{0}_traffic'.format(cache.name)] = {
88 + 'options': [None, 'Traffic', 'KB', family,
89 + 'nginx_plus.cache_traffic', 'stacked'],
90 + 'lines': [
91 + ['_'.join([cache.name, 'hit_bytes']), 'served', 'absolute', 1, 1024],
92 + ['_'.join([cache.name, 'miss_bytes_written']), 'written', 'absolute', 1, 1024],
93 + ['_'.join([cache.name, 'miss_bytes']), 'bypass', 'absolute', 1, 1024]
94 + ]
95 + }
96 + charts['{0}_memory_usage'.format(cache.name)] = {
97 + 'options': [None, 'Memory Usage', '%', family,
98 + 'nginx_plus.cache_memory_usage', 'area'],
99 + 'lines': [
100 + ['_'.join([cache.name, 'memory_usage']), 'usage', 'absolute', 1, 100],
101 + ]
102 + }
103 + return charts
104 +
105 +
106 +def web_zone_charts(wz):
107 + charts = OrderedDict()
108 + family = 'web zone {name}'.format(name=wz.real_name)
109 +
110 + # Processing
111 + charts['zone_{name}_processing'.format(name=wz.name)] = {
112 + 'options': [None, 'Zone "{name}" Processing'.format(name=wz.name), 'requests', family,
113 + 'nginx_plus.web_zone_processing', 'line'],
114 + 'lines': [
115 + ['_'.join([wz.name, 'processing']), 'processing']
116 + ]
117 + }
118 + # Requests
119 + charts['zone_{name}_requests'.format(name=wz.name)] = {
120 + 'options': [None, 'Zone "{name}" Requests'.format(name=wz.name), 'requests/s', family,
121 + 'nginx_plus.web_zone_requests', 'line'],
122 + 'lines': [
123 + ['_'.join([wz.name, 'requests']), 'requests', 'incremental']
124 + ]
125 + }
126 + # Response Codes
127 + charts['zone_{name}_responses'.format(name=wz.name)] = {
128 + 'options': [None, 'Zone "{name}" Responses'.format(name=wz.name), 'requests/s', family,
129 + 'nginx_plus.web_zone_responses', 'stacked'],
130 + 'lines': [
131 + ['_'.join([wz.name, 'responses_2xx']), '2xx', 'incremental'],
132 + ['_'.join([wz.name, 'responses_5xx']), '5xx', 'incremental'],
133 + ['_'.join([wz.name, 'responses_3xx']), '3xx', 'incremental'],
134 + ['_'.join([wz.name, 'responses_4xx']), '4xx', 'incremental'],
135 + ['_'.join([wz.name, 'responses_1xx']), '1xx', 'incremental']
136 + ]
137 + }
138 + # Traffic
139 + charts['zone_{name}_net'.format(name=wz.name)] = {
140 + 'options': [None, 'Zone "{name}" Traffic'.format(name=wz.name), 'kilobits/s', family,
141 + 'nginx_plus.zone_net', 'area'],
142 + 'lines': [
143 + ['_'.join([wz.name, 'received']), 'received', 'incremental', 1, 1000],
144 + ['_'.join([wz.name, 'sent']), 'sent', 'incremental', -1, 1000]
145 + ]
146 + }
147 + return charts
148 +
149 +
150 +def web_upstream_charts(wu):
151 + def dimensions(value, a='absolute', m=1, d=1):
152 + dims = list()
153 + for p in wu:
154 + dims.append(['_'.join([wu.name, p.server, value]), p.real_server, a, m, d])
155 + return dims
156 +
157 + charts = OrderedDict()
158 + family = 'web upstream {name}'.format(name=wu.real_name)
159 +
160 + # Requests
161 + charts['web_upstream_{name}_requests'.format(name=wu.name)] = {
162 + 'options': [None, 'Peers Requests', 'requests/s', family,
163 + 'nginx_plus.web_upstream_requests', 'line'],
164 + 'lines': dimensions('requests', 'incremental')
165 + }
166 + # Responses Codes
167 + charts['web_upstream_{name}_all_responses'.format(name=wu.name)] = {
168 + 'options': [None, 'All Peers Responses', 'responses/s', family,
169 + 'nginx_plus.web_upstream_all_responses', 'stacked'],
170 + 'lines': [
171 + ['_'.join([wu.name, 'responses_2xx']), '2xx', 'incremental'],
172 + ['_'.join([wu.name, 'responses_5xx']), '5xx', 'incremental'],
173 + ['_'.join([wu.name, 'responses_3xx']), '3xx', 'incremental'],
174 + ['_'.join([wu.name, 'responses_4xx']), '4xx', 'incremental'],
175 + ['_'.join([wu.name, 'responses_1xx']), '1xx', 'incremental'],
176 + ]
177 + }
178 + for peer in wu:
179 + charts['web_upstream_{0}_{1}_responses'.format(wu.name, peer.id)] = {
180 + 'options': [None, 'Peer "{0}" Responses'.format(peer.real_server), 'responses/s', family,
181 + 'nginx_plus.web_upstream_peer_responses'.format(wu.name), 'stacked'],
182 + 'lines': [
183 + ['_'.join([wu.name, peer.server, 'responses_2xx']), '2xx', 'incremental'],
184 + ['_'.join([wu.name, peer.server, 'responses_5xx']), '5xx', 'incremental'],
185 + ['_'.join([wu.name, peer.server, 'responses_3xx']), '3xx', 'incremental'],
186 + ['_'.join([wu.name, peer.server, 'responses_4xx']), '4xx', 'incremental'],
187 + ['_'.join([wu.name, peer.server, 'responses_1xx']), '1xx', 'incremental']
188 + ]
189 + }
190 + # Connections
191 + charts['web_upstream_{name}_connections'.format(name=wu.name)] = {
192 + 'options': [None, 'Peers Connections', 'active', family,
193 + 'nginx_plus.web_upstream_connections', 'line'],
194 + 'lines': dimensions('active')
195 + }
196 + charts['web_upstream_{name}_connections_usage'.format(name=wu.name)] = {
197 + 'options': [None, 'Peers Connections Usage', '%', family,
198 + 'nginx_plus.web_upstream_connections_usage', 'line'],
199 + 'lines': dimensions('connections_usage', d=100)
200 + }
201 + # Traffic
202 + charts['web_upstream_{0}_all_net'.format(wu.name)] = {
203 + 'options': [None, 'All Peers Traffic', 'kilobits/s', family,
204 + 'nginx_plus.web_upstream_all_net', 'area'],
205 + 'lines': [
206 + ['{0}_received'.format(wu.name), 'received', 'incremental', 1, 1000],
207 + ['{0}_sent'.format(wu.name), 'sent', 'incremental', -1, 1000]
208 + ]
209 + }
210 + for peer in wu:
211 + charts['web_upstream_{0}_{1}_net'.format(wu.name, peer.id)] = {
212 + 'options': [None, 'Peer "{0}" Traffic'.format(peer.real_server), 'kilobits/s', family,
213 + 'nginx_plus.web_upstream_peer_traffic'.format(wu.name), 'area'],
214 + 'lines': [
215 + ['{0}_{1}_received'.format(wu.name, peer.server), 'received', 'incremental', 1, 1000],
216 + ['{0}_{1}_sent'.format(wu.name, peer.server), 'sent', 'incremental', -1, 1000]
217 + ]
218 + }
219 + # Response Time
220 + for peer in wu:
221 + charts['web_upstream_{0}_{1}_timings'.format(wu.name, peer.id)] = {
222 + 'options': [None, 'Peer "{0}" Timings'.format(peer.real_server), 'ms', family,
223 + 'nginx_plus.web_upstream_peer_timings', 'line'],
224 + 'lines': [
225 + ['_'.join([wu.name, peer.server, 'header_time']), 'header'],
226 + ['_'.join([wu.name, peer.server, 'response_time']), 'response']
227 + ]
228 + }
229 + # Memory Usage
230 + charts['web_upstream_{name}_memory_usage'.format(name=wu.name)] = {
231 + 'options': [None, 'Memory Usage', '%', family,
232 + 'nginx_plus.web_upstream_memory_usage', 'area'],
233 + 'lines': [
234 + ['_'.join([wu.name, 'memory_usage']), 'usage', 'absolute', 1, 100]
235 + ]
236 + }
237 + # State
238 + charts['web_upstream_{name}_status'.format(name=wu.name)] = {
239 + 'options': [None, 'Peers Status', 'state', family,
240 + 'nginx_plus.web_upstream_status', 'line'],
241 + 'lines': dimensions('state')
242 + }
243 + # Downtime
244 + charts['web_upstream_{name}_downtime'.format(name=wu.name)] = {
245 + 'options': [None, 'Peers Downtime', 'seconds', family,
246 + 'nginx_plus.web_upstream_peer_downtime', 'line'],
247 + 'lines': dimensions('downtime', d=1000)
248 + }
249 +
250 + return charts
251 +
252 +
253 +METRICS = dict(
254 + SERVER=[
255 + 'processes.respawned',
256 + 'connections.accepted',
257 + 'connections.dropped',
258 + 'connections.active',
259 + 'connections.idle',
260 + 'ssl.handshakes',
261 + 'ssl.handshakes_failed',
262 + 'ssl.session_reuses',
263 + 'requests.total',
264 + 'requests.current',
265 + 'slabs.SSL.pages.free',
266 + 'slabs.SSL.pages.used'
267 + ],
268 + WEB_ZONE=[
269 + 'processing',
270 + 'requests',
271 + 'responses.1xx',
272 + 'responses.2xx',
273 + 'responses.3xx',
274 + 'responses.4xx',
275 + 'responses.5xx',
276 + 'discarded',
277 + 'received',
278 + 'sent'
279 + ],
280 + WEB_UPSTREAM_PEER=[
281 + 'id',
282 + 'server',
283 + 'name',
284 + 'state',
285 + 'active',
286 + 'max_conns',
287 + 'requests',
288 + 'header_time', # alive only
289 + 'response_time', # alive only
290 + 'responses.1xx',
291 + 'responses.2xx',
292 + 'responses.3xx',
293 + 'responses.4xx',
294 + 'responses.5xx',
295 + 'sent',
296 + 'received',
297 + 'downtime'
298 + ],
299 + WEB_UPSTREAM_SUMMARY=[
300 + 'responses.1xx',
301 + 'responses.2xx',
302 + 'responses.3xx',
303 + 'responses.4xx',
304 + 'responses.5xx',
305 + 'sent',
306 + 'received'
307 + ],
308 + CACHE=[
309 + 'hit.bytes', # served
310 + 'miss.bytes_written', # written
311 + 'miss.bytes' # bypass
312 +
313 + ]
314 +)
315 +
316 +BAD_SYMBOLS = re.compile(r'[:/.-]+')
317 +
318 +
319 +class Cache:
320 + key = 'caches'
321 + charts = cache_charts
322 +
323 + def __init__(self, **kw):
324 + self.real_name = kw['name']
325 + self.name = BAD_SYMBOLS.sub('_', self.real_name)
326 +
327 + def memory_usage(self, data):
328 + used = data['slabs'][self.real_name]['pages']['used']
329 + free = data['slabs'][self.real_name]['pages']['free']
330 + return used / float(free + used) * 1e4
331 +
332 + def get_data(self, raw_data):
333 + zone_data = raw_data['caches'][self.real_name]
334 + data = parse_json(zone_data, METRICS['CACHE'])
335 + data['memory_usage'] = self.memory_usage(raw_data)
336 + return dict(('_'.join([self.name, k]), v) for k, v in data.items())
337 +
338 +
339 +class WebZone:
340 + key = 'server_zones'
341 + charts = web_zone_charts
342 +
343 + def __init__(self, **kw):
344 + self.real_name = kw['name']
345 + self.name = BAD_SYMBOLS.sub('_', self.real_name)
346 +
347 + def get_data(self, raw_data):
348 + zone_data = raw_data['server_zones'][self.real_name]
349 + data = parse_json(zone_data, METRICS['WEB_ZONE'])
350 + return dict(('_'.join([self.name, k]), v) for k, v in data.items())
351 +
352 +
353 +class WebUpstream:
354 + key = 'upstreams'
355 + charts = web_upstream_charts
356 +
357 + def __init__(self, **kw):
358 + self.real_name = kw['name']
359 + self.name = BAD_SYMBOLS.sub('_', self.real_name)
360 + self.peers = OrderedDict()
361 +
362 + peers = kw['response']['upstreams'][self.real_name]['peers']
363 + for peer in peers:
364 + self.add_peer(peer['id'], peer['server'])
365 +
366 + def __iter__(self):
367 + return iter(self.peers.values())
368 +
369 + def add_peer(self, idx, server):
370 + peer = WebUpstreamPeer(idx, server)
371 + self.peers[peer.real_server] = peer
372 + return peer
373 +
374 + def peers_stats(self, peers):
375 + data = dict()
376 + for peer in self.peers.values():
377 + if not peer.active:
378 + continue
379 + try:
380 + data.update(peer.get_data(peers[peer.id]))
381 + except KeyError:
382 + peer.active = False
383 + return data
384 +
385 + def memory_usage(self, data):
386 + used = data['slabs'][self.real_name]['pages']['used']
387 + free = data['slabs'][self.real_name]['pages']['free']
388 + return used / float(free + used) * 1e4
389 +
390 + def summary_stats(self, data):
391 + rv = defaultdict(int)
392 + for metric in METRICS['WEB_UPSTREAM_SUMMARY']:
393 + for peer in self.peers.values():
394 + if peer.active:
395 + metric = '_'.join(metric.split('.'))
396 + rv[metric] += data['_'.join([peer.server, metric])]
397 + return rv
398 +
399 + def get_data(self, raw_data):
400 + data = dict()
401 + peers = raw_data['upstreams'][self.real_name]['peers']
402 + data.update(self.peers_stats(peers))
403 + data.update(self.summary_stats(data))
404 + data['memory_usage'] = self.memory_usage(raw_data)
405 + return dict(('_'.join([self.name, k]), v) for k, v in data.items())
406 +
407 +
408 +class WebUpstreamPeer:
409 + def __init__(self, idx, server):
410 + self.id = idx
411 + self.real_server = server
412 + self.server = BAD_SYMBOLS.sub('_', self.real_server)
413 + self.active = True
414 +
415 + def get_data(self, raw):
416 + data = dict(header_time=0, response_time=0, max_conns=0)
417 + data.update(parse_json(raw, METRICS['WEB_UPSTREAM_PEER']))
418 + data['connections_usage'] = 0 if not data['max_conns'] else data['active'] / float(data['max_conns']) * 1e4
419 + data['state'] = int(data['state'] == 'up')
420 + return dict(('_'.join([self.server, k]), v) for k, v in data.items())
421 +
422 +
423 +class Service(UrlService):
424 + def __init__(self, configuration=None, name=None):
425 + UrlService.__init__(self, configuration=configuration, name=name)
426 + self.order = ORDER
427 + self.definitions = CHARTS
428 + self.objects = dict()
429 +
430 + def check(self):
431 + if not self.url:
432 + self.error('URL is not defined')
433 + return None
434 +
435 + self._manager = self._build_manager()
436 + if not self._manager:
437 + return None
438 +
439 + raw_data = self._get_raw_data()
440 + if not raw_data:
441 + return None
442 + response = loads(raw_data)
443 +
444 + for obj_cls in [WebZone, WebUpstream, Cache]:
445 + for obj_name in response[obj_cls.key]:
446 + obj = obj_cls(name=obj_name, response=response)
447 + self.objects[obj.real_name] = obj
448 + charts = obj_cls.charts(obj)
449 + for chart in charts:
450 + self.order.append(chart)
451 + self.definitions[chart] = charts[chart]
452 +
453 + return True
454 +
455 + def _get_data(self):
456 + """
457 + Format data received from http request
458 + :return: dict
459 + """
460 + raw_data = self._get_raw_data()
461 + if not raw_data:
462 + return None
463 + response = loads(raw_data)
464 +
465 + data = parse_json(response, METRICS['SERVER'])
466 + data['ssl_memory_usage'] = data['slabs_SSL_pages_used'] / float(data['slabs_SSL_pages_free']) * 1e4
467 +
468 + for obj_name in self.objects:
469 + try:
470 + obj = self.objects[obj_name]
471 + except KeyError:
472 + continue
473 + else:
474 + data.update(obj.get_data(response))
475 +
476 + return data
477 +
478 +
479 +def parse_json(raw_data, metrics):
480 + data = dict()
481 + for metric in metrics:
482 + value = raw_data
483 + metrics_list = metric.split('.')
484 + try:
485 + for m in metrics_list:
486 + value = value[m]
487 + except KeyError:
488 + continue
489 + data['_'.join(metrics_list)] = value
490 + return data