@cryptotaxi247 / netdata-1 / commits / 52fd55a9d

service inheritance

paulfantom committed Jun 17, 2016 at 14:20 UTC 52fd55a9dded964026d89d7b8128d99431c68993
2 files changed +244 -132
python.d/mysql.chart.py
+196 -132
@@ -1,32 +1,38 @@
1 +# Description: MySQL netdata python.d plugin
2 +# Author: Pawel Krupa (paulfantom)
3
4 NAME = "mysql.chart.py"
3 -from sys import stderr
5 +import sys
6 +
7 +# import 3rd party library to handle MySQL communication
8 try:
9 import MySQLdb
6 - stderr.write(NAME + ": using MySQLdb")
10 # https://github.com/PyMySQL/mysqlclient-python
11 + sys.stderr.write(NAME + ": using MySQLdb")
12 except ImportError:
13 try:
14 import pymysql as MySQLdb
15 # https://github.com/PyMySQL/PyMySQL
12 - stderr.write(NAME + ": using pymysql")
16 + sys.stderr.write(NAME + ": using pymysql")
17 except ImportError:
14 - stderr.write(NAME + ": You need to install PyMySQL module to use mysql.chart.py plugin\n")
18 + sys.stderr.write(NAME + ": You need to install PyMySQL module to use mysql.chart.py plugin\n")
19
20 +# default configuration (overriden by python.d.plugin)
21 config = [
22 {
23 'name' : 'local',
24 'user' : 'root',
25 'password' : '',
21 - 'socket' : '/var/run/mysqld/mysqld.sock'
26 + 'socket' : '/var/run/mysqld/mysqld.sock',
27 + 'update_every' : 3,
28 + 'retries' : 4
29 }
30 ]
31
25 -update_every = 3
26 -priority = 90000
27 -
28 -#query = "SHOW GLOBAL STATUS WHERE value REGEX '^[0-9]'"
32 +# query executed on MySQL server
33 QUERY = "SHOW GLOBAL STATUS"
34 +
35 +# charts order (can be overriden if you want less charts, or different order)
36 ORDER = ['net',
37 'queries',
38 'handlers',
@@ -63,23 +69,32 @@ ORDER = ['net',
69 'binlog_stmt_cache',
70 'connection_errors']
71
72 +# charts definitions in format:
73 +# CHARTS = {
74 +# 'chart_name_in_netdata': (
75 +# "parameters defining chart (passed to CHART statement)",
76 +# [ # dimensions (lines) definitions
77 +# ("dimension_name", "dimension parameters (passed to DIMENSION statement)", "additional parameter (optional)")
78 +# ])
79 +# }
80 +
81 CHARTS = {
82 'net' : (
83 "'' 'mysql Bandwidth' 'kilobits/s' bandwidth mysql.net area",
69 - (
84 + [
85 ("Bytes_received", "in incremental 8 1024"),
86 ("Bytes_sent", "out incremental -8 1024")
72 - )),
87 + ]),
88 'queries' : (
89 "'' 'mysql Queries' 'queries/s' queries mysql.queries line",
75 - (
90 + [
91 ("Queries", "queries incremental 1 1"),
92 ("Questions", "questions incremental 1 1"),
93 ("Slow_queries", "slow_queries incremental -1 1")
79 - )),
94 + ]),
95 'handlers' : (
96 "'' 'mysql Handlers' 'handlers/s' handlers mysql.handlers line",
82 - (
97 + [
98 ("Handler_commit", "commit incremental 1 1"),
99 ("Handler_delete", "delete incremental 1 1"),
100 ("Handler_prepare", "prepare incremental 1 1"),
@@ -94,274 +109,312 @@ CHARTS = {
109 ("Handler_savepoint_rollback", "savepoint_rollback incremental 1 1"),
110 ("Handler_update", "update incremental 1 1"),
111 ("Handler_write", "write incremental 1 1")
97 - )),
112 + ]),
113 'table_locks' : (
114 "'' 'mysql Tables Locks' 'locks/s' locks mysql.table_locks line",
100 - (
115 + [
116 ("Table_locks_immediate", "immediate incremental 1 1"),
117 ("Table_locks_waited", "waited incremental -1 1")
103 - )),
118 + ]),
119 'join_issues' : (
120 "'' 'mysql Select Join Issues' 'joins/s' issues mysql.join_issues line",
106 - (
121 + [
122 ("Select_full_join", "full_join incremental 1 1"),
123 ("Select_full_range_join", "full_range_join incremental 1 1"),
124 ("Select_range", "range incremental 1 1"),
125 ("Select_range_check", "range_check incremental 1 1"),
126 ("Select_scan", "scan incremental 1 1"),
112 - )),
127 + ]),
128 'sort_issues' : (
129 "'' 'mysql Sort Issues' 'issues/s' issues mysql.sort.issues line",
115 - (
130 + [
131 ("Sort_merge_passes", "merge_passes incremental 1 1"),
132 ("Sort_range", "range incremental 1 1"),
133 ("Sort_scan", "scan incremental 1 1"),
119 - )),
134 + ]),
135 'tmp' : (
136 "'' 'mysql Tmp Operations' 'counter' temporaries mysql.tmp line",
122 - (
137 + [
138 ("Created_tmp_disk_tables", "disk_tables incremental 1 1"),
139 ("Created_tmp_files", "files incremental 1 1"),
140 ("Created_tmp_tables", "tables incremental 1 1"),
126 - )),
141 + ]),
142 'connections' : (
143 "'' 'mysql Connections' 'connections/s' connections mysql.connections line",
129 - (
144 + [
145 ("Connections", "all incremental 1 1"),
146 ("Aborted_connects", "aborted incremental 1 1"),
132 - )),
147 + ]),
148 'binlog_cache' : (
149 "'' 'mysql Binlog Cache' 'transactions/s' binlog mysql.binlog_cache line",
135 - (
150 + [
151 ("Binlog_cache_disk_use", "disk incremental 1 1"),
152 ("Binlog_cache_use", "all incremental 1 1"),
138 - )),
153 + ]),
154 'threads' : (
155 "'' 'mysql Threads' 'threads' threads mysql.threads line",
141 - (
156 + [
157 ("Threads_connected", "connected absolute 1 1"),
158 ("Threads_created", "created incremental 1 1"),
159 ("Threads_cached", "cached absolute -1 1"),
160 ("Threads_running", "running absolute 1 1"),
146 - )),
161 + ]),
162 'thread_cache_misses' : (
163 "'' 'mysql Threads Cache Misses' 'misses' threads mysql.thread_cache_misses area",
149 - (
164 + [
165 ("Thread_cache_misses", "misses misses absolute 1 100"),
151 - )),
166 + ]),
167 'innodb_io' : (
168 "'' 'mysql InnoDB I/O Bandwidth' 'kilobytes/s' innodb mysql.innodb_io area",
154 - (
169 + [
170 ("Innodb_data_read", "read incremental 1 1024"),
171 ("Innodb_data_written", "write incremental -1 1024"),
157 - )),
172 + ]),
173 'innodb_io_ops' : (
174 "'' 'mysql InnoDB I/O Operations' 'operations/s' innodb mysql.innodb_io_ops line",
160 - (
175 + [
176 ("Innodb_data_reads", "reads incremental 1 1"),
177 ("Innodb_data_writes", "writes incremental -1 1"),
178 ("Innodb_data_fsyncs", "fsyncs incremental 1 1"),
164 - )),
179 + ]),
180 'innodb_io_pending_ops' : (
181 "'' 'mysql InnoDB Pending I/O Operations' 'operations' innodb mysql.innodb_io_pending_ops line",
167 - (
182 + [
183 ("Innodb_data_pending_reads", "reads absolute 1 1"),
184 ("Innodb_data_pending_writes", "writes absolute -1 1"),
185 ("Innodb_data_pending_fsyncs", "fsyncs absolute 1 1"),
171 - )),
186 + ]),
187 'innodb_log' : (
188 "'' 'mysql InnoDB Log Operations' 'operations/s' innodb mysql.innodb_log line",
174 - (
189 + [
190 ("Innodb_log_waits", "waits incremental 1 1"),
191 ("Innodb_log_write_requests", "write_requests incremental -1 1"),
192 ("Innodb_log_writes", "incremental -1 1"),
178 - )),
193 + ]),
194 'innodb_os_log' : (
195 "'' 'mysql InnoDB OS Log Operations' 'operations' innodb mysql.innodb_os_log line",
181 - (
196 + [
197 ("Innodb_os_log_fsyncs", "fsyncs incremental 1 1"),
198 ("Innodb_os_log_pending_fsyncs", "pending_fsyncs absolute 1 1"),
199 ("Innodb_os_log_pending_writes", "pending_writes absolute -1 1"),
185 - )),
200 + ]),
201 'innodb_os_log_io' : (
202 "'' 'mysql InnoDB OS Log Bandwidth' 'kilobytes/s' innodb mysql.innodb_os_log_io area",
188 - (
203 + [
204 ("Innodb_os_log_written", "write incremental -1 1024"),
190 - )),
205 + ]),
206 'innodb_cur_row_lock' : (
207 "'' 'mysql InnoDB Current Row Locks' 'operations' innodb mysql.innodb_cur_row_lock area",
193 - (
208 + [
209 ("Innodb_row_lock_current_waits", "current_waits absolute 1 1"),
195 - )),
210 + ]),
211 'innodb_rows' : (
212 "'' 'mysql InnoDB Row Operations' 'operations/s' innodb mysql.innodb_rows area",
198 - (
213 + [
214 ("Innodb_rows_inserted", "read incremental 1 1"),
215 ("Innodb_rows_read", "deleted incremental -1 1"),
216 ("Innodb_rows_updated", "inserted incremental 1 1"),
217 ("Innodb_rows_deleted", "updated incremental -1 1"),
203 - )),
218 + ]),
219 'innodb_buffer_pool_pages' : (
220 "'' 'mysql InnoDB Buffer Pool Pages' 'pages' innodb mysql.innodb_buffer_pool_pages line",
206 - (
221 + [
222 ("Innodb_buffer_pool_pages_data", "data absolute 1 1"),
223 ("Innodb_buffer_pool_pages_dirty", "dirty absolute -1 1"),
224 ("Innodb_buffer_pool_pages_free", "free absolute 1 1"),
225 ("Innodb_buffer_pool_pages_flushed", "flushed incremental -1 1"),
226 ("Innodb_buffer_pool_pages_misc", "misc absolute -1 1"),
227 ("Innodb_buffer_pool_pages_total", "total absolute 1 1"),
213 - )),
228 + ]),
229 'innodb_buffer_pool_bytes' : (
230 "'' 'mysql InnoDB Buffer Pool Bytes' 'MB' innodb mysql.innodb_buffer_pool_bytes area",
216 - (
231 + [
232 ("Innodb_buffer_pool_bytes_data", "data absolute 1"),
233 ("Innodb_buffer_pool_bytes_dirty", "dirty absolute -1"),
219 - )),
234 + ]),
235 'innodb_buffer_pool_read_ahead' : (
236 "'' 'mysql InnoDB Buffer Pool Read Ahead' 'operations/s' innodb mysql.innodb_buffer_pool_read_ahead area",
222 - (
237 + [
238 ("Innodb_buffer_pool_read_ahead", "all incremental 1 1"),
239 ("Innodb_buffer_pool_read_ahead_evicted", "evicted incremental -1 1"),
240 ("Innodb_buffer_pool_read_ahead_rnd", "random incremental 1 1"),
226 - )),
241 + ]),
242 'innodb_buffer_pool_reqs' : (
243 "'' 'mysql InnoDB Buffer Pool Requests' 'requests/s' innodb mysql.innodb_buffer_pool_reqs area",
229 - (
244 + [
245 ("Innodb_buffer_pool_read_requests", "reads incremental 1 1"),
246 ("Innodb_buffer_pool_write_requests", "writes incremental -1 1"),
232 - )),
247 + ]),
248 'innodb_buffer_pool_ops' : (
249 "'' 'mysql InnoDB Buffer Pool Operations' 'operations/s' innodb mysql.innodb_buffer_pool_ops area",
235 - (
250 + [
251 ("Innodb_buffer_pool_reads", "'disk reads' incremental 1 1"),
252 ("Innodb_buffer_pool_wait_free", "'wait free' incremental -1 1"),
238 - )),
253 + ]),
254 'qcache_ops' : (
255 "'' 'mysql QCache Operations' 'queries/s' qcache mysql.qcache_ops line",
241 - (
256 + [
257 ("Qcache_hits", "hits incremental 1 1"),
258 ("Qcache_lowmem_prunes", "'lowmem prunes' incremental -1 1"),
259 ("Qcache_inserts", "inserts incremental 1 1"),
260 ("Qcache_not_cached", "'not cached' incremental -1 1"),
246 - )),
261 + ]),
262 'qcache' : (
263 "'' 'mysql QCache Queries in Cache' 'queries' qcache mysql.qcache line",
249 - (
264 + [
265 ("Qcache_queries_in_cache", "queries absolute 1 1"),
251 - )),
266 + ]),
267 'qcache_freemem' : (
268 "'' 'mysql QCache Free Memory' 'MB' qcache mysql.qcache_freemem area",
254 - (
269 + [
270 ("Qcache_free_memory", "free absolute 1"),
256 - )),
271 + ]),
272 'qcache_memblocks' : (
273 "'' 'mysql QCache Memory Blocks' 'blocks' qcache mysql.qcache_memblocks line",
259 - (
274 + [
275 ("Qcache_free_blocks", "free absolute 1"),
276 ("Qcache_total_blocks", "total absolute 1 1"),
262 - )),
277 + ]),
278 'key_blocks' : (
279 "'' 'mysql MyISAM Key Cache Blocks' 'blocks' myisam mysql.key_blocks line",
265 - (
280 + [
281 ("Key_blocks_unused", "unused absolute 1 1"),
282 ("Key_blocks_used", "used absolute -1 1"),
283 ("Key_blocks_not_flushed", "'not flushed' absolute 1 1"),
269 - )),
284 + ]),
285 'key_requests' : (
286 "'' 'mysql MyISAM Key Cache Requests' 'requests/s' myisam mysql.key_requests area",
272 - (
287 + [
288 ("Key_read_requests", "reads incremental 1 1"),
289 ("Key_write_requests", "writes incremental -1 1"),
275 - )),
290 + ]),
291 'key_disk_ops' : (
292 "'' 'mysql MyISAM Key Cache Disk Operations' 'operations/s' myisam mysql.key_disk_ops area",
278 - (
293 + [
294 ("Key_reads", "reads incremental 1 1"),
295 ("Key_writes", "writes incremental -1 1"),
281 - )),
296 + ]),
297 'files' : (
298 "'' 'mysql Open Files' 'files' files mysql.files line",
284 - (
299 + [
300 ("Open_files", "files absolute 1 1"),
286 - )),
301 + ]),
302 'files_rate' : (
303 "'' 'mysql Opened Files Rate' 'files/s' files mysql.files_rate line",
289 - (
304 + [
305 ("Opened_files", "files incremental 1 1"),
291 - )),
306 + ]),
307 'binlog_stmt_cache' : (
308 "'' 'mysql Binlog Statement Cache' 'statements/s' binlog mysql.binlog_stmt_cache line",
294 - (
309 + [
310 ("Binlog_stmt_cache_disk_use", "disk incremental 1 1"),
311 ("Binlog_stmt_cache_use", "all incremental 1 1"),
297 - )),
312 + ]),
313 'connection_errors' : (
314 "'' 'mysql Connection Errors' 'connections/s' connections mysql.connection_errors line",
300 - (
315 + [
316 ("Connection_errors_accept", "accept incremental 1 1"),
317 ("Connection_errors_internal", "internal incremental 1 1"),
318 ("Connection_errors_max_connections", "max incremental 1 1"),
319 ("Connection_errors_peer_address", "peer_addr incremental 1 1"),
320 ("Connection_errors_select", "select incremental 1 1"),
321 ("Connection_errors_tcpwrap", "tcpwrap incremental 1 1")
307 - ))
322 + ])
323 }
324
325
311 -def Service(object):
312 - def __init__(self,config=None):
313 - if config is None:
314 - pass # TODO use defaults
315 - if 'name' not in config:
316 - from random import randint
317 - config['name'] = "srv_"+str(randint(0,99)
318 - if 'user' not in config:
319 - config['user'] = 'root'
320 - if 'password' not in config:
321 - config['password'] = ''
322 - if 'my.cnf' in config:
323 - config['socket'] = ''
324 - config['host'] = ''
325 - config['port'] = 0
326 - elif 'socket' in config:
327 - config['my.cnf'] = ''
328 - config['host'] = ''
329 - config['port'] = 0
330 - elif 'host' in config:
331 - config['my.cnf'] = ''
332 - config['socket'] = ''
333 - if 'port' in config:
334 - config['port'] = int(config['port'])
335 - else:
336 - config['port'] = 3306
326 +class BaseService(object):
327 + def __init__(self,configuration,update_every,priority,retries):
328 + if configuration is None:
329 + # use defaults
330 + configuration = config
331 + self.error(NAME+": no configuration supplied. using defaults.")
332
338 - self.config = config
339 - self.connection = None
340 - self.mysql_def = {}
333 + self._parse_base_config(configuration)
334
342 - def check(self):
335 + def _parse_base_config(self,config,update_every,priority,retries):
336 + # parse configuration options to run this Service
337 try:
344 - self.connection = self._connect()
345 - return True
346 - except RuntimeError:
347 - self.connection = None
348 - return False
338 + self.update_every = int(config['update_every'])
339 + except (KeyError, ValueError):
340 + self.update_every = update_every
341 + try:
342 + self.priority = int(config['priority'])
343 + except (KeyError, ValueError):
344 + self.priority = priority
345 + try:
346 + self.retries = int(config['retries'])
347 + except (KeyError, ValueError):
348 + self.retries = retries
349 + self.retries_left = self.retries
350
350 -
351 + def error(self, msg, exception=""):
352 + if exception != "":
353 + exception = " " + str(exception).replace("\n"," ")
354 + sys.stderr.write(str(msg)+exception+"\n")
355 + sys.stderr.flush()
356 +
357 + def check(self):
358 + # TODO notify about not overriden function
359 + self.error("Where is your check()?")
360 + return False
361 +
362 + def create(self):
363 + # TODO notify about not overriden function
364 + self.error("Where is your create()?")
365 + return False
366 +
367 + def update(self):
368 + # TODO notify about not overriden function
369 + self.error("Where is your update()?")
370 + return False
371 +
372 +
373 +class Service(BaseService):
374 + def __init__(self,configuration=None,update_every=3,priority=90000,retries=2):
375 + super().__init__(*args,**kwargs)
376 + self.configuration = self._parse_config(configuration)
377 + self.connection = None
378 + self.defs = None
379 +
380 + def _parse_config(configuration):
381 + # parse configuration to collect data from mysql server
382 + if 'name' not in configuration:
383 + configuration['name'] = 'local'
384 + if 'user' not in configuration:
385 + configuration['user'] = 'root'
386 + if 'password' not in configuration:
387 + configuration['password'] = ''
388 + if 'my.cnf' in configuration:
389 + configuration['socket'] = ''
390 + configuration['host'] = ''
391 + configuration['port'] = 0
392 + elif 'socket' in configuration:
393 + configuration['my.cnf'] = ''
394 + configuration['host'] = ''
395 + configuration['port'] = 0
396 + elif 'host' in configuration:
397 + configuration['my.cnf'] = ''
398 + configuration['socket'] = ''
399 + if 'port' in configuration:
400 + configuration['port'] = int(configuration['port'])
401 + else:
402 + configuration['port'] = 3306
403 +
404 + return configuration
405
406 def _connect(self):
407 try:
354 - self.connection = MySQLdb.connect(user=self.config['user'],
355 - passwd=self.config['password'],
356 - read_default_file=self.config['my.cnf'],
357 - unix_socket=self.config['socket'],
358 - host=self.config['host'],
359 - port=self.config['port'],
360 - connect_timeout=int(update_every))
408 + self.connection = MySQLdb.connect(user=self.configuration['user'],
409 + passwd=self.configuration['password'],
410 + read_default_file=self.configuration['my.cnf'],
411 + unix_socket=self.configuration['socket'],
412 + host=self.configuration['host'],
413 + port=self.configuration['port'],
414 + connect_timeout=self.configuration['update_every'])
415 except Exception as e:
362 - stderr.write(NAME + " has problem connecting to server: "+str(e).replace("\n"," ")+"\n")
416 + self.error(NAME + " has problem connecting to server:", e)
417 raise RuntimeError #stop creating module, need to catch it in supervisor
364 -
418
419 def _get_data(self):
420 try:
@@ -369,22 +422,28 @@ def Service(object):
422 cursor.execute(QUERY)
423 raw_data = cursor.fetchall()
424 except Exception as e:
372 - stderr.write(NAME + ": cannot execute query." + str(e) + "\n")
425 + self.error(NAME + ": cannot execute query.", e)
426 self.connection.close()
427 return None
428
429 return dict(raw_data)
430
378 -
431 + def check(self):
432 + try:
433 + self.connection = self._connect()
434 + return True
435 + except RuntimeError:
436 + self.connection = None
437 + return False
438
439 def create(self):
440 for name in ORDER:
441 self.mysql_def[name] = []
442 for line in CHARTS[name][1]:
384 - self.mysql_def[name].append(line[0])
443 + self.defs[name].append(line[0])
444
445 idx = 0
387 - data = self._get_data(self.config)
446 + data = self._get_data(self.configuration)
447 for name in ORDER:
448 header = "CHART mysql_" + \
449 str(srv['name']) + "." + \
@@ -406,17 +465,16 @@ def Service(object):
465 return False
466 return True
467
409 -
468 def update(self,interval):
411 - data = self._get_data(self.config)
469 + data = self._get_data(self.configuration)
470 if data is None:
471 return False
472 try:
473 data['Thread cache misses'] = int( int(data['Threads_created']) * 10000 / int(data['Connections']))
474 except Exception:
475 pass
418 - for chart, dimensions in self._mysql_def.items():
419 - header = "BEGIN mysql_" + str(self.config['name']) + "." + chart + " " + str(interval) + '\n'
476 + for chart, dimensions in self.defs.items():
477 + header = "BEGIN mysql_" + str(self.configuration['name']) + "." + chart + " " + str(interval) + '\n'
478 lines = ""
479 for d in dimensions:
480 try:
@@ -427,3 +485,9 @@ def Service(object):
485 print(header + lines + "END")
486
487 return True
488 +
489 +if __name__ == "__main__":
490 + my = Service(config)
491 + my.check()
492 + my.create()
493 + my.update(1)
python.d/python_modules/base.py new
+48
@@ -0,0 +1,48 @@
1 +# Description: base for netdata python.d plugins
2 +# Author: Pawel Krupa (paulfantom)
3 +
4 +class BaseService(object):
5 + def __init__(self,configuration,update_every,priority,retries):
6 + if configuration is None:
7 + # use defaults
8 + configuration = config
9 + self.error(NAME+": no configuration supplied. using defaults.")
10 +
11 + self._parse_base_config(configuration)
12 +
13 + def _parse_base_config(self,config,update_every,priority,retries):
14 + # parse configuration options to run this Service
15 + try:
16 + self.update_every = int(config['update_every'])
17 + except (KeyError, ValueError):
18 + self.update_every = update_every
19 + try:
20 + self.priority = int(config['priority'])
21 + except (KeyError, ValueError):
22 + self.priority = priority
23 + try:
24 + self.retries = int(config['retries'])
25 + except (KeyError, ValueError):
26 + self.retries = retries
27 + self.retries_left = self.retries
28 +
29 + def error(self, msg, exception=""):
30 + if exception != "":
31 + exception = " " + str(exception).replace("\n"," ")
32 + sys.stderr.write(str(msg)+exception+"\n")
33 + sys.stderr.flush()
34 +
35 + def check(self):
36 + # TODO notify about not overriden function
37 + self.error("Where is your check()?")
38 + return False
39 +
40 + def create(self):
41 + # TODO notify about not overriden function
42 + self.error("Where is your create()?")
43 + return False
44 +
45 + def update(self):
46 + # TODO notify about not overriden function
47 + self.error("Where is your update()?")
48 + return False