@cryptotaxi247 / netdata-1 / commits / d14e84a17

Comments, comments, comments.

paulfantom committed Jun 19, 2016 at 21:12 UTC d14e84a17f12fc3da9b72bbe484e7f15004de4ae

3 files changed +163 -65

plugins.d/python.d.plugin

+104 -15

@@ -42,12 +42,22 @@ BASE_CONFIG = {'update_every': 10,

44 class PythonCharts(object):

45 + """

46 + Main class used to control every python module.

47 + """

48 def __init__(self,

49 interval=None,

50 modules=None,

51 modules_path='../python.d/',

52 modules_configs='../conf.d/',

53 modules_disabled=None):

54 + """

55 + :param interval: int

56 + :param modules: list

57 + :param modules_path: str

58 + :param modules_configs: str

59 + :param modules_disabled: list

60 + """

62 if modules is None:

63 modules = []

@@ -65,14 +75,20 @@ class PythonCharts(object):

75 configured_modules = self._load_configs(loaded_modules)

77 # good economy and prosperity:

68 - self.jobs = self._create_jobs(configured_modules)

78 + self.jobs = self._create_jobs(configured_modules) # type: list

79 if DEBUG_FLAG and interval is not None:

80 for job in self.jobs:

81 job.create_timetable(interval)

83 @staticmethod

84 def _import_module(path, name=None):

75 - # try to import module using only its path

85 + """

86 + Try to import module using only its path.

87 + :param path: str

88 + :param name: str

89 + :return: object

90 + """

91 +

92 if name is None:

93 name = path.split('/')[-1]

94 if name[-len(MODULE_EXTENSION):] != MODULE_EXTENSION:

@@ -83,12 +99,19 @@ class PythonCharts(object):

99 return importlib.machinery.SourceFileLoader(name, path).load_module()

100 else:

101 return imp.load_source(name, path)

86 - # return importlib.import_module(path, name)

102 except Exception as e:

103 debug(str(e))

104 return None

105

106 def _load_modules(self, path, modules, disabled):

107 + """

108 + Load modules from 'modules' list or dynamically every file from 'path' (only .chart.py files)

109 + :param path: str

110 + :param modules: list

111 + :param disabled: list

112 + :return: list

113 + """

114 +

115 # check if plugin directory exists

116 if not os.path.isdir(path):

117 debug("cannot find charts directory ", path)

@@ -122,7 +145,14 @@ class PythonCharts(object):

145 return loaded

146

147 def _load_configs(self, modules):

125 - # function loads configuration files to modules

148 + """

149 + Append configuration in list named `config` to every module.

150 + For multi-job modules `config` list is created in _parse_config,

151 + otherwise it is created here based on BASE_CONFIG prototype with None as identifier.

152 + :param modules: list

153 + :return: list

154 + """

155 +

156 for mod in modules:

157 configfile = self.configs + mod.__name__ + ".conf"

158 if os.path.isfile(configfile):

@@ -150,6 +180,19 @@ class PythonCharts(object):

180

181 @staticmethod

182 def _parse_config(module, config):

183 + """

184 + Parse configuration file or extract configuration from module file.

185 + Example of returned dictionary:

186 + config = {'name': {

187 + 'update_every': 2,

188 + 'retries': 3,

189 + 'priority': 30000

190 + 'other_val': 123}}

191 + :param module: object

192 + :param config: dict

193 + :return: dict

194 + """

195 +

196 # get default values

197 defaults = {}

198 for key in BASE_CONFIG:

@@ -187,11 +230,11 @@ class PythonCharts(object):

230

231 @staticmethod

232 def _create_jobs(modules):

190 - # module store a definition of Service class

191 - # module store configuration in module.config

192 - # configs are list of dicts or a dict

193 - # one dict is one service

194 - # iterate over list of modules and inside one loop iterate over configs

233 + """

234 + Create jobs based on module.config dictionary and module.Service class definition.

235 + :param modules: list

236 + :return: list

237 + """

238 jobs = []

239 for module in modules:

240 for name in module.config:

@@ -212,10 +255,16 @@ class PythonCharts(object):

255 if name is not None:

256 job.execution_name += "_" + name

257 jobs.append(job)

258 +

259 return [j for j in jobs if j is not None]

260

261 def _stop(self, job, reason=None):

218 - # modifies self.jobs

262 + """

263 + Stop specified job and remove it from self.jobs list

264 + Also notifies user about job failure if DEBUG_FLAG is set

265 + :param job: object

266 + :param reason: str

267 + """

268 self.jobs.remove(job)

269 if reason is None:

270 return

@@ -240,7 +289,11 @@ class PythonCharts(object):

289 debug(job.execution_name, "is " + reason)

290

291 def check(self):

243 - # try to execute check() on every job

292 + """

293 + Tries to execute check() on every job.

294 + This cannot fail thus it is catching every exception

295 + If job.check() fails job is stopped

296 + """

297 for job in self.jobs:

298 try:

299 if not job.check():

@@ -251,7 +304,12 @@ class PythonCharts(object):

304 self._stop(job, "misbehaving. Reason: " + str(e))

305

306 def create(self):

254 - # try to execute create() on every job

307 + """

308 + Tries to execute create() on every job.

309 + This cannot fail thus it is catching every exception.

310 + If job.create() fails job is stopped.

311 + This is also creating job run time chart.

312 + """

313 for job in self.jobs:

314 try:

315 if not job.create():

@@ -274,7 +332,14 @@ class PythonCharts(object):

332 self._stop(job, "misbehaving. Reason: " + str(e))

333

334 def _update_job(self, job):

277 - # try to execute update() on every job and draw run time graph

335 + """

336 + Tries to execute update() on specified job.

337 + This cannot fail thus it is catching every exception.

338 + If job.update() returns False, number of retries is decremented. If there are no more retries, job is stopped.

339 + Job is also stopped if it throws an exception.

340 + This is also updating job run time chart.

341 + :param job: object

342 + """

343 t_start = time.time()

344 # check if it is time to execute job update() function

345 if job.timetable['next'] > t_start:

@@ -285,7 +350,11 @@ class PythonCharts(object):

350 else:

351 since_last = int((t_start - job.timetable['last']) * 1000000)

352 if not job.update(since_last):

288 - self._stop(job, "update failed")

353 + if job.retries <= 0:

354 + self._stop(job, "update failed")

355 + else:

356 + job.retries -= 1

357 + job.timetable['next'] += job.timetable['freq']

358 return

359 except AttributeError:

360 self._stop(job, "no update")

@@ -304,7 +373,10 @@ class PythonCharts(object):

373 self.first_run = False

374

375 def update(self):

307 - # run updates (this will stay forever and ever and ever forever and ever it'll be the one...)

376 + """

377 + Tries to execute update() on every job by using _update_job()

378 + This will stay forever and ever and ever forever and ever it'll be the one...

379 + """

380 self.first_run = True

381 while True:

382 next_runs = []

@@ -322,6 +394,11 @@ class PythonCharts(object):

394

395

396 def read_config(path):

397 + """

398 + Read YAML configuration from specified file

399 + :param path: str

400 + :return: dict

401 + """

402 try:

403 with open(path, 'r') as stream:

404 config = yaml.load(stream)

@@ -335,6 +412,9 @@ def read_config(path):

412

413

414 def debug(*args):

415 + """

416 + Print message on stderr.

417 + """

418 if not DEBUG_FLAG:

419 return

420 sys.stderr.write(PROGRAM + ":")

@@ -345,6 +425,12 @@ def debug(*args):

425

426

427 def parse_cmdline(directory, *commands):

428 + """

429 + Parse parameters from command line.

430 + :param directory: str

431 + :param commands: list of str

432 + :return: dict

433 + """

434 global DEBUG_FLAG

435 interval = None

436

@@ -375,6 +461,9 @@ def parse_cmdline(directory, *commands):

461

462 # if __name__ == '__main__':

463 def run():

464 + """

465 + Main program.

466 + """

467 global PROGRAM, DEBUG_FLAG

468 PROGRAM = sys.argv[0].split('/')[-1].split('.plugin')[0]

469 # parse env variables

python.d/example.chart.py

+4 -1

@@ -19,7 +19,10 @@ class Service(BaseService):

19 return True

21 def create(self):

22 - print("CHART example.python_random '' 'A random number' 'random number' random random line "+str(self.priority)+" "+str(self.update_every))

22 + print("CHART example.python_random '' 'A random number' 'random number' random random line " +

23 + str(self.priority) +

24 + " " +

25 + str(self.update_every))

26 print("DIMENSION random1 '' absolute 1 1")

27 return True

python.d/mysql.chart.py

+55 -49

@@ -2,16 +2,19 @@

2 # Author: Pawel Krupa (paulfantom)

4 import sys

5 +

6 NAME = "mysql.chart.py"

8 # import 3rd party library to handle MySQL communication

9 try:

10 import MySQLdb

11 +

12 # https://github.com/PyMySQL/mysqlclient-python

13 sys.stderr.write(NAME + ": using MySQLdb\n")

14 except ImportError:

15 try:

16 import pymysql as MySQLdb

17 +

18 # https://github.com/PyMySQL/PyMySQL

19 sys.stderr.write(NAME + ": using pymysql\n")

20 except ImportError:

@@ -23,12 +26,12 @@ from base import BaseService

26 # default configuration (overridden by python.d.plugin)

27 config = {

28 'local': {

26 - 'user' : 'root',

27 - 'password' : '',

28 - 'socket' : '/var/run/mysqld/mysqld.sock',

29 - 'update_every' : 3,

30 - 'retries' : 4,

31 - 'priority' : 100

29 + 'user': 'root',

30 + 'password': '',

31 + 'socket': '/var/run/mysqld/mysqld.sock',

32 + 'update_every': 3,

33 + 'retries': 4,

34 + 'priority': 100

35 }

36 }

@@ -87,20 +90,20 @@ ORDER = ['net',

90 # }

92 CHARTS = {

90 - 'net' : (

93 + 'net': (

94 "'' 'mysql Bandwidth' 'kilobits/s' bandwidth mysql.net area",

95 [

96 ("Bytes_received", "in incremental 8 1024"),

94 - ("Bytes_sent", "out incremental -8 1024")

97 + ("Bytes_sent", "out incremental -8 1024")

98 ]),

96 - 'queries' : (

99 + 'queries': (

100 "'' 'mysql Queries' 'queries/s' queries mysql.queries line",

101 [

99 - ("Queries", "queries incremental 1 1"),

100 - ("Questions", "questions incremental 1 1"),

102 + ("Queries", "queries incremental 1 1"),

103 + ("Questions", "questions incremental 1 1"),

104 ("Slow_queries", "slow_queries incremental -1 1")

105 ]),

103 - 'handlers' : (

106 + 'handlers': (

107 "'' 'mysql Handlers' 'handlers/s' handlers mysql.handlers line",

108 [

109 ("Handler_commit", "commit incremental 1 1"),

@@ -118,13 +121,13 @@ CHARTS = {

121 ("Handler_update", "update incremental 1 1"),

122 ("Handler_write", "write incremental 1 1")

123 ]),

121 - 'table_locks' : (

124 + 'table_locks': (

125 "'' 'mysql Tables Locks' 'locks/s' locks mysql.table_locks line",

126 [

127 ("Table_locks_immediate", "immediate incremental 1 1"),

128 ("Table_locks_waited", "waited incremental -1 1")

129 ]),

127 - 'join_issues' : (

130 + 'join_issues': (

131 "'' 'mysql Select Join Issues' 'joins/s' issues mysql.join_issues line",

132 [

133 ("Select_full_join", "full_join incremental 1 1"),

@@ -133,33 +136,33 @@ CHARTS = {

136 ("Select_range_check", "range_check incremental 1 1"),

137 ("Select_scan", "scan incremental 1 1"),

138 ]),

136 - 'sort_issues' : (

139 + 'sort_issues': (

140 "'' 'mysql Sort Issues' 'issues/s' issues mysql.sort.issues line",

141 [

142 ("Sort_merge_passes", "merge_passes incremental 1 1"),

143 ("Sort_range", "range incremental 1 1"),

144 ("Sort_scan", "scan incremental 1 1"),

145 ]),

143 - 'tmp' : (

146 + 'tmp': (

147 "'' 'mysql Tmp Operations' 'counter' temporaries mysql.tmp line",

148 [

149 ("Created_tmp_disk_tables", "disk_tables incremental 1 1"),

150 ("Created_tmp_files", "files incremental 1 1"),

151 ("Created_tmp_tables", "tables incremental 1 1"),

152 ]),

150 - 'connections' : (

153 + 'connections': (

154 "'' 'mysql Connections' 'connections/s' connections mysql.connections line",

155 [

156 ("Connections", "all incremental 1 1"),

157 ("Aborted_connects", "aborted incremental 1 1"),

158 ]),

156 - 'binlog_cache' : (

159 + 'binlog_cache': (

160 "'' 'mysql Binlog Cache' 'transactions/s' binlog mysql.binlog_cache line",

161 [

162 ("Binlog_cache_disk_use", "disk incremental 1 1"),

163 ("Binlog_cache_use", "all incremental 1 1"),

164 ]),

162 - 'threads' : (

165 + 'threads': (

166 "'' 'mysql Threads' 'threads' threads mysql.threads line",

167 [

168 ("Threads_connected", "connected absolute 1 1"),

@@ -167,56 +170,56 @@ CHARTS = {

170 ("Threads_cached", "cached absolute -1 1"),

171 ("Threads_running", "running absolute 1 1"),

172 ]),

170 - 'thread_cache_misses' : (

173 + 'thread_cache_misses': (

174 "'' 'mysql Threads Cache Misses' 'misses' threads mysql.thread_cache_misses area",

175 [

176 ("Thread_cache_misses", "misses misses absolute 1 100"),

177 ]),

175 - 'innodb_io' : (

178 + 'innodb_io': (

179 "'' 'mysql InnoDB I/O Bandwidth' 'kilobytes/s' innodb mysql.innodb_io area",

180 [

181 ("Innodb_data_read", "read incremental 1 1024"),

182 ("Innodb_data_written", "write incremental -1 1024"),

183 ]),

181 - 'innodb_io_ops' : (

184 + 'innodb_io_ops': (

185 "'' 'mysql InnoDB I/O Operations' 'operations/s' innodb mysql.innodb_io_ops line",

186 [

187 ("Innodb_data_reads", "reads incremental 1 1"),

188 ("Innodb_data_writes", "writes incremental -1 1"),

189 ("Innodb_data_fsyncs", "fsyncs incremental 1 1"),

190 ]),

188 - 'innodb_io_pending_ops' : (

191 + 'innodb_io_pending_ops': (

192 "'' 'mysql InnoDB Pending I/O Operations' 'operations' innodb mysql.innodb_io_pending_ops line",

193 [

194 ("Innodb_data_pending_reads", "reads absolute 1 1"),

195 ("Innodb_data_pending_writes", "writes absolute -1 1"),

196 ("Innodb_data_pending_fsyncs", "fsyncs absolute 1 1"),

197 ]),

195 - 'innodb_log' : (

198 + 'innodb_log': (

199 "'' 'mysql InnoDB Log Operations' 'operations/s' innodb mysql.innodb_log line",

200 [

201 ("Innodb_log_waits", "waits incremental 1 1"),

202 ("Innodb_log_write_requests", "write_requests incremental -1 1"),

203 ("Innodb_log_writes", "incremental -1 1"),

204 ]),

202 - 'innodb_os_log' : (

205 + 'innodb_os_log': (

206 "'' 'mysql InnoDB OS Log Operations' 'operations' innodb mysql.innodb_os_log line",

207 [

208 ("Innodb_os_log_fsyncs", "fsyncs incremental 1 1"),

209 ("Innodb_os_log_pending_fsyncs", "pending_fsyncs absolute 1 1"),

210 ("Innodb_os_log_pending_writes", "pending_writes absolute -1 1"),

211 ]),

209 - 'innodb_os_log_io' : (

212 + 'innodb_os_log_io': (

213 "'' 'mysql InnoDB OS Log Bandwidth' 'kilobytes/s' innodb mysql.innodb_os_log_io area",

214 [

215 ("Innodb_os_log_written", "write incremental -1 1024"),

216 ]),

214 - 'innodb_cur_row_lock' : (

217 + 'innodb_cur_row_lock': (

218 "'' 'mysql InnoDB Current Row Locks' 'operations' innodb mysql.innodb_cur_row_lock area",

219 [

220 ("Innodb_row_lock_current_waits", "current_waits absolute 1 1"),

221 ]),

219 - 'innodb_rows' : (

222 + 'innodb_rows': (

223 "'' 'mysql InnoDB Row Operations' 'operations/s' innodb mysql.innodb_rows area",

224 [

225 ("Innodb_rows_inserted", "read incremental 1 1"),

@@ -224,7 +227,7 @@ CHARTS = {

227 ("Innodb_rows_updated", "inserted incremental 1 1"),

228 ("Innodb_rows_deleted", "updated incremental -1 1"),

229 ]),

227 - 'innodb_buffer_pool_pages' : (

230 + 'innodb_buffer_pool_pages': (

231 "'' 'mysql InnoDB Buffer Pool Pages' 'pages' innodb mysql.innodb_buffer_pool_pages line",

232 [

233 ("Innodb_buffer_pool_pages_data", "data absolute 1 1"),

@@ -234,32 +237,32 @@ CHARTS = {

237 ("Innodb_buffer_pool_pages_misc", "misc absolute -1 1"),

238 ("Innodb_buffer_pool_pages_total", "total absolute 1 1"),

239 ]),

237 - 'innodb_buffer_pool_bytes' : (

240 + 'innodb_buffer_pool_bytes': (

241 "'' 'mysql InnoDB Buffer Pool Bytes' 'MB' innodb mysql.innodb_buffer_pool_bytes area",

242 [

243 ("Innodb_buffer_pool_bytes_data", "data absolute 1"),

244 ("Innodb_buffer_pool_bytes_dirty", "dirty absolute -1"),

245 ]),

243 - 'innodb_buffer_pool_read_ahead' : (

246 + 'innodb_buffer_pool_read_ahead': (

247 "'' 'mysql InnoDB Buffer Pool Read Ahead' 'operations/s' innodb mysql.innodb_buffer_pool_read_ahead area",

248 [

249 ("Innodb_buffer_pool_read_ahead", "all incremental 1 1"),

250 ("Innodb_buffer_pool_read_ahead_evicted", "evicted incremental -1 1"),

251 ("Innodb_buffer_pool_read_ahead_rnd", "random incremental 1 1"),

252 ]),

250 - 'innodb_buffer_pool_reqs' : (

253 + 'innodb_buffer_pool_reqs': (

254 "'' 'mysql InnoDB Buffer Pool Requests' 'requests/s' innodb mysql.innodb_buffer_pool_reqs area",

255 [

256 ("Innodb_buffer_pool_read_requests", "reads incremental 1 1"),

257 ("Innodb_buffer_pool_write_requests", "writes incremental -1 1"),

258 ]),

256 - 'innodb_buffer_pool_ops' : (

259 + 'innodb_buffer_pool_ops': (

260 "'' 'mysql InnoDB Buffer Pool Operations' 'operations/s' innodb mysql.innodb_buffer_pool_ops area",

261 [

262 ("Innodb_buffer_pool_reads", "'disk reads' incremental 1 1"),

263 ("Innodb_buffer_pool_wait_free", "'wait free' incremental -1 1"),

264 ]),

262 - 'qcache_ops' : (

265 + 'qcache_ops': (

266 "'' 'mysql QCache Operations' 'queries/s' qcache mysql.qcache_ops line",

267 [

268 ("Qcache_hits", "hits incremental 1 1"),

@@ -267,58 +270,58 @@ CHARTS = {

270 ("Qcache_inserts", "inserts incremental 1 1"),

271 ("Qcache_not_cached", "'not cached' incremental -1 1"),

272 ]),

270 - 'qcache' : (

273 + 'qcache': (

274 "'' 'mysql QCache Queries in Cache' 'queries' qcache mysql.qcache line",

275 [

276 ("Qcache_queries_in_cache", "queries absolute 1 1"),

277 ]),

275 - 'qcache_freemem' : (

278 + 'qcache_freemem': (

279 "'' 'mysql QCache Free Memory' 'MB' qcache mysql.qcache_freemem area",

280 [

281 ("Qcache_free_memory", "free absolute 1"),

282 ]),

280 - 'qcache_memblocks' : (

283 + 'qcache_memblocks': (

284 "'' 'mysql QCache Memory Blocks' 'blocks' qcache mysql.qcache_memblocks line",

285 [

286 ("Qcache_free_blocks", "free absolute 1"),

287 ("Qcache_total_blocks", "total absolute 1 1"),

288 ]),

286 - 'key_blocks' : (

289 + 'key_blocks': (

290 "'' 'mysql MyISAM Key Cache Blocks' 'blocks' myisam mysql.key_blocks line",

291 [

292 ("Key_blocks_unused", "unused absolute 1 1"),

293 ("Key_blocks_used", "used absolute -1 1"),

294 ("Key_blocks_not_flushed", "'not flushed' absolute 1 1"),

295 ]),

293 - 'key_requests' : (

296 + 'key_requests': (

297 "'' 'mysql MyISAM Key Cache Requests' 'requests/s' myisam mysql.key_requests area",

298 [

299 ("Key_read_requests", "reads incremental 1 1"),

300 ("Key_write_requests", "writes incremental -1 1"),

301 ]),

299 - 'key_disk_ops' : (

302 + 'key_disk_ops': (

303 "'' 'mysql MyISAM Key Cache Disk Operations' 'operations/s' myisam mysql.key_disk_ops area",

304 [

305 ("Key_reads", "reads incremental 1 1"),

306 ("Key_writes", "writes incremental -1 1"),

307 ]),

305 - 'files' : (

308 + 'files': (

309 "'' 'mysql Open Files' 'files' files mysql.files line",

310 [

311 ("Open_files", "files absolute 1 1"),

312 ]),

310 - 'files_rate' : (

313 + 'files_rate': (

314 "'' 'mysql Opened Files Rate' 'files/s' files mysql.files_rate line",

315 [

316 ("Opened_files", "files incremental 1 1"),

317 ]),

315 - 'binlog_stmt_cache' : (

318 + 'binlog_stmt_cache': (

319 "'' 'mysql Binlog Statement Cache' 'statements/s' binlog mysql.binlog_stmt_cache line",

320 [

321 ("Binlog_stmt_cache_disk_use", "disk incremental 1 1"),

322 ("Binlog_stmt_cache_use", "all incremental 1 1"),

323 ]),

321 - 'connection_errors' : (

324 + 'connection_errors': (

325 "'' 'mysql Connection Errors' 'connections/s' connections mysql.connection_errors line",

326 [

327 ("Connection_errors_accept", "accept incremental 1 1"),

@@ -332,14 +335,14 @@ CHARTS = {

335

336

337 class Service(BaseService):

335 - def __init__(self,configuration=None,name=None):

338 + def __init__(self, configuration=None, name=None):

339 super(self.__class__, self).__init__(configuration=configuration)

340 self.name = name

341 self.configuration = self._parse_config(configuration)

342 self.connection = None

343 self.defs = {}

344

342 - def _parse_config(self,configuration):

345 + def _parse_config(self, configuration):

346 # parse configuration to collect data from mysql server

347 if self.name is None:

348 self.name = 'local'

@@ -376,11 +379,14 @@ class Service(BaseService):

379 connect_timeout=self.configuration['update_every'])

380 except Exception as e:

381 self.error(NAME + " has problem connecting to server:", e)

379 - raise RuntimeError # stop creating module, need to catch it in supervisor

382 + raise RuntimeError

383

384 def _get_data(self):

385 if self.connection is None:

383 - self._connect()

386 + try:

387 + self._connect()

388 + except RuntimeError:

389 + return None

390 try:

391 with self.connection.cursor() as cursor:

392 cursor.execute(QUERY)

@@ -420,7 +426,7 @@ class Service(BaseService):

426 # check if server has this data point

427 for line in CHARTS[name][1]:

428 if line[0] in data:

423 - content += "DIMENSION " + line[0] + " " + line[1] + "\n"

429 + content += "DIMENSION " + line[0] + " " + line[1] + "\n"

430 if len(content) > 0:

431 print(header)

432 print(content)