@cryptotaxi247 / netdata-1 / commits / 159d817bc

mongodb_plugin: initial version added

Ilya committed Feb 20, 2017 at 14:38 UTC 159d817bc2abf7a4dc0bceb6c3153a2485c59c89
1 file changed +370
python.d/mongodb.chart.py new
+370
@@ -0,0 +1,370 @@
1 +# -*- coding: utf-8 -*-
2 +# Description: mongodb netdata python.d module
3 +# Author: l2isbad
4 +
5 +from base import SimpleService
6 +try:
7 + from pymongo import MongoClient
8 + from pymongo.errors import PyMongoError
9 + PYMONGO = True
10 +except ImportError:
11 + PYMONGO = False
12 +
13 +# default module values (can be overridden per job in `config`)
14 +# update_every = 2
15 +priority = 60000
16 +retries = 60
17 +
18 +# charts order (can be overridden if you want less charts, or different order)
19 +ORDER = ['read_operations', 'write_operations', 'active_clients', 'journaling_transactions',
20 + 'journaling_volume', 'background_flush_average', 'background_flush_last', 'background_flush_rate',
21 + 'wiredtiger_read', 'wiredtiger_write', 'cursors', 'connections', 'memory', 'page_faults',
22 + 'queued_requests', 'record_moves', 'wiredtiger_cache', 'wiredtiger_pages_evicted', 'asserts',
23 + 'dbstats_objects', 'tcmalloc_generic', 'tcmalloc_metrics', 'command_total_rate', 'command_failed_rate']
24 +
25 +CHARTS = {
26 + 'read_operations': {
27 + 'options': [None, "Received read requests", "requests/s", 'throughput metrics',
28 + 'mongodb.read_operations', 'line'],
29 + 'lines': [
30 + ['readWriteOper_query', 'query', 'incremental'],
31 + ['readWriteOper_getmore', 'getmore', 'incremental']
32 + ]},
33 + 'write_operations': {
34 + 'options': [None, "Received write requests", "requests/s", 'throughput metrics',
35 + 'mongodb.write_operations', 'line'],
36 + 'lines': [
37 + ['readWriteOper_insert', 'insert', 'incremental'],
38 + ['readWriteOper_update', 'update', 'incremental'],
39 + ['readWriteOper_delete', 'delete', 'incremental']
40 + ]},
41 + 'active_clients': {
42 + 'options': [None, "Clients with read or write operations in progress or queued", "clients",
43 + 'throughput metrics', 'mongodb.active_clients', 'line'],
44 + 'lines': [
45 + ['activeClients_readers', 'readers', 'absolute'],
46 + ['activeClients_writers', 'writers', 'absolute']
47 + ]},
48 + 'journaling_transactions': {
49 + 'options': [None, "Transactions that have been written to the journal", "commits",
50 + 'database performance', 'mongodb.journaling_transactions', 'line'],
51 + 'lines': [
52 + ['journalTrans_commits', 'commits', 'absolute']
53 + ]},
54 + 'journaling_volume': {
55 + 'options': [None, "Volume of data written to the journal", "MB", 'database performance',
56 + 'mongodb.journaling_volume', 'line'],
57 + 'lines': [
58 + ['journalTrans_journaled', 'volume', 'absolute', 1, 100]
59 + ]},
60 + 'background_flush_average': {
61 + 'options': [None, "Average time taken by flushes to execute", "ms", 'database performance',
62 + 'mongodb.background_flush_average', 'line'],
63 + 'lines': [
64 + ['background_flush_average', 'time', 'absolute', 1, 100]
65 + ]},
66 + 'background_flush_last': {
67 + 'options': [None, "Time taken by the last flush operation to execute", "ms", 'database performance',
68 + 'mongodb.background_flush_last', 'line'],
69 + 'lines': [
70 + ['background_flush_last', 'time', 'absolute', 1, 100]
71 + ]},
72 + 'background_flush_rate': {
73 + 'options': [None, "Flushes rate", "flushes", 'database performance', 'mongodb.background_flush_rate', 'line'],
74 + 'lines': [
75 + ['background_flush_rate', 'flushes', 'incremental', 1, 1]
76 + ]},
77 + 'wiredtiger_read': {
78 + 'options': [None, "Read tickets in use and remaining", "tickets", 'database performance',
79 + 'mongodb.wiredtiger_read', 'stacked'],
80 + 'lines': [
81 + ['wiredTigerRead_available', 'available', 'absolute', 1, 1],
82 + ['wiredTigerRead_out', 'inuse', 'absolute', 1, 1]
83 + ]},
84 + 'wiredtiger_write': {
85 + 'options': [None, "Write tickets in use and remaining", "tickets", 'database performance',
86 + 'mongodb.wiredtiger_write', 'stacked'],
87 + 'lines': [
88 + ['wiredTigerWrite_available', 'available', 'absolute', 1, 1],
89 + ['wiredTigerWrite_out', 'inuse', 'absolute', 1, 1]
90 + ]},
91 + 'cursors': {
92 + 'options': [None, "Currently openned cursors, cursors with timeout disabled and timed out cursors",
93 + "cursors", 'database performance', 'mongodb.cursors', 'stacked'],
94 + 'lines': [
95 + ['cursor_total', 'openned', 'absolute', 1, 1],
96 + ['cursor_noTimeout', 'notimeout', 'absolute', 1, 1],
97 + ['cursor_timedOut', 'timedout', 'incremental', 1, 1]
98 + ]},
99 + 'connections': {
100 + 'options': [None, "Currently connected clients and unused connections", "connections",
101 + 'resource utilization', 'mongodb.connections', 'stacked'],
102 + 'lines': [
103 + ['connections_available', 'unused', 'absolute', 1, 1],
104 + ['connections_current', 'connected', 'absolute', 1, 1]
105 + ]},
106 + 'memory': {
107 + 'options': [None, "Memory metrics", "MB", 'resource utilization', 'mongodb.memory', 'stacked'],
108 + 'lines': [
109 + ['memory_virtual', 'virtual', 'absolute', 1, 1],
110 + ['memory_resident', 'resident', 'absolute', 1, 1],
111 + ['memory_mapped', 'mapped', 'absolute', 1, 1]
112 + ]},
113 + 'page_faults': {
114 + 'options': [None, "Number of times MongoDB had to fetch data from disk", "request/s",
115 + 'resource utilization', 'mongodb.page_faults', 'line'],
116 + 'lines': [
117 + ['page_faults', 'page_faults', 'incremental', 1, 1]
118 + ]},
119 + 'queued_requests': {
120 + 'options': [None, "Currently queued read and wrire requests", "requests", 'resource saturation',
121 + 'mongodb.queued_requests', 'line'],
122 + 'lines': [
123 + ['currentQueue_readers', 'readers', 'absolute', 1, 1],
124 + ['currentQueue_writers', 'writers', 'absolute', 1, 1]
125 + ]},
126 + 'record_moves': {
127 + 'options': [None, "Number of times documents had to be moved on-disk", "number",
128 + 'resource saturation', 'mongodb.record_moves', 'line'],
129 + 'lines': [
130 + ['record_moves', 'moves', 'incremental', 1, 1]
131 + ]},
132 + 'asserts': {
133 + 'options': [None, "Number of message, warning, regular, corresponding to errors generated"
134 + " by users assertions raised", "number", 'errors (asserts)', 'mongodb.asserts', 'line'],
135 + 'lines': [
136 + ['errors_msg', 'msg', 'incremental', 1, 1],
137 + ['errors_warning', 'warning', 'incremental', 1, 1],
138 + ['errors_regular', 'regular', 'incremental', 1, 1],
139 + ['errors_user', 'user', 'incremental', 1, 1]
140 + ]},
141 + 'wiredtiger_cache': {
142 + 'options': [None, "Amount of space taken by cached data/dirty data in the cache and maximum cache size",
143 + "KB", 'resource utilization', 'mongodb.wiredtiger_cache', 'stacked'],
144 + 'lines': [
145 + ['wiredTiger_bytes_in_cache', 'cached', 'absolute', 1, 1024],
146 + ['wiredTiger_dirty_in_cache', 'dirty', 'absolute', 1, 1024],
147 + ['wiredTiger_maximum_in_conf', 'maximum', 'absolute', 1, 1024]
148 + ]},
149 + 'wiredtiger_pages_evicted': {
150 + 'options': [None, "Pages evicted from the cache",
151 + "pages", 'resource utilization', 'mongodb.wiredtiger_pages_evicted', 'stacked'],
152 + 'lines': [
153 + ['wiredTiger_unmodified_pages_evicted', 'unmodified', 'absolute', 1, 1],
154 + ['wiredTiger_modified_pages_evicted', 'modified', 'absolute', 1, 1]
155 + ]},
156 + 'dbstats_objects': {
157 + 'options': [None, "Number of documents in the database among all the collections", "documents",
158 + 'storage size metrics', 'mongodb.dbstats_objects', 'stacked'],
159 + 'lines': [
160 + ]},
161 + 'tcmalloc_generic': {
162 + 'options': [None, "Tcmalloc generic metrics", "MB", 'tcmalloc', 'mongodb.tcmalloc_generic', 'stacked'],
163 + 'lines': [
164 + ['current_allocated_bytes', 'allocated', 'absolute', 1, 1048576],
165 + ['heap_size', 'heap_size', 'absolute', 1, 1048576]
166 + ]},
167 + 'tcmalloc_metrics': {
168 + 'options': [None, "Tcmalloc metrics", "KB", 'tcmalloc', 'mongodb.tcmalloc_metrics', 'stacked'],
169 + 'lines': [
170 + ['central_cache_free_bytes', 'central_cache_free', 'absolute', 1, 1024],
171 + ['current_total_thread_cache_bytes', 'current_total_thread_cache', 'absolute', 1, 1024],
172 + ['pageheap_free_bytes', 'pageheap_free', 'absolute', 1, 1024],
173 + ['pageheap_unmapped_bytes', 'pageheap_unmapped', 'absolute', 1, 1024],
174 + ['thread_cache_free_bytes', 'thread_cache_free', 'absolute', 1, 1024],
175 + ['transfer_cache_free_bytes', 'transfer_cache_free', 'absolute', 1, 1024]
176 + ]},
177 + 'command_total_rate': {
178 + 'options': [None, "Commands total rate", "commands/s", 'commands', 'mongodb.command_total_rate', 'stacked'],
179 + 'lines': [
180 + ['count_total', 'count', 'incremental', 1, 1],
181 + ['createIndexes_total', 'createIndexes', 'incremental', 1, 1],
182 + ['delete_total', 'delete', 'incremental', 1, 1],
183 + ['eval_total', 'eval', 'incremental', 1, 1],
184 + ['findAndModify_total', 'findAndModify', 'incremental', 1, 1],
185 + ['insert_total', 'insert', 'incremental', 1, 1],
186 + ['update_total', 'update', 'incremental', 1, 1]
187 + ]},
188 + 'command_failed_rate': {
189 + 'options': [None, "Commands failed rate", "commands/s", 'commands', 'mongodb.command_failed_rate', 'stacked'],
190 + 'lines': [
191 + ['count_failed', 'count', 'incremental', 1, 1],
192 + ['createIndexes_failed', 'createIndexes', 'incremental', 1, 1],
193 + ['delete_dailed', 'delete', 'incremental', 1, 1],
194 + ['eval_failed', 'eval', 'incremental', 1, 1],
195 + ['findAndModify_failed', 'findAndModify', 'incremental', 1, 1],
196 + ['insert_failed', 'insert', 'incremental', 1, 1],
197 + ['update_failed', 'update', 'incremental', 1, 1]
198 + ]}
199 +}
200 +
201 +
202 +class Service(SimpleService):
203 + def __init__(self, configuration=None, name=None):
204 + SimpleService.__init__(self, configuration=configuration, name=name)
205 + self.user = self.configuration.get('user')
206 + self.password = self.configuration.get('pass')
207 + self.host = self.configuration.get('host', '127.0.0.1')
208 + self.port = self.configuration.get('port', 27017)
209 + self.timeout = self.configuration.get('timeout', 100)
210 +
211 + def check(self):
212 + if not PYMONGO:
213 + self.error('Pymongo module is needed to use mongodb.chart.py')
214 + return False
215 +
216 + self.connection, server_status, error = self._create_connection()
217 + if error:
218 + self.error(error)
219 + return False
220 +
221 + self._create_charts(server_status)
222 +
223 + return True
224 +
225 + def _create_charts(self, server_status):
226 +
227 + self.order = ORDER[:]
228 + self.definitions = CHARTS
229 + self.ss = dict()
230 +
231 + for elem in ['dur', 'backgroundFlushing', 'wiredTiger', 'tcmalloc', 'cursor', 'commands']:
232 + self.ss[elem] = in_server_status(elem, server_status)
233 +
234 + if not self.ss['dur']:
235 + self.order.remove('journaling_transactions')
236 + self.order.remove('journaling_volume')
237 +
238 + if not self.ss['backgroundFlushing']:
239 + self.order.remove('background_flush_average')
240 + self.order.remove('background_flush_last')
241 +
242 + if not self.ss['cursor']:
243 + self.order.remove('cursors')
244 +
245 + if not self.ss['wiredTiger']:
246 + self.order.remove('wiredtiger_write')
247 + self.order.remove('wiredtiger_read')
248 + self.order.remove('wiredtiger_cache')
249 +
250 + if not self.ss['tcmalloc']:
251 + self.order.remove('tcmalloc_generic')
252 + self.order.remove('tcmalloc_metrics')
253 +
254 + if not self.ss['commands']:
255 + self.order.remove('command_total_rate')
256 + self.order.remove('command_failed_rate')
257 +
258 + self.databases = self.connection.database_names()
259 +
260 + for dbase in self.databases:
261 + self.order.append('_'.join([dbase, 'dbstats']))
262 + self.definitions['_'.join([dbase, 'dbstats'])] = {
263 + 'options': [None, "%s: size of all documents, indexes, extents" % dbase, "KB",
264 + 'storage size metrics', 'mongodb.dbstats', 'line'],
265 + 'lines': [
266 + ['_'.join([dbase, 'dataSize']), 'documents', 'absolute', 1, 1024],
267 + ['_'.join([dbase, 'indexSize']), 'indexes', 'absolute', 1, 1024],
268 + ['_'.join([dbase, 'storageSize']), 'extents', 'absolute', 1, 1024]
269 + ]}
270 + self.definitions['dbstats_objects']['lines'].append(['_'.join([dbase, 'objects']), dbase, 'absolute'])
271 +
272 +
273 + def _get_raw_data(self):
274 + raw_data = dict()
275 +
276 + try:
277 + raw_data['serverStatus'] = self.connection.admin.command('serverStatus')
278 + for dbase in self.databases:
279 + raw_data[dbase] = self.connection[dbase].command('dbStats')
280 + except PyMongoError:
281 + return None
282 + return raw_data
283 +
284 + def _get_data(self):
285 + """
286 + :return: dict
287 + """
288 + raw_data = self._get_raw_data()
289 +
290 + if not raw_data:
291 + return None
292 +
293 + to_netdata = dict()
294 + server_status = raw_data['serverStatus']
295 +
296 + to_netdata.update(update_dict_key(server_status['opcounters'], 'readWriteOper'))
297 + to_netdata.update(update_dict_key(server_status['globalLock']['activeClients'], 'activeClients'))
298 + to_netdata.update(update_dict_key(server_status['connections'], 'connections'))
299 + to_netdata.update(update_dict_key(server_status['mem'], 'memory'))
300 + to_netdata.update(update_dict_key(server_status['globalLock']['currentQueue'], 'currentQueue'))
301 + to_netdata.update(update_dict_key(server_status['asserts'], 'errors'))
302 + to_netdata['page_faults'] = server_status['extra_info']['page_faults']
303 + to_netdata['record_moves'] = server_status['metrics']['record']['moves']
304 +
305 + if self.ss['dur']:
306 + to_netdata['journalTrans_commits'] = server_status['dur']['commits']
307 + to_netdata['journalTrans_journaled'] = int(server_status['dur']['journaledMB'] * 100)
308 +
309 + if self.ss['backgroundFlushing']:
310 + to_netdata['background_flush_average'] = int(server_status['backgroundFlushing']['average_ms'] * 100)
311 + to_netdata['background_flush_last'] = int(server_status['backgroundFlushing']['last_ms'] * 100)
312 + to_netdata['background_flush_rate'] = server_status['backgroundFlushing']['flushes']
313 +
314 + if self.ss['cursor']:
315 + to_netdata['cursor_timedOut'] = server_status['metrics']['cursor']['timedOut']
316 + to_netdata.update(update_dict_key(server_status['metrics']['cursor']['open'], 'cursor'))
317 +
318 + if self.ss['wiredTiger']:
319 + wired_tiger = server_status['wiredTiger']
320 + to_netdata.update(update_dict_key(server_status['wiredTiger']['concurrentTransactions']['read'],
321 + 'wiredTigerRead'))
322 + to_netdata.update(update_dict_key(server_status['wiredTiger']['concurrentTransactions']['write'],
323 + 'wiredTigerWrite'))
324 + to_netdata['wiredTiger_bytes_in_cache'] = wired_tiger['cache']['bytes currently in the cache']
325 + to_netdata['wiredTiger_maximum_in_conf'] = wired_tiger['cache']['maximum bytes configured']
326 + to_netdata['wiredTiger_dirty_in_cache'] = wired_tiger['cache']['tracked dirty bytes in the cache']
327 + to_netdata['wiredTiger_unmodified_pages_evicted'] = wired_tiger['cache']['unmodified pages evicted']
328 + to_netdata['wiredTiger_modified_pages_evicted'] = wired_tiger['cache']['modified pages evicted']
329 +
330 + if self.ss['tcmalloc']:
331 + to_netdata.update(server_status['tcmalloc']['generic'])
332 + to_netdata.update(dict([(k, v) for k, v in server_status['tcmalloc']['tcmalloc'].items()
333 + if int_or_float(v)]))
334 +
335 + if self.ss['commands']:
336 + for elem in ['count', 'createIndexes', 'delete', 'eval', 'findAndModify', 'insert', 'update']:
337 + to_netdata.update(update_dict_key(server_status['metrics']['commands'][elem], elem))
338 +
339 + for dbase in self.databases:
340 + dbase_dbstats = raw_data[dbase]
341 + dbase_dbstats = dict([(k, v) for k, v in dbase_dbstats.items() if int_or_float(v)])
342 + to_netdata.update(update_dict_key(dbase_dbstats, dbase))
343 +
344 + return to_netdata
345 +
346 + def _create_connection(self):
347 + conn_vars = {'host': self.host, 'port': self.port}
348 + if 'server_selection_timeout' in dir(MongoClient):
349 + conn_vars.update({'serverselectiontimeoutms': self.timeout})
350 + try:
351 + connection = MongoClient(**conn_vars)
352 + if self.user and self.password:
353 + connection.admin.authenticate(name=self.user, password=self.password)
354 + server_status = connection.admin.command('serverStatus')
355 + except PyMongoError as error:
356 + return None, None, str(error)
357 + else:
358 + return connection, server_status, None
359 +
360 +
361 +def update_dict_key(collection, string):
362 + return dict([('_'.join([string, k]), int(round(v))) for k, v in collection.items()])
363 +
364 +
365 +def int_or_float(value):
366 + return isinstance(value, int) or isinstance(value, float)
367 +
368 +
369 +def in_server_status(elem, server_status):
370 + return elem in server_status or elem in server_status['metrics']