Add python.d module ntp
Sven Mäder committed
Feb 13, 2018 at 20:53 UTC
1d37333eb8d0e6d9cb8fadc3ba6e3b0040144749
2 files changed
+503
conf.d/python.d/ntp.conf
new
+73
@@ -0,0 +1,73 @@
1
+# netdata python.d.plugin configuration for ntp
2
+#
3
+# This file is in YaML format. Generally the format is:
4
+#
5
+# name: value
6
+#
7
+# There are 2 sections:
8
+# - global variables
9
+# - one or more JOBS
10
+#
11
+# JOBS allow you to collect values from multiple sources.
12
+# Each source will have its own set of charts.
13
+#
14
+# JOB parameters have to be indented (using spaces only, example below).
15
+
16
+# ----------------------------------------------------------------------
17
+# Global Variables
18
+# These variables set the defaults for all JOBs, however each JOB
19
+# may define its own, overriding the defaults.
20
+
21
+# update_every sets the default data collection frequency.
22
+# If unset, the python.d.plugin default is used.
23
+# update_every: 1
24
+
25
+# priority controls the order of charts at the netdata dashboard.
26
+# Lower numbers move the charts towards the top of the page.
27
+# If unset, the default for python.d.plugin is used.
28
+# priority: 60000
29
+
30
+# retries sets the number of retries to be made in case of failures.
31
+# If unset, the default for python.d.plugin is used.
32
+# Attempts to restore the service are made once every update_every
33
+# and only if the module has collected values in the past.
34
+# retries: 60
35
+
36
+# ----------------------------------------------------------------------
37
+# JOBS (data collection sources)
38
+#
39
+# The default JOBS share the same *name*. JOBS with the same name
40
+# are mutually exclusive. Only one of them will be allowed running at
41
+# any time. This allows autodetection to try several alternatives and
42
+# pick the one that works.
43
+#
44
+# Any number of jobs is supported.
45
+#
46
+# All python.d.plugin JOBS (for all its modules) support a set of
47
+# predefined parameters. These are:
48
+#
49
+# job_name:
50
+# name: myname # the JOB's name as it will appear at the
51
+# # dashboard (by default is the job_name)
52
+# # JOBs sharing a name are mutually exclusive
53
+# update_every: 1 # the JOB's data collection frequency
54
+# priority: 60000 # the JOB's order on the dashboard
55
+# retries: 60 # the JOB's number of restoration attempts
56
+#
57
+# Additionally to the above, ntp also supports the following:
58
+#
59
+# host: 'localhost' # the host to query
60
+# port: '123' # the UDP port where `ntpd` listens
61
+# peer_filter: '127\..*' # regex to exclude peers by ip or name
62
+# # by default local peers are hidden
63
+# # use `'.*'` to disable the peers submenu.
64
+# peer_names: yes # show peer names instead of ip addresses
65
+# # by default names are shown, if the dns name
66
+# # of the peer matches the local host's fqdn,
67
+# # the matching domain part is hidden.
68
+# # Set to `no` to show ip addresses.
69
+# # Dots `.` are replaced by dashes `-`.
70
+#
71
+# ----------------------------------------------------------------------
72
+# AUTO-DETECTION JOBS
73
+# only one of them will run (they have the same name)
python.d/ntp.chart.py
new
+430
@@ -0,0 +1,430 @@
1
+# -*- coding: utf-8 -*-
2
+# Description: ntp netdata python.d module
3
+# Author: Sven Mäder (rda0)
4
+
5
+import socket
6
+import struct
7
+import re
8
+
9
+from itertools import cycle
10
+from base import SocketService
11
+
12
+# default module values
13
+update_every = 1
14
+priority = 60000
15
+retries = 60
16
+
17
+# NTP Control Message Protocol constants
18
+MODE = 6
19
+HEADER_FORMAT = '!BBHHHHH'
20
+HEADER_LEN = 12
21
+OPCODES = {
22
+ 'readstat': 1,
23
+ 'readvar': 2
24
+}
25
+
26
+# Maximal dimension precision
27
+PRECISION = 1000000
28
+
29
+# Static charts
30
+ORDER = [
31
+ 'sys_offset',
32
+ 'sys_jitter',
33
+ 'sys_frequency',
34
+ 'sys_wander',
35
+ 'sys_rootdelay',
36
+ 'sys_rootdisp',
37
+ 'sys_stratum',
38
+ 'sys_tc',
39
+ 'sys_precision'
40
+]
41
+
42
+CHARTS = {
43
+ 'sys_offset': {
44
+ 'options': [None, "Combined offset of server relative to this host", "ms", 'system', 'ntp.sys_offset', 'area'],
45
+ 'lines': [
46
+ ['offset', 'offset', 'absolute', 1, PRECISION]
47
+ ]},
48
+ 'sys_jitter': {
49
+ 'options': [None, "Combined system jitter and clock jitter", "ms", 'system', 'ntp.sys_jitter', 'line'],
50
+ 'lines': [
51
+ ['sys_jitter', 'system', 'absolute', 1, PRECISION],
52
+ ['clk_jitter', 'clock', 'absolute', 1, PRECISION]
53
+ ]},
54
+ 'sys_frequency': {
55
+ 'options': [None, "Frequency offset relative to hardware clock", "ppm", 'system', 'ntp.sys_frequency', 'area'],
56
+ 'lines': [
57
+ ['frequency', 'frequency', 'absolute', 1, PRECISION]
58
+ ]},
59
+ 'sys_wander': {
60
+ 'options': [None, "Clock frequency wander", "ppm", 'system', 'ntp.sys_wander', 'area'],
61
+ 'lines': [
62
+ ['clk_wander', 'clock', 'absolute', 1, PRECISION]
63
+ ]},
64
+ 'sys_rootdelay': {
65
+ 'options': [None, "Total roundtrip delay to the primary reference clock", "ms", 'system', 'ntp.sys_rootdelay', 'area'],
66
+ 'lines': [
67
+ ['rootdelay', 'delay', 'absolute', 1, PRECISION]
68
+ ]},
69
+ 'sys_rootdisp': {
70
+ 'options': [None, "Total root dispersion to the primary reference clock", "ms", 'system', 'ntp.sys_rootdisp', 'area'],
71
+ 'lines': [
72
+ ['rootdisp', 'dispersion', 'absolute', 1, PRECISION]
73
+ ]},
74
+ 'sys_stratum': {
75
+ 'options': [None, "Stratum (1-15)", "1", 'system', 'ntp.sys_stratum', 'line'],
76
+ 'lines': [
77
+ ['stratum', 'stratum', 'absolute', 1, PRECISION]
78
+ ]},
79
+ 'sys_tc': {
80
+ 'options': [None, "Time constant and poll exponent (3-17)", "log2 s", 'system', 'ntp.sys_tc', 'line'],
81
+ 'lines': [
82
+ ['tc', 'current', 'absolute', 1, PRECISION],
83
+ ['mintc', 'minimum', 'absolute', 1, PRECISION]
84
+ ]},
85
+ 'sys_precision': {
86
+ 'options': [None, "Precision", "log2 s", 'system', 'ntp.sys_precision', 'line'],
87
+ 'lines': [
88
+ ['precision', 'precision', 'absolute', 1, PRECISION]
89
+ ]}
90
+}
91
+
92
+# Dynamic charts templates
93
+PEER_PREFIX = 'peer'
94
+
95
+PEER_DIMENSIONS = [
96
+ ['offset', 'Filter offset', 'ms'],
97
+ ['delay', 'Filter delay', 'ms'],
98
+ ['dispersion', 'Filter dispersion', 'ms'],
99
+ ['jitter', 'Filter jitter', 'ms'],
100
+ ['xleave', 'Interleave delay', 'ms'],
101
+ ['rootdelay', 'Total roundtrip delay to the primary reference clock', 'ms'],
102
+ ['rootdisp', 'Total root dispersion to the primary reference clock', 'ms'],
103
+ ['stratum', 'Stratum (1-15)', '1'],
104
+ ['hmode', 'Host mode (1-6)', '1'],
105
+ ['pmode', 'Peer mode (1-5)', '1'],
106
+ ['hpoll', 'Host poll exponent', 'log2 s'],
107
+ ['ppoll', 'Peer poll exponent', 'log2 s'],
108
+ ['precision', 'Precision', 'log2 s']
109
+]
110
+
111
+
112
+class Peer(object):
113
+ """
114
+ Class to hold peer data required in _get_data
115
+ """
116
+ def __init__(self, peer_id, name, request):
117
+ self.id = peer_id
118
+ self.name = name
119
+ self.request = request
120
+
121
+
122
+class Service(SocketService):
123
+ def __init__(self, configuration=None, name=None):
124
+ SocketService.__init__(self, configuration=configuration, name=name)
125
+ self.port = 'ntp'
126
+ self.dgram_socket = True
127
+ self.request_systemvars = None
128
+ self.peers = None
129
+ self.peer_error = 0
130
+ self.regex_srcadr = re.compile(r'srcadr=([A-Za-z0-9.-]+)')
131
+ self.regex_data = re.compile(r'([a-z_]+)=([0-9-]+(?:\.[0-9]+)?)(?=,)')
132
+ self.order = None
133
+ self.definitions = None
134
+
135
+ try:
136
+ peer_filter = r'^((0\.0\.0\.0)|(' + str(self.configuration['peer_filter']) + r'))$'
137
+ except (KeyError, TypeError):
138
+ self.error('error parsing peer_filter')
139
+ peer_filter = r'^((0\.0\.0\.0)|(127\..*))$'
140
+ self.regex_peer_filter = re.compile(peer_filter)
141
+
142
+ try:
143
+ self.peer_names = bool((self.configuration['peer_names']))
144
+ except (KeyError, TypeError):
145
+ self.peer_names = True
146
+
147
+ def create_charts(self):
148
+ """
149
+ Creates the charts dynamically.
150
+ Checks ntp for available peers.
151
+ Adds all peers whith valid data.
152
+ """
153
+ # Create systemvars charts
154
+ self.order = ORDER
155
+ self.definitions = CHARTS
156
+
157
+ # Get peer ids
158
+ self.request = self.get_header(0, 'readstat')
159
+ peer_ids = self.get_peer_ids(self._get_raw_data_bytes())
160
+
161
+ # Get peers
162
+ peers = self.get_peers(peer_ids)
163
+
164
+ # Create peer charts
165
+ if peers:
166
+ charts = dict()
167
+
168
+ for dimension in PEER_DIMENSIONS:
169
+ chart_id = '_'.join([PEER_PREFIX, dimension[0]])
170
+ context = '.'.join(['ntp', chart_id])
171
+ title = dimension[1]
172
+ units = dimension[2]
173
+ lines = list()
174
+
175
+ for peer in peers:
176
+ unique_dimension_id = '_'.join([peer.name, dimension[0]])
177
+ line = [unique_dimension_id, peer.name, 'absolute', 1, PRECISION]
178
+ lines.append(line)
179
+ charts[chart_id] = dict()
180
+ charts[chart_id]['options'] = [None, title, units, 'peers', context, 'line']
181
+ charts[chart_id]['lines'] = lines
182
+
183
+ self.order += ['_'.join([PEER_PREFIX, d[0]]) for d in PEER_DIMENSIONS]
184
+ self.definitions.update(charts)
185
+ self.peers = cycle(peers)
186
+ else:
187
+ self.peers = None
188
+
189
+ def get_peers(self, peer_ids):
190
+ """
191
+ Figures out the possible local domain name.
192
+ Queries each peer once to get data for charts.
193
+ Replace the peer srcadr with the possible hostname.
194
+ Returns all peers whith valid data.
195
+ """
196
+ if peer_ids:
197
+ peer_ids.sort()
198
+ domain = None
199
+
200
+ if self.peer_names:
201
+ try:
202
+ hostname = socket.gethostname()
203
+ fqdn = socket.getfqdn()
204
+ if fqdn.startswith(hostname):
205
+ domain = fqdn[len(hostname):]
206
+ except socket.error:
207
+ self.error('Error getting local domain')
208
+
209
+ # Get peer data
210
+ peers = list()
211
+
212
+ for peer_id in peer_ids:
213
+ request = self.get_header(peer_id, 'readvar')
214
+ self.request = request
215
+ raw = self._get_raw_data()
216
+ if not raw:
217
+ continue
218
+
219
+ data = self.get_data_from_raw(raw)
220
+ if not data:
221
+ continue
222
+
223
+ match_srcadr = self.regex_srcadr.search(raw)
224
+ if not match_srcadr:
225
+ continue
226
+
227
+ name = match_srcadr.group(1)
228
+ match_peer_filter = self.regex_peer_filter.search(name)
229
+ if match_peer_filter:
230
+ continue
231
+
232
+ if domain:
233
+ try:
234
+ name = socket.gethostbyaddr(name)[0]
235
+ if len(name) > len(domain) and name.endswith(domain):
236
+ name = name[:-len(domain)]
237
+ except (IndexError, socket.error):
238
+ self.error('Failed to reverse lookup address')
239
+
240
+ name = name.replace('.', '-')
241
+ match_peer_filter = self.regex_peer_filter.search(name)
242
+ if match_peer_filter:
243
+ continue
244
+
245
+ peers.append(Peer(peer_id, name, request))
246
+
247
+ return peers
248
+
249
+ def check(self):
250
+ """
251
+ Checks if we can get valid systemvars.
252
+ If not, returns None to disable module.
253
+ """
254
+ self._parse_config()
255
+
256
+ self.request_systemvars = self.get_header(0, 'readvar')
257
+ self.request = self.request_systemvars
258
+ raw_systemvars = self._get_raw_data()
259
+
260
+ if not self.get_data_from_raw(raw_systemvars):
261
+ return None
262
+
263
+ self.create_charts()
264
+
265
+ return True
266
+
267
+ def _get_data(self):
268
+ """
269
+ Gets systemvars data on each update.
270
+ Gets peervars data for only one peer on each update.
271
+ Total amount of _get_raw_data invocations per update = 2
272
+ """
273
+ data = dict()
274
+
275
+ self.request = self.request_systemvars
276
+ raw_systemvars = self._get_raw_data()
277
+ data.update(self.get_data_from_raw(raw_systemvars))
278
+
279
+ if self.peers:
280
+ peer = next(self.peers)
281
+ self.request = peer.request
282
+ raw_peervars = self._get_raw_data()
283
+ data.update(self.get_data_from_raw(raw_peervars, peer))
284
+
285
+ if not data:
286
+ self.error("No data received")
287
+ return None
288
+
289
+ return data
290
+
291
+ def get_data_from_raw(self, raw, peer=None):
292
+ """
293
+ Extracts key=value pairs with float/integer from ntp response packet data.
294
+ """
295
+ data = dict()
296
+ try:
297
+ data_list = self.regex_data.findall(raw)
298
+
299
+ for data_point in data_list:
300
+ key, value = data_point
301
+ if peer:
302
+ dimension = '_'.join([peer.name, key])
303
+ else:
304
+ dimension = key
305
+ data[dimension] = int(float(value) * PRECISION)
306
+ except (ValueError, AttributeError, TypeError):
307
+ self.error("Invalid data received")
308
+ return None
309
+
310
+ # If peer returns no valid data, probably due to ntpd restart,
311
+ # then wait 10 seconds and re-initialize the peers and charts
312
+ if not data and peer:
313
+ self.error('Peer error: No data received')
314
+ self.peer_error += 1
315
+
316
+ if (self.peer_error * self.update_every) > 10:
317
+ self.error('Peer error count exceeded, re-creating charts.')
318
+ self.create_charts()
319
+ self.peer_error = 0
320
+ self.create()
321
+
322
+ return data
323
+
324
+ def _receive_bytes(self):
325
+ """
326
+ Receive data from socket
327
+ :return: bytestr
328
+ """
329
+ data = b""
330
+ while True:
331
+ self.debug('receiving response')
332
+ try:
333
+ buf = self._sock.recv(4096)
334
+ except Exception as error:
335
+ self._socket_error('failed to receive response: {0}'.format(error))
336
+ self._disconnect()
337
+ break
338
+
339
+ if buf is None or len(buf) == 0: # handle server disconnect
340
+ if data == b"":
341
+ self._socket_error('unexpectedly disconnected')
342
+ else:
343
+ self.debug('server closed the connection')
344
+ self._disconnect()
345
+ break
346
+
347
+ self.debug('received data')
348
+ data += buf
349
+ if self._check_raw_data(data):
350
+ break
351
+
352
+ self.debug('final response: {0}'.format(data))
353
+ return data
354
+
355
+ def _get_raw_data_bytes(self):
356
+ """
357
+ Get raw data with low-level "socket" module.
358
+ :return: bytestr
359
+ """
360
+ if self._sock is None:
361
+ self._connect()
362
+ if self._sock is None:
363
+ return None
364
+
365
+ # Send request if it is needed
366
+ if not self._send():
367
+ return None
368
+
369
+ data = self._receive_bytes()
370
+
371
+ if not self._keep_alive:
372
+ self._disconnect()
373
+
374
+ return data
375
+
376
+ def get_header(self, associd=0, operation='readvar'):
377
+ """
378
+ Constructs the NTP Control Message header:
379
+ 0 1 2 3
380
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
381
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
382
+ |LI | VN |Mode |R|E|M| OpCode | Sequence Number |
383
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
384
+ | Status | Association ID |
385
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
386
+ | Offset | Count |
387
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
388
+ """
389
+ try:
390
+ opcode = OPCODES[operation]
391
+ except KeyError:
392
+ self.error('Invalid operation: {0}'.format(operation))
393
+ return None
394
+ version = 2
395
+ sequence = 1
396
+ status = 0
397
+ offset = 0
398
+ count = 0
399
+ try:
400
+ header = struct.pack(HEADER_FORMAT, (version << 3 | MODE), opcode,
401
+ sequence, status, associd, offset, count)
402
+ return header
403
+ except struct.error:
404
+ self.error('error packing header: {0}'.format(struct.error))
405
+ return None
406
+
407
+ def get_peer_ids(self, res):
408
+ """
409
+ Unpack the NTP Control Message header
410
+ Get data length from header
411
+ Get list of association ids returned in the readstat response
412
+ """
413
+ try:
414
+ count = struct.unpack(HEADER_FORMAT, res[:HEADER_LEN])[6]
415
+ except struct.error:
416
+ self.error('error unpacking header: {0}'.format(struct.error))
417
+ return list()
418
+ if not count:
419
+ self.debug('empty data field in NTP control packet')
420
+ return list()
421
+
422
+ data_end = HEADER_LEN + count
423
+ data = res[HEADER_LEN:data_end]
424
+ data_format = ''.join(['!', 'H' * int(count / 2)])
425
+
426
+ try:
427
+ return list(struct.unpack(data_format, data))[::2]
428
+ except struct.error:
429
+ self.error('error unpacking data: {0}'.format(struct.error))
430
+ return list()