Add per-thread stats reporting for the unbound plugin.
Each of the three regular metric sets queries, request list, and recursion timings) also have per-thread versions reported by Unbound if threading is enabled. This commit adds an option (`per_thread`) to enable reporting this data in the unbound module. It produces one chart per-thread per-set of metrics, all of which are grouped below the main charts by their metric set, and displayed at reduced height (because it's _a lot_ of charts on most systems, since having at least one thread per core is pretty normal for Unbound configurations). Detection is done only at startup, during the check to see if the plugin will run or not, which greatly simplifies the data collection code. It also fixes a few typos in the Unbound documentation and config file, and removes some bogus family overrides in the plugin itself.
Austin S. Hemmelgarn committed
Jun 6, 2018 at 10:46 UTC
591e64af16763cb518e03d6b7cea96fe6221a788
4 files changed
+128
-10
conf.d/python.d/unbound.conf
+3
-2
@@ -69,10 +69,11 @@
69
# # of a TCP connection
70
# tls_key_file: /path/to/key # The key file to use for authentication
71
# tls_cert_file: /path/to/key # The certificate to use for authentication
72
-# extended: false @ Whether to collect extended stats or not
72
+# extended: false # Whether to collect extended stats or not
73
+# per_thread: false # Whether to show charts for per-thread stats
74
#
75
# In addition to the above, you can set the following to try and
75
-# auto-detect settings based on the unbound configuration:
76
+# auto-detect most settings based on the unbound configuration:
77
#
78
# ubconf: /etc/unbound/unbound.conf
79
#
python.d/README.md
+9
-3
@@ -2349,20 +2349,26 @@ that either the certificate and key are readable by Netdata (if you're
2349
using the regular control interface), or that the socket is accessible
2350
to Netdata (if you're using a UNIX socket for the contorl interface).
2351
2352
-By default, for the local system, every thing can be auto-detected
2353
-assumign Unbound is configured correctly and has been told to listen
2352
+By default, for the local system, everything can be auto-detected
2353
+assuming Unbound is configured correctly and has been told to listen
2354
on the loopback interface or a UNIX socket. This is done by looking
2355
up info in the Unbound config file specified by the `ubconf` key.
2356
2357
To enable extended stats for a given job, add `extended: yes` to the
2358
definition.
2359
2360
-A basic local configuration with extended statistics looks like this:
2360
+You can also enable per-thread charts for a given job by adding
2361
+`per_thread: yes` to the definition. Note that the numbe rof threads
2362
+is only checked on startup.
2363
+
2364
+A basic local configuration with extended statistics and per-thread
2365
+charts looks like this:
2366
2367
```yaml
2368
local:
2369
ubconf: /etc/unbound/unbound.conf
2370
extended: yes
2371
+ per_thread: yes
2372
```
2373
2374
While it's a bit more complicated to set up correctly, it is recommended
python.d/unbound.chart.py
+104
-5
@@ -3,6 +3,7 @@
3
# Author: Austin S. Hemmelgarn (Ferroin)
4
5
import os
6
+import sys
7
8
from copy import deepcopy
9
@@ -62,6 +63,40 @@ EXTENDED_CHARTS = {
63
}
64
}
65
66
+# This is used as a templates for the per-thread charts.
67
+PER_THREAD_CHARTS = {
68
+ '_queries': {
69
+ 'options': [None, '{0} Queries Processed', 'queries', 'Queries Processed', 'unbound.threads.queries', 'line'],
70
+ 'lines': [
71
+ ['{0}_ratelimit', 'Ratelimited', 'absolute', 1, 1],
72
+ ['{0}_cachemiss', 'Cache Miss', 'absolute', 1, 1],
73
+ ['{0}_cachehit', 'Cache Hit', 'absolute', 1, 1],
74
+ ['{0}_expired', 'Expired', 'absolute', 1, 1],
75
+ ['{0}_prefetch', 'Prefetched', 'absolute', 1, 1],
76
+ ['{0}_recursive', 'Recursive', 'absolute', 1, 1]
77
+ ]
78
+ },
79
+ '_reqlist': {
80
+ 'options': [None, '{0} Request List', 'items', 'Request List', 'unbound.threads.reqlist', 'line'],
81
+ 'lines': [
82
+ ['{0}_reqlist_avg', 'Average Size', 'absolute', 1, 1],
83
+ ['{0}_reqlist_max', 'Maximum Size', 'absolute', 1, 1],
84
+ ['{0}_reqlist_overwritten', 'Overwritten Requests', 'absolute', 1, 1],
85
+ ['{0}_reqlist_exceeded', 'Overruns', 'absolute', 1, 1],
86
+ ['{0}_reqlist_current', 'Current Size', 'absolute', 1, 1],
87
+ ['{0}_reqlist_user', 'User Requests', 'absolute', 1, 1]
88
+ ]
89
+ },
90
+ '_recursion': {
91
+ 'options': [None, '{0} Recursion Timings', 'seconds', 'Recursive Timings', 'unbound.threads.recursion', 'line'],
92
+ 'lines': [
93
+ ['{0}_recursive_avg', 'Average', 'absolute', 1, PRECISION],
94
+ ['{0}_recursive_med', 'Median', 'absolute', 1, PRECISION]
95
+ ]
96
+ }
97
+}
98
+
99
+
100
# This maps the Unbound stat names to our names and precision requiremnets.
101
STAT_MAP = {
102
'total.num.queries_ip_ratelimited': ('ratelimit', 1),
@@ -86,6 +121,24 @@ STAT_MAP = {
121
'dnscrypt_nonce.cache.count': ('cache_dnscn', 1)
122
}
123
124
+# Same as above, but for per-thread stats.
125
+PER_THREAD_STAT_MAP = {
126
+ '{0}.num.queries_ip_ratelimited': ('{0}_ratelimit', 1),
127
+ '{0}.num.cachehits': ('{0}_cachehit', 1),
128
+ '{0}.num.cachemiss': ('{0}_cachemiss', 1),
129
+ '{0}.num.zero_ttl': ('{0}_expired', 1),
130
+ '{0}.num.prefetch': ('{0}_prefetch', 1),
131
+ '{0}.num.recursivereplies': ('{0}_recursive', 1),
132
+ '{0}.requestlist.avg': ('{0}_reqlist_avg', 1),
133
+ '{0}.requestlist.max': ('{0}_reqlist_max', 1),
134
+ '{0}.requestlist.overwritten': ('{0}_reqlist_overwritten', 1),
135
+ '{0}.requestlist.exceeded': ('{0}_reqlist_exceeded', 1),
136
+ '{0}.requestlist.current.all': ('{0}_reqlist_current', 1),
137
+ '{0}.requestlist.current.user': ('{0}_reqlist_user', 1),
138
+ '{0}.recursion.time.avg': ('{0}_recursive_avg', PRECISION),
139
+ '{0}.recursion.time.median': ('{0}_recursive_med', PRECISION)
140
+}
141
+
142
143
class Service(SocketService):
144
def __init__(self, configuration=None, name=None):
@@ -97,25 +150,24 @@ class Service(SocketService):
150
SocketService.__init__(self, configuration, name)
151
self.ext = self.configuration.get('extended', None)
152
self.ubconf = self.configuration.get('ubconf', None)
153
+ self.perthread = self.configuration.get('per_thread', False)
154
+ self.threads = None
155
self.order = deepcopy(ORDER)
156
self.definitions = deepcopy(CHARTS)
157
self.request = 'UBCT1 stats\n'
158
self._parse_config()
159
self._auto_config()
160
self.debug('Extended stats: {0}'.format(self.ext))
161
+ self.debug('Per-thread stats: {0}'.format(self.perthread))
162
if self.ext:
163
self.order = self.order + EXTENDED_ORDER
164
self.definitions.update(EXTENDED_CHARTS)
165
if self.unix_socket:
166
self.debug('Using unix socket: {0}'.format(self.unix_socket))
111
- for key in self.definitions:
112
- self.definitions[key]['options'][4] = 'Local'
167
else:
168
self.debug('Connecting to: {0}:{1}'.format(self.host, self.port))
169
self.debug('Using key: {0}'.format(self.key))
170
self.debug('Using certificate: {0}'.format(self.cert))
117
- for key in self.definitions:
118
- self.definitions[key]['options'][4] = self.host
171
172
def _auto_config(self):
173
if self.ubconf and os.access(self.ubconf, os.R_OK):
@@ -140,11 +192,58 @@ class Service(SocketService):
192
if not self.port:
193
self.port = 8953
194
195
+ def _generate_perthread_charts(self):
196
+ # TODO: THis could probably be more efficient, but it's only
197
+ # run once, so it probably doesn't really matter.
198
+ for thread in range(0, self.threads):
199
+ shortname = 'thread{0}'.format(thread)
200
+ longname = 'Thread {0}'.format(thread)
201
+ charts = dict()
202
+ order = [[], [], []]
203
+ statmap = dict()
204
+ count = 0
205
+ for item in PER_THREAD_CHARTS:
206
+ chartname = '{0}{1}'.format(shortname, item)
207
+ order[count].append(chartname)
208
+ count += 1
209
+ charts[chartname] = deepcopy(PER_THREAD_CHARTS[item])
210
+ charts[chartname]['options'][1] = charts[chartname]['options'][1].format(longname)
211
+ for line in range(0, len(charts[chartname]['lines'])):
212
+ charts[chartname]['lines'][line][0] = charts[chartname]['lines'][line][0].format(shortname)
213
+ order = order[0] + order[1] + order[2]
214
+ for chart in order:
215
+ params = [chart] + charts[chart]['options']
216
+ dimensions = charts[chart]['lines']
217
+ new_chart = self.charts.add_chart(params)
218
+ for dimension in dimensions:
219
+ new_chart.add_dimension(dimension)
220
+ for key, value in PER_THREAD_STAT_MAP.items():
221
+ STAT_MAP[key.format(shortname)] = (value[0].format(shortname), value[1])
222
+
223
def check(self):
144
- # We need to check that auth works, otherwise there's no point.
224
+ # Check if authentication is working.
225
self._connect()
226
result = bool(self._sock)
227
self._disconnect()
228
+ # If auth works, and we need per-thread charts, query the server
229
+ # to see how many threads it's using. This somewhat abuses the
230
+ # SocketService API to get the data we need.
231
+ if result and self.perthread:
232
+ tmp = self.request
233
+ if sys.version_info[0] < 3:
234
+ self.request = 'UBCT1 status\n'
235
+ else:
236
+ self.request = b'UBCT1 status\n'
237
+ raw = self._get_raw_data()
238
+ for line in raw.splitlines():
239
+ if line.startswith('threads'):
240
+ self.threads = int(line.split()[1])
241
+ self._generate_perthread_charts()
242
+ break
243
+ if self.threads is None:
244
+ self.info('Unable to auto-detect thread counts, disabling per-thread stats.')
245
+ self.perthread = False
246
+ self.request = tmp
247
return result
248
249
@staticmethod
web/dashboard_info.js
+12
@@ -2107,5 +2107,17 @@ netdataDashboard.context = {
2107
info: 'The number of items in each of the various caches.'
2108
}
2109
2110
+ 'unbound.threads.queries': {
2111
+ height: 0.2
2112
+ }
2113
+
2114
+ 'unbound.threads.reqlist': {
2115
+ height: 0.2
2116
+ }
2117
+
2118
+ 'unbound.threads.recursion': {
2119
+ height: 0.2
2120
+ }
2121
+
2122
// ------------------------------------------------------------------------
2123
};