/collectors/python.d/varnish: collect smf metrics (#7926)
* /collectors/python.d/varnish: collect per smf g_bytes g_space * /collectors/python.d/varnish: update README.md
Ilya Mashchenko committed
Jan 31, 2020 at 16:19 UTC
43bc627b1dc2e667ad2e6e8a0c2ff1e0b512dcdf
2 files changed
+122
-97
collectors/python.d.plugin/varnish/README.md
+26
-65
@@ -1,80 +1,41 @@
1
# varnish
2
3
-Module uses the `varnishstat` command to provide varnish cache statistics.
3
+This module will monitor [`Varnish Cache`](https://varnish-cache.org/) performance metrics.
4
5
-It produces:
5
+It uses the `varnishstat` command to provide Varnish Cache statistics.
6
7
-1. **Connections Statistics** in connections/s
7
+## Requirements
8
9
- - accepted
10
- - dropped
9
+- `netdata` user must be a member of the `varnish` group
10
12
-2. **Client Requests** in requests/s
11
+## Charts
12
14
- - received
13
+This module produces the following charts:
14
16
-3. **All History Hit Rate Ratio** in percent
15
+- Connections Statistics in `connections/s`
16
+- Client Requests in `requests/s`
17
+- All History Hit Rate Ratio in `percent`
18
+- Current Poll Hit Rate Ratio in `percent`
19
+- Expired Objects in `expired/s`
20
+- Least Recently Used Nuked Objects in `nuked/s`
21
+- Number Of Threads In All Pools in `pools`
22
+- Threads Statistics in `threads/s`
23
+- Current Queue Length in `requests`
24
+- Backend Connections Statistics in `connections/s`
25
+- Requests To The Backend in `requests/s`
26
+- ESI Statistics in `problems/s`
27
+- Memory Usage in `MiB`
28
+- Uptime in `seconds`
29
18
- - hit
19
- - miss
20
- - hitpass
30
+For every backend (VBE):
31
22
-4. **Current Poll Hit Rate Ratio** in percent
32
+- Backend Response Statistics in `kilobits/s`
33
24
- - hit
25
- - miss
26
- - hitpass
34
+For every disk (SMF):
35
28
-5. **Expired Objects** in expired/s
36
+- Disk Usage in `KiB`
37
30
- - objects
31
-
32
-6. **Least Recently Used Nuked Objects** in nuked/s
33
-
34
- - objects
35
-
36
-7. **Number Of Threads In All Pools** in threads
37
-
38
- - threads
39
-
40
-8. **Threads Statistics** in threads/s
41
-
42
- - created
43
- - failed
44
- - limited
45
-
46
-9. **Current Queue Length** in requests
47
-
48
- - in queue
49
-
50
-10. **Backend Connections Statistics** in connections/s
51
-
52
- - successful
53
- - unhealthy
54
- - reused
55
- - closed
56
- - resycled
57
- - failed
58
-
59
-11. **Requests To The Backend** in requests/s
60
-
61
- - received
62
-
63
-12. **ESI Statistics** in problems/s
64
-
65
- - errors
66
- - warnings
67
-
68
-13. **Memory Usage** in MB
69
-
70
- - free
71
- - allocated
72
-
73
-14. **Uptime** in seconds
74
-
75
- - uptime
76
-
77
-## configuration
38
+## Configuration
39
40
Only one parameter is supported:
41
@@ -82,7 +43,7 @@ Only one parameter is supported:
43
instance_name: 'name'
44
```
45
85
-The name of the varnishd instance to get logs from. If not specified, the host name is used.
46
+The name of the `varnishd` instance to get logs from. If not specified, the host name is used.
47
48
---
49
collectors/python.d.plugin/varnish/varnish.chart.py
+96
-32
@@ -135,6 +135,44 @@ CHARTS = {
135
}
136
}
137
138
+
139
+def backend_charts_template(name):
140
+ order = [
141
+ '{0}_response_statistics'.format(name),
142
+ ]
143
+
144
+ charts = {
145
+ order[0]: {
146
+ 'options': [None, 'Backend "{0}"'.format(name), 'kilobits/s', 'backend response statistics',
147
+ 'varnish.backend', 'area'],
148
+ 'lines': [
149
+ ['{0}_beresp_hdrbytes'.format(name), 'header', 'incremental', 8, 1000],
150
+ ['{0}_beresp_bodybytes'.format(name), 'body', 'incremental', -8, 1000]
151
+ ]
152
+ },
153
+ }
154
+
155
+ return order, charts
156
+
157
+
158
+def disk_charts_template(name):
159
+ order = [
160
+ 'disk_{0}_usage'.format(name),
161
+ ]
162
+
163
+ charts = {
164
+ order[0]: {
165
+ 'options': [None, 'Disk "{0}" Usage'.format(name), 'KiB', 'disk usage', 'varnish.disk_usage', 'stacked'],
166
+ 'lines': [
167
+ ['{0}.g_space'.format(name), 'free', 'absolute', 1, 1 << 10],
168
+ ['{0}.g_bytes'.format(name), 'allocated', 'absolute', 1, 1 << 10]
169
+ ]
170
+ },
171
+ }
172
+
173
+ return order, charts
174
+
175
+
176
VARNISHSTAT = 'varnishstat'
177
178
re_version = re.compile(r'varnish-(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)')
@@ -188,6 +226,8 @@ class Service(ExecutableService):
226
self.instance_name = configuration.get('instance_name')
227
self.parser = Parser()
228
self.command = None
229
+ self.collected_vbe = set()
230
+ self.collected_smf = set()
231
232
def create_command(self):
233
varnishstat = find_binary(VARNISHSTAT)
@@ -206,10 +246,7 @@ class Service(ExecutableService):
246
ver = parse_varnish_version(reply)
247
if not ver:
248
self.error("failed to parse reply from '{0}', used regex :'{1}', reply : {2}".format(
209
- ' '.join(command),
210
- re_version.pattern,
211
- reply,
212
- ))
249
+ ' '.join(command), re_version.pattern, reply))
250
return False
251
252
if self.instance_name:
@@ -241,9 +278,6 @@ class Service(ExecutableService):
278
self.error('cant parse the output...')
279
return False
280
244
- if self.parser.re_backend:
245
- backends = [b[0] for b in self.parser.backend_stats(reply)[::2]]
246
- self.create_backends_charts(backends)
281
return True
282
283
def get_data(self):
@@ -260,11 +294,11 @@ class Service(ExecutableService):
294
if not server_stats:
295
return None
296
263
- if self.parser.re_backend:
264
- backend_stats = self.parser.backend_stats(raw)
265
- data.update(dict(('_'.join([name, param]), value) for name, param, value in backend_stats))
297
+ stats = dict((param, value) for _, param, value in server_stats)
298
+ data.update(stats)
299
267
- data.update(dict((param, value) for _, param, value in server_stats))
300
+ self.get_vbe_backends(data, raw)
301
+ self.get_smf_disks(server_stats)
302
303
# varnish 5 uses default.g_bytes and default.g_space
304
data['memory_allocated'] = data.get('s0.g_bytes') or data.get('default.g_bytes')
@@ -272,27 +306,57 @@ class Service(ExecutableService):
306
307
return data
308
275
- def create_backends_charts(self, backends):
276
- for backend in backends:
277
- chart_name = ''.join([backend, '_response_statistics'])
278
- title = 'Backend "{0}"'.format(backend.capitalize())
279
- hdr_bytes = ''.join([backend, '_beresp_hdrbytes'])
280
- body_bytes = ''.join([backend, '_beresp_bodybytes'])
281
-
282
- chart = {
283
- chart_name:
284
- {
285
- 'options': [None, title, 'kilobits/s', 'backend response statistics',
286
- 'varnish.backend', 'area'],
287
- 'lines': [
288
- [hdr_bytes, 'header', 'incremental', 8, 1000],
289
- [body_bytes, 'body', 'incremental', -8, 1000]
290
- ]
291
- }
292
- }
293
-
294
- self.order.insert(0, chart_name)
295
- self.definitions.update(chart)
309
+ def get_vbe_backends(self, data, raw):
310
+ if not self.parser.re_backend:
311
+ return
312
+ stats = self.parser.backend_stats(raw)
313
+ if not stats:
314
+ return
315
+
316
+ for (name, param, value) in stats:
317
+ data['_'.join([name, param])] = value
318
+ if name in self.collected_vbe:
319
+ continue
320
+ self.collected_vbe.add(name)
321
+ self.add_backend_charts(name)
322
+
323
+ def get_smf_disks(self, server_stats):
324
+ # [('SMF.', 'ssdStorage.c_req', '47686'),
325
+ # ('SMF.', 'ssdStorage.c_fail', '0'),
326
+ # ('SMF.', 'ssdStorage.c_bytes', '668102656'),
327
+ # ('SMF.', 'ssdStorage.c_freed', '140980224'),
328
+ # ('SMF.', 'ssdStorage.g_alloc', '39753'),
329
+ # ('SMF.', 'ssdStorage.g_bytes', '527122432'),
330
+ # ('SMF.', 'ssdStorage.g_space', '53159968768'),
331
+ # ('SMF.', 'ssdStorage.g_smf', '40130'),
332
+ # ('SMF.', 'ssdStorage.g_smf_frag', '311'),
333
+ # ('SMF.', 'ssdStorage.g_smf_large', '66')]
334
+ disks = [name for typ, name, _ in server_stats if typ.startswith('SMF') and name.endswith('g_space')]
335
+ if not disks:
336
+ return
337
+ for disk in disks:
338
+ disk = disk.split('.')[0] # ssdStorage
339
+ if disk in self.collected_smf:
340
+ continue
341
+ self.collected_smf.add(disk)
342
+ self.add_disk_charts(disk)
343
+
344
+ def add_backend_charts(self, backend_name):
345
+ self.add_charts(backend_name, backend_charts_template)
346
+
347
+ def add_disk_charts(self, disk_name):
348
+ self.add_charts(disk_name, disk_charts_template)
349
+
350
+ def add_charts(self, name, charts_template):
351
+ order, charts = charts_template(name)
352
+
353
+ for chart_name in order:
354
+ params = [chart_name] + charts[chart_name]['options']
355
+ dimensions = charts[chart_name]['lines']
356
+
357
+ new_chart = self.charts.add_chart(params)
358
+ for dimension in dimensions:
359
+ new_chart.add_dimension(dimension)
360
361
362
def parse_varnish_version(lines):