Add a module for ZFS pool state (#11071)
Co-authored-by: Ilya Mashchenko <ilya@netdata.cloud>
Vladimir Kobal committed
May 5, 2021 at 20:33 UTC
3f93df7aa3995d676a86f96cef061eb57ba14f50
7 files changed
+269
collectors/all.h
+2
@@ -180,6 +180,8 @@
180
#define NETDATA_CHART_PRIO_ZFS_HASH_ELEMENTS 2800
181
#define NETDATA_CHART_PRIO_ZFS_HASH_CHAINS 2810
182
183
+#define NETDATA_CHART_PRIO_ZFS_POOL_STATE 2820
184
+
185
186
// SOFTIRQs
187
collectors/proc.plugin/README.md
+2
@@ -26,6 +26,8 @@ custom_edit_url: https://github.com/netdata/netdata/edit/master/collectors/proc.
26
- `/proc/loadavg` (system load and total processes running)
27
- `/proc/pressure/{cpu,memory,io}` (pressure stall information)
28
- `/proc/sys/kernel/random/entropy_avail` (random numbers pool availability - used in cryptography)
29
+- `/proc/spl/kstat/zfs/arcstats` (status of ZFS adaptive replacement cache)
30
+- `/proc/spl/kstat/zfs/pool/state` (state of ZFS pools)
31
- `/sys/class/power_supply` (power supply properties)
32
- `/sys/class/infiniband` (infiniband interconnect)
33
- `ipc` (IPC semaphores and message queues)
collectors/proc.plugin/plugin_proc.c
+3
@@ -66,6 +66,9 @@ static struct proc_module {
66
67
// ZFS metrics
68
{.name = "/proc/spl/kstat/zfs/arcstats", .dim = "zfs_arcstats", .func = do_proc_spl_kstat_zfs_arcstats},
69
+ {.name = "/proc/spl/kstat/zfs/pool/state",
70
+ .dim = "zfs_pool_state",
71
+ .func = do_proc_spl_kstat_zfs_pool_state},
72
73
// BTRFS metrics
74
{.name = "/sys/fs/btrfs", .dim = "btrfs", .func = do_sys_fs_btrfs},
collectors/proc.plugin/plugin_proc.h
+1
@@ -51,6 +51,7 @@ extern int do_proc_uptime(int update_every, usec_t dt);
51
extern int do_proc_sys_devices_system_edac_mc(int update_every, usec_t dt);
52
extern int do_proc_sys_devices_system_node(int update_every, usec_t dt);
53
extern int do_proc_spl_kstat_zfs_arcstats(int update_every, usec_t dt);
54
+extern int do_proc_spl_kstat_zfs_pool_state(int update_every, usec_t dt);
55
extern int do_sys_fs_btrfs(int update_every, usec_t dt);
56
extern int do_proc_net_sockstat(int update_every, usec_t dt);
57
extern int do_proc_net_sockstat6(int update_every, usec_t dt);
collectors/proc.plugin/proc_spl_kstat_zfs.c
+220
@@ -4,6 +4,10 @@
4
#include "zfs_common.h"
5
6
#define ZFS_PROC_ARCSTATS "/proc/spl/kstat/zfs/arcstats"
7
+#define ZFS_PROC_POOLS "/proc/spl/kstat/zfs"
8
+
9
+#define STATE_SIZE 8
10
+#define MAX_CHART_ID 256
11
12
extern struct arcstats arcstats;
13
@@ -194,3 +198,219 @@ int do_proc_spl_kstat_zfs_arcstats(int update_every, usec_t dt) {
198
199
return 0;
200
}
201
+
202
+struct zfs_pool {
203
+ RRDSET *st;
204
+
205
+ RRDDIM *rd_online;
206
+ RRDDIM *rd_degraded;
207
+ RRDDIM *rd_faulted;
208
+ RRDDIM *rd_offline;
209
+ RRDDIM *rd_removed;
210
+ RRDDIM *rd_unavail;
211
+
212
+ int updated;
213
+ int disabled;
214
+
215
+ int online;
216
+ int degraded;
217
+ int faulted;
218
+ int offline;
219
+ int removed;
220
+ int unavail;
221
+};
222
+
223
+struct deleted_zfs_pool {
224
+ char *name;
225
+ struct deleted_zfs_pool *next;
226
+} *deleted_zfs_pools = NULL;
227
+
228
+DICTIONARY *zfs_pools = NULL;
229
+
230
+void disable_zfs_pool_state(struct zfs_pool *pool)
231
+{
232
+ if (pool->st)
233
+ rrdset_is_obsolete(pool->st);
234
+
235
+ pool->st = NULL;
236
+
237
+ pool->rd_online = NULL;
238
+ pool->rd_degraded = NULL;
239
+ pool->rd_faulted = NULL;
240
+ pool->rd_offline = NULL;
241
+ pool->rd_removed = NULL;
242
+ pool->rd_unavail = NULL;
243
+
244
+ pool->disabled = 1;
245
+}
246
+
247
+int update_zfs_pool_state_chart(char *name, void *pool_p, void *update_every_p)
248
+{
249
+ struct zfs_pool *pool = (struct zfs_pool *)pool_p;
250
+ int update_every = *(int *)update_every_p;
251
+
252
+ if (pool->updated) {
253
+ pool->updated = 0;
254
+
255
+ if (!pool->disabled) {
256
+ if (unlikely(!pool->st)) {
257
+ char chart_id[MAX_CHART_ID + 1];
258
+ snprintf(chart_id, MAX_CHART_ID, "state_%s", name);
259
+
260
+ pool->st = rrdset_create_localhost(
261
+ "zfspool",
262
+ chart_id,
263
+ NULL,
264
+ name,
265
+ "zfspool.state",
266
+ "ZFS pool state",
267
+ "boolean",
268
+ PLUGIN_PROC_NAME,
269
+ ZFS_PROC_POOLS,
270
+ NETDATA_CHART_PRIO_ZFS_POOL_STATE,
271
+ update_every,
272
+ RRDSET_TYPE_LINE);
273
+
274
+ pool->rd_online = rrddim_add(pool->st, "online", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
275
+ pool->rd_degraded = rrddim_add(pool->st, "degraded", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
276
+ pool->rd_faulted = rrddim_add(pool->st, "faulted", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
277
+ pool->rd_offline = rrddim_add(pool->st, "offline", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
278
+ pool->rd_removed = rrddim_add(pool->st, "removed", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
279
+ pool->rd_unavail = rrddim_add(pool->st, "unavail", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
280
+ } else
281
+ rrdset_next(pool->st);
282
+
283
+ rrddim_set_by_pointer(pool->st, pool->rd_online, pool->online);
284
+ rrddim_set_by_pointer(pool->st, pool->rd_degraded, pool->degraded);
285
+ rrddim_set_by_pointer(pool->st, pool->rd_faulted, pool->faulted);
286
+ rrddim_set_by_pointer(pool->st, pool->rd_offline, pool->offline);
287
+ rrddim_set_by_pointer(pool->st, pool->rd_removed, pool->removed);
288
+ rrddim_set_by_pointer(pool->st, pool->rd_unavail, pool->unavail);
289
+ rrdset_done(pool->st);
290
+ }
291
+ } else {
292
+ disable_zfs_pool_state(pool);
293
+ struct deleted_zfs_pool *new = calloc(1, sizeof(struct deleted_zfs_pool));
294
+ new->name = strdupz(name);
295
+ new->next = deleted_zfs_pools;
296
+ deleted_zfs_pools = new;
297
+ }
298
+
299
+ return 0;
300
+}
301
+
302
+int do_proc_spl_kstat_zfs_pool_state(int update_every, usec_t dt)
303
+{
304
+ (void)dt;
305
+
306
+ static int do_zfs_pool_state = -1;
307
+ static char *dirname = NULL;
308
+
309
+ int pool_found = 0, state_file_found = 0;
310
+
311
+ if (unlikely(do_zfs_pool_state == -1)) {
312
+ char filename[FILENAME_MAX + 1];
313
+ snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/spl/kstat/zfs");
314
+ dirname = config_get("plugin:proc:" ZFS_PROC_POOLS, "directory to monitor", filename);
315
+
316
+ zfs_pools = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
317
+
318
+ do_zfs_pool_state = 1;
319
+ }
320
+
321
+ if (likely(do_zfs_pool_state)) {
322
+ DIR *dir = opendir(dirname);
323
+ if (unlikely(!dir)) {
324
+ error("Cannot read directory '%s'", dirname);
325
+ return 1;
326
+ }
327
+
328
+ struct dirent *de = NULL;
329
+ while (likely(de = readdir(dir))) {
330
+ if (likely(
331
+ de->d_type == DT_DIR && ((de->d_name[0] == '.' && de->d_name[1] == '\0') ||
332
+ (de->d_name[0] == '.' && de->d_name[1] == '.' && de->d_name[2] == '\0'))))
333
+ continue;
334
+
335
+ if (unlikely(de->d_type == DT_LNK || de->d_type == DT_DIR)) {
336
+ pool_found = 1;
337
+
338
+ struct zfs_pool *pool = dictionary_get(zfs_pools, de->d_name);
339
+
340
+ if (unlikely(!pool)) {
341
+ struct zfs_pool new_zfs_pool = {};
342
+ pool = dictionary_set(zfs_pools, de->d_name, &new_zfs_pool, sizeof(struct zfs_pool));
343
+ };
344
+
345
+ pool->updated = 1;
346
+
347
+ if (pool->disabled) {
348
+ state_file_found = 1;
349
+ continue;
350
+ }
351
+
352
+ pool->online = 0;
353
+ pool->degraded = 0;
354
+ pool->faulted = 0;
355
+ pool->offline = 0;
356
+ pool->removed = 0;
357
+ pool->unavail = 0;
358
+
359
+ char filename[FILENAME_MAX + 1];
360
+ snprintfz(
361
+ filename, FILENAME_MAX, "%s%s/%s/state", netdata_configured_host_prefix, dirname, de->d_name);
362
+
363
+ char state[STATE_SIZE + 1];
364
+ int ret = read_file(filename, state, STATE_SIZE);
365
+
366
+ if (!ret) {
367
+ state_file_found = 1;
368
+
369
+ // ZFS pool states are described at https://openzfs.github.io/openzfs-docs/man/8/zpoolconcepts.8.html?#Device_Failure_and_Recovery
370
+ if (!strcmp(state, "ONLINE\n")) {
371
+ pool->online = 1;
372
+ } else if (!strcmp(state, "DEGRADED\n")) {
373
+ pool->degraded = 1;
374
+ } else if (!strcmp(state, "FAULTED\n")) {
375
+ pool->faulted = 1;
376
+ } else if (!strcmp(state, "OFFLINE\n")) {
377
+ pool->offline = 1;
378
+ } else if (!strcmp(state, "REMOVED\n")) {
379
+ pool->removed = 1;
380
+ } else if (!strcmp(state, "UNAVAIL\n")) {
381
+ pool->unavail = 1;
382
+ } else {
383
+ disable_zfs_pool_state(pool);
384
+
385
+ char *c = strchr(state, '\n');
386
+ if (c)
387
+ *c = '\0';
388
+ error("ZFS POOLS: Undefined state %s for zpool %s, disabling the chart", state, de->d_name);
389
+ }
390
+ }
391
+ }
392
+ }
393
+
394
+ closedir(dir);
395
+ }
396
+
397
+ if (do_zfs_pool_state && pool_found && !state_file_found) {
398
+ info("ZFS POOLS: State files not found. Disabling the module.");
399
+ do_zfs_pool_state = 0;
400
+ }
401
+
402
+ if (do_zfs_pool_state)
403
+ dictionary_get_all_name_value(zfs_pools, update_zfs_pool_state_chart, &update_every);
404
+
405
+ while (deleted_zfs_pools) {
406
+ struct deleted_zfs_pool *current_pool = deleted_zfs_pools;
407
+ dictionary_del(zfs_pools, current_pool->name);
408
+
409
+ deleted_zfs_pools = deleted_zfs_pools->next;
410
+
411
+ freez(current_pool->name);
412
+ freez(current_pool);
413
+ }
414
+
415
+ return 0;
416
+}
health/health.d/zfs.conf
+28
@@ -11,3 +11,31 @@ component: File system
11
delay: down 1h multiplier 1.5 max 2h
12
info: number of times ZFS had to limit the ARC growth in the last 10 minutes
13
to: sysadmin
14
+
15
+# ZFS pool state
16
+
17
+ template: zfs_pool_state_warn
18
+ on: zfspool.state
19
+ class: System
20
+component: File system
21
+ type: Errors
22
+ calc: $degraded
23
+ units: boolean
24
+ every: 10s
25
+ warn: $this > 0
26
+ delay: down 1m multiplier 1.5 max 1h
27
+ info: ZFS pool $family state is degraded
28
+ to: sysadmin
29
+
30
+ template: zfs_pool_state_crit
31
+ on: zfspool.state
32
+ class: System
33
+component: File system
34
+ type: Errors
35
+ calc: $faulted + $unavail
36
+ units: boolean
37
+ every: 10s
38
+ crit: $this > 0
39
+ delay: down 1m multiplier 1.5 max 1h
40
+ info: ZFS pool $family state is faulted or unavail
41
+ to: sysadmin
web/gui/dashboard_info.js
+13
@@ -163,6 +163,12 @@ netdataDashboard.menu = {
163
info: 'Performance metrics of the ZFS filesystem. The following charts visualize all metrics reported by <a href="https://github.com/zfsonlinux/zfs/blob/master/cmd/arcstat/arcstat" target="_blank">arcstat.py</a> and <a href="https://github.com/zfsonlinux/zfs/blob/master/cmd/arc_summary/arc_summary3" target="_blank">arc_summary.py</a>.'
164
},
165
166
+ 'zfspool': {
167
+ title: 'ZFS pools',
168
+ icon: '<i class="fas fa-database"></i>',
169
+ info: 'State of ZFS pools.'
170
+ },
171
+
172
'btrfs': {
173
title: 'BTRFS filesystem',
174
icon: '<i class="fas fa-folder-open"></i>',
@@ -1519,6 +1525,13 @@ netdataDashboard.context = {
1525
info: 'inodes (or index nodes) are filesystem objects (e.g. files and directories). On many types of file system implementations, the maximum number of inodes is fixed at filesystem creation, limiting the maximum number of files the filesystem can hold. It is possible for a device to run out of inodes. When this happens, new files cannot be created on the device, even though there may be free space available.'
1526
},
1527
1528
+ // ------------------------------------------------------------------------
1529
+ // ZFS pools
1530
+ 'zfspool.state': {
1531
+ info: 'ZFS pool state. The overall health of a pool, as reported by <code>zpool status</code>, is determined by the aggregate state of all devices within the pool. ' +
1532
+ 'For details, see <a href="https://openzfs.github.io/openzfs-docs/man/8/zpoolconcepts.8.html?#Device_Failure_and_Recovery" target="_blank"> ZFS documentation</a>.'
1533
+ },
1534
+
1535
// ------------------------------------------------------------------------
1536
// MYSQL
1537