Collect additional BTRFS metrics (#14636)
* Add commit_stats metrics to BTRFS section * Add error_stats metrics (per device) to BTRFS section * Simplify commit stats variables and chart ids/names * Add basic BTRFS error alarms. Configured to trip whenever any of the error dimensions is non-zero. * Add chart descriptions for new charts. * Remove duplicate code * Comment out some debugging code * Always create error stats dimensions, even if zero * Show rate of commits and commit duration instead of totals * Change current commit metrics to absolute from incremental * Change commits dimension to absolute and add separate commits time share chart * Rename 'device_' rrdlabels to 'filesystem_' * Replace all snprintf() calls with snprintfz() * Fix codacy warning * Provide separate error charts for each filesystem device * Accept code review suggestions for more descriptive context and labels Co-authored-by: Ilya Mashchenko <ilya@netdata.cloud> * Add 'device' prefix to id, name, title of errors chart * Add 'device_id' label to device_errors * Update health.d/btrfs.conf to match new errors charts * Remove commented out code * Do not disable all BTRFS metrics collection if only commit_stats is missing * Do not disable all BTRFS metrics collection if only error_stats is missing * Fix bug of BTRFS device add/remove not being detected properly * Fix double free() error when deleting a device * Update dashboard info with bold tags Co-authored-by: Ilya Mashchenko <ilya@netdata.cloud> --------- Co-authored-by: Austin S. Hemmelgarn <austin@netdata.cloud> Co-authored-by: Ilya Mashchenko <ilya@netdata.cloud>
Dimitris P committed
Apr 12, 2023 at 15:23 UTC
52f2df7ef56242a47821b0657f311f19490fe9ef
4 files changed
+567
-42
collectors/all.h
+4
@@ -182,6 +182,10 @@
182
#define NETDATA_CHART_PRIO_BTRFS_DATA 2401
183
#define NETDATA_CHART_PRIO_BTRFS_METADATA 2402
184
#define NETDATA_CHART_PRIO_BTRFS_SYSTEM 2403
185
+#define NETDATA_CHART_PRIO_BTRFS_COMMITS 2404
186
+#define NETDATA_CHART_PRIO_BTRFS_COMMITS_PERC_TIME 2405
187
+#define NETDATA_CHART_PRIO_BTRFS_COMMIT_TIMINGS 2406
188
+#define NETDATA_CHART_PRIO_BTRFS_ERRORS 2407
189
190
// ZFS
191
collectors/proc.plugin/sys_fs_btrfs.c
+472
-42
@@ -15,6 +15,26 @@ typedef struct btrfs_disk {
15
struct btrfs_disk *next;
16
} BTRFS_DISK;
17
18
+typedef struct btrfs_device {
19
+ int id;
20
+ int exists;
21
+
22
+ char *error_stats_filename;
23
+ RRDSET *st_error_stats;
24
+ RRDDIM *rd_write_errs;
25
+ RRDDIM *rd_read_errs;
26
+ RRDDIM *rd_flush_errs;
27
+ RRDDIM *rd_corruption_errs;
28
+ RRDDIM *rd_generation_errs;
29
+ collected_number write_errs;
30
+ collected_number read_errs;
31
+ collected_number flush_errs;
32
+ collected_number corruption_errs;
33
+ collected_number generation_errs;
34
+
35
+ struct btrfs_device *next;
36
+} BTRFS_DEVICE;
37
+
38
typedef struct btrfs_node {
39
int exists;
40
int logged_error;
@@ -24,10 +44,6 @@ typedef struct btrfs_node {
44
45
char *label;
46
27
- // unsigned long long int sectorsize;
28
- // unsigned long long int nodesize;
29
- // unsigned long long int quota_override;
30
-
47
#define declare_btrfs_allocation_section_field(SECTION, FIELD) \
48
char *allocation_ ## SECTION ## _ ## FIELD ## _filename; \
49
unsigned long long int allocation_ ## SECTION ## _ ## FIELD;
@@ -73,19 +89,133 @@ typedef struct btrfs_node {
89
declare_btrfs_allocation_section_field(system, disk_total)
90
declare_btrfs_allocation_section_field(system, disk_used)
91
92
+ // --------------------------------------------------------------------
93
+ // commit stats
94
+
95
+ char *commit_stats_filename;
96
+
97
+ RRDSET *st_commits;
98
+ RRDDIM *rd_commits;
99
+ long long commits_total;
100
+ collected_number commits_new;
101
+
102
+ RRDSET *st_commits_percentage_time;
103
+ RRDDIM *rd_commits_percentage_time;
104
+ long long commit_timings_total;
105
+ long long commits_percentage_time;
106
+
107
+ RRDSET *st_commit_timings;
108
+ RRDDIM *rd_commit_timings_last;
109
+ RRDDIM *rd_commit_timings_max;
110
+ collected_number commit_timings_last;
111
+ collected_number commit_timings_max;
112
+
113
BTRFS_DISK *disks;
114
115
+ BTRFS_DEVICE *devices;
116
+
117
struct btrfs_node *next;
118
} BTRFS_NODE;
119
120
static BTRFS_NODE *nodes = NULL;
121
122
+static inline int collect_btrfs_error_stats(BTRFS_DEVICE *device){
123
+ char buffer[120 + 1];
124
+
125
+ int ret = read_file(device->error_stats_filename, buffer, 120);
126
+ if(unlikely(ret)) {
127
+ collector_error("BTRFS: failed to read '%s'", device->error_stats_filename);
128
+ device->write_errs = 0;
129
+ device->read_errs = 0;
130
+ device->flush_errs = 0;
131
+ device->corruption_errs = 0;
132
+ device->generation_errs = 0;
133
+ return ret;
134
+ }
135
+
136
+ char *p = buffer;
137
+ while(p){
138
+ char *val = mystrsep(&p, "\n");
139
+ if(unlikely(!val || !*val)) break;
140
+ char *key = mystrsep(&val, " ");
141
+
142
+ if(!strcmp(key, "write_errs")) device->write_errs = str2ull(val, NULL);
143
+ else if(!strcmp(key, "read_errs")) device->read_errs = str2ull(val, NULL);
144
+ else if(!strcmp(key, "flush_errs")) device->flush_errs = str2ull(val, NULL);
145
+ else if(!strcmp(key, "corruption_errs")) device->corruption_errs = str2ull(val, NULL);
146
+ else if(!strcmp(key, "generation_errs")) device->generation_errs = str2ull(val, NULL);
147
+ }
148
+ return 0;
149
+}
150
+
151
+static inline int collect_btrfs_commits_stats(BTRFS_NODE *node, int update_every){
152
+ char buffer[120 + 1];
153
+
154
+ int ret = read_file(node->commit_stats_filename, buffer, 120);
155
+ if(unlikely(ret)) {
156
+ collector_error("BTRFS: failed to read '%s'", node->commit_stats_filename);
157
+ node->commits_total = 0;
158
+ node->commits_new = 0;
159
+ node->commit_timings_last = 0;
160
+ node->commit_timings_max = 0;
161
+ node->commit_timings_total = 0;
162
+ node->commits_percentage_time = 0;
163
+
164
+ return ret;
165
+ }
166
+
167
+ char *p = buffer;
168
+ while(p){
169
+ char *val = mystrsep(&p, "\n");
170
+ if(unlikely(!val || !*val)) break;
171
+ char *key = mystrsep(&val, " ");
172
+
173
+ if(!strcmp(key, "commits")){
174
+ long long commits_total_new = str2ull(val, NULL);
175
+ if(likely(node->commits_total)){
176
+ if((node->commits_new = commits_total_new - node->commits_total))
177
+ node->commits_total = commits_total_new;
178
+ } else node->commits_total = commits_total_new;
179
+ }
180
+ else if(!strcmp(key, "last_commit_ms")) node->commit_timings_last = str2ull(val, NULL);
181
+ else if(!strcmp(key, "max_commit_ms")) node->commit_timings_max = str2ull(val, NULL);
182
+ else if(!strcmp(key, "total_commit_ms")) {
183
+ long long commit_timings_total_new = str2ull(val, NULL);
184
+ if(likely(node->commit_timings_total)){
185
+ long time_delta = commit_timings_total_new - node->commit_timings_total;
186
+ if(time_delta){
187
+ node->commits_percentage_time = time_delta * 10 / update_every;
188
+ node->commit_timings_total = commit_timings_total_new;
189
+ } else node->commits_percentage_time = 0;
190
+
191
+ } else node->commit_timings_total = commit_timings_total_new;
192
+ }
193
+ }
194
+ return 0;
195
+}
196
+
197
+static inline void btrfs_free_commits_stats(BTRFS_NODE *node){
198
+ if(node->st_commits){
199
+ rrdset_is_obsolete(node->st_commits);
200
+ rrdset_is_obsolete(node->st_commit_timings);
201
+ }
202
+ freez(node->commit_stats_filename);
203
+ node->commit_stats_filename = NULL;
204
+}
205
+
206
static inline void btrfs_free_disk(BTRFS_DISK *d) {
207
freez(d->name);
208
freez(d->size_filename);
209
freez(d);
210
}
211
212
+static inline void btrfs_free_device(BTRFS_DEVICE *d) {
213
+ if(d->st_error_stats)
214
+ rrdset_is_obsolete(d->st_error_stats);
215
+ freez(d->error_stats_filename);
216
+ freez(d);
217
+}
218
+
219
static inline void btrfs_free_node(BTRFS_NODE *node) {
220
// collector_info("BTRFS: destroying '%s'", node->id);
221
@@ -110,12 +240,20 @@ static inline void btrfs_free_node(BTRFS_NODE *node) {
240
freez(node->allocation_system_bytes_used_filename);
241
freez(node->allocation_system_total_bytes_filename);
242
243
+ btrfs_free_commits_stats(node);
244
+
245
while(node->disks) {
246
BTRFS_DISK *d = node->disks;
247
node->disks = node->disks->next;
248
btrfs_free_disk(d);
249
}
250
251
+ while(node->devices) {
252
+ BTRFS_DEVICE *d = node->devices;
253
+ node->devices = node->devices->next;
254
+ btrfs_free_device(d);
255
+ }
256
+
257
freez(node->label);
258
freez(node->id);
259
freez(node);
@@ -227,8 +365,106 @@ static inline int find_btrfs_disks(BTRFS_NODE *node, const char *path) {
365
return 0;
366
}
367
368
+static inline int find_btrfs_devices(BTRFS_NODE *node, const char *path) {
369
+ char filename[FILENAME_MAX + 1];
370
+
371
+ BTRFS_DEVICE *d;
372
+ for(d = node->devices ; d ; d = d->next)
373
+ d->exists = 0;
374
+
375
+ DIR *dir = opendir(path);
376
+ if (!dir) {
377
+ if(!node->logged_error) {
378
+ collector_error("BTRFS: Cannot open directory '%s'.", path);
379
+ node->logged_error = 1;
380
+ }
381
+ return 1;
382
+ }
383
+ node->logged_error = 0;
384
+
385
+ struct dirent *de = NULL;
386
+ while ((de = readdir(dir))) {
387
+ if (de->d_type != DT_DIR
388
+ || !strcmp(de->d_name, ".")
389
+ || !strcmp(de->d_name, "..")
390
+ ) {
391
+ // collector_info("BTRFS: ignoring '%s'", de->d_name);
392
+ continue;
393
+ }
394
+
395
+ collector_info("BTRFS: device found '%s'", de->d_name);
396
+
397
+ // --------------------------------------------------------------------
398
+ // search for it
399
+
400
+ for(d = node->devices ; d ; d = d->next) {
401
+ if(str2ll(de->d_name, NULL) == d->id){
402
+ collector_info("BTRFS: existing device id '%d'", d->id);
403
+ break;
404
+ }
405
+ }
406
+
407
+ // --------------------------------------------------------------------
408
+ // did we find it?
409
+
410
+ if(!d) {
411
+ d = callocz(sizeof(BTRFS_DEVICE), 1);
412
+
413
+ d->id = str2ll(de->d_name, NULL);
414
+ collector_info("BTRFS: new device with id '%d'", d->id);
415
+
416
+ snprintfz(filename, FILENAME_MAX, "%s/%d/error_stats", path, d->id);
417
+ d->error_stats_filename = strdupz(filename);
418
+ collector_info("BTRFS: error_stats_filename '%s'", filename);
419
+
420
+ // link it
421
+ d->next = node->devices;
422
+ node->devices = d;
423
+ }
424
+
425
+ d->exists = 1;
426
+
427
+
428
+ // --------------------------------------------------------------------
429
+ // update the values
430
+
431
+ if(unlikely(collect_btrfs_error_stats(d)))
432
+ d->exists = 0; // 'd' will be garbaged collected in loop below
433
+ }
434
+ closedir(dir);
435
+
436
+ // ------------------------------------------------------------------------
437
+ // cleanup
438
+
439
+ BTRFS_DEVICE *last = NULL;
440
+ d = node->devices;
441
+
442
+ while(d) {
443
+ if(unlikely(!d->exists)) {
444
+ if(unlikely(node->devices == d)) {
445
+ node->devices = d->next;
446
+ btrfs_free_device(d);
447
+ d = node->devices;
448
+ last = NULL;
449
+ }
450
+ else {
451
+ last->next = d->next;
452
+ btrfs_free_device(d);
453
+ d = last->next;
454
+ }
455
+
456
+ continue;
457
+ }
458
+
459
+ last = d;
460
+ d = d->next;
461
+ }
462
+
463
+ return 0;
464
+}
465
+
466
231
-static inline int find_all_btrfs_pools(const char *path) {
467
+static inline int find_all_btrfs_pools(const char *path, int update_every) {
468
static int logged_error = 0;
469
char filename[FILENAME_MAX + 1];
470
@@ -274,6 +510,10 @@ static inline int find_all_btrfs_pools(const char *path) {
510
snprintfz(filename, FILENAME_MAX, "%s/%s/devices", path, de->d_name);
511
find_btrfs_disks(node, filename);
512
513
+ // update devices
514
+ snprintfz(filename, FILENAME_MAX, "%s/%s/devinfo", path, de->d_name);
515
+ find_btrfs_devices(node, filename);
516
+
517
continue;
518
}
519
@@ -306,27 +546,6 @@ static inline int find_all_btrfs_pools(const char *path) {
546
node->label = strdupz(node->id);
547
}
548
309
- //snprintfz(filename, FILENAME_MAX, "%s/%s/sectorsize", path, de->d_name);
310
- //if(read_single_number_file(filename, &node->sectorsize) != 0) {
311
- // collector_error("BTRFS: failed to read '%s'", filename);
312
- // btrfs_free_node(node);
313
- // continue;
314
- //}
315
-
316
- //snprintfz(filename, FILENAME_MAX, "%s/%s/nodesize", path, de->d_name);
317
- //if(read_single_number_file(filename, &node->nodesize) != 0) {
318
- // collector_error("BTRFS: failed to read '%s'", filename);
319
- // btrfs_free_node(node);
320
- // continue;
321
- //}
322
-
323
- //snprintfz(filename, FILENAME_MAX, "%s/%s/quota_override", path, de->d_name);
324
- //if(read_single_number_file(filename, &node->quota_override) != 0) {
325
- // collector_error("BTRFS: failed to read '%s'", filename);
326
- // btrfs_free_node(node);
327
- // continue;
328
- //}
329
-
549
// --------------------------------------------------------------------
550
// macros to simplify our life
551
@@ -381,6 +600,15 @@ static inline int find_all_btrfs_pools(const char *path) {
600
init_btrfs_allocation_section_field(system, disk_total);
601
init_btrfs_allocation_section_field(system, disk_used);
602
603
+ // --------------------------------------------------------------------
604
+ // commit stats
605
+
606
+ snprintfz(filename, FILENAME_MAX, "%s/%s/commit_stats", path, de->d_name);
607
+ if(!node->commit_stats_filename) node->commit_stats_filename = strdupz(filename);
608
+ if(unlikely(collect_btrfs_commits_stats(node, update_every))){
609
+ collector_error("BTRFS: failed to collect commit stats for '%s'", node->id);
610
+ btrfs_free_commits_stats(node);
611
+ }
612
613
// --------------------------------------------------------------------
614
// find all disks related to this node
@@ -389,6 +617,11 @@ static inline int find_all_btrfs_pools(const char *path) {
617
snprintfz(filename, FILENAME_MAX, "%s/%s/devices", path, de->d_name);
618
find_btrfs_disks(node, filename);
619
620
+ // --------------------------------------------------------------------
621
+ // find all devices related to this node
622
+
623
+ snprintfz(filename, FILENAME_MAX, "%s/%s/devinfo", path, de->d_name);
624
+ find_btrfs_devices(node, filename);
625
626
// --------------------------------------------------------------------
627
// link it
@@ -431,8 +664,8 @@ static inline int find_all_btrfs_pools(const char *path) {
664
}
665
666
static void add_labels_to_btrfs(BTRFS_NODE *n, RRDSET *st) {
434
- rrdlabels_add(st->rrdlabels, "device", n->id, RRDLABEL_SRC_AUTO);
435
- rrdlabels_add(st->rrdlabels, "device_label", n->label, RRDLABEL_SRC_AUTO);
667
+ rrdlabels_add(st->rrdlabels, "filesystem_uuid", n->id, RRDLABEL_SRC_AUTO);
668
+ rrdlabels_add(st->rrdlabels, "filesystem_label", n->label, RRDLABEL_SRC_AUTO);
669
}
670
671
int do_sys_fs_btrfs(int update_every, usec_t dt) {
@@ -440,7 +673,9 @@ int do_sys_fs_btrfs(int update_every, usec_t dt) {
673
, do_allocation_disks = CONFIG_BOOLEAN_AUTO
674
, do_allocation_system = CONFIG_BOOLEAN_AUTO
675
, do_allocation_data = CONFIG_BOOLEAN_AUTO
443
- , do_allocation_metadata = CONFIG_BOOLEAN_AUTO;
676
+ , do_allocation_metadata = CONFIG_BOOLEAN_AUTO
677
+ , do_commit_stats = CONFIG_BOOLEAN_AUTO
678
+ , do_error_stats = CONFIG_BOOLEAN_AUTO;
679
680
static usec_t refresh_delta = 0, refresh_every = 60 * USEC_PER_SEC;
681
static char *btrfs_path = NULL;
@@ -461,12 +696,14 @@ int do_sys_fs_btrfs(int update_every, usec_t dt) {
696
do_allocation_data = config_get_boolean_ondemand("plugin:proc:/sys/fs/btrfs", "data allocation", do_allocation_data);
697
do_allocation_metadata = config_get_boolean_ondemand("plugin:proc:/sys/fs/btrfs", "metadata allocation", do_allocation_metadata);
698
do_allocation_system = config_get_boolean_ondemand("plugin:proc:/sys/fs/btrfs", "system allocation", do_allocation_system);
699
+ do_commit_stats = config_get_boolean_ondemand("plugin:proc:/sys/fs/btrfs", "commit stats", do_commit_stats);
700
+ do_error_stats = config_get_boolean_ondemand("plugin:proc:/sys/fs/btrfs", "error stats", do_error_stats);
701
}
702
703
refresh_delta += dt;
704
if(refresh_delta >= refresh_every) {
705
refresh_delta = 0;
469
- find_all_btrfs_pools(btrfs_path);
706
+ find_all_btrfs_pools(btrfs_path, update_every);
707
}
708
709
BTRFS_NODE *node;
@@ -526,6 +763,25 @@ int do_sys_fs_btrfs(int update_every, usec_t dt) {
763
}
764
}
765
766
+ if(do_commit_stats != CONFIG_BOOLEAN_NO && node->commit_stats_filename) {
767
+ if (unlikely(collect_btrfs_commits_stats(node, update_every))) {
768
+ collector_error("BTRFS: failed to collect commit stats for '%s'", node->id);
769
+ btrfs_free_commits_stats(node);
770
+ }
771
+ }
772
+
773
+ if(do_error_stats != CONFIG_BOOLEAN_NO) {
774
+ for(BTRFS_DEVICE *d = node->devices ; d ; d = d->next) {
775
+ if(unlikely(collect_btrfs_error_stats(d))){
776
+ collector_error("BTRFS: failed to collect error stats for '%s', devid:'%d'", node->id, d->id);
777
+ /* make it refresh btrfs at the next iteration,
778
+ * btrfs_free_device(d) will be called in
779
+ * find_btrfs_devices() as part of the garbage collection */
780
+ refresh_delta = refresh_every;
781
+ }
782
+ }
783
+ }
784
+
785
// --------------------------------------------------------------------
786
// allocation/disks
787
@@ -537,9 +793,9 @@ int do_sys_fs_btrfs(int update_every, usec_t dt) {
793
if(unlikely(!node->st_allocation_disks)) {
794
char id[RRD_ID_LENGTH_MAX + 1], name[RRD_ID_LENGTH_MAX + 1], title[200 + 1];
795
540
- snprintf(id, RRD_ID_LENGTH_MAX, "disk_%s", node->id);
541
- snprintf(name, RRD_ID_LENGTH_MAX, "disk_%s", node->label);
542
- snprintf(title, 200, "BTRFS Physical Disk Allocation");
796
+ snprintfz(id, RRD_ID_LENGTH_MAX, "disk_%s", node->id);
797
+ snprintfz(name, RRD_ID_LENGTH_MAX, "disk_%s", node->label);
798
+ snprintfz(title, 200, "BTRFS Physical Disk Allocation");
799
800
netdata_fix_chart_id(id);
801
netdata_fix_chart_name(name);
@@ -596,9 +852,9 @@ int do_sys_fs_btrfs(int update_every, usec_t dt) {
852
if(unlikely(!node->st_allocation_data)) {
853
char id[RRD_ID_LENGTH_MAX + 1], name[RRD_ID_LENGTH_MAX + 1], title[200 + 1];
854
599
- snprintf(id, RRD_ID_LENGTH_MAX, "data_%s", node->id);
600
- snprintf(name, RRD_ID_LENGTH_MAX, "data_%s", node->label);
601
- snprintf(title, 200, "BTRFS Data Allocation");
855
+ snprintfz(id, RRD_ID_LENGTH_MAX, "data_%s", node->id);
856
+ snprintfz(name, RRD_ID_LENGTH_MAX, "data_%s", node->label);
857
+ snprintfz(title, 200, "BTRFS Data Allocation");
858
859
netdata_fix_chart_id(id);
860
netdata_fix_chart_name(name);
@@ -640,9 +896,9 @@ int do_sys_fs_btrfs(int update_every, usec_t dt) {
896
if(unlikely(!node->st_allocation_metadata)) {
897
char id[RRD_ID_LENGTH_MAX + 1], name[RRD_ID_LENGTH_MAX + 1], title[200 + 1];
898
643
- snprintf(id, RRD_ID_LENGTH_MAX, "metadata_%s", node->id);
644
- snprintf(name, RRD_ID_LENGTH_MAX, "metadata_%s", node->label);
645
- snprintf(title, 200, "BTRFS Metadata Allocation");
899
+ snprintfz(id, RRD_ID_LENGTH_MAX, "metadata_%s", node->id);
900
+ snprintfz(name, RRD_ID_LENGTH_MAX, "metadata_%s", node->label);
901
+ snprintfz(title, 200, "BTRFS Metadata Allocation");
902
903
netdata_fix_chart_id(id);
904
netdata_fix_chart_name(name);
@@ -686,9 +942,9 @@ int do_sys_fs_btrfs(int update_every, usec_t dt) {
942
if(unlikely(!node->st_allocation_system)) {
943
char id[RRD_ID_LENGTH_MAX + 1], name[RRD_ID_LENGTH_MAX + 1], title[200 + 1];
944
689
- snprintf(id, RRD_ID_LENGTH_MAX, "system_%s", node->id);
690
- snprintf(name, RRD_ID_LENGTH_MAX, "system_%s", node->label);
691
- snprintf(title, 200, "BTRFS System Allocation");
945
+ snprintfz(id, RRD_ID_LENGTH_MAX, "system_%s", node->id);
946
+ snprintfz(name, RRD_ID_LENGTH_MAX, "system_%s", node->label);
947
+ snprintfz(title, 200, "BTRFS System Allocation");
948
949
netdata_fix_chart_id(id);
950
netdata_fix_chart_name(name);
@@ -718,6 +974,180 @@ int do_sys_fs_btrfs(int update_every, usec_t dt) {
974
rrddim_set_by_pointer(node->st_allocation_system, node->rd_allocation_system_used, node->allocation_system_bytes_used);
975
rrdset_done(node->st_allocation_system);
976
}
977
+
978
+ // --------------------------------------------------------------------
979
+ // commit_stats
980
+
981
+ if(do_commit_stats == CONFIG_BOOLEAN_YES || (do_commit_stats == CONFIG_BOOLEAN_AUTO &&
982
+ (node->commits_total ||
983
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
984
+ do_commit_stats = CONFIG_BOOLEAN_YES;
985
+
986
+ if(unlikely(!node->st_commits)) {
987
+ char id[RRD_ID_LENGTH_MAX + 1], name[RRD_ID_LENGTH_MAX + 1], title[200 + 1];
988
+
989
+ snprintfz(id, RRD_ID_LENGTH_MAX, "commits_%s", node->id);
990
+ snprintfz(name, RRD_ID_LENGTH_MAX, "commits_%s", node->label);
991
+ snprintfz(title, 200, "BTRFS Commits");
992
+
993
+ netdata_fix_chart_id(id);
994
+ netdata_fix_chart_name(name);
995
+
996
+ node->st_commits = rrdset_create_localhost(
997
+ "btrfs"
998
+ , id
999
+ , name
1000
+ , node->label
1001
+ , "btrfs.commits"
1002
+ , title
1003
+ , "commits"
1004
+ , PLUGIN_PROC_NAME
1005
+ , PLUGIN_PROC_MODULE_BTRFS_NAME
1006
+ , NETDATA_CHART_PRIO_BTRFS_COMMITS
1007
+ , update_every
1008
+ , RRDSET_TYPE_LINE
1009
+ );
1010
+
1011
+ node->rd_commits = rrddim_add(node->st_commits, "commits", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1012
+
1013
+ add_labels_to_btrfs(node, node->st_commits);
1014
+ }
1015
+
1016
+ rrddim_set_by_pointer(node->st_commits, node->rd_commits, node->commits_new);
1017
+ rrdset_done(node->st_commits);
1018
+
1019
+ if(unlikely(!node->st_commits_percentage_time)) {
1020
+ char id[RRD_ID_LENGTH_MAX + 1], name[RRD_ID_LENGTH_MAX + 1], title[200 + 1];
1021
+
1022
+ snprintfz(id, RRD_ID_LENGTH_MAX, "commits_perc_time_%s", node->id);
1023
+ snprintfz(name, RRD_ID_LENGTH_MAX, "commits_perc_time_%s", node->label);
1024
+ snprintfz(title, 200, "BTRFS Commits Time Share");
1025
+
1026
+ netdata_fix_chart_id(id);
1027
+ netdata_fix_chart_name(name);
1028
+
1029
+ node->st_commits_percentage_time = rrdset_create_localhost(
1030
+ "btrfs"
1031
+ , id
1032
+ , name
1033
+ , node->label
1034
+ , "btrfs.commits_perc_time"
1035
+ , title
1036
+ , "percentage"
1037
+ , PLUGIN_PROC_NAME
1038
+ , PLUGIN_PROC_MODULE_BTRFS_NAME
1039
+ , NETDATA_CHART_PRIO_BTRFS_COMMITS_PERC_TIME
1040
+ , update_every
1041
+ , RRDSET_TYPE_LINE
1042
+ );
1043
+
1044
+ node->rd_commits_percentage_time = rrddim_add(node->st_commits_percentage_time, "commits", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
1045
+
1046
+ add_labels_to_btrfs(node, node->st_commits_percentage_time);
1047
+ }
1048
+
1049
+ rrddim_set_by_pointer(node->st_commits_percentage_time, node->rd_commits_percentage_time, node->commits_percentage_time);
1050
+ rrdset_done(node->st_commits_percentage_time);
1051
+
1052
+
1053
+ if(unlikely(!node->st_commit_timings)) {
1054
+ char id[RRD_ID_LENGTH_MAX + 1], name[RRD_ID_LENGTH_MAX + 1], title[200 + 1];
1055
+
1056
+ snprintfz(id, RRD_ID_LENGTH_MAX, "commit_timings_%s", node->id);
1057
+ snprintfz(name, RRD_ID_LENGTH_MAX, "commit_timings_%s", node->label);
1058
+ snprintfz(title, 200, "BTRFS Commit Timings");
1059
+
1060
+ netdata_fix_chart_id(id);
1061
+ netdata_fix_chart_name(name);
1062
+
1063
+ node->st_commit_timings = rrdset_create_localhost(
1064
+ "btrfs"
1065
+ , id
1066
+ , name
1067
+ , node->label
1068
+ , "btrfs.commit_timings"
1069
+ , title
1070
+ , "ms"
1071
+ , PLUGIN_PROC_NAME
1072
+ , PLUGIN_PROC_MODULE_BTRFS_NAME
1073
+ , NETDATA_CHART_PRIO_BTRFS_COMMIT_TIMINGS
1074
+ , update_every
1075
+ , RRDSET_TYPE_LINE
1076
+ );
1077
+
1078
+ node->rd_commit_timings_last = rrddim_add(node->st_commit_timings, "last", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1079
+ node->rd_commit_timings_max = rrddim_add(node->st_commit_timings, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1080
+
1081
+ add_labels_to_btrfs(node, node->st_commit_timings);
1082
+ }
1083
+
1084
+ rrddim_set_by_pointer(node->st_commit_timings, node->rd_commit_timings_last, node->commit_timings_last);
1085
+ rrddim_set_by_pointer(node->st_commit_timings, node->rd_commit_timings_max, node->commit_timings_max);
1086
+ rrdset_done(node->st_commit_timings);
1087
+ }
1088
+
1089
+ // --------------------------------------------------------------------
1090
+ // error_stats per device
1091
+
1092
+ if(do_error_stats == CONFIG_BOOLEAN_YES || (do_error_stats == CONFIG_BOOLEAN_AUTO &&
1093
+ (node->devices ||
1094
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
1095
+ do_error_stats = CONFIG_BOOLEAN_YES;
1096
+
1097
+ for(BTRFS_DEVICE *d = node->devices ; d ; d = d->next) {
1098
+
1099
+ if(unlikely(!d->st_error_stats)) {
1100
+ char id[RRD_ID_LENGTH_MAX + 1], name[RRD_ID_LENGTH_MAX + 1], title[200 + 1];
1101
+
1102
+ snprintfz(id, RRD_ID_LENGTH_MAX, "device_errors_dev%d_%s", d->id, node->id);
1103
+ snprintfz(name, RRD_ID_LENGTH_MAX, "device_errors_dev%d_%s", d->id, node->label);
1104
+ snprintfz(title, 200, "BTRFS Device Errors");
1105
+
1106
+ netdata_fix_chart_id(id);
1107
+ netdata_fix_chart_name(name);
1108
+
1109
+ d->st_error_stats = rrdset_create_localhost(
1110
+ "btrfs"
1111
+ , id
1112
+ , name
1113
+ , node->label
1114
+ , "btrfs.device_errors"
1115
+ , title
1116
+ , "errors"
1117
+ , PLUGIN_PROC_NAME
1118
+ , PLUGIN_PROC_MODULE_BTRFS_NAME
1119
+ , NETDATA_CHART_PRIO_BTRFS_ERRORS
1120
+ , update_every
1121
+ , RRDSET_TYPE_LINE
1122
+ );
1123
+
1124
+ char rd_id[RRD_ID_LENGTH_MAX + 1];
1125
+ snprintfz(rd_id, RRD_ID_LENGTH_MAX, "write_errs");
1126
+ d->rd_write_errs = rrddim_add(d->st_error_stats, rd_id, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1127
+ snprintfz(rd_id, RRD_ID_LENGTH_MAX, "read_errs");
1128
+ d->rd_read_errs = rrddim_add(d->st_error_stats, rd_id, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1129
+ snprintfz(rd_id, RRD_ID_LENGTH_MAX, "flush_errs");
1130
+ d->rd_flush_errs = rrddim_add(d->st_error_stats, rd_id, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1131
+ snprintfz(rd_id, RRD_ID_LENGTH_MAX, "corruption_errs");
1132
+ d->rd_corruption_errs = rrddim_add(d->st_error_stats, rd_id, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1133
+ snprintfz(rd_id, RRD_ID_LENGTH_MAX, "generation_errs");
1134
+ d->rd_generation_errs = rrddim_add(d->st_error_stats, rd_id, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1135
+
1136
+ char dev_id[5];
1137
+ snprintfz(dev_id, 4, "%d", d->id);
1138
+ rrdlabels_add(d->st_error_stats->rrdlabels, "device_id", dev_id, RRDLABEL_SRC_AUTO);
1139
+ add_labels_to_btrfs(node, d->st_error_stats);
1140
+ }
1141
+
1142
+ rrddim_set_by_pointer(d->st_error_stats, d->rd_write_errs, d->write_errs);
1143
+ rrddim_set_by_pointer(d->st_error_stats, d->rd_read_errs, d->read_errs);
1144
+ rrddim_set_by_pointer(d->st_error_stats, d->rd_flush_errs, d->flush_errs);
1145
+ rrddim_set_by_pointer(d->st_error_stats, d->rd_corruption_errs, d->corruption_errs);
1146
+ rrddim_set_by_pointer(d->st_error_stats, d->rd_generation_errs, d->generation_errs);
1147
+
1148
+ rrdset_done(d->st_error_stats);
1149
+ }
1150
+ }
1151
}
1152
1153
return 0;
health/health.d/btrfs.conf
+75
@@ -66,3 +66,78 @@ component: File system
66
delay: up 1m down 15m multiplier 1.5 max 1h
67
info: utilization of BTRFS system space
68
to: sysadmin
69
+
70
+ template: btrfs_device_read_errors
71
+ on: btrfs.device_errors
72
+ class: Errors
73
+ type: System
74
+component: File system
75
+ os: *
76
+ hosts: *
77
+ families: *
78
+ units: errors
79
+ lookup: max -10m every 1m of read_errs
80
+ warn: $this > 0
81
+ delay: up 1m down 15m multiplier 1.5 max 1h
82
+ info: number of encountered BTRFS read errors
83
+ to: sysadmin
84
+
85
+ template: btrfs_device_write_errors
86
+ on: btrfs.device_errors
87
+ class: Errors
88
+ type: System
89
+component: File system
90
+ os: *
91
+ hosts: *
92
+ families: *
93
+ units: errors
94
+ lookup: max -10m every 1m of write_errs
95
+ warn: $this > 0
96
+ delay: up 1m down 15m multiplier 1.5 max 1h
97
+ info: number of encountered BTRFS write errors
98
+ to: sysadmin
99
+
100
+ template: btrfs_device_flush_errors
101
+ on: btrfs.device_errors
102
+ class: Errors
103
+ type: System
104
+component: File system
105
+ os: *
106
+ hosts: *
107
+ families: *
108
+ units: errors
109
+ lookup: max -10m every 1m of flush_errs
110
+ warn: $this > 0
111
+ delay: up 1m down 15m multiplier 1.5 max 1h
112
+ info: number of encountered BTRFS flush errors
113
+ to: sysadmin
114
+
115
+ template: btrfs_device_corruption_errors
116
+ on: btrfs.device_errors
117
+ class: Errors
118
+ type: System
119
+component: File system
120
+ os: *
121
+ hosts: *
122
+ families: *
123
+ units: errors
124
+ lookup: max -10m every 1m of corruption_errs
125
+ warn: $this > 0
126
+ delay: up 1m down 15m multiplier 1.5 max 1h
127
+ info: number of encountered BTRFS corruption errors
128
+ to: sysadmin
129
+
130
+ template: btrfs_device_generation_errors
131
+ on: btrfs.device_errors
132
+ class: Errors
133
+ type: System
134
+component: File system
135
+ os: *
136
+ hosts: *
137
+ families: *
138
+ units: errors
139
+ lookup: max -10m every 1m of generation_errs
140
+ warn: $this > 0
141
+ delay: up 1m down 15m multiplier 1.5 max 1h
142
+ info: number of encountered BTRFS generation errors
143
+ to: sysadmin
web/gui/dashboard_info.js
+16
@@ -6445,6 +6445,22 @@ netdataDashboard.context = {
6445
info: 'Logical disk usage for BTRFS system. System chunks store information about the allocation of other chunks. The disk space reported here is the usable allocation (i.e. after any striping or replication). The values reported here should be relatively small compared to Data and Metadata, and will scale with the volume size and overall space usage.'
6446
},
6447
6448
+ 'btrfs.commits': {
6449
+ info: 'Tracks filesystem wide commits. Commits mark fully consistent synchronization points for the filesystem, and are triggered automatically when certain events happen or when enough time has elapsed since the last commit.'
6450
+ },
6451
+
6452
+ 'btrfs.commits_perc_time': {
6453
+ info: 'Tracks commits time share. The reported time share metrics are valid only when BTRFS commit interval is longer than Netdata\'s <b>update_every</b> interval.'
6454
+ },
6455
+
6456
+ 'btrfs.commit_timings': {
6457
+ info: 'Tracks timing information for commits. <b>last</b> dimension metrics are valid only when BTRFS commit interval is longer than Netdata\'s <b>update_every</b> interval.'
6458
+ },
6459
+
6460
+ 'btrfs.device_errors': {
6461
+ info: 'Tracks per-device error counts. Five types of errors are tracked: read errors, write errors, flush errors, corruption errors, and generation errors. <b>Read</b>, <b>write</b>, and <b>flush</b> are errors reported by the underlying block device when trying to perform the associated operations on behalf of BTRFS. <b>Corruption</b> errors count checksum mismatches, which usually are a result of either at-rest data corruption or hardware problems. <b>Generation</b> errors count generational mismatches within the internal data structures of the volume, and are also usually indicative of at-rest data corruption or hardware problems. Note that errors reported here may not trigger an associated IO error in userspace, as BTRFS has relatively robust error recovery that allows it to return correct data in most multi-device setups.'
6462
+ },
6463
+
6464
// ------------------------------------------------------------------------
6465
// RabbitMQ
6466