eBPF NFS monitoring (#11313)
Add new filesystem monitoring.
thiagoftsm committed
Jul 8, 2021 at 17:50 UTC
ea9721f83c8ecd7d501efa505a95a5cb6557ec33
7 files changed
+56
-24
collectors/ebpf.plugin/README.md
+3
@@ -327,9 +327,12 @@ filesystems are monitored.
327
```conf
328
[filesystem]
329
ext4dist = yes
330
+ nfsdist = yes
331
xfsdist = yes
332
```
333
334
+The ebpf program `nfsdist` monitors only `nfs` mount points.
335
+
336
## Troubleshooting
337
338
If the eBPF collector does not work, you can troubleshoot it by running the `ebpf.plugin` command and investigating its
collectors/ebpf.plugin/ebpf.d/filesystem.conf
+1
@@ -14,4 +14,5 @@
14
# All filesystems are named as 'NAMEdist' where NAME is the filesystem name while 'dist' is a reference for distribution.
15
[filesystem]
16
ext4dist = yes
17
+ nfsdist = yes
18
xfsdist = yes
collectors/ebpf.plugin/ebpf_filesystem.c
+28
-18
@@ -10,6 +10,7 @@ struct config fs_config = { .first_section = NULL,
10
11
ebpf_filesystem_partitions_t localfs[] =
12
{{.filesystem = "ext4",
13
+ .optional_filesystem = NULL,
14
.family = "EXT4",
15
.objects = NULL,
16
.probe_links = NULL,
@@ -17,13 +18,23 @@ ebpf_filesystem_partitions_t localfs[] =
18
.enabled = CONFIG_BOOLEAN_YES,
19
.addresses = {.function = NULL, .addr = 0}},
20
{.filesystem = "xfs",
21
+ .optional_filesystem = NULL,
22
.family = "XFS",
23
.objects = NULL,
24
.probe_links = NULL,
25
.flags = NETDATA_FILESYSTEM_FLAG_NO_PARTITION,
26
.enabled = CONFIG_BOOLEAN_YES,
27
.addresses = {.function = NULL, .addr = 0}},
28
+ {.filesystem = "nfs",
29
+ .optional_filesystem = "nfs4",
30
+ .family = "NFS",
31
+ .objects = NULL,
32
+ .probe_links = NULL,
33
+ .flags = NETDATA_FILESYSTEM_ATTR_CHARTS,
34
+ .enabled = CONFIG_BOOLEAN_YES,
35
+ .addresses = {.function = NULL, .addr = 0}},
36
{.filesystem = NULL,
37
+ .optional_filesystem = NULL,
38
.family = NULL,
39
.objects = NULL,
40
.probe_links = NULL,
@@ -73,15 +84,13 @@ static void ebpf_obsolete_fs_charts()
84
EBPF_COMMON_DIMENSION_CALL, efp->family_name,
85
NULL, NETDATA_EBPF_CHART_TYPE_STACKED, efp->hwrite.order);
86
76
- ebpf_write_chart_obsolete(NETDATA_FILESYSTEM_FAMILY, efp->hopen.name,
77
- efp->hopen.title,
87
+ ebpf_write_chart_obsolete(NETDATA_FILESYSTEM_FAMILY, efp->hopen.name, efp->hopen.title,
88
EBPF_COMMON_DIMENSION_CALL, efp->family_name,
89
NULL, NETDATA_EBPF_CHART_TYPE_STACKED, efp->hopen.order);
90
81
- ebpf_write_chart_obsolete(NETDATA_FILESYSTEM_FAMILY, efp->hsync.name,
82
- efp->hsync.title,
91
+ ebpf_write_chart_obsolete(NETDATA_FILESYSTEM_FAMILY, efp->hadditional.name, efp->hadditional.title,
92
EBPF_COMMON_DIMENSION_CALL, efp->family_name,
84
- NULL, NETDATA_EBPF_CHART_TYPE_STACKED, efp->hsync.order);
93
+ NULL, NETDATA_EBPF_CHART_TYPE_STACKED, efp->hadditional.order);
94
}
95
efp->flags = flags;
96
}
@@ -141,13 +150,13 @@ static void ebpf_create_fs_charts()
150
filesystem_publish_aggregated, NETDATA_EBPF_HIST_MAX_BINS);
151
order++;
152
144
- snprintfz(title, 255, "%s latency for each sync request.", efp->filesystem);
145
- snprintfz(chart_name, 63, "%s_sync_latency", efp->filesystem);
146
- efp->hsync.name = strdupz(chart_name);
147
- efp->hsync.title = strdupz(title);
148
- efp->hsync.order = order;
149
- ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY, efp->hsync.name,
150
- title,
153
+ char *type = (efp->flags & NETDATA_FILESYSTEM_ATTR_CHARTS) ? "attribute" : "sync";
154
+ snprintfz(title, 255, "%s latency for each %s request.", efp->filesystem, type);
155
+ snprintfz(chart_name, 63, "%s_%s_latency", efp->filesystem, type);
156
+ efp->hadditional.name = strdupz(chart_name);
157
+ efp->hadditional.title = strdupz(title);
158
+ efp->hadditional.order = order;
159
+ ebpf_create_chart(NETDATA_FILESYSTEM_FAMILY, efp->hadditional.name, title,
160
EBPF_COMMON_DIMENSION_CALL, family,
161
NULL, NETDATA_EBPF_CHART_TYPE_STACKED, order, ebpf_create_global_dimension,
162
filesystem_publish_aggregated, NETDATA_EBPF_HIST_MAX_BINS);
@@ -244,7 +253,8 @@ static int ebpf_read_local_partitions()
253
254
for (i = 0; localfs[i].filesystem; i++) {
255
ebpf_filesystem_partitions_t *w = &localfs[i];
247
- if (w->enabled && !strcmp(fs, w->filesystem)) {
256
+ if (w->enabled && (!strcmp(fs, w->filesystem) ||
257
+ (w->optional_filesystem && !strcmp(fs, w->optional_filesystem)))) {
258
localfs[i].flags |= NETDATA_FILESYSTEM_LOAD_EBPF_PROGRAM;
259
localfs[i].flags &= ~NETDATA_FILESYSTEM_REMOVE_CHARTS;
260
count++;
@@ -313,8 +323,8 @@ void ebpf_filesystem_cleanup_ebpf_data()
323
freez(efp->hopen.name);
324
freez(efp->hopen.title);
325
316
- freez(efp->hsync.name);
317
- freez(efp->hsync.title);
326
+ freez(efp->hadditional.name);
327
+ freez(efp->hadditional.title);
328
329
struct bpf_link **probe_links = efp->probe_links;
330
size_t j = 0 ;
@@ -385,7 +395,7 @@ static inline netdata_ebpf_histogram_t *select_hist(ebpf_filesystem_partitions_t
395
return &efp->hopen;
396
} else if (id < NETDATA_KEY_CALLS_SYNC ){
397
*idx = id - NETDATA_KEY_CALLS_OPEN;
388
- return &efp->hsync;
398
+ return &efp->hadditional;
399
}
400
401
return NULL;
@@ -503,8 +513,8 @@ static void ebpf_histogram_send_data()
513
write_histogram_chart(NETDATA_FILESYSTEM_FAMILY, efp->hopen.name,
514
efp->hopen.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS);
515
506
- write_histogram_chart(NETDATA_FILESYSTEM_FAMILY, efp->hsync.name,
507
- efp->hsync.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS);
516
+ write_histogram_chart(NETDATA_FILESYSTEM_FAMILY, efp->hadditional.name,
517
+ efp->hadditional.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS);
518
}
519
}
520
}
collectors/ebpf.plugin/ebpf_filesystem.h
+4
-2
@@ -28,7 +28,8 @@ enum netdata_filesystem_flags {
28
NETDATA_FILESYSTEM_FLAG_HAS_PARTITION = 2,
29
NETDATA_FILESYSTEM_FLAG_CHART_CREATED = 4,
30
NETDATA_FILESYSTEM_FILL_ADDRESS_TABLE = 8,
31
- NETDATA_FILESYSTEM_REMOVE_CHARTS = 16
31
+ NETDATA_FILESYSTEM_REMOVE_CHARTS = 16,
32
+ NETDATA_FILESYSTEM_ATTR_CHARTS = 32
33
};
34
35
enum netdata_filesystem_table {
@@ -38,6 +39,7 @@ enum netdata_filesystem_table {
39
40
typedef struct ebpf_filesystem_partitions {
41
char *filesystem;
42
+ char *optional_filesystem;
43
char *family;
44
char *family_name;
45
struct bpf_object *objects;
@@ -46,7 +48,7 @@ typedef struct ebpf_filesystem_partitions {
48
netdata_ebpf_histogram_t hread;
49
netdata_ebpf_histogram_t hwrite;
50
netdata_ebpf_histogram_t hopen;
49
- netdata_ebpf_histogram_t hsync;
51
+ netdata_ebpf_histogram_t hadditional;
52
53
uint32_t flags;
54
uint32_t enabled;
packaging/ebpf.checksums
+3
-3
@@ -1,3 +1,3 @@
1
-1e2441471f2f399776e1b61fa0e55df6937e4e120a003bda73b510a2b99df583 netdata-kernel-collector-glibc-v0.7.2.4.tar.xz
2
-76f7c342e2eb50d306b6a12d42be32f42ae74fe36ea48d0b5021973aae9be666 netdata-kernel-collector-musl-v0.7.2.4.tar.xz
3
-8a57cec811512c0fc9f8c06bcf2e4c01553d7fdd0a8123c578fd0372f5ea768f netdata-kernel-collector-static-v0.7.2.4.tar.xz
1
+131825cfef1880b7264d8fde815026fc9326b8e1dffb515d2ac3f8e3eef9af29 netdata-kernel-collector-glibc-v0.7.3.tar.xz
2
+09d70c4000897c1ec99d41841abddf613f20c39b6e822efc838ffef3c56dda8b netdata-kernel-collector-musl-v0.7.3.tar.xz
3
+b80dfe18288103c01d5222cf122ac81d1f5a8f60f8b3f61fc86c18d7176d5f61 netdata-kernel-collector-static-v0.7.3.tar.xz
packaging/ebpf.version
+1
-1
@@ -1 +1 @@
1
-v0.7.2.4
1
+v0.7.3
web/gui/dashboard_info.js
+16
@@ -3515,6 +3515,22 @@ netdataDashboard.context = {
3515
info: 'Latency is the time it takes for an event to be completed. Netdata is attaching a kprobe for when the function <code>xfs_file_sync</code> is called and another for when it finishes the execution. We calculate the difference between the calling and return times, we get the logarithmic for the final result and we sum one value to the respective bin. Based on the eBPF <a href="https://github.com/iovisor/bcc/blob/master/tools/xfsdist_example.txt" target="_blank">xfsdist</a> from BCC tools.'
3516
},
3517
3518
+ 'filesystem.nfs_read_latency': {
3519
+ info: 'Latency is the time it takes for an event to be completed. Netdata is attaching a kprobe for when the function <code>nfs_file_read</code> is called and another for when it finishes the execution. We calculate the difference between the calling and return times, we get the logarithmic for the final result and we sum one value to the respective bin. Based on the eBPF <a href="https://github.com/iovisor/bcc/blob/master/tools/nfsdist_example.txt" target="_blank">nfsdist</a> from BCC tools.'
3520
+ },
3521
+
3522
+ 'filesystem.nfs_write_latency': {
3523
+ info: 'Latency is the time it takes for an event to be completed. Netdata is attaching a kprobe for when the function <code>nfs_file_write</code> is called and another for when it finishes the execution. We calculate the difference between the calling and return times, we get the logarithmic for the final result and we sum one value to the respective bin. Based on the eBPF <a href="https://github.com/iovisor/bcc/blob/master/tools/nfsdist_example.txt" target="_blank">nfsdist</a> from BCC tools.'
3524
+ },
3525
+
3526
+ 'filesystem.nfs_open_latency': {
3527
+ info: 'Latency is the time it takes for an event to be completed. Netdata is attaching a kprobe for when functions <code>nfs_file_open</code> and <code>nfs4_file_open</code> are called and another for when they finish the execution. We calculate the difference between the calling and return times, we get the logarithmic for the final result and we sum one value to the respective bin. Based on the eBPF <a href="https://github.com/iovisor/bcc/blob/master/tools/nfsdist_example.txt" target="_blank">nfsdist</a> from BCC tools.'
3528
+ },
3529
+
3530
+ 'filesystem.nfs_attribute_latency': {
3531
+ info: 'Latency is the time it takes for an event to be completed. Netdata is attaching a kprobe for when the function <code>nfs_getattr</code> is called and another for when it finishes the execution. We calculate the difference between the calling and return times, we get the logarithmic for the final result and we sum one value to the respective bin. Based on the eBPF <a href="https://github.com/iovisor/bcc/blob/master/tools/nfsdist_example.txt" target="_blank">nfsdist</a> from BCC tools.'
3532
+ },
3533
+
3534
// ------------------------------------------------------------------------
3535
// eBPF
3536