@cryptotaxi247 / netdata-1 / commits / 37686c8d6

Ebpf disk latency (#11276)

Add disk monitoring independent of filesystem.

thiagoftsm committed Jul 2, 2021 at 12:28 UTC 37686c8d682dc5235f8c1b790f484242fc194401
19 files changed +1145 -57
CMakeLists.txt
+2
@@ -490,6 +490,8 @@ set(EBPF_PROCESS_PLUGIN_FILES
490 collectors/ebpf.plugin/ebpf_cachestat.h
491 collectors/ebpf.plugin/ebpf_dcstat.c
492 collectors/ebpf.plugin/ebpf_dcstat.h
493 + collectors/ebpf.plugin/ebpf_disk.c
494 + collectors/ebpf.plugin/ebpf_disk.h
495 collectors/ebpf.plugin/ebpf_filesystem.c
496 collectors/ebpf.plugin/ebpf_filesystem.h
497 collectors/ebpf.plugin/ebpf_process.c
Makefile.am
+2
@@ -289,6 +289,8 @@ EBPF_PLUGIN_FILES = \
289 collectors/ebpf.plugin/ebpf_cachestat.h \
290 collectors/ebpf.plugin/ebpf_dcstat.c \
291 collectors/ebpf.plugin/ebpf_dcstat.h \
292 + collectors/ebpf.plugin/ebpf_disk.c \
293 + collectors/ebpf.plugin/ebpf_disk.h \
294 collectors/ebpf.plugin/ebpf_filesystem.c \
295 collectors/ebpf.plugin/ebpf_filesystem.h \
296 collectors/ebpf.plugin/ebpf_process.c \
collectors/all.h
+1
@@ -116,6 +116,7 @@
116 #define NETDATA_CHART_PRIO_DISK_SVCTM 2070
117 #define NETDATA_CHART_PRIO_DISK_MOPS 2080
118 #define NETDATA_CHART_PRIO_DISK_IOTIME 2090
119 +#define NETDATA_CHART_PRIO_DISK_LATENCY 2095
120 #define NETDATA_CHART_PRIO_BCACHE_CACHE_ALLOC 2120
121 #define NETDATA_CHART_PRIO_BCACHE_HIT_RATIO 2120
122 #define NETDATA_CHART_PRIO_BCACHE_RATES 2121
collectors/ebpf.plugin/Makefile.am
+1
@@ -34,6 +34,7 @@ dist_ebpfconfig_DATA = \
34 ebpf.d/ebpf_kernel_reject_list.txt \
35 ebpf.d/cachestat.conf \
36 ebpf.d/dcstat.conf \
37 + ebpf.d/disk.conf \
38 ebpf.d/filesystem.conf \
39 ebpf.d/network.conf \
40 ebpf.d/process.conf \
collectors/ebpf.plugin/README.md
+2
@@ -216,6 +216,7 @@ The eBPF collector enables and runs the following eBPF programs by default:
216 - `dcstat` : This eBPF program creates charts that show information about file access using directory cache. It appends
217 `kprobes` for `lookup_fast()` and `d_lookup()` to identify if files are inside directory cache, outside and
218 files are not found.
219 +- `disk` : This eBPF program creates charts that show information about disk latency independent of filesystem.
220 - `filesystem`: This eBPF program creates charts that show latency information for selected filesystem.
221 - `process`: This eBPF program creates charts that show information about process creation, calls to open files.
222 When in `return` mode, it also creates charts showing errors when these operations are executed.
@@ -240,6 +241,7 @@ The following configuration files are available:
241
242 - `cachestat.conf`: Configuration for the `cachestat` thread.
243 - `dcstat.conf`: Configuration for the `dcstat` thread.
244 +- `disk.conf`: Configuration for the `disk` thread.
245 - `filesystem.conf`: Configuration for the `filesystem` thread.
246 - `process.conf`: Configuration for the `process` thread.
247 - `network.conf`: Configuration for the `network viewer` thread. This config file overwrites the global options and
collectors/ebpf.plugin/ebpf.c
+50
@@ -115,6 +115,11 @@ ebpf_module_t ebpf_modules[] = {
115 .optional = 0, .apps_routine = NULL, .maps = NULL,
116 .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &fs_config,
117 .config_file = NETDATA_SYNC_CONFIG_FILE},
118 + { .thread_name = "disk", .config_name = "disk", .enabled = 0, .start_routine = ebpf_disk_thread,
119 + .update_time = 1, .global_charts = 1, .apps_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY,
120 + .optional = 0, .apps_routine = NULL, .maps = NULL,
121 + .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &disk_config,
122 + .config_file = NETDATA_SYNC_CONFIG_FILE},
123 { .thread_name = NULL, .enabled = 0, .start_routine = NULL, .update_time = 1,
124 .global_charts = 0, .apps_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY,
125 .optional = 0, .apps_routine = NULL, .maps = NULL, .pid_map_size = 0, .names = NULL,
@@ -509,6 +514,31 @@ void ebpf_create_charts_on_apps(char *id, char *title, char *units, char *family
514 }
515 }
516
517 +/**
518 + * Call the necessary functions to create a name.
519 + *
520 + * @param family family name
521 + * @param name chart name
522 + * @param hist0 histogram values
523 + * @param dimensions dimension values.
524 + * @param end number of bins that will be sent to Netdata.
525 + *
526 + * @return It returns a variable tha maps the charts that did not have zero values.
527 + */
528 +void write_histogram_chart(char *family, char *name, const netdata_idx_t *hist, char **dimensions, uint32_t end)
529 +{
530 + write_begin_chart(family, name);
531 +
532 + uint32_t i;
533 + for (i = 0; i < end; i++) {
534 + write_chart_dimension(dimensions[i], (long long) hist[i]);
535 + }
536 +
537 + write_end_chart();
538 +
539 + fflush(stdout);
540 +}
541 +
542 /*****************************************************************
543 *
544 * FUNCTIONS TO DEFINE OPTIONS
@@ -658,6 +688,8 @@ void ebpf_print_help()
688 "\n"
689 " --dcstat or -d Enable charts related to directory cache.\n"
690 "\n"
691 + " --disk or -k Enable charts related to disk monitoring.\n"
692 + "\n"
693 " --filesystem or -i Enable chart related to filesystem run time.\n"
694 "\n"
695 " --net or -n Enable network viewer charts.\n"
@@ -982,6 +1014,13 @@ static void read_collector_values(int *disable_apps)
1014 started++;
1015 }
1016
1017 + enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "disk",
1018 + CONFIG_BOOLEAN_NO);
1019 + if (enabled) {
1020 + ebpf_enable_chart(EBPF_MODULE_DISK_IDX, *disable_apps);
1021 + started++;
1022 + }
1023 +
1024 if (!started){
1025 ebpf_enable_all_charts(*disable_apps);
1026 // Read network viewer section
@@ -1066,6 +1105,7 @@ static void parse_args(int argc, char **argv)
1105 {"all", no_argument, 0, 'a' },
1106 {"cachestat", no_argument, 0, 'c' },
1107 {"dcstat", no_argument, 0, 'd' },
1108 + {"disk", no_argument, 0, 'k' },
1109 {"filesystem", no_argument, 0, 'i' },
1110 {"net", no_argument, 0, 'n' },
1111 {"process", no_argument, 0, 'p' },
@@ -1139,6 +1179,14 @@ static void parse_args(int argc, char **argv)
1179 ebpf_enable_chart(EBPF_MODULE_FILESYSTEM_IDX, disable_apps);
1180 #ifdef NETDATA_INTERNAL_CHECKS
1181 info("EBPF enabling \"filesystem\" chart, because it was started with the option \"--filesystem\" or \"-i\".");
1182 +#endif
1183 + break;
1184 + }
1185 + case 'k': {
1186 + enabled = 1;
1187 + ebpf_enable_chart(EBPF_MODULE_DISK_IDX, disable_apps);
1188 +#ifdef NETDATA_INTERNAL_CHECKS
1189 + info("EBPF enabling \"disk\" chart, because it was started with the option \"--disk\" or \"-k\".");
1190 #endif
1191 break;
1192 }
@@ -1464,6 +1512,8 @@ int main(int argc, char **argv)
1512 NULL, NULL, ebpf_modules[EBPF_MODULE_VFS_IDX].start_routine},
1513 {"EBPF FILESYSTEM" , NULL, NULL, 1,
1514 NULL, NULL, ebpf_modules[EBPF_MODULE_FILESYSTEM_IDX].start_routine},
1515 + {"EBPF DISK" , NULL, NULL, 1,
1516 + NULL, NULL, ebpf_modules[EBPF_MODULE_DISK_IDX].start_routine},
1517 {NULL , NULL, NULL, 0,
1518 NULL, NULL, NULL}
1519 };
collectors/ebpf.plugin/ebpf.d.conf
+3
@@ -26,6 +26,8 @@
26 # The eBPF collector enables and runs the following eBPF programs by default:
27 #
28 # `cachestat` : Make charts for kernel functions related to page cache.
29 +# `dcstat` : Make charts for kernel functions related to directory cache.
30 +# `disk` : Monitor I/O latencies for disks
31 # `filesystem`: Monitor calls for functions used to manipulate specific filesystems
32 # `process` : This eBPF program creates charts that show information about process creation, and file manipulation.
33 # `socket` : This eBPF program creates charts with information about `TCP` and `UDP` functions, including the
@@ -37,6 +39,7 @@
39 [ebpf programs]
40 cachestat = no
41 dcstat = no
42 + disk = no
43 filesystem = no
44 process = yes
45 socket = yes
collectors/ebpf.plugin/ebpf.d/disk.conf new
+13
@@ -0,0 +1,13 @@
1 +# The `ebpf load mode` option accepts the following values :
2 +# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
3 +# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
4 +# new charts for the return of these functions, such as errors.
5 +#
6 +# The eBPF collector also creates charts for each running application through an integration with the `apps plugin`.
7 +# If you want to disable the integration with `apps.plugin` along with the above charts, change the setting `apps` to
8 +# 'no'.
9 +#
10 +[global]
11 + ebpf load mode = entry
12 + update every = 2
13 +
collectors/ebpf.plugin/ebpf.h
+3 -1
@@ -81,7 +81,8 @@ enum ebpf_module_indexes {
81 EBPF_MODULE_DCSTAT_IDX,
82 EBPF_MODULE_SWAP_IDX,
83 EBPF_MODULE_VFS_IDX,
84 - EBPF_MODULE_FILESYSTEM_IDX
84 + EBPF_MODULE_FILESYSTEM_IDX,
85 + EBPF_MODULE_DISK_IDX
86 };
87
88 // Copied from musl header
@@ -226,6 +227,7 @@ extern collected_number get_value_from_structure(char *basis, size_t offset);
227 extern void ebpf_update_pid_table(ebpf_local_maps_t *pid, ebpf_module_t *em);
228 extern void ebpf_write_chart_obsolete(char *type, char *id, char *title, char *units, char *family,
229 char *charttype, char *context, int order);
230 +extern void write_histogram_chart(char *family, char *name, const netdata_idx_t *hist, char **dimensions, uint32_t end);
231
232 #define EBPF_MAX_SYNCHRONIZATION_TIME 300
233
collectors/ebpf.plugin/ebpf_apps.h
+1
@@ -20,6 +20,7 @@
20
21 #include "ebpf_process.h"
22 #include "ebpf_dcstat.h"
23 +#include "ebpf_disk.h"
24 #include "ebpf_filesystem.h"
25 #include "ebpf_cachestat.h"
26 #include "ebpf_sync.h"
collectors/ebpf.plugin/ebpf_disk.c new
+837
@@ -0,0 +1,837 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include <sys/resource.h>
4 +#include <stdlib.h>
5 +
6 +#include "ebpf.h"
7 +#include "ebpf_disk.h"
8 +
9 +struct config disk_config = { .first_section = NULL,
10 + .last_section = NULL,
11 + .mutex = NETDATA_MUTEX_INITIALIZER,
12 + .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
13 + .rwlock = AVL_LOCK_INITIALIZER } };
14 +
15 +static ebpf_local_maps_t disk_maps[] = {{.name = "tbl_disk_iocall", .internal_input = NETDATA_DISK_HISTOGRAM_LENGTH,
16 + .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
17 + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED},
18 + {.name = NULL, .internal_input = 0, .user_input = 0,
19 + .type = NETDATA_EBPF_MAP_CONTROLLER,
20 + .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED}};
21 +static ebpf_data_t disk_data;
22 +
23 +static avl_tree_lock disk_tree;
24 +netdata_ebpf_disks_t *disk_list = NULL;
25 +
26 +char *tracepoint_block_type = { "block"} ;
27 +char *tracepoint_block_issue = { "block_rq_issue" };
28 +char *tracepoint_block_rq_complete = { "block_rq_complete" };
29 +
30 +static struct bpf_link **probe_links = NULL;
31 +static struct bpf_object *objects = NULL;
32 +
33 +static int was_block_issue_enabled = 0;
34 +static int was_block_rq_complete_enabled = 0;
35 +
36 +static char **dimensions = NULL;
37 +static netdata_syscall_stat_t disk_aggregated_data[NETDATA_EBPF_HIST_MAX_BINS];
38 +static netdata_publish_syscall_t disk_publish_aggregated[NETDATA_EBPF_HIST_MAX_BINS];
39 +
40 +static int read_thread_closed = 1;
41 +
42 +static netdata_idx_t *disk_hash_values = NULL;
43 +static struct netdata_static_thread disk_threads = {"DISK KERNEL",
44 + NULL, NULL, 1, NULL,
45 + NULL, NULL };
46 +
47 +ebpf_publish_disk_t *plot_disks = NULL;
48 +pthread_mutex_t plot_mutex;
49 +
50 +/*****************************************************************
51 + *
52 + * FUNCTIONS TO MANIPULATE HARD DISKS
53 + *
54 + *****************************************************************/
55 +
56 +/**
57 + * Parse start
58 + *
59 + * Parse start address of disk
60 + *
61 + * @param w structure where data is stored
62 + * @param filename variable used to store value
63 + *
64 + * @return It returns 0 on success and -1 otherwise
65 + */
66 +static inline int ebpf_disk_parse_start(netdata_ebpf_disks_t *w, char *filename)
67 +{
68 + char content[FILENAME_MAX + 1];
69 + int fd = open(filename, O_RDONLY, 0);
70 + if (fd < 0) {
71 + return -1;
72 + }
73 +
74 + ssize_t file_length = read(fd, content, 4095);
75 + if (file_length > 0) {
76 + if (file_length > FILENAME_MAX)
77 + file_length = FILENAME_MAX;
78 +
79 + content[file_length] = '\0';
80 + w->start = strtoul(content, NULL, 10);
81 + }
82 + close(fd);
83 +
84 + return 0;
85 +}
86 +
87 +/**
88 + * Parse uevent
89 + *
90 + * Parse uevent file
91 + *
92 + * @param w structure where data is stored
93 + * @param filename variable used to store value
94 + *
95 + * @return It returns 0 on success and -1 otherwise
96 + */
97 +static inline int ebpf_parse_uevent(netdata_ebpf_disks_t *w, char *filename)
98 +{
99 + char content[FILENAME_MAX + 1];
100 + int fd = open(filename, O_RDONLY, 0);
101 + if (fd < 0) {
102 + return -1;
103 + }
104 +
105 + ssize_t file_length = read(fd, content, FILENAME_MAX);
106 + if (file_length > 0) {
107 + if (file_length > FILENAME_MAX)
108 + file_length = FILENAME_MAX;
109 +
110 + content[file_length] = '\0';
111 +
112 + char *s = strstr(content, "PARTNAME=EFI");
113 + if (s) {
114 + w->main->boot_partition = w;
115 + w->flags |= NETDATA_DISK_HAS_EFI;
116 + w->boot_chart = strdupz("disk_bootsector");
117 + }
118 + }
119 + close(fd);
120 +
121 + return 0;
122 +}
123 +
124 +/**
125 + * Parse Size
126 + *
127 + * @param w structure where data is stored
128 + * @param filename variable used to store value
129 + *
130 + * @return It returns 0 on success and -1 otherwise
131 + */
132 +static inline int ebpf_parse_size(netdata_ebpf_disks_t *w, char *filename)
133 +{
134 + char content[FILENAME_MAX + 1];
135 + int fd = open(filename, O_RDONLY, 0);
136 + if (fd < 0) {
137 + return -1;
138 + }
139 +
140 + ssize_t file_length = read(fd, content, FILENAME_MAX);
141 + if (file_length > 0) {
142 + if (file_length > FILENAME_MAX)
143 + file_length = FILENAME_MAX;
144 +
145 + content[file_length] = '\0';
146 + w->end = w->start + strtoul(content, NULL, 10) -1;
147 + }
148 + close(fd);
149 +
150 + return 0;
151 +}
152 +
153 +/**
154 + * Read Disk information
155 + *
156 + * Read disk information from /sys/block
157 + *
158 + * @param w structure where data is stored
159 + * @param name disk name
160 + */
161 +static void ebpf_read_disk_info(netdata_ebpf_disks_t *w, char *name)
162 +{
163 + static netdata_ebpf_disks_t *main_disk = NULL;
164 + static uint32_t key = 0;
165 + char *path = { "/sys/block" };
166 + char disk[NETDATA_DISK_NAME_LEN + 1];
167 + char filename[FILENAME_MAX + 1];
168 + snprintfz(disk, NETDATA_DISK_NAME_LEN, "%s", name);
169 + size_t length = strlen(disk);
170 + if (!length) {
171 + return;
172 + }
173 +
174 + length--;
175 + size_t curr = length;
176 + while (isdigit((int)disk[length])) {
177 + disk[length--] = '\0';
178 + }
179 +
180 + // We are looking for partition information, if it is a device we will ignore it.
181 + if (curr == length) {
182 + main_disk = w;
183 + key = MKDEV(w->major, w->minor);
184 + w->bootsector_key = key;
185 + return;
186 + }
187 + w->bootsector_key = key;
188 + w->main = main_disk;
189 +
190 + snprintfz(filename, FILENAME_MAX, "%s/%s/%s/uevent", path, disk, name);
191 + if (ebpf_parse_uevent(w, filename))
192 + return;
193 +
194 + snprintfz(filename, FILENAME_MAX, "%s/%s/%s/start", path, disk, name);
195 + if (ebpf_disk_parse_start(w, filename))
196 + return;
197 +
198 + snprintfz(filename, FILENAME_MAX, "%s/%s/%s/size", path, disk, name);
199 + ebpf_parse_size(w, filename);
200 +}
201 +
202 +/**
203 + * New encode dev
204 + *
205 + * New encode algorithm extracted from https://elixir.bootlin.com/linux/v5.10.8/source/include/linux/kdev_t.h#L39
206 + *
207 + * @param major driver major number
208 + * @param minor driver minor number
209 + *
210 + * @return
211 + */
212 +static inline uint32_t netdata_new_encode_dev(uint32_t major, uint32_t minor) {
213 + return (minor & 0xff) | (major << 8) | ((minor & ~0xff) << 12);
214 +}
215 +
216 +/**
217 + * Compare disks
218 + *
219 + * Compare major and minor values to add disks to tree.
220 + *
221 + * @param a pointer to netdata_ebpf_disks
222 + * @param b pointer to netdata_ebpf_disks
223 + *
224 + * @return It returns 0 case the values are equal, 1 case a is bigger than b and -1 case a is smaller than b.
225 +*/
226 +static int ebpf_compare_disks(void *a, void *b)
227 +{
228 + netdata_ebpf_disks_t *ptr1 = a;
229 + netdata_ebpf_disks_t *ptr2 = b;
230 +
231 + if (ptr1->dev > ptr2->dev)
232 + return 1;
233 + if (ptr1->dev < ptr2->dev)
234 + return -1;
235 +
236 + return 0;
237 +}
238 +
239 +/**
240 + * Update listen table
241 + *
242 + * Update link list when it is necessary.
243 + *
244 + * @param name disk name
245 + * @param major major disk identifier
246 + * @param minor minor disk identifier
247 + * @param current_time current timestamp
248 + */
249 +static void update_disk_table(char *name, int major, int minor, time_t current_time)
250 +{
251 + netdata_ebpf_disks_t find;
252 + netdata_ebpf_disks_t *w;
253 + size_t length;
254 +
255 + uint32_t dev = netdata_new_encode_dev(major, minor);
256 + find.dev = dev;
257 + netdata_ebpf_disks_t *ret = (netdata_ebpf_disks_t *) avl_search_lock(&disk_tree, (avl_t *)&find);
258 + if (ret) { // Disk is already present
259 + ret->flags |= NETDATA_DISK_IS_HERE;
260 + ret->last_update = current_time;
261 + return;
262 + }
263 +
264 + netdata_ebpf_disks_t *update_next = disk_list;
265 + if (likely(disk_list)) {
266 + netdata_ebpf_disks_t *move = disk_list;
267 + while (move) {
268 + if (dev == move->dev)
269 + return;
270 +
271 + update_next = move;
272 + move = move->next;
273 + }
274 +
275 + w = callocz(1, sizeof(netdata_ebpf_disks_t));
276 + length = strlen(name);
277 + if (length >= NETDATA_DISK_NAME_LEN)
278 + length = NETDATA_DISK_NAME_LEN;
279 +
280 + strncpy(w->family, name, length);
281 + w->family[length] = '\0';
282 + w->major = major;
283 + w->minor = minor;
284 + w->dev = netdata_new_encode_dev(major, minor);
285 + update_next->next = w;
286 + } else {
287 + disk_list = callocz(1, sizeof(netdata_ebpf_disks_t));
288 + length = strlen(name);
289 + if (length >= NETDATA_DISK_NAME_LEN)
290 + length = NETDATA_DISK_NAME_LEN;
291 +
292 + strncpy(disk_list->family, name, length);
293 + disk_list->family[length] = '\0';
294 + disk_list->major = major;
295 + disk_list->minor = minor;
296 + disk_list->dev = netdata_new_encode_dev(major, minor);
297 +
298 + w = disk_list;
299 + }
300 +
301 + ebpf_read_disk_info(w, name);
302 +
303 + netdata_ebpf_disks_t *check;
304 + check = (netdata_ebpf_disks_t *) avl_insert_lock(&disk_tree, (avl_t *)w);
305 + if (check != w)
306 + error("Internal error, cannot insert the AVL tree.");
307 +
308 +#ifdef NETDATA_INTERNAL_CHECKS
309 + info("The Latency is monitoring the hard disk %s (Major = %d, Minor = %d, Device = %u)", name, major, minor,w->dev);
310 +#endif
311 +
312 + w->flags |= NETDATA_DISK_IS_HERE;
313 +}
314 +
315 +/**
316 + * Read Local Disks
317 + *
318 + * Parse /proc/partitions to get block disks used to measure latency.
319 + *
320 + * @return It returns 0 on success and -1 otherwise
321 + */
322 +static int read_local_disks()
323 +{
324 + char filename[FILENAME_MAX + 1];
325 + snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, NETDATA_EBPF_PROC_PARTITIONS);
326 + procfile *ff = procfile_open(filename, " \t:", PROCFILE_FLAG_DEFAULT);
327 + if (!ff)
328 + return -1;
329 +
330 + ff = procfile_readall(ff);
331 + if (!ff)
332 + return -1;
333 +
334 + size_t lines = procfile_lines(ff), l;
335 + time_t current_time = now_realtime_sec();
336 + for(l = 2; l < lines ;l++) {
337 + size_t words = procfile_linewords(ff, l);
338 + // This is header or end of file
339 + if (unlikely(words < 4))
340 + continue;
341 +
342 + int major = (int)strtol(procfile_lineword(ff, l, 0), NULL, 10);
343 + // The main goal of this thread is to measure block devices, so any block device with major number
344 + // smaller than 7 according /proc/devices is not "important".
345 + if (major > 7) {
346 + int minor = (int)strtol(procfile_lineword(ff, l, 1), NULL, 10);
347 + update_disk_table(procfile_lineword(ff, l, 3), major, minor, current_time);
348 + }
349 + }
350 +
351 + procfile_close(ff);
352 +
353 + return 0;
354 +}
355 +
356 +/**
357 + * Update disks
358 + *
359 + * @param em main thread structure
360 + */
361 +void ebpf_update_disks(ebpf_module_t *em)
362 +{
363 + static time_t update_time = 0;
364 + time_t curr = now_realtime_sec();
365 + if (curr < update_time)
366 + return;
367 +
368 + update_time = curr + 5 * em->update_time;
369 +
370 + (void)read_local_disks();
371 +}
372 +
373 +/*****************************************************************
374 + *
375 + * FUNCTIONS TO CLOSE THE THREAD
376 + *
377 + *****************************************************************/
378 +
379 +/**
380 + * Disk disable tracepoints
381 + *
382 + * Disable tracepoints when the plugin was responsible to enable it.
383 + */
384 +static void ebpf_disk_disable_tracepoints()
385 +{
386 + char *default_message = { "Cannot disable the tracepoint" };
387 + if (!was_block_issue_enabled) {
388 + if (ebpf_disable_tracing_values(tracepoint_block_type, tracepoint_block_issue))
389 + error("%s %s/%s.", default_message, tracepoint_block_type, tracepoint_block_issue);
390 + }
391 +
392 + if (!was_block_rq_complete_enabled) {
393 + if (ebpf_disable_tracing_values(tracepoint_block_type, tracepoint_block_rq_complete))
394 + error("%s %s/%s.", default_message, tracepoint_block_type, tracepoint_block_rq_complete);
395 + }
396 +}
397 +
398 +/**
399 + * Cleanup plot disks
400 + *
401 + * Clean disk list
402 + */
403 +static void ebpf_cleanup_plot_disks()
404 +{
405 + ebpf_publish_disk_t *move = plot_disks, *next;
406 + while (move) {
407 + next = move->next;
408 +
409 + freez(move);
410 +
411 + move = next;
412 + }
413 +}
414 +
415 +/**
416 + * Cleanup Disk List
417 + */
418 +static void ebpf_cleanup_disk_list()
419 +{
420 + netdata_ebpf_disks_t *move = disk_list;
421 + while (move) {
422 + netdata_ebpf_disks_t *next = move->next;
423 +
424 + freez(move->histogram.name);
425 + freez(move->boot_chart);
426 + freez(move);
427 +
428 + move = next;
429 + }
430 +}
431 +
432 +/**
433 + * Clean up the main thread.
434 + *
435 + * @param ptr thread data.
436 + */
437 +static void ebpf_disk_cleanup(void *ptr)
438 +{
439 + ebpf_disk_disable_tracepoints();
440 +
441 + ebpf_module_t *em = (ebpf_module_t *)ptr;
442 + if (!em->enabled)
443 + return;
444 +
445 + heartbeat_t hb;
446 + heartbeat_init(&hb);
447 + uint32_t tick = 2 * USEC_PER_MS;
448 + while (!read_thread_closed) {
449 + usec_t dt = heartbeat_next(&hb, tick);
450 + UNUSED(dt);
451 + }
452 +
453 + if (dimensions)
454 + ebpf_histogram_dimension_cleanup(dimensions, NETDATA_EBPF_HIST_MAX_BINS);
455 +
456 + freez(disk_hash_values);
457 + freez(disk_threads.thread);
458 + pthread_mutex_destroy(&plot_mutex);
459 +
460 + ebpf_cleanup_plot_disks();
461 + ebpf_cleanup_disk_list();
462 +
463 + if (probe_links) {
464 + struct bpf_program *prog;
465 + size_t i = 0 ;
466 + bpf_object__for_each_program(prog, objects) {
467 + bpf_link__destroy(probe_links[i]);
468 + i++;
469 + }
470 + bpf_object__close(objects);
471 + }
472 +}
473 +
474 +/*****************************************************************
475 + *
476 + * MAIN LOOP
477 + *
478 + *****************************************************************/
479 +
480 +/**
481 + * Fill Plot list
482 + *
483 + * @param ptr a pointer for current disk
484 + */
485 +static void ebpf_fill_plot_disks(netdata_ebpf_disks_t *ptr)
486 +{
487 + pthread_mutex_lock(&plot_mutex);
488 + ebpf_publish_disk_t *w;
489 + if (likely(plot_disks)) {
490 + ebpf_publish_disk_t *move = plot_disks, *store = plot_disks;
491 + while (move) {
492 + if (move->plot == ptr) {
493 + pthread_mutex_unlock(&plot_mutex);
494 + return;
495 + }
496 +
497 + store = move;
498 + move = move->next;
499 + }
500 +
501 + w = callocz(1, sizeof(ebpf_publish_disk_t));
502 + w->plot = ptr;
503 + store->next = w;
504 + } else {
505 + plot_disks = callocz(1, sizeof(ebpf_publish_disk_t));
506 + plot_disks->plot = ptr;
507 + }
508 + pthread_mutex_unlock(&plot_mutex);
509 +
510 + ptr->flags |= NETDATA_DISK_ADDED_TO_PLOT_LIST;
511 +}
512 +
513 +/**
514 + * Read hard disk table
515 + *
516 + * @param table file descriptor for table
517 + *
518 + * Read the table with number of calls for all functions
519 + */
520 +static void read_hard_disk_tables(int table)
521 +{
522 + netdata_idx_t *values = disk_hash_values;
523 + block_key_t key = {};
524 + block_key_t next_key = {};
525 +
526 + netdata_ebpf_disks_t *ret = NULL;
527 +
528 + while (bpf_map_get_next_key(table, &key, &next_key) == 0) {
529 + int test = bpf_map_lookup_elem(table, &key, values);
530 + if (test < 0) {
531 + key = next_key;
532 + continue;
533 + }
534 +
535 + netdata_ebpf_disks_t find;
536 + find.dev = key.dev;
537 +
538 + if (likely(ret)) {
539 + if (find.dev != ret->dev)
540 + ret = (netdata_ebpf_disks_t *)avl_search_lock(&disk_tree, (avl_t *)&find);
541 + } else
542 + ret = (netdata_ebpf_disks_t *)avl_search_lock(&disk_tree, (avl_t *)&find);
543 +
544 + // Disk was inserted after we parse /proc/partitions
545 + if (!ret) {
546 + if (read_local_disks()) {
547 + key = next_key;
548 + continue;
549 + }
550 +
551 + ret = (netdata_ebpf_disks_t *)avl_search_lock(&disk_tree, (avl_t *)&find);
552 + if (!ret) {
553 + // We should never reach this point, but we are adding it to keep a safe code
554 + key = next_key;
555 + continue;
556 + }
557 + }
558 +
559 + uint64_t total = 0;
560 + int i;
561 + int end = (running_on_kernel < NETDATA_KERNEL_V4_15) ? 1 : ebpf_nprocs;
562 + for (i = 0; i < end; i++) {
563 + total += values[i];
564 + }
565 +
566 + ret->histogram.histogram[key.bin] = total;
567 +
568 + if (!(ret->flags & NETDATA_DISK_ADDED_TO_PLOT_LIST))
569 + ebpf_fill_plot_disks(ret);
570 +
571 + key = next_key;
572 + }
573 +}
574 +
575 +/**
576 + * Disk read hash
577 + *
578 + * This is the thread callback.
579 + * This thread is necessary, because we cannot freeze the whole plugin to read the data on very busy socket.
580 + *
581 + * @param ptr It is a NULL value for this thread.
582 + *
583 + * @return It always returns NULL.
584 + */
585 +void *ebpf_disk_read_hash(void *ptr)
586 +{
587 + heartbeat_t hb;
588 + heartbeat_init(&hb);
589 +
590 + ebpf_module_t *em = (ebpf_module_t *)ptr;
591 +
592 + usec_t step = NETDATA_LATENCY_DISK_SLEEP_MS * em->update_time;
593 + while (!close_ebpf_plugin) {
594 + usec_t dt = heartbeat_next(&hb, step);
595 + (void)dt;
596 +
597 + read_hard_disk_tables(disk_maps[NETDATA_DISK_READ].map_fd);
598 + }
599 +
600 + return NULL;
601 +}
602 +
603 +/**
604 + * Obsolete Hard Disk charts
605 + *
606 + * @param w the structure with necessary information to create the chart
607 + *
608 + * Make Hard disk charts and fill chart name
609 + */
610 +static void ebpf_obsolete_hd_charts(netdata_ebpf_disks_t *w)
611 +{
612 + ebpf_write_chart_obsolete(w->histogram.name, w->family, w->histogram.title, EBPF_COMMON_DIMENSION_CALL,
613 + w->family, "disk.latency_io", NETDATA_EBPF_CHART_TYPE_STACKED,
614 + w->histogram.order);
615 +
616 + w->flags = 0;
617 +}
618 +
619 +/**
620 + * Create Hard Disk charts
621 + *
622 + * @param w the structure with necessary information to create the chart
623 + *
624 + * Make Hard disk charts and fill chart name
625 + */
626 +static void ebpf_create_hd_charts(netdata_ebpf_disks_t *w)
627 +{
628 + int order = NETDATA_CHART_PRIO_DISK_LATENCY;
629 + char *family = w->family;
630 +
631 + w->histogram.name = strdupz("disk_latency_io");
632 + w->histogram.title = NULL;
633 + w->histogram.order = order;
634 +
635 + ebpf_create_chart(w->histogram.name, family, "Disk latency", EBPF_COMMON_DIMENSION_CALL,
636 + family, "disk.latency_io", NETDATA_EBPF_CHART_TYPE_STACKED, order,
637 + ebpf_create_global_dimension, disk_publish_aggregated, NETDATA_EBPF_HIST_MAX_BINS);
638 + order++;
639 +
640 + w->flags |= NETDATA_DISK_CHART_CREATED;
641 +}
642 +
643 +/**
644 + * Remove pointer from plot
645 + *
646 + * Remove pointer from plot list when the disk is not present.
647 + */
648 +static void ebpf_remove_pointer_from_plot_disk(ebpf_module_t *em)
649 +{
650 + time_t current_time = now_realtime_sec();
651 + time_t limit = 10 * em->update_time;
652 + pthread_mutex_lock(&plot_mutex);
653 + ebpf_publish_disk_t *move = plot_disks, *prev = plot_disks;
654 + while (move) {
655 + netdata_ebpf_disks_t *ned = move->plot;
656 + uint32_t flags = ned->flags;
657 +
658 + if (!(flags & NETDATA_DISK_IS_HERE) && ((current_time - ned->last_update) > limit)) {
659 + ebpf_obsolete_hd_charts(ned);
660 + if (move == plot_disks) {
661 + freez(move);
662 + plot_disks = NULL;
663 + break;
664 + } else {
665 + prev->next = move->next;
666 + ebpf_publish_disk_t *clean = move;
667 + move = move->next;
668 + freez(clean);
669 + continue;
670 + }
671 + }
672 +
673 + prev = move;
674 + move = move->next;
675 + }
676 + pthread_mutex_unlock(&plot_mutex);
677 +}
678 +
679 +/**
680 + * Send Hard disk data
681 + *
682 + * Send hard disk information to Netdata.
683 + */
684 +static void ebpf_latency_send_hd_data()
685 +{
686 + pthread_mutex_lock(&plot_mutex);
687 + if (!plot_disks) {
688 + pthread_mutex_unlock(&plot_mutex);
689 + return;
690 + }
691 +
692 + ebpf_publish_disk_t *move = plot_disks;
693 + while (move) {
694 + netdata_ebpf_disks_t *ned = move->plot;
695 + uint32_t flags = ned->flags;
696 + if (!(flags & NETDATA_DISK_CHART_CREATED)) {
697 + ebpf_create_hd_charts(ned);
698 + }
699 +
700 + if ((flags & NETDATA_DISK_CHART_CREATED)) {
701 + write_histogram_chart(ned->histogram.name, ned->family,
702 + ned->histogram.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS);
703 + }
704 +
705 + ned->flags &= ~NETDATA_DISK_IS_HERE;
706 +
707 + move = move->next;
708 + }
709 + pthread_mutex_unlock(&plot_mutex);
710 +}
711 +
712 +/**
713 +* Main loop for this collector.
714 +*/
715 +static void disk_collector(ebpf_module_t *em)
716 +{
717 + disk_hash_values = callocz(ebpf_nprocs, sizeof(netdata_idx_t));
718 + disk_threads.thread = mallocz(sizeof(netdata_thread_t));
719 + disk_threads.start_routine = ebpf_disk_read_hash;
720 +
721 + netdata_thread_create(disk_threads.thread, disk_threads.name, NETDATA_THREAD_OPTION_JOINABLE,
722 + ebpf_disk_read_hash, em);
723 +
724 +
725 + read_thread_closed = 0;
726 + while (!close_ebpf_plugin) {
727 + pthread_mutex_lock(&collect_data_mutex);
728 + pthread_cond_wait(&collect_data_cond_var, &collect_data_mutex);
729 +
730 + pthread_mutex_lock(&lock);
731 + ebpf_remove_pointer_from_plot_disk(em);
732 + ebpf_latency_send_hd_data();
733 +
734 + pthread_mutex_unlock(&lock);
735 + pthread_mutex_unlock(&collect_data_mutex);
736 +
737 + ebpf_update_disks(em);
738 + }
739 + read_thread_closed = 1;
740 +}
741 +
742 +/*****************************************************************
743 + *
744 + * EBPF DISK THREAD
745 + *
746 + *****************************************************************/
747 +
748 +/**
749 + * Enable tracepoints
750 + *
751 + * Enable necessary tracepoints for thread.
752 + *
753 + * @return It returns 0 on success and -1 otherwise
754 + */
755 +static int ebpf_disk_enable_tracepoints()
756 +{
757 + int test = ebpf_is_tracepoint_enabled(tracepoint_block_type, tracepoint_block_issue);
758 + if (test == -1)
759 + return -1;
760 + else if (!test) {
761 + if (ebpf_enable_tracing_values(tracepoint_block_type, tracepoint_block_issue))
762 + return -1;
763 + }
764 + was_block_issue_enabled = test;
765 +
766 + test = ebpf_is_tracepoint_enabled(tracepoint_block_type, tracepoint_block_rq_complete);
767 + if (test == -1)
768 + return -1;
769 + else if (!test) {
770 + if (ebpf_enable_tracing_values(tracepoint_block_type, tracepoint_block_rq_complete))
771 + return -1;
772 + }
773 + was_block_rq_complete_enabled = test;
774 +
775 + return 0;
776 +}
777 +
778 +/**
779 + * Disk thread
780 + *
781 + * Thread used to generate disk charts.
782 + *
783 + * @param ptr a pointer to `struct ebpf_module`
784 + *
785 + * @return It always return NULL
786 + */
787 +void *ebpf_disk_thread(void *ptr)
788 +{
789 + netdata_thread_cleanup_push(ebpf_disk_cleanup, ptr);
790 +
791 + ebpf_module_t *em = (ebpf_module_t *)ptr;
792 + em->maps = disk_maps;
793 +
794 + fill_ebpf_data(&disk_data);
795 +
796 + if (!em->enabled)
797 + goto enddisk;
798 +
799 + if (ebpf_update_kernel(&disk_data)) {
800 + goto enddisk;
801 + }
802 +
803 + if (ebpf_disk_enable_tracepoints()) {
804 + em->enabled = CONFIG_BOOLEAN_NO;
805 + goto enddisk;
806 + }
807 +
808 + avl_init_lock(&disk_tree, ebpf_compare_disks);
809 + if (read_local_disks()) {
810 + em->enabled = CONFIG_BOOLEAN_NO;
811 + goto enddisk;
812 + }
813 +
814 + if (pthread_mutex_init(&plot_mutex, NULL)) {
815 + error("Cannot initialize local mutex");
816 + goto enddisk;
817 + }
818 +
819 + probe_links = ebpf_load_program(ebpf_plugin_dir, em, kernel_string, &objects, disk_data.map_fd);
820 + if (!probe_links) {
821 + goto enddisk;
822 + }
823 +
824 + int algorithms[NETDATA_EBPF_HIST_MAX_BINS];
825 + ebpf_fill_algorithms(algorithms, NETDATA_EBPF_HIST_MAX_BINS, NETDATA_EBPF_INCREMENTAL_IDX);
826 + dimensions = ebpf_fill_histogram_dimension(NETDATA_EBPF_HIST_MAX_BINS);
827 +
828 + ebpf_global_labels(disk_aggregated_data, disk_publish_aggregated, dimensions, dimensions, algorithms,
829 + NETDATA_EBPF_HIST_MAX_BINS);
830 +
831 + disk_collector(em);
832 +
833 +enddisk:
834 + netdata_thread_cleanup_pop(1);
835 +
836 + return NULL;
837 +}
collectors/ebpf.plugin/ebpf_disk.h new
+72
@@ -0,0 +1,72 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_EBPF_DISK_H
4 +#define NETDATA_EBPF_DISK_H 1
5 +
6 +#include "libnetdata/avl/avl.h"
7 +#include "libnetdata/ebpf/ebpf.h"
8 +
9 +#define NETDATA_EBPF_PROC_PARTITIONS "/proc/partitions"
10 +
11 +#define NETDATA_LATENCY_DISK_SLEEP_MS 650000ULL
12 +
13 +// Decode function extracted from: https://elixir.bootlin.com/linux/v5.10.8/source/include/linux/kdev_t.h#L7
14 +#define MINORBITS 20
15 +#define MKDEV(ma,mi) (((ma) << MINORBITS) | (mi))
16 +
17 +enum netdata_latency_disks_flags {
18 + NETDATA_DISK_ADDED_TO_PLOT_LIST = 1,
19 + NETDATA_DISK_CHART_CREATED = 2,
20 + NETDATA_DISK_IS_HERE = 4,
21 + NETDATA_DISK_HAS_EFI = 8
22 +};
23 +
24 +/*
25 + * The definition (DISK_NAME_LEN) has been a stable value since Kernel 3.0,
26 + * I decided to bring it as internal definition, to avoid include linux/genhd.h.
27 + */
28 +#define NETDATA_DISK_NAME_LEN 32
29 +typedef struct netdata_ebpf_disks {
30 + // Search
31 + avl_t avl;
32 + uint32_t dev;
33 + uint32_t major;
34 + uint32_t minor;
35 + uint32_t bootsector_key;
36 + uint64_t start; // start sector
37 + uint64_t end; // end sector
38 +
39 + // Print information
40 + char family[NETDATA_DISK_NAME_LEN + 1];
41 + char *boot_chart;
42 +
43 + netdata_ebpf_histogram_t histogram;
44 +
45 + uint32_t flags;
46 + time_t last_update;
47 +
48 + struct netdata_ebpf_disks *main;
49 + struct netdata_ebpf_disks *boot_partition;
50 + struct netdata_ebpf_disks *next;
51 +} netdata_ebpf_disks_t;
52 +
53 +enum ebpf_disk_tables {
54 + NETDATA_DISK_READ
55 +};
56 +
57 +typedef struct block_key {
58 + uint32_t bin;
59 + uint32_t dev;
60 +} block_key_t;
61 +
62 +typedef struct netdata_ebpf_publish_disk {
63 + netdata_ebpf_disks_t *plot;
64 + struct netdata_ebpf_publish_disk *next;
65 +} ebpf_publish_disk_t;
66 +
67 +extern struct config disk_config;
68 +
69 +extern void *ebpf_disk_thread(void *ptr);
70 +
71 +#endif /* NETDATA_EBPF_DISK_H */
72 +
collectors/ebpf.plugin/ebpf_filesystem.c
+20 -43
@@ -36,8 +36,8 @@ struct netdata_static_thread filesystem_threads = {"EBPF FS READ",
36 NULL, NULL };
37
38 static int read_thread_closed = 1;
39 -static netdata_syscall_stat_t filesystem_aggregated_data[NETDATA_FILESYSTEM_MAX_BINS];
40 -static netdata_publish_syscall_t filesystem_publish_aggregated[NETDATA_FILESYSTEM_MAX_BINS];
39 +static netdata_syscall_stat_t filesystem_aggregated_data[NETDATA_EBPF_HIST_MAX_BINS];
40 +static netdata_publish_syscall_t filesystem_publish_aggregated[NETDATA_EBPF_HIST_MAX_BINS];
41
42 char **dimensions = NULL;
43 static netdata_idx_t *filesystem_hash_values = NULL;
@@ -114,7 +114,7 @@ static void ebpf_create_fs_charts()
114 title,
115 EBPF_COMMON_DIMENSION_CALL, family,
116 NULL, NETDATA_EBPF_CHART_TYPE_STACKED, order, ebpf_create_global_dimension,
117 - filesystem_publish_aggregated, NETDATA_FILESYSTEM_MAX_BINS);
117 + filesystem_publish_aggregated, NETDATA_EBPF_HIST_MAX_BINS);
118 order++;
119
120 snprintfz(title, 255, "%s latency for each write request.", efp->filesystem);
@@ -126,7 +126,7 @@ static void ebpf_create_fs_charts()
126 title,
127 EBPF_COMMON_DIMENSION_CALL, family,
128 NULL, NETDATA_EBPF_CHART_TYPE_STACKED, order, ebpf_create_global_dimension,
129 - filesystem_publish_aggregated, NETDATA_FILESYSTEM_MAX_BINS);
129 + filesystem_publish_aggregated, NETDATA_EBPF_HIST_MAX_BINS);
130 order++;
131
132 snprintfz(title, 255, "%s latency for each open request.", efp->filesystem);
@@ -138,7 +138,7 @@ static void ebpf_create_fs_charts()
138 title,
139 EBPF_COMMON_DIMENSION_CALL, family,
140 NULL, NETDATA_EBPF_CHART_TYPE_STACKED, order, ebpf_create_global_dimension,
141 - filesystem_publish_aggregated, NETDATA_FILESYSTEM_MAX_BINS);
141 + filesystem_publish_aggregated, NETDATA_EBPF_HIST_MAX_BINS);
142 order++;
143
144 snprintfz(title, 255, "%s latency for each sync request.", efp->filesystem);
@@ -150,7 +150,7 @@ static void ebpf_create_fs_charts()
150 title,
151 EBPF_COMMON_DIMENSION_CALL, family,
152 NULL, NETDATA_EBPF_CHART_TYPE_STACKED, order, ebpf_create_global_dimension,
153 - filesystem_publish_aggregated, NETDATA_FILESYSTEM_MAX_BINS);
153 + filesystem_publish_aggregated, NETDATA_EBPF_HIST_MAX_BINS);
154 order++;
155 efp->flags |= NETDATA_FILESYSTEM_FLAG_CHART_CREATED;
156 }
@@ -197,10 +197,10 @@ int ebpf_filesystem_initialize_ebpf_data(ebpf_module_t *em)
197 em->thread_name = saved_name;
198
199 if (!dimensions) {
200 - dimensions = ebpf_fill_histogram_dimension(NETDATA_FILESYSTEM_MAX_BINS);
200 + dimensions = ebpf_fill_histogram_dimension(NETDATA_EBPF_HIST_MAX_BINS);
201
202 - memset(filesystem_aggregated_data, 0 , NETDATA_FILESYSTEM_MAX_BINS * sizeof(netdata_syscall_stat_t));
203 - memset(filesystem_publish_aggregated, 0 , NETDATA_FILESYSTEM_MAX_BINS * sizeof(netdata_publish_syscall_t));
202 + memset(filesystem_aggregated_data, 0 , NETDATA_EBPF_HIST_MAX_BINS * sizeof(netdata_syscall_stat_t));
203 + memset(filesystem_publish_aggregated, 0 , NETDATA_EBPF_HIST_MAX_BINS * sizeof(netdata_publish_syscall_t));
204
205 filesystem_hash_values = callocz(ebpf_nprocs, sizeof(netdata_idx_t));
206 }
@@ -351,7 +351,8 @@ static void ebpf_filesystem_cleanup(void *ptr)
351 ebpf_cleanup_publish_syscall(filesystem_publish_aggregated);
352
353 ebpf_filesystem_cleanup_ebpf_data();
354 - ebpf_histogram_dimension_cleanup(dimensions, NETDATA_FILESYSTEM_MAX_BINS);
354 + if (dimensions)
355 + ebpf_histogram_dimension_cleanup(dimensions, NETDATA_EBPF_HIST_MAX_BINS);
356 freez(filesystem_hash_values);
357 }
358
@@ -421,8 +422,8 @@ static void read_filesystem_table(ebpf_filesystem_partitions_t *efp)
422 total += values[i];
423 }
424
424 - if (idx >= NETDATA_FILESYSTEM_MAX_BINS)
425 - idx = NETDATA_FILESYSTEM_MAX_BINS - 1;
425 + if (idx >= NETDATA_EBPF_HIST_MAX_BINS)
426 + idx = NETDATA_EBPF_HIST_MAX_BINS - 1;
427 w->histogram[idx] = total;
428 }
429 }
@@ -481,30 +482,6 @@ void *ebpf_filesystem_read_hash(void *ptr)
482 return NULL;
483 }
484
484 -/**
485 - * Call the necessary functions to create a name.
486 - *
487 - * @param family family name
488 - * @param name chart name
489 - * @param hist0 histogram values
490 - * @param end number of bins that will be sent to Netdata.
491 - *
492 - * @return It returns a variable tha maps the charts that did not have zero values.
493 - */
494 -static void write_histogram_chart(char *family, char *name, const netdata_idx_t *hist, uint32_t end)
495 -{
496 - write_begin_chart(family, name);
497 -
498 - uint32_t i;
499 - for (i = 0; i < end; i++) {
500 - write_chart_dimension(dimensions[i], (long long) hist[i]);
501 - }
502 -
503 - write_end_chart();
504 -
505 - fflush(stdout);
506 -}
507 -
485 /**
486 * Send Hard disk data
487 *
@@ -518,16 +495,16 @@ static void ebpf_histogram_send_data()
495 ebpf_filesystem_partitions_t *efp = &localfs[i];
496 if ((efp->flags & test) == NETDATA_FILESYSTEM_FLAG_HAS_PARTITION) {
497 write_histogram_chart(NETDATA_FILESYSTEM_FAMILY, efp->hread.name,
521 - efp->hread.histogram, NETDATA_FILESYSTEM_MAX_BINS);
498 + efp->hread.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS);
499
500 write_histogram_chart(NETDATA_FILESYSTEM_FAMILY, efp->hwrite.name,
524 - efp->hwrite.histogram, NETDATA_FILESYSTEM_MAX_BINS);
501 + efp->hwrite.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS);
502
503 write_histogram_chart(NETDATA_FILESYSTEM_FAMILY, efp->hopen.name,
527 - efp->hopen.histogram, NETDATA_FILESYSTEM_MAX_BINS);
504 + efp->hopen.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS);
505
506 write_histogram_chart(NETDATA_FILESYSTEM_FAMILY, efp->hsync.name,
530 - efp->hsync.histogram, NETDATA_FILESYSTEM_MAX_BINS);
507 + efp->hsync.histogram, dimensions, NETDATA_EBPF_HIST_MAX_BINS);
508 }
509 }
510 }
@@ -612,10 +589,10 @@ void *ebpf_filesystem_thread(void *ptr)
589 goto endfilesystem;
590 }
591
615 - int algorithms[NETDATA_FILESYSTEM_MAX_BINS];
616 - ebpf_fill_algorithms(algorithms, NETDATA_FILESYSTEM_MAX_BINS, NETDATA_EBPF_INCREMENTAL_IDX);
592 + int algorithms[NETDATA_EBPF_HIST_MAX_BINS];
593 + ebpf_fill_algorithms(algorithms, NETDATA_EBPF_HIST_MAX_BINS, NETDATA_EBPF_INCREMENTAL_IDX);
594 ebpf_global_labels(filesystem_aggregated_data, filesystem_publish_aggregated, dimensions, dimensions,
618 - algorithms, NETDATA_FILESYSTEM_MAX_BINS);
595 + algorithms, NETDATA_EBPF_HIST_MAX_BINS);
596
597 pthread_mutex_lock(&lock);
598 ebpf_create_fs_charts();
collectors/ebpf.plugin/ebpf_filesystem.h
-9
@@ -5,7 +5,6 @@
5
6 #include "ebpf.h"
7
8 -#define NETDATA_FILESYSTEM_MAX_BINS 24UL
8 #define NETDATA_FS_MAX_DIST_NAME 64UL
9
10 #define NETDATA_FILESYSTEM_CONFIG_NAME "filesystem"
@@ -32,14 +31,6 @@ enum netdata_filesystem_flags {
31 NETDATA_FILESYSTEM_REMOVE_CHARTS = 16
32 };
33
35 -typedef struct netdata_ebpf_histogram {
36 - char *name;
37 - char *title;
38 - int order;
39 - uint64_t histogram[NETDATA_FILESYSTEM_MAX_BINS];
40 -} netdata_ebpf_histogram_t;
41 -
42 -
34 enum netdata_filesystem_table {
35 NETDATA_MAIN_FS_TABLE,
36 NETDATA_ADDR_FS_TABLE
libnetdata/ebpf/ebpf.c
+111
@@ -701,3 +701,114 @@ void ebpf_histogram_dimension_cleanup(char **ptr, size_t length)
701 }
702 freez(ptr);
703 }
704 +
705 +//----------------------------------------------------------------------------------------------------------------------
706 +
707 +/**
708 + * Open tracepoint path
709 + *
710 + * @param filename pointer to store the path
711 + * @param length file length
712 + * @param subsys is the name of your subsystem.
713 + * @param eventname is the name of the event to trace.
714 + * @param flags flags used with syscall open
715 + *
716 + * @return it returns a positive value on success and a negative otherwise.
717 + */
718 +static inline int ebpf_open_tracepoint_path(char *filename, size_t length, char *subsys, char *eventname, int flags)
719 +{
720 + snprintfz(filename, length, "%s/events/%s/%s/enable", NETDATA_DEBUGFS, subsys, eventname);
721 + return open(filename, flags, 0);
722 +}
723 +
724 +/**
725 + * Is tracepoint enabled
726 + *
727 + * Check whether the tracepoint is enabled.
728 + *
729 + * @param subsys is the name of your subsystem.
730 + * @param eventname is the name of the event to trace.
731 + *
732 + * @return it returns 1 when it is enabled, 0 when it is disabled and -1 on error.
733 + */
734 +int ebpf_is_tracepoint_enabled(char *subsys, char *eventname)
735 +{
736 + char text[FILENAME_MAX + 1];
737 + int fd = ebpf_open_tracepoint_path(text, FILENAME_MAX, subsys, eventname, O_RDONLY);
738 + if (fd < 0) {
739 + return -1;
740 + }
741 +
742 + ssize_t length = read(fd, text, 1);
743 + if (length != 1) {
744 + close(fd);
745 + return -1;
746 + }
747 + close(fd);
748 +
749 + return (text[0] == '1') ? CONFIG_BOOLEAN_YES : CONFIG_BOOLEAN_NO;
750 +}
751 +
752 +/**
753 + * Change Tracing values
754 + *
755 + * Change value for specific tracepoint enabling or disabling it according value given.
756 + *
757 + * @param subsys is the name of your subsystem.
758 + * @param eventname is the name of the event to trace.
759 + * @param value a value to enable (1) or disable (0) a tracepoint.
760 + *
761 + * @return It returns 0 on success and -1 otherwise
762 + */
763 +static int ebpf_change_tracing_values(char *subsys, char *eventname, char *value)
764 +{
765 + if (strcmp("0", value) && strcmp("1", value)) {
766 + error("Invalid value given to either enable or disable a tracepoint.");
767 + return -1;
768 + }
769 +
770 + char filename[1024];
771 + int fd = ebpf_open_tracepoint_path(filename, 1023, subsys, eventname, O_WRONLY);
772 + if (fd < 0) {
773 + return -1;
774 + }
775 +
776 + ssize_t written = write(fd, value, strlen(value));
777 + if (written < 0) {
778 + close(fd);
779 + return -1;
780 + }
781 +
782 + close(fd);
783 + return 0;
784 +}
785 +
786 +/**
787 + * Enable tracing values
788 + *
789 + * Enable a tracepoint on a system
790 + *
791 + * @param subsys is the name of your subsystem.
792 + * @param eventname is the name of the event to trace.
793 + *
794 + * @return It returns 0 on success and -1 otherwise
795 + */
796 +int ebpf_enable_tracing_values(char *subsys, char *eventname)
797 +{
798 + return ebpf_change_tracing_values(subsys, eventname, "1");
799 +}
800 +
801 +/**
802 + * Disable tracing values
803 + *
804 + * Disable tracing points enabled by collector
805 + *
806 + * @param subsys is the name of your subsystem.
807 + * @param eventname is the name of the event to trace.
808 + *
809 + * @return It returns 0 on success and -1 otherwise
810 + */
811 +int ebpf_disable_tracing_values(char *subsys, char *eventname)
812 +{
813 + return ebpf_change_tracing_values(subsys, eventname, "0");
814 +}
libnetdata/ebpf/ebpf.h
+19
@@ -184,6 +184,25 @@ extern void ebpf_update_names(ebpf_specify_name_t *opt, ebpf_module_t *em);
184 extern void ebpf_load_addresses(ebpf_addresses_t *fa, int fd);
185 extern void ebpf_fill_algorithms(int *algorithms, size_t length, int algorithm);
186 extern char **ebpf_fill_histogram_dimension(size_t maximum);
187 +
188 +// Histogram
189 +#define NETDATA_EBPF_HIST_MAX_BINS 24UL
190 +#define NETDATA_DISK_MAX 256U
191 +#define NETDATA_DISK_HISTOGRAM_LENGTH (NETDATA_DISK_MAX * NETDATA_EBPF_HIST_MAX_BINS)
192 +
193 +typedef struct netdata_ebpf_histogram {
194 + char *name;
195 + char *title;
196 + int order;
197 + uint64_t histogram[NETDATA_EBPF_HIST_MAX_BINS];
198 +} netdata_ebpf_histogram_t;
199 +
200 extern void ebpf_histogram_dimension_cleanup(char **ptr, size_t length);
201
202 +// Tracepoint helpers
203 +// For more information related to tracepoints read https://www.kernel.org/doc/html/latest/trace/tracepoints.html
204 +extern int ebpf_is_tracepoint_enabled(char *subsys, char *eventname);
205 +extern int ebpf_enable_tracing_values(char *subsys, char *eventname);
206 +extern int ebpf_disable_tracing_values(char *subsys, char *eventname);
207 +
208 #endif /* NETDATA_EBPF_H */
packaging/ebpf.checksums
+3 -3
@@ -1,3 +1,3 @@
1 -a2849a54f43f9419419298a0e06de7da52b1ecb74e3c5e788540f7fdd380862a netdata-kernel-collector-glibc-v0.7.1.1.tar.xz
2 -15fb92966fef3fe8b6f6f08f15c57270b5b37071b52834073097076c4897c0e1 netdata-kernel-collector-musl-v0.7.1.1.tar.xz
3 -bdfa64cfc9a9d457252dd824435ac5e7f107c9a30fc6d915b294ae9fb1cd5205 netdata-kernel-collector-static-v0.7.1.1.tar.xz
1 +1e2441471f2f399776e1b61fa0e55df6937e4e120a003bda73b510a2b99df583 netdata-kernel-collector-glibc-v0.7.2.4.tar.xz
2 +76f7c342e2eb50d306b6a12d42be32f42ae74fe36ea48d0b5021973aae9be666 netdata-kernel-collector-musl-v0.7.2.4.tar.xz
3 +8a57cec811512c0fc9f8c06bcf2e4c01553d7fdd0a8123c578fd0372f5ea768f netdata-kernel-collector-static-v0.7.2.4.tar.xz
packaging/ebpf.version
+1 -1
@@ -1 +1 @@
1 -v0.7.1.1
1 +v0.7.2.4
web/gui/dashboard_info.js
+4
@@ -1507,6 +1507,10 @@ netdataDashboard.context = {
1507 height: 0.5,
1508 info: 'The average service time for completed I/O operations. This metric is calculated using the total busy time of the disk and the number of completed operations. If the disk is able to execute multiple parallel operations the reporting average service time will be misleading.'
1509 },
1510 + 'disk.latency_io': {
1511 + height: 0.5,
1512 + info: 'Disk I/O latency is the time it takes for an I/O request to be completed. Latency is the single most important metric to focus on when it comes to storage performance, under most circumstances. For hard drives, an average latency somewhere between 10 to 20 ms can be considered acceptable. For SSD (Solid State Drives), depending on the workload it should never reach higher than 1-3 ms. In most cases, workloads will experience less than 1ms latency numbers. The dimensions refer to time intervals. This chart is based on the <a href="https://github.com/cloudflare/ebpf_exporter/blob/master/examples/bio-tracepoints.yaml" target="_blank">bio_tracepoints</a> tool of the ebpf_exporter.'
1513 + },
1514 'disk.avgsz': {
1515 height: 0.5,
1516 info: 'The average I/O operation size.'