@cryptotaxi247 / netdata-1 / commits / 65f4f12e8

New thread for ebpf.plugin

Add monitoring to `sync` syscall

thiagoftsm committed Mar 17, 2021 at 10:22 UTC 65f4f12e8606a14d2c133569890e294275c2e056
16 files changed +322 -37
CMakeLists.txt
+2
@@ -487,6 +487,8 @@ set(EBPF_PROCESS_PLUGIN_FILES
487 collectors/ebpf.plugin/ebpf_process.h
488 collectors/ebpf.plugin/ebpf_socket.c
489 collectors/ebpf.plugin/ebpf_socket.h
490 + collectors/ebpf.plugin/ebpf_sync.c
491 + collectors/ebpf.plugin/ebpf_sync.h
492 collectors/ebpf.plugin/ebpf_apps.c
493 collectors/ebpf.plugin/ebpf_apps.h
494 )
Makefile.am
+2
@@ -295,6 +295,8 @@ EBPF_PLUGIN_FILES = \
295 collectors/ebpf.plugin/ebpf_process.h \
296 collectors/ebpf.plugin/ebpf_socket.c \
297 collectors/ebpf.plugin/ebpf_socket.h \
298 + collectors/ebpf.plugin/ebpf_sync.c \
299 + collectors/ebpf.plugin/ebpf_sync.h \
300 collectors/ebpf.plugin/ebpf.h \
301 collectors/ebpf.plugin/ebpf_apps.c \
302 collectors/ebpf.plugin/ebpf_apps.h \
collectors/ebpf.plugin/README.md
+1
@@ -194,6 +194,7 @@ The eBPF collector enables and runs the following eBPF programs by default:
194 When in `return` mode, it also creates charts showing errors when these operations are executed.
195 - `network viewer`: This eBPF program creates charts with information about `TCP` and `UDP` functions, including the
196 bandwidth consumed by each.
197 +- `sync`: Montitor calls for syscall sync(2).
198
199 ### `[network connections]`
200
collectors/ebpf.plugin/ebpf.c
+44 -1
@@ -86,6 +86,9 @@ ebpf_module_t ebpf_modules[] = {
86 { .thread_name = "cachestat", .config_name = "cachestat", .enabled = 0, .start_routine = ebpf_cachestat_thread,
87 .update_time = 1, .global_charts = 1, .apps_charts = 1, .mode = MODE_ENTRY,
88 .optional = 0, .apps_routine = ebpf_cachestat_create_apps_charts },
89 + { .thread_name = "sync", .config_name = "sync", .enabled = 0, .start_routine = ebpf_sync_thread,
90 + .update_time = 1, .global_charts = 1, .apps_charts = 1, .mode = MODE_ENTRY,
91 + .optional = 0, .apps_routine = NULL },
92 { .thread_name = NULL, .enabled = 0, .start_routine = NULL, .update_time = 1,
93 .global_charts = 0, .apps_charts = 1, .mode = MODE_ENTRY,
94 .optional = 0, .apps_routine = NULL },
@@ -321,6 +324,25 @@ void write_err_chart(char *name, char *family, netdata_publish_syscall_t *move,
324 write_end_chart();
325 }
326
327 +/**
328 + * Write charts
329 + *
330 + * Write the current information to publish the charts.
331 + *
332 + * @param family chart family
333 + * @param chart chart id
334 + * @param dim dimension name
335 + * @param v1 value.
336 + */
337 +void ebpf_one_dimension_write_charts(char *family, char *chart, char *dim, long long v1)
338 +{
339 + write_begin_chart(family, chart);
340 +
341 + write_chart_dimension(dim, v1);
342 +
343 + write_end_chart();
344 +}
345 +
346 /**
347 * Call the necessary functions to create a chart.
348 *
@@ -606,6 +628,8 @@ void ebpf_print_help()
628 "\n"
629 " --return or -r Run the collector in return mode.\n"
630 "\n",
631 + " --sync or -s Enable chart related to sync run time.\n"
632 + "\n"
633 VERSION,
634 (year >= 116) ? year + 1900 : 2020);
635 }
@@ -1701,6 +1725,14 @@ static void read_collector_values(int *disable_apps)
1725 started++;
1726 }
1727
1728 + enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "sync",
1729 + CONFIG_BOOLEAN_YES);
1730 +
1731 + if (enabled) {
1732 + ebpf_enable_chart(EBPF_MODULE_SYNC_IDX, *disable_apps);
1733 + started++;
1734 + }
1735 +
1736 if (!started){
1737 ebpf_enable_all_charts(*disable_apps);
1738 // Read network viewer section
@@ -1783,6 +1815,7 @@ static void parse_args(int argc, char **argv)
1815 {"net", no_argument, 0, 'n' },
1816 {"process", no_argument, 0, 'p' },
1817 {"return", no_argument, 0, 'r' },
1818 + {"sync", no_argument, 0, 's' },
1819 {0, 0, 0, 0}
1820 };
1821
@@ -1797,7 +1830,7 @@ static void parse_args(int argc, char **argv)
1830 }
1831
1832 while (1) {
1800 - int c = getopt_long(argc, argv, "hvgcanpr", long_options, &option_index);
1833 + int c = getopt_long(argc, argv, "hvgcanprs", long_options, &option_index);
1834 if (c == -1)
1835 break;
1836
@@ -1856,6 +1889,14 @@ static void parse_args(int argc, char **argv)
1889 ebpf_set_thread_mode(MODE_RETURN);
1890 #ifdef NETDATA_INTERNAL_CHECKS
1891 info("EBPF running in \"return\" mode, because it was started with the option \"--return\" or \"-r\".");
1892 +#endif
1893 + break;
1894 + }
1895 + case 's': {
1896 + enabled = 1;
1897 + ebpf_enable_chart(EBPF_MODULE_SYNC_IDX, disable_apps);
1898 +#ifdef NETDATA_INTERNAL_CHECKS
1899 + info("EBPF enabling \"sync\" chart, because it was started with the option \"--sync\" or \"-s\".");
1900 #endif
1901 break;
1902 }
@@ -1983,6 +2024,8 @@ int main(int argc, char **argv)
2024 NULL, NULL, ebpf_modules[EBPF_MODULE_SOCKET_IDX].start_routine},
2025 {"EBPF CACHESTAT" , NULL, NULL, 1,
2026 NULL, NULL, ebpf_modules[EBPF_MODULE_CACHESTAT_IDX].start_routine},
2027 + {"EBPF SYNC" , NULL, NULL, 1,
2028 + NULL, NULL, ebpf_modules[EBPF_MODULE_SYNC_IDX].start_routine},
2029 {NULL , NULL, NULL, 0,
2030 NULL, NULL, NULL}
2031 };
collectors/ebpf.plugin/ebpf.conf
+2
@@ -24,10 +24,12 @@
24 # files removed.
25 # `socket` : This eBPF program creates charts with information about `TCP` and `UDP` functions, including the
26 # bandwidth consumed by each.
27 +# `sync` : Montitor calls for syscall sync(2).
28 [ebpf programs]
29 cachestat = no
30 process = yes
31 socket = yes
32 + sync = yes
33 network connections = no
34
35 #
collectors/ebpf.plugin/ebpf.h
+4 -1
@@ -73,7 +73,8 @@ extern ebpf_module_t ebpf_modules[];
73 enum ebpf_module_indexes {
74 EBPF_MODULE_PROCESS_IDX,
75 EBPF_MODULE_SOCKET_IDX,
76 - EBPF_MODULE_CACHESTAT_IDX
76 + EBPF_MODULE_CACHESTAT_IDX,
77 + EBPF_MODULE_SYNC_IDX
78 };
79
80 // Copied from musl header
@@ -87,6 +88,7 @@ enum ebpf_module_indexes {
88
89 // Chart defintions
90 #define NETDATA_EBPF_FAMILY "ebpf"
91 +#define NETDATA_EBPF_MEMORY_GROUP "mem"
92
93 // Log file
94 #define NETDATA_DEVELOPER_LOG_FILE "developer.log"
@@ -203,6 +205,7 @@ extern char *ebpf_algorithms[];
205 extern void ebpf_process_create_apps_charts(struct ebpf_module *em, void *ptr);
206 extern void ebpf_socket_create_apps_charts(struct ebpf_module *em, void *ptr);
207 extern void ebpf_cachestat_create_apps_charts(struct ebpf_module *em, void *root);
208 +extern void ebpf_one_dimension_write_charts(char *family, char *chart, char *dim, long long v1);
209 extern collected_number get_value_from_structure(char *basis, size_t offset);
210 extern struct pid_stat *root_of_pids;
211 extern ebpf_process_stat_t *global_process_stat;
collectors/ebpf.plugin/ebpf_apps.h
+1
@@ -19,6 +19,7 @@
19
20 #include "ebpf_process.h"
21 #include "ebpf_cachestat.h"
22 +#include "ebpf_sync.h"
23
24 #define MAX_COMPARE_NAME 100
25 #define MAX_NAME 100
collectors/ebpf.plugin/ebpf_cachestat.c
+11 -28
@@ -90,25 +90,6 @@ static void ebpf_cachestat_cleanup(void *ptr)
90 *
91 *****************************************************************/
92
93 -/**
94 - * Write charts
95 - *
96 - * Write the current information to publish the charts.
97 - *
98 - * @param family chart family
99 - * @param chart chart id
100 - * @param dim dimension name
101 - * @param v1 value.
102 - */
103 -static inline void cachestat_write_charts(char *family, char *chart, char *dim, long long v1)
104 -{
105 - write_begin_chart(family, chart);
106 -
107 - write_chart_dimension(dim, v1);
108 -
109 - write_end_chart();
110 -}
111 -
93 /**
94 * Update publish
95 *
@@ -398,19 +379,21 @@ static void cachestat_send_global(netdata_publish_cachestat_t *publish)
379 netdata_publish_syscall_t *ptr = cachestat_counter_publish_aggregated;
380 // The algorithm sets this value to zero sometimes, we are not written them to have a smooth chart
381 if (publish->ratio) {
401 - cachestat_write_charts(NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_HIT_RATIO_CHART,
402 - ptr[NETDATA_CACHESTAT_IDX_RATIO].dimension, publish->ratio);
382 + ebpf_one_dimension_write_charts(
383 + NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_HIT_RATIO_CHART, ptr[NETDATA_CACHESTAT_IDX_RATIO].dimension,
384 + publish->ratio);
385 }
386
405 - cachestat_write_charts(NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_DIRTY_CHART,
406 - ptr[NETDATA_CACHESTAT_IDX_DIRTY].dimension,
407 - cachestat_hash_values[NETDATA_KEY_CALLS_MARK_BUFFER_DIRTY]);
387 + ebpf_one_dimension_write_charts(
388 + NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_DIRTY_CHART, ptr[NETDATA_CACHESTAT_IDX_DIRTY].dimension,
389 + cachestat_hash_values[NETDATA_KEY_CALLS_MARK_BUFFER_DIRTY]);
390
409 - cachestat_write_charts(NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_HIT_CHART,
410 - ptr[NETDATA_CACHESTAT_IDX_HIT].dimension, publish->hit);
391 + ebpf_one_dimension_write_charts(
392 + NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_HIT_CHART, ptr[NETDATA_CACHESTAT_IDX_HIT].dimension, publish->hit);
393
412 - cachestat_write_charts(NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_MISSES_CHART,
413 - ptr[NETDATA_CACHESTAT_IDX_MISS].dimension, publish->miss);
394 + ebpf_one_dimension_write_charts(
395 + NETDATA_EBPF_MEMORY_GROUP, NETDATA_CACHESTAT_MISSES_CHART, ptr[NETDATA_CACHESTAT_IDX_MISS].dimension,
396 + publish->miss);
397 }
398
399 /**
collectors/ebpf.plugin/ebpf_cachestat.h
-2
@@ -3,8 +3,6 @@
3 #ifndef NETDATA_EBPF_CACHESTAT_H
4 #define NETDATA_EBPF_CACHESTAT_H 1
5
6 -#define NETDATA_EBPF_MEMORY_GROUP "mem"
7 -
6 // charts
7 #define NETDATA_CACHESTAT_HIT_RATIO_CHART "cachestat_ratio"
8 #define NETDATA_CACHESTAT_DIRTY_CHART "cachestat_dirties"
collectors/ebpf.plugin/ebpf_kernel_reject_list.txt
+1 -1
@@ -1 +1 @@
1 -Ubuntu 4.18.0-13.
1 +Ubuntu 4.18.0
collectors/ebpf.plugin/ebpf_sync.c new
+211
@@ -0,0 +1,211 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "ebpf.h"
4 +#include "ebpf_sync.h"
5 +
6 +static ebpf_data_t sync_data;
7 +
8 +static struct bpf_link **probe_links = NULL;
9 +static struct bpf_object *objects = NULL;
10 +
11 +static char *sync_counter_dimension_name[NETDATA_SYNC_END] = { "sync" };
12 +static netdata_syscall_stat_t sync_counter_aggregated_data;
13 +static netdata_publish_syscall_t sync_counter_publish_aggregated;
14 +
15 +static int read_thread_closed = 1;
16 +
17 +static int *map_fd = NULL;
18 +static netdata_idx_t sync_hash_values = 0;
19 +
20 +struct netdata_static_thread sync_threads = {"SYNC KERNEL", NULL, NULL, 1,
21 + NULL, NULL, NULL};
22 +
23 +/*****************************************************************
24 + *
25 + * DATA THREAD
26 + *
27 + *****************************************************************/
28 +
29 +/**
30 + * Read global table
31 + *
32 + * Read the table with number of calls for all functions
33 + */
34 +static void read_global_table()
35 +{
36 + uint32_t idx = NETDATA_SYNC_CALL;
37 + netdata_idx_t stored;
38 + int fd = map_fd[NETDATA_SYNC_GLOBLAL_TABLE];
39 +
40 + if (!bpf_map_lookup_elem(fd, &idx, &stored)) {
41 + sync_hash_values = stored;
42 + }
43 +}
44 +
45 +/**
46 + * Sync read hash
47 + *
48 + * This is the thread callback.
49 + *
50 + * @param ptr It is a NULL value for this thread.
51 + *
52 + * @return It always returns NULL.
53 + */
54 +void *ebpf_sync_read_hash(void *ptr)
55 +{
56 + UNUSED(ptr);
57 + read_thread_closed = 0;
58 +
59 + heartbeat_t hb;
60 + heartbeat_init(&hb);
61 + usec_t step = NETDATA_EBPF_SYNC_SLEEP_MS;
62 +
63 + while (!close_ebpf_plugin) {
64 + usec_t dt = heartbeat_next(&hb, step);
65 + (void)dt;
66 +
67 + read_global_table();
68 + }
69 + read_thread_closed = 1;
70 +
71 + return NULL;
72 +}
73 +
74 +/**
75 + * Send global
76 + *
77 + * Send global charts to Netdata
78 + */
79 +static void sync_send_global()
80 +{
81 + ebpf_one_dimension_write_charts(NETDATA_EBPF_MEMORY_GROUP, NETDATA_EBPF_SYNC_CHART,
82 + sync_counter_publish_aggregated.dimension, sync_hash_values);
83 +}
84 +
85 +/**
86 +* Main loop for this collector.
87 +*/
88 +static void sync_collector(ebpf_module_t *em)
89 +{
90 + sync_threads.thread = mallocz(sizeof(netdata_thread_t));
91 + sync_threads.start_routine = ebpf_sync_read_hash;
92 +
93 + map_fd = sync_data.map_fd;
94 +
95 + netdata_thread_create(sync_threads.thread, sync_threads.name, NETDATA_THREAD_OPTION_JOINABLE,
96 + ebpf_sync_read_hash, em);
97 +
98 + while (!close_ebpf_plugin) {
99 + pthread_mutex_lock(&collect_data_mutex);
100 + pthread_cond_wait(&collect_data_cond_var, &collect_data_mutex);
101 +
102 + pthread_mutex_lock(&lock);
103 +
104 + sync_send_global();
105 +
106 + pthread_mutex_unlock(&lock);
107 + pthread_mutex_unlock(&collect_data_mutex);
108 + }
109 +}
110 +
111 +
112 +/*****************************************************************
113 + *
114 + * CLEANUP THREAD
115 + *
116 + *****************************************************************/
117 +
118 +/**
119 + * Clean up the main thread.
120 + *
121 + * @param ptr thread data.
122 + */
123 +static void ebpf_sync_cleanup(void *ptr)
124 +{
125 + ebpf_module_t *em = (ebpf_module_t *)ptr;
126 + if (!em->enabled)
127 + return;
128 +
129 + heartbeat_t hb;
130 + heartbeat_init(&hb);
131 + uint32_t tick = 2*USEC_PER_MS;
132 + while (!read_thread_closed) {
133 + usec_t dt = heartbeat_next(&hb, tick);
134 + UNUSED(dt);
135 + }
136 +
137 + freez(sync_threads.thread);
138 +
139 + struct bpf_program *prog;
140 + size_t i = 0 ;
141 + bpf_object__for_each_program(prog, objects) {
142 + bpf_link__destroy(probe_links[i]);
143 + i++;
144 + }
145 + bpf_object__close(objects);
146 +}
147 +
148 +/*****************************************************************
149 + *
150 + * MAIN THREAD
151 + *
152 + *****************************************************************/
153 +
154 +/**
155 + * Create global charts
156 + *
157 + * Call ebpf_create_chart to create the charts for the collector.
158 + */
159 +static void ebpf_create_sync_charts()
160 +{
161 + ebpf_create_chart(NETDATA_EBPF_MEMORY_GROUP, NETDATA_EBPF_SYNC_CHART,
162 + "Monitor calls for <a href=\"https://linux.die.net/man/2/sync\">sync(2)</a> syscall.",
163 + EBPF_COMMON_DIMENSION_CALL, NETDATA_EBPF_SYNC_SUBMENU, NULL, 21300,
164 + ebpf_create_global_dimension, &sync_counter_publish_aggregated, 1);
165 +}
166 +
167 +/**
168 + * Sync thread
169 + *
170 + * Thread used to make sync thread
171 + *
172 + * @param ptr a pointer to `struct ebpf_module`
173 + *
174 + * @return It always return NULL
175 + */
176 +void *ebpf_sync_thread(void *ptr)
177 +{
178 + netdata_thread_cleanup_push(ebpf_sync_cleanup, ptr);
179 +
180 + ebpf_module_t *em = (ebpf_module_t *)ptr;
181 + fill_ebpf_data(&sync_data);
182 +
183 + if (!em->enabled)
184 + goto endsync;
185 +
186 + if (ebpf_update_kernel(&sync_data)) {
187 + pthread_mutex_unlock(&lock);
188 + goto endsync;
189 + }
190 +
191 + probe_links = ebpf_load_program(ebpf_plugin_dir, em, kernel_string, &objects, sync_data.map_fd);
192 + if (!probe_links) {
193 + pthread_mutex_unlock(&lock);
194 + goto endsync;
195 + }
196 +
197 + int algorithm = NETDATA_EBPF_INCREMENTAL_IDX;
198 + ebpf_global_labels(&sync_counter_aggregated_data, &sync_counter_publish_aggregated,
199 + sync_counter_dimension_name, sync_counter_dimension_name,
200 + &algorithm, NETDATA_SYNC_END);
201 +
202 + pthread_mutex_lock(&lock);
203 + ebpf_create_sync_charts();
204 + pthread_mutex_unlock(&lock);
205 +
206 + sync_collector(em);
207 +
208 +endsync:
209 + netdata_thread_cleanup_pop(1);
210 + return NULL;
211 +}
collectors/ebpf.plugin/ebpf_sync.h new
+25
@@ -0,0 +1,25 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_EBPF_SYNC_H
4 +#define NETDATA_EBPF_SYNC_H 1
5 +
6 +// charts
7 +#define NETDATA_EBPF_SYNC_CHART "sync"
8 +#define NETDATA_EBPF_SYNC_SUBMENU "synchronization"
9 +
10 +#define NETDATA_EBPF_SYNC_SLEEP_MS 800000ULL
11 +
12 +enum netdata_sync_charts {
13 + NETDATA_SYNC_CALL,
14 +
15 + // Keep this as last and don't skip numbers as it is used as element counter
16 + NETDATA_SYNC_END
17 +};
18 +
19 +enum netdata_sync_table {
20 + NETDATA_SYNC_GLOBLAL_TABLE
21 +};
22 +
23 +extern void *ebpf_sync_thread(void *ptr);
24 +
25 +#endif /* NETDATA_EBPF_SYNC_H */
health/health.d/synchronization.conf new
+10
@@ -0,0 +1,10 @@
1 + alarm: sync_freq
2 + on: mem.sync
3 + lookup: sum -1m
4 + units: calls
5 + plugin: ebpf.plugin
6 + every: 1m
7 + warn: $this > 6
8 + delay: down 10m multiplier 1.5 max 1h
9 + info: Your OS is making a considerable number of synchronization to file descriptor in a short period of time.
10 + to: sysadmin
packaging/ebpf.checksums
+3 -3
@@ -1,3 +1,3 @@
1 -eb25cf47d7881ad4cafdd570ca1da1d3f95259fcaf75804f5e89181673dee571 netdata-kernel-collector-glibc-v0.5.7.tar.xz
2 -85aca7f06dff433cced967a62826e2778ddb541a99c33d4d7ec220e88a550bce netdata-kernel-collector-musl-v0.5.7.tar.xz
3 -b5cb0625c461ba16fa50e8886b3da2281e6d75e82f6a0a7f3817bf0a13b3b04e netdata-kernel-collector-static-v0.5.7.tar.xz
1 +fd93c6cda92c1eda6bedb384dd79095bb1ba7e87493153ec455efa9cb6365b48 netdata-kernel-collector-glibc-v0.6.0.tar.xz
2 +a19e776eb714f2ed08eb4e43bfd9798a06fad8f1b15421e18b22a36583f41a6f netdata-kernel-collector-musl-v0.6.0.tar.xz
3 +582b2c5dec077266c5f993f3a735cc46f09448cd6b24e6397daafde97616b113 netdata-kernel-collector-static-v0.6.0.tar.xz
packaging/ebpf.version
+1 -1
@@ -1 +1 @@
1 -v0.5.7
1 +v0.6.0
web/gui/dashboard_info.js
+4
@@ -1034,6 +1034,10 @@ netdataDashboard.context = {
1034 info: 'When the processor needs to read or write a location in main memory, it checks for a corresponding entry in the page cache. If the entry is not there, a page cache miss has occurred and the cache allocates a new entry and copies in data for the main memory. Misses count page insertions to the memory not related to writing.'
1035 },
1036
1037 + 'mem.sync': {
1038 + info: 'System calls for <code>sync()</code> which flushes file system buffers to storage devices. These calls can cause performance perturbations, and it can be useful to know if they are happening and how frequently. Based on the eBPF <a href="https://github.com/iovisor/bcc/blob/master/tools/syncsnoop.py" target="_blank">syncsnoop</a> from BCC tools.'
1039 + },
1040 +
1041 // ------------------------------------------------------------------------
1042 // network interfaces
1043