@cryptotaxi247 / netdata-1 / commits / fa12e3bb0

Bring flexible adjust for eBPF hash tables (#10962)

Give possibility for users to set hash table size.

thiagoftsm committed Apr 15, 2021 at 18:23 UTC fa12e3bb0f0cc368b97fd59e4b99073ae647d976
15 files changed +183 -26
collectors/ebpf.plugin/README.md
+15 -6
@@ -149,6 +149,7 @@ accepts the following values: ​
149 new charts for the return of these functions, such as errors. Monitoring function returns can help in debugging
150 software, such as failing to close file descriptors or creating zombie processes.
151 - `update every`: Number of seconds used for eBPF to send data for Netdata.
152 +- `pid table size`: Defines the maximum number of PIDs stored inside the application hash table.
153
154 #### Integration with `apps.plugin`
155
@@ -187,6 +188,11 @@ If you want to _disable_ the integration with `apps.plugin` along with the above
188 apps = yes
189 ```
190
191 +When the integration is enabled, eBPF collector allocates memory for each process running. The total
192 + allocated memory has direct relationship with the kernel version. When the eBPF plugin is running on kernels newer than `4.15`,
193 + it uses per-cpu maps to speed up the update of hash tables. This also implies storing data for the same PID
194 + for each processor it runs.
195 +
196 #### `[ebpf programs]`
197
198 The eBPF collector enables and runs the following eBPF programs by default:
@@ -347,13 +353,16 @@ mount these filesystems on startup. More information can be found in the [ftrace
353
354 ## Performance
355
350 -Because eBPF monitoring is complex, we are evaluating the performance of this new collector in various real-world
351 -conditions, across various system loads, and when monitoring complex applications.
356 +eBPF monitoring is complex and produces a large volume of metrics. We've discovered scenarios where the eBPF plugin
357 +significantly increases kernel memory usage by several hundred MB.
358 +
359 +If your node is experiencing high memory usage and there is no obvious culprit to be found in the `apps.mem` chart,
360 +consider testing for high kernel memory usage by [disabling eBPF monitoring](#configuration). Next,
361 +[restart Netdata](/docs/configure/start-stop-restart.md) with `sudo systemctl restart netdata` to see if system
362 +memory usage (see the `system.ram` chart) has dropped significantly.
363
353 -Our [initial testing](https://github.com/netdata/netdata/issues/8195) shows the performance of the eBPF collector is
354 -nearly identical to our [apps.plugin collector](/collectors/apps.plugin/README.md), despite collecting and displaying
355 -much more sophisticated metrics. You can now use the eBPF to gather deeper insights without affecting the performance of
356 -your complex applications at any load.
364 +Beginning with `v1.31`, kernel memory usage is configurable via the [`pid table size` setting](#ebpf-load-mode)
365 +in `ebpf.conf`.
366
367 ## SELinux
368
collectors/ebpf.plugin/ebpf.c
+28 -7
@@ -77,19 +77,22 @@ pthread_cond_t collect_data_cond_var;
77 ebpf_module_t ebpf_modules[] = {
78 { .thread_name = "process", .config_name = "process", .enabled = 0, .start_routine = ebpf_process_thread,
79 .update_time = 1, .global_charts = 1, .apps_charts = 1, .mode = MODE_ENTRY,
80 - .optional = 0, .apps_routine = ebpf_process_create_apps_charts },
80 + .optional = 0, .apps_routine = ebpf_process_create_apps_charts, .maps = NULL,
81 + .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE},
82 { .thread_name = "socket", .config_name = "socket", .enabled = 0, .start_routine = ebpf_socket_thread,
83 .update_time = 1, .global_charts = 1, .apps_charts = 1, .mode = MODE_ENTRY,
83 - .optional = 0, .apps_routine = ebpf_socket_create_apps_charts },
84 + .optional = 0, .apps_routine = ebpf_socket_create_apps_charts, .maps = NULL,
85 + .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE},
86 { .thread_name = "cachestat", .config_name = "cachestat", .enabled = 0, .start_routine = ebpf_cachestat_thread,
85 - .update_time = 1, .global_charts = 1, .apps_charts = 1, .mode = MODE_ENTRY,
86 - .optional = 0, .apps_routine = ebpf_cachestat_create_apps_charts },
87 + .update_time = 1, .global_charts = 1, .apps_charts = 1, .mode = MODE_ENTRY,
88 + .optional = 0, .apps_routine = ebpf_cachestat_create_apps_charts, .maps = NULL,
89 + .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE},
90 { .thread_name = "sync", .config_name = "sync", .enabled = 0, .start_routine = ebpf_sync_thread,
88 - .update_time = 1, .global_charts = 1, .apps_charts = 1, .mode = MODE_ENTRY,
89 - .optional = 0, .apps_routine = NULL },
91 + .update_time = 1, .global_charts = 1, .apps_charts = 1, .mode = MODE_ENTRY,
92 + .optional = 0, .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE },
93 { .thread_name = NULL, .enabled = 0, .start_routine = NULL, .update_time = 1,
94 .global_charts = 0, .apps_charts = 1, .mode = MODE_ENTRY,
92 - .optional = 0, .apps_routine = NULL },
95 + .optional = 0, .apps_routine = NULL, .maps = NULL, .pid_map_size = 0 },
96 };
97
98 // Link with apps.plugin
@@ -773,6 +776,22 @@ static void ebpf_update_interval()
776 }
777 }
778
779 +/**
780 + * Update PID table size
781 + *
782 + * Update default size with value from user
783 + */
784 +static void ebpf_update_table_size()
785 +{
786 + int i;
787 + uint32_t value = (uint32_t) appconfig_get_number(&collector_config, EBPF_GLOBAL_SECTION,
788 + EBPF_CFG_PID_SIZE, ND_EBPF_DEFAULT_PID_SIZE);
789 + for (i = 0; ebpf_modules[i].thread_name; i++) {
790 + ebpf_modules[i].pid_map_size = value;
791 + }
792 +}
793 +
794 +
795 /**
796 * Read collector values
797 *
@@ -793,6 +812,8 @@ static void read_collector_values(int *disable_apps)
812
813 ebpf_update_interval();
814
815 + ebpf_update_table_size();
816 +
817 // This is kept to keep compatibility
818 uint32_t enabled = appconfig_get_boolean(&collector_config, EBPF_GLOBAL_SECTION, "disable apps",
819 CONFIG_BOOLEAN_NO);
collectors/ebpf.plugin/ebpf.d.conf
+4
@@ -11,10 +11,14 @@
11 # 'no'.
12 #
13 # The `update every` option defines the number of seconds used to read data from kernel and send to netdata
14 +#
15 +# The `pid table size` defines the maximum number of PIDs stored in the application hash tables.
16 +#
17 [global]
18 ebpf load mode = entry
19 apps = yes
20 update every = 1
21 + pid table size = 32768
22
23 #
24 # eBPF Programs
collectors/ebpf.plugin/ebpf.d/cachestat.conf
+2
@@ -7,8 +7,10 @@
7 # If you want to disable the integration with `apps.plugin` along with the above charts, change the setting `apps` to
8 # 'no'.
9 #
10 +# The `pid table size` defines the maximum number of PIDs stored inside the application hash table.
11 #
12 [global]
13 ebpf load mode = entry
14 apps = yes
15 update every = 2
16 + pid table size = 32768
collectors/ebpf.plugin/ebpf.d/network.conf
+10 -1
@@ -7,11 +7,20 @@
7 # If you want to disable the integration with `apps.plugin` along with the above charts, change the setting `apps` to
8 # 'no'.
9 #
10 -#
10 +# The following options change the hash table size:
11 +# `bandwidth table size`: Maximum number of connections monitored
12 +# `ipv4 connection table size`: Maximum number of IPV4 connections monitored
13 +# `ipv6 connection table size`: Maximum number of IPV6 connections monitored
14 +# `udp connection table size`: Maximum number of UDP connections monitored
15 +#
16 [global]
17 ebpf load mode = entry
18 apps = yes
19 update every = 1
20 + bandwidth table size = 16384
21 + ipv4 connection table size = 16384
22 + ipv6 connection table size = 16384
23 + udp connection table size = 4096
24
25 #
26 # Network Connection
collectors/ebpf.plugin/ebpf.d/process.conf
+2
@@ -7,8 +7,10 @@
7 # If you want to disable the integration with `apps.plugin` along with the above charts, change the setting `apps` to
8 # 'no'.
9 #
10 +# The `pid table size` defines the maximum number of PIDs stored inside the hash table.
11 #
12 [global]
13 ebpf load mode = entry
14 apps = yes
15 update every = 1
16 + pid table size = 32768
collectors/ebpf.plugin/ebpf.h
+1
@@ -215,6 +215,7 @@ extern void ebpf_socket_create_apps_charts(struct ebpf_module *em, void *ptr);
215 extern void ebpf_cachestat_create_apps_charts(struct ebpf_module *em, void *root);
216 extern void ebpf_one_dimension_write_charts(char *family, char *chart, char *dim, long long v1);
217 extern collected_number get_value_from_structure(char *basis, size_t offset);
218 +extern void ebpf_update_pid_table(ebpf_local_maps_t *pid, ebpf_module_t *em);
219
220 #define EBPF_MAX_SYNCHRONIZATION_TIME 300
221
collectors/ebpf.plugin/ebpf_cachestat.c
+6
@@ -24,6 +24,10 @@ struct netdata_static_thread cachestat_threads = {"CACHESTAT KERNEL",
24 NULL, NULL, 1, NULL,
25 NULL, NULL};
26
27 +static ebpf_local_maps_t cachestat_maps[] = {{.name = "cstat_pid", .internal_input = ND_EBPF_DEFAULT_PID_SIZE,
28 + .user_input = 0},
29 + {.name = NULL, .internal_input = 0, .user_input = 0}};
30 +
31 static int *map_fd = NULL;
32
33 struct config cachestat_config = { .first_section = NULL,
@@ -608,9 +612,11 @@ void *ebpf_cachestat_thread(void *ptr)
612 netdata_thread_cleanup_push(ebpf_cachestat_cleanup, ptr);
613
614 ebpf_module_t *em = (ebpf_module_t *)ptr;
615 + em->maps = cachestat_maps;
616 fill_ebpf_data(&cachestat_data);
617
618 ebpf_update_module(em, &cachestat_config, NETDATA_CACHESTAT_CONFIG_FILE);
619 + ebpf_update_pid_table(&cachestat_maps[0], em);
620
621 if (!em->enabled)
622 goto endcachestat;
collectors/ebpf.plugin/ebpf_process.c
+6
@@ -18,6 +18,10 @@ static char *process_id_names[NETDATA_KEY_PUBLISH_PROCESS_END] = { "do_sys_open"
18 "release_task", "_do_fork", "sys_clone" };
19 static char *status[] = { "process", "zombie" };
20
21 +static ebpf_local_maps_t process_maps[] = {{.name = "tbl_pid_stats", .internal_input = ND_EBPF_DEFAULT_PID_SIZE,
22 + .user_input = 0},
23 + {.name = NULL, .internal_input = 0, .user_input = 0}};
24 +
25 static netdata_idx_t *process_hash_values = NULL;
26 static netdata_syscall_stat_t process_aggregated_data[NETDATA_KEY_PUBLISH_PROCESS_END];
27 static netdata_publish_syscall_t process_publish_aggregated[NETDATA_KEY_PUBLISH_PROCESS_END];
@@ -1029,6 +1033,7 @@ void *ebpf_process_thread(void *ptr)
1033 netdata_thread_cleanup_push(ebpf_process_cleanup, ptr);
1034
1035 ebpf_module_t *em = (ebpf_module_t *)ptr;
1036 + em->maps = process_maps;
1037 process_enabled = em->enabled;
1038 fill_ebpf_data(&process_data);
1039
@@ -1041,6 +1046,7 @@ void *ebpf_process_thread(void *ptr)
1046 }
1047
1048 ebpf_update_module(em, &process_config, NETDATA_PROCESS_CONFIG_FILE);
1049 + ebpf_update_pid_table(&process_maps[0], em);
1050
1051 set_local_pointers();
1052 probe_links = ebpf_load_program(ebpf_plugin_dir, em, kernel_string, &objects, process_data.map_fd);
collectors/ebpf.plugin/ebpf_socket.c
+35
@@ -16,6 +16,20 @@ static char *socket_dimension_names[NETDATA_MAX_SOCKET_VECTOR] = { "sent", "rece
16 static char *socket_id_names[NETDATA_MAX_SOCKET_VECTOR] = { "tcp_sendmsg", "tcp_cleanup_rbuf", "tcp_close",
17 "udp_sendmsg", "udp_recvmsg", "tcp_retransmit_skb" };
18
19 +static ebpf_local_maps_t socket_maps[] = {{.name = "tbl_bandwidth",
20 + .internal_input = NETDATA_COMPILED_CONNECTIONS_ALLOWED,
21 + .user_input = NETDATA_MAXIMUM_CONNECTIONS_ALLOWED},
22 + {.name = "tbl_conn_ipv4",
23 + .internal_input = NETDATA_COMPILED_CONNECTIONS_ALLOWED,
24 + .user_input = NETDATA_MAXIMUM_CONNECTIONS_ALLOWED},
25 + {.name = "tbl_conn_ipv6",
26 + .internal_input = NETDATA_COMPILED_CONNECTIONS_ALLOWED,
27 + .user_input = NETDATA_MAXIMUM_CONNECTIONS_ALLOWED},
28 + {.name = "tbl_nv_udp_conn_stats",
29 + .internal_input = NETDATA_COMPILED_UDP_CONNECTIONS_ALLOWED,
30 + .user_input = NETDATA_MAXIMUM_UDP_CONNECTIONS_ALLOWED},
31 + {.name = NULL, .internal_input = 0, .user_input = 0}};
32 +
33 static netdata_idx_t *socket_hash_values = NULL;
34 static netdata_syscall_stat_t socket_aggregated_data[NETDATA_MAX_SOCKET_VECTOR];
35 static netdata_publish_syscall_t socket_publish_aggregated[NETDATA_MAX_SOCKET_VECTOR];
@@ -2807,6 +2821,25 @@ void parse_service_name_section(struct config *cfg)
2821 }
2822 }
2823
2824 +void parse_table_size_options(struct config *cfg)
2825 +{
2826 + socket_maps[NETDATA_SOCKET_TABLE_BANDWIDTH].user_input = (uint32_t) appconfig_get_number(cfg,
2827 + EBPF_GLOBAL_SECTION,
2828 + EBPF_CONFIG_BANDWIDTH_SIZE, NETDATA_MAXIMUM_CONNECTIONS_ALLOWED);
2829 +
2830 + socket_maps[NETDATA_SOCKET_TABLE_IPV4].user_input = (uint32_t) appconfig_get_number(cfg,
2831 + EBPF_GLOBAL_SECTION,
2832 + EBPF_CONFIG_IPV4_SIZE, NETDATA_MAXIMUM_CONNECTIONS_ALLOWED);
2833 +
2834 + socket_maps[NETDATA_SOCKET_TABLE_IPV6].user_input = (uint32_t) appconfig_get_number(cfg,
2835 + EBPF_GLOBAL_SECTION,
2836 + EBPF_CONFIG_IPV6_SIZE, NETDATA_MAXIMUM_CONNECTIONS_ALLOWED);
2837 +
2838 + socket_maps[NETDATA_SOCKET_TABLE_UDP].user_input = (uint32_t) appconfig_get_number(cfg,
2839 + EBPF_GLOBAL_SECTION,
2840 + EBPF_CONFIG_UDP_SIZE, NETDATA_MAXIMUM_UDP_CONNECTIONS_ALLOWED);
2841 +}
2842 +
2843 /**
2844 * Socket thread
2845 *
@@ -2826,11 +2859,13 @@ void *ebpf_socket_thread(void *ptr)
2859 avl_init_lock(&outbound_vectors.tree, compare_sockets);
2860
2861 ebpf_module_t *em = (ebpf_module_t *)ptr;
2862 + em->maps = socket_maps;
2863 fill_ebpf_data(&socket_data);
2864
2865 ebpf_update_module(em, &socket_config, NETDATA_NETWORK_CONFIG_FILE);
2866 parse_network_viewer_section(&socket_config);
2867 parse_service_name_section(&socket_config);
2868 + parse_table_size_options(&socket_config);
2869
2870 if (!em->enabled)
2871 goto endsocket;
collectors/ebpf.plugin/ebpf_socket.h
+15
@@ -24,8 +24,19 @@
24 #define EBPF_CONFIG_RESOLVE_SERVICE "resolve service names"
25 #define EBPF_CONFIG_PORTS "ports"
26 #define EBPF_CONFIG_HOSTNAMES "hostnames"
27 +#define EBPF_CONFIG_BANDWIDTH_SIZE "bandwidth table size"
28 +#define EBPF_CONFIG_IPV4_SIZE "ipv4 connection table size"
29 +#define EBPF_CONFIG_IPV6_SIZE "ipv6 connection table size"
30 +#define EBPF_CONFIG_UDP_SIZE "udp connection table size"
31 #define EBPF_MAXIMUM_DIMENSIONS "maximum dimensions"
32
33 +enum ebpf_socket_table_list {
34 + NETDATA_SOCKET_TABLE_BANDWIDTH,
35 + NETDATA_SOCKET_TABLE_IPV4,
36 + NETDATA_SOCKET_TABLE_IPV6,
37 + NETDATA_SOCKET_TABLE_UDP
38 +};
39 +
40 enum ebpf_socket_publish_index {
41 NETDATA_IDX_TCP_SENDMSG,
42 NETDATA_IDX_TCP_CLEANUP_RBUF,
@@ -94,6 +105,10 @@ typedef enum ebpf_socket_idx {
105 // Port range
106 #define NETDATA_MINIMUM_PORT_VALUE 1
107 #define NETDATA_MAXIMUM_PORT_VALUE 65535
108 +#define NETDATA_COMPILED_CONNECTIONS_ALLOWED 65535U
109 +#define NETDATA_MAXIMUM_CONNECTIONS_ALLOWED 16384U
110 +#define NETDATA_COMPILED_UDP_CONNECTIONS_ALLOWED 8192U
111 +#define NETDATA_MAXIMUM_UDP_CONNECTIONS_ALLOWED 4096U
112
113 #define NETDATA_MINIMUM_IPV4_CIDR 0
114 #define NETDATA_MAXIMUM_IPV4_CIDR 32
libnetdata/ebpf/ebpf.c
+44 -8
@@ -295,25 +295,58 @@ static int select_file(char *name, const char *program, size_t length, int mode,
295 return ret;
296 }
297
298 +void ebpf_update_pid_table(ebpf_local_maps_t *pid, ebpf_module_t *em)
299 +{
300 + pid->user_input = em->pid_map_size;
301 +}
302 +
303 +void ebpf_update_map_sizes(struct bpf_object *program, ebpf_module_t *em)
304 +{
305 + struct bpf_map *map;
306 + ebpf_local_maps_t *maps = em->maps;
307 + if (!maps)
308 + return;
309 +
310 + bpf_map__for_each(map, program)
311 + {
312 + const char *map_name = bpf_map__name(map);
313 + int i = 0; ;
314 + while (maps[i].name) {
315 + ebpf_local_maps_t *w = &maps[i];
316 + if (w->user_input != w->internal_input && !strcmp(w->name, map_name)) {
317 +#ifdef NETDATA_INTERNAL_CHECKS
318 + info("Changing map %s from size %u to %u ", map_name, w->internal_input, w->user_input);
319 +#endif
320 + bpf_map__resize(map, w->user_input);
321 + }
322 + i++;
323 + }
324 + }
325 +}
326 +
327 struct bpf_link **ebpf_load_program(char *plugins_dir, ebpf_module_t *em, char *kernel_string, struct bpf_object **obj, int *map_fd)
328 {
329 char lpath[4096];
330 char lname[128];
302 - int prog_fd;
331
332 int test = select_file(lname, em->thread_name, (size_t)127, em->mode, kernel_string);
333 if (test < 0 || test > 127)
334 return NULL;
335
336 snprintf(lpath, 4096, "%s/ebpf.d/%s", plugins_dir, lname);
309 - // We are using BPF_PROG_TYPE_UNSPEC instead a specific type for bpf_prog_load to define the type
310 - // according the eBPF program loaded
311 - if (bpf_prog_load(lpath, BPF_PROG_TYPE_UNSPEC, obj, &prog_fd)) {
312 - em->enabled = CONFIG_BOOLEAN_NO;
313 - info("Cannot load program: %s", lpath);
337 + *obj = bpf_object__open_file(lpath, NULL);
338 + if (libbpf_get_error(obj)) {
339 + error("Cannot open BPF object %s", lpath);
340 + bpf_object__close(*obj);
341 + return NULL;
342 + }
343 +
344 + ebpf_update_map_sizes(*obj, em);
345 +
346 + if (bpf_object__load(*obj)) {
347 + error("ERROR: loading BPF object file failed %s\n", lpath);
348 + bpf_object__close(*obj);
349 return NULL;
315 - } else {
316 - info("The eBPF program %s was loaded with success.", em->thread_name);
350 }
351
352 struct bpf_map *map;
@@ -368,6 +401,9 @@ void ebpf_update_module_using_config(ebpf_module_t *modules, struct config *cfg)
401
402 modules->apps_charts = appconfig_get_boolean(cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_APPLICATION,
403 CONFIG_BOOLEAN_YES);
404 +
405 + modules->pid_map_size = (uint32_t)appconfig_get_number(cfg, EBPF_GLOBAL_SECTION, EBPF_CFG_PID_SIZE,
406 + modules->pid_map_size);
407 }
408
409
libnetdata/ebpf/ebpf.h
+11
@@ -15,6 +15,7 @@
15 #define EBPF_CFG_LOAD_MODE_RETURN "return"
16
17 #define EBPF_CFG_UPDATE_EVERY "update every"
18 +#define EBPF_CFG_PID_SIZE "pid table size"
19 #define EBPF_CFG_APPLICATION "apps"
20
21 /**
@@ -95,6 +96,14 @@ typedef enum {
96 MODE_ENTRY // This attaches kprobe when the function is called
97 } netdata_run_mode_t;
98
99 +#define ND_EBPF_DEFAULT_PID_SIZE 32768U
100 +
101 +typedef struct ebpf_local_maps {
102 + char *name;
103 + uint32_t internal_input;
104 + uint32_t user_input;
105 +} ebpf_local_maps_t;
106 +
107 typedef struct ebpf_module {
108 const char *thread_name;
109 const char *config_name;
@@ -107,6 +116,8 @@ typedef struct ebpf_module {
116 uint32_t thread_id;
117 int optional;
118 void (*apps_routine)(struct ebpf_module *em, void *ptr);
119 + ebpf_local_maps_t *maps;
120 + uint32_t pid_map_size;
121 } ebpf_module_t;
122
123 #define NETDATA_MAX_PROBES 64
packaging/ebpf.checksums
+3 -3
@@ -1,3 +1,3 @@
1 -380e31fe143e7b53bcebaaf03a04d143ae82e13318b264461ebb5d3ac9026ae5 netdata-kernel-collector-glibc-v0.6.1.tar.xz
2 -5a196ab8a00d307a4f6a5c213178bd62e5720173f433afc6e77dfa911fb6ca56 netdata-kernel-collector-musl-v0.6.1.tar.xz
3 -683e6676c1eee0cd4a7da5be953e94052e780de1ca375146a488d62593220c46 netdata-kernel-collector-static-v0.6.1.tar.xz
1 +1442027d53cf11e1b086ec837659a498a9a2738ef43e44b32a2a0171d057544a netdata-kernel-collector-glibc-v0.6.3.tar.xz
2 +0863b06e78bb3a596cb1f68d13560301f563683cb174fd27b1e34c232e6f3c22 netdata-kernel-collector-musl-v0.6.3.tar.xz
3 +571dddd2b3b06d9f53cc24384ffbd88e2bb662ad953acaef46c76249186fe3b6 netdata-kernel-collector-static-v0.6.3.tar.xz
packaging/ebpf.version
+1 -1
@@ -1 +1 @@
1 -v0.6.1
1 +v0.6.3