@cryptotaxi247 / netdata-1 / commits / cb98702e1

Work to find leaks easily (#20106)

Costa Tsaousis committed Apr 15, 2025 at 11:09 UTC cb98702e117f90e13e1ddebeee96a06c7f6f4ce1
53 files changed +2689 -887
CMakeLists.txt
+14 -2
@@ -219,7 +219,7 @@ mark_as_advanced(ENABLE_SENTRY)
219 option(BUILD_FOR_PACKAGING "Include component files for native packages" False)
220 mark_as_advanced(BUILD_FOR_PACKAGING)
221
222 -cmake_dependent_option(ENABLE_LIBBACKTRACE "Use libbacktrace for stack traces in log output" True "OS_LINUX OR OS_WINDOWS" False)
222 +cmake_dependent_option(ENABLE_LIBBACKTRACE "Use libbacktrace for stack traces in log output" True "OS_LINUX OR OS_WINDOWS OR OS_FREEBSD" False)
223 mark_as_advanced(ENABLE_LIBBACKTRACE)
224 cmake_dependent_option(ENABLE_LIBUNWIND "Use libunwind for stack traces in log output" False "NOT ENABLE_LIBBACKTRACE" False)
225 mark_as_advanced(ENABLE_LIBUNWIND)
@@ -843,6 +843,8 @@ set(LIBNETDATA_FILES
843 src/libnetdata/datetime/rfc3339.c
844 src/libnetdata/datetime/rfc3339.h
845 src/libnetdata/dictionary/dictionary.c
846 + src/libnetdata/dictionary/dictionary-debug.c
847 + src/libnetdata/dictionary/dictionary-debug.h
848 src/libnetdata/dictionary/dictionary.h
849 src/libnetdata/eval/eval.c
850 src/libnetdata/eval/eval.h
@@ -984,6 +986,17 @@ set(LIBNETDATA_FILES
986 src/libnetdata/os/setproctitle.h
987 src/libnetdata/paths/paths.c
988 src/libnetdata/paths/paths.h
989 + src/libnetdata/stacktrace/stacktrace.h
990 + src/libnetdata/stacktrace/stacktrace-common.h
991 + src/libnetdata/stacktrace/stacktrace-common.c
992 + src/libnetdata/stacktrace/stacktrace-array.h
993 + src/libnetdata/stacktrace/stacktrace-array.c
994 + src/libnetdata/stacktrace/stacktrace-libbacktrace.c
995 + src/libnetdata/stacktrace/stacktrace-libunwind.c
996 + src/libnetdata/stacktrace/stacktrace-backtrace.c
997 + src/libnetdata/stacktrace/stacktrace-none.c
998 + src/libnetdata/stacktrace/stacktrace-log.c
999 + src/libnetdata/stacktrace/stacktrace-unittest.c
1000 src/libnetdata/json/json-c-parser-inline.c
1001 src/libnetdata/parsers/duration.h
1002 src/libnetdata/parsers/timeframe.c
@@ -1064,7 +1077,6 @@ set(LIBNETDATA_FILES
1077 src/libnetdata/locks/benchmark.h
1078 src/libnetdata/locks/benchmark-rw.c
1079 src/libnetdata/locks/benchmark-rw.h
1067 - src/libnetdata/log/nd_log-stacktrace.c
1080 src/libnetdata/memory/nd-mallocz.c
1081 src/libnetdata/memory/nd-mallocz.h
1082 src/libnetdata/memory/nd-mmap.c
packaging/makeself/jobs/70-netdata-git.install.sh
+4 -5
@@ -7,12 +7,12 @@
7 cd "${NETDATA_SOURCE_PATH}" || exit 1
8
9 if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then
10 - export CFLAGS="${TUNING_FLAGS} -ffunction-sections -fdata-sections -static -O2 -funroll-loops -DNETDATA_STATIC_BUILD=1 -I/libunwind-static/include -I/openssl-static/include -I/libnetfilter-acct-static/include/libnetfilter_acct -I/curl-local/include/curl -I/usr/include/libmnl -pipe"
10 + export CFLAGS="${TUNING_FLAGS} -ffunction-sections -fdata-sections -static -O2 -funroll-loops -DNETDATA_STATIC_BUILD=1 -I/openssl-static/include -I/libnetfilter-acct-static/include/libnetfilter_acct -I/curl-local/include/curl -I/usr/include/libmnl -pipe"
11 else
12 - export CFLAGS="${TUNING_FLAGS} -static -O1 -pipe -ggdb -Wall -Wextra -Wformat-signedness -DNETDATA_STATIC_BUILD=1 -DNETDATA_INTERNAL_CHECKS=1 -I/libunwind-static/include -I/openssl-static/include -I/libnetfilter-acct-static/include/libnetfilter_acct -I/curl-local/include/curl -I/usr/include/libmnl"
12 + export CFLAGS="${TUNING_FLAGS} -static -O1 -pipe -ggdb -Wall -Wextra -Wformat-signedness -DNETDATA_STATIC_BUILD=1 -DNETDATA_INTERNAL_CHECKS=1 -I/openssl-static/include -I/libnetfilter-acct-static/include/libnetfilter_acct -I/curl-local/include/curl -I/usr/include/libmnl"
13 fi
14
15 -export LDFLAGS="-Wl,--gc-sections -static -L/libuwnind-static/lib -L/openssl-static/lib64 -L/libnetfilter-acct-static/lib -lnetfilter_acct -L/usr/lib -lmnl -L/usr/lib -lzstd -L/curl-local/lib"
15 +export LDFLAGS="-Wl,--gc-sections -static -L/openssl-static/lib64 -L/libnetfilter-acct-static/lib -lnetfilter_acct -L/usr/lib -lmnl -L/usr/lib -lzstd -L/curl-local/lib"
16
17 # We export this to 'yes', installer sets this to .environment.
18 # The updater consumes this one, so that it can tell whether it should update a static install or a non-static one
@@ -21,13 +21,12 @@ export IS_NETDATA_STATIC_BINARY="yes"
21 # Set eBPF LIBC to "static" to bundle the `-static` variant of the kernel-collector
22 export EBPF_LIBC="static"
23 export PKG_CONFIG="pkg-config --static"
24 -export PKG_CONFIG_PATH="/libunwind-static/lib/pkgconfig:/openssl-static/lib64/pkgconfig:/libnetfilter-acct-static/lib/pkgconfig:/usr/lib/pkgconfig:/curl-local/lib/pkgconfig"
24 +export PKG_CONFIG_PATH="/openssl-static/lib64/pkgconfig:/libnetfilter-acct-static/lib/pkgconfig:/usr/lib/pkgconfig:/curl-local/lib/pkgconfig"
25
26 NETDATA_BUILD_DIR="$(build_path netdata)"
27 export NETDATA_BUILD_DIR
28
29 case "${BUILDARCH}" in
30 - armv7l|armv6l) export NETDATA_CMAKE_OPTIONS="-DENABLE_LIBBACKTRACE=OFF -DENABLE_LIBUNWIND=ON" ;;
30 *) export NETDATA_CMAKE_OPTIONS="-DENABLE_LIBBACKTRACE=On"
31 esac
32
packaging/utils/compile-and-run-windows.sh
+6 -2
@@ -90,7 +90,7 @@ if [ $RUN_AS_SERVICE -eq 1 ]; then
90 sc delete "Netdata" || echo "delete Failed, ok"
91 fi
92
93 -rm -f /opt/netdata/usr/bin/*.dll || echo "deleting old .dll files failed, ok"
93 +rm -f /opt/netdata/usr/bin/*.dll /opt/netdata/usr/libexec/netdata/plugins.d/*.dll || echo "deleting old .dll files failed, ok"
94 ninja -v -C "${build}" install
95
96 # register the event log publisher
@@ -108,9 +108,13 @@ if [ $RUN_AS_SERVICE -eq 1 ]; then
108 sed -e 's|\s\+| |g' -e 's|^ ||g' |\
109 cut -d ' ' -f 3 |\
110 while read x; do
111 - cp $x /opt/netdata/usr/bin/
111 + cp "$x" /opt/netdata/usr/bin/
112 done
113
114 + for x in bash.exe sh.exe; do
115 + cp "/usr/bin/$x" /opt/netdata/usr/bin/
116 + done
117 +
118 echo
119 echo "Registering Netdata service..."
120 sc create "Netdata" binPath= "$(cygpath.exe -w /opt/netdata/usr/bin/netdata.exe)" start= auto
src/collectors/cgroups.plugin/cgroup-internals.h
+2
@@ -394,6 +394,8 @@ int cgroup_function_cgroup_top(BUFFER *wb, const char *function, BUFFER *payload
394 int cgroup_function_systemd_top(BUFFER *wb, const char *function, BUFFER *payload, const char *source);
395
396 void cgroup_netdev_link_init(void);
397 +void cgroup_netdev_link_destroy(void);
398 +
399 const DICTIONARY_ITEM *cgroup_netdev_get(struct cgroup *cg);
400 void cgroup_netdev_delete(struct cgroup *cg);
401
src/collectors/cgroups.plugin/cgroup-top.c
+9
@@ -10,10 +10,19 @@ struct cgroup_netdev_link {
10
11 static DICTIONARY *cgroup_netdev_link_dict = NULL;
12
13 +void netdev_renames_destroy(void);
14 +
15 void cgroup_netdev_link_init(void) {
16 cgroup_netdev_link_dict = dictionary_create_advanced(DICT_OPTION_FIXED_SIZE|DICT_OPTION_DONT_OVERWRITE_VALUE, NULL, sizeof(struct cgroup_netdev_link));
17 }
18
19 +void cgroup_netdev_link_destroy(void) {
20 + dictionary_destroy(cgroup_netdev_link_dict);
21 + cgroup_netdev_link_dict = NULL;
22 +
23 + netdev_renames_destroy();
24 +}
25 +
26 const DICTIONARY_ITEM *cgroup_netdev_get(struct cgroup *cg) {
27 if(!cg->cgroup_netdev_link) {
28 struct cgroup_netdev_link t = {
src/collectors/cgroups.plugin/sys_fs_cgroup.c
+6 -4
@@ -1380,6 +1380,10 @@ void *cgroups_main(void *ptr) {
1380 goto exit;
1381 }
1382
1383 + // we register this only on localhost
1384 + // for the other nodes, the origin server should register it
1385 + cgroup_netdev_link_init();
1386 +
1387 discovery_thread.exited = 0;
1388
1389 if (uv_mutex_init(&discovery_thread.mutex)) {
@@ -1397,10 +1401,6 @@ void *cgroups_main(void *ptr) {
1401 goto exit;
1402 }
1403
1400 - // we register this only on localhost
1401 - // for the other nodes, the origin server should register it
1402 - cgroup_netdev_link_init();
1403 -
1404 rrd_function_add_inline(localhost, NULL, "containers-vms", 10,
1405 RRDFUNCTIONS_PRIORITY_DEFAULT / 2, RRDFUNCTIONS_VERSION_DEFAULT,
1406 RRDFUNCTIONS_CGTOP_HELP,
@@ -1459,6 +1459,8 @@ void *cgroups_main(void *ptr) {
1459 uv_mutex_unlock(&cgroup_root_mutex);
1460 }
1461
1462 + // uv_thread_join(&discovery_thread.thread);
1463 +
1464 exit:
1465 return NULL;
1466 }
src/collectors/proc.plugin/proc_net_dev_renames.c
+10 -4
@@ -41,16 +41,22 @@ static bool dictionary_netdev_rename_conflict_cb(const DICTIONARY_ITEM *item __m
41 return true;
42 }
43
44 +static SPINLOCK netdev_renames_spinlock = SPINLOCK_INITIALIZER;
45 void netdev_renames_init(void) {
45 - static SPINLOCK spinlock = SPINLOCK_INITIALIZER;
46 -
47 - spinlock_lock(&spinlock);
46 + spinlock_lock(&netdev_renames_spinlock);
47 if(!netdev_renames) {
48 netdev_renames = dictionary_create_advanced(DICT_OPTION_FIXED_SIZE | DICT_OPTION_DONT_OVERWRITE_VALUE, NULL, sizeof(struct rename_task));
49 dictionary_register_conflict_callback(netdev_renames, dictionary_netdev_rename_conflict_cb, NULL);
50 dictionary_register_delete_callback(netdev_renames, dictionary_netdev_rename_delete_cb, NULL);
51 }
53 - spinlock_unlock(&spinlock);
52 + spinlock_unlock(&netdev_renames_spinlock);
53 +}
54 +
55 +void netdev_renames_destroy(void) {
56 + spinlock_lock(&netdev_renames_spinlock);
57 + dictionary_destroy(netdev_renames);
58 + netdev_renames = NULL;
59 + spinlock_unlock(&netdev_renames_spinlock);
60 }
61
62 void cgroup_rename_task_add(
src/collectors/proc.plugin/proc_net_dev_renames.h
+1
@@ -27,6 +27,7 @@ struct rename_task {
27 } while(0)
28
29 void netdev_renames_init(void);
30 +void netdev_renames_destroy(void);
31
32 void cgroup_netdev_reset_all(void);
33 void cgroup_netdev_release(const DICTIONARY_ITEM *link);
src/daemon/buildinfo.c
+1 -1
@@ -1303,7 +1303,7 @@ __attribute__((constructor)) void initialize_build_info(void) {
1303 #ifdef HAVE_LIBMNL
1304 build_info_set_status(BIB_LIB_LIBMNL, true);
1305 #endif
1306 - build_info_set_value(BIB_LIB_STACKTRACE, capture_stack_trace_backend());
1306 + build_info_set_value(BIB_LIB_STACKTRACE, stacktrace_backend());
1307
1308 #ifdef ENABLE_PLUGIN_APPS
1309 build_info_set_status(BIB_PLUGIN_APPS, true);
src/daemon/daemon-shutdown.c
+15 -8
@@ -16,12 +16,13 @@
16 // External configuration structures that need cleanup
17 extern struct config netdata_config;
18 extern struct config cloud_config;
19 -extern void inicfg_free(struct config *root);
19 +
20 // Functions to free various configurations
21 -extern void claim_config_free(void);
22 -extern void stream_config_free(void);
23 -extern void exporting_config_free(void);
24 -extern void rrd_functions_inflight_destroy(void);
21 +void claim_config_free(void);
22 +void rrd_functions_inflight_destroy(void);
23 +void cgroup_netdev_link_destroy(void);
24 +void bearer_tokens_destroy(void);
25 +void alerts_by_x_cleanup(void);
26
27 static bool abort_on_fatal = true;
28
@@ -349,12 +350,17 @@ static void netdata_cleanup_and_exit(EXIT_REASON reason, bool abnormal, bool exi
350 dyncfg_shutdown();
351 rrd_functions_inflight_destroy();
352 health_plugin_destroy();
353 + cgroup_netdev_link_destroy();
354 + bearer_tokens_destroy();
355
356 fprintf(stderr, "Cleaning up destroyed dictionaries...\n");
354 - size_t dictionaries_referenced = cleanup_destroyed_dictionaries();
357 + size_t dictionaries_referenced = cleanup_destroyed_dictionaries(true);
358 if(dictionaries_referenced)
359 fprintf(stderr, "WARNING: There are %zu dictionaries with references in them, that cannot be destroyed.\n",
360 dictionaries_referenced);
361 +
362 + // Always report dictionary allocations during ASAN builds
363 + dictionary_print_still_allocated_stacktraces();
364
365 #ifdef ENABLE_DBENGINE
366 // destroy the caches in reverse order (extent and open depend on main cache)
@@ -392,10 +398,11 @@ static void netdata_cleanup_and_exit(EXIT_REASON reason, bool abnormal, bool exi
398 stream_config_free();
399 inicfg_free(&cloud_config);
400 inicfg_free(&netdata_config);
395 -
401 +
402 fprintf(stderr, "Cleaning up worker utilization...\n");
403 worker_utilization_cleanup();
398 -
404 +
405 + alerts_by_x_cleanup();
406 size_t strings_referenced = string_destroy();
407 if(strings_referenced)
408 fprintf(stderr, "WARNING: STRING has %zu strings still allocated.\n",
src/daemon/daemon.c
+2 -2
@@ -428,7 +428,7 @@ int become_daemon(int dont_fork, const char *user) {
428 // the child
429 gettid_uncached();
430 nd_initialize_signals(false);
431 - capture_stack_trace_flush();
431 + stacktrace_flush();
432
433 // become session leader
434 if (setsid() < 0) {
@@ -451,7 +451,7 @@ int become_daemon(int dont_fork, const char *user) {
451 // the child
452 gettid_uncached();
453 nd_initialize_signals(false);
454 - capture_stack_trace_flush();
454 + stacktrace_flush();
455 }
456
457 // generate our pid file
src/daemon/main.c
+5
@@ -404,6 +404,7 @@ int netdata_main(int argc, char **argv) {
404 if (dyncfg_unittest()) return 1;
405 if (unittest_waiting_queue()) return 1;
406 if (uuidmap_unittest()) return 1;
407 + if (stacktrace_unittest()) return 1;
408 #ifdef OS_WINDOWS
409 if (perflibnamestest_main()) return 1;
410 #endif
@@ -454,6 +455,10 @@ int netdata_main(int argc, char **argv) {
455 unittest_running = true;
456 return uuid_unittest();
457 }
458 + else if(strcmp(optarg, "stacktracetest") == 0) {
459 + unittest_running = true;
460 + return stacktrace_unittest();
461 + }
462 #ifdef OS_WINDOWS
463 else if(strcmp(optarg, "perflibdump") == 0) {
464 return windows_perflib_dump(optind + 1 > argc ? NULL : argv[optind]);
src/daemon/signal-handler.c
+1 -1
@@ -173,7 +173,7 @@ void nd_initialize_signals(bool chain_existing) {
173 signals_block_all_except_deadly();
174
175 // Set the signal handler name for stack trace filtering
176 - capture_stack_trace_set_signal_handler_function("nd_signal_handler");
176 + stacktrace_set_signal_handler_function("nd_signal_handler");
177
178 struct sigaction act;
179 memset(&act, 0, sizeof(struct sigaction));
src/daemon/status-file.c
+8 -8
@@ -709,7 +709,7 @@ static void daemon_status_file_migrate_once(void) {
709 session_status.reliability++;
710 }
711
712 - safecpy(session_status.stack_traces, capture_stack_trace_backend());
712 + safecpy(session_status.stack_traces, stacktrace_backend());
713
714 fill_dmi_info(&session_status);
715
@@ -1320,7 +1320,7 @@ static void daemon_status_file_save_twice_if_we_can_get_stack_trace(BUFFER *wb,
1320 // IMPORTANT: NO LOCKS OR ALLOCATIONS HERE, THIS FUNCTION IS CALLED FROM SIGNAL HANDLERS
1321 // THIS FUNCTION MUST USE ONLY ASYNC-SIGNAL-SAFE OPERATIONS
1322
1323 - if(capture_stack_trace_available())
1323 + if(stacktrace_available())
1324 set_stack_trace_message_if_empty(&session_status, STACK_TRACE_INFO_PREFIX "will now attempt to get stack trace - if you see this message, we couldn't get it.");
1325 else
1326 set_stack_trace_message_if_empty(&session_status, STACK_TRACE_INFO_PREFIX "no stack trace backend available");
@@ -1332,11 +1332,11 @@ static void daemon_status_file_save_twice_if_we_can_get_stack_trace(BUFFER *wb,
1332 return;
1333
1334 buffer_flush(wb);
1335 -
1336 - capture_stack_trace(wb);
1335 +
1336 + stacktrace_capture(wb);
1337
1338 // Store the first netdata function from the stack trace if available
1339 - const char *first_nd_fn = capture_stack_trace_root_cause_function();
1339 + const char *first_nd_fn = stacktrace_root_cause_function();
1340 if (first_nd_fn && *first_nd_fn &&
1341 (!ds->fatal.function[0] || strncmp(ds->fatal.function, "thread:", 7) == 0))
1342 safecpy(ds->fatal.function, first_nd_fn);
@@ -1487,14 +1487,14 @@ bool daemon_status_file_deadly_signal_received(EXIT_REASON reason, SIGNAL_CODE c
1487 }
1488 }
1489
1490 - bool safe_to_get_stack_trace = reason != EXIT_REASON_SIGABRT || capture_stack_trace_is_async_signal_safe();
1491 - bool get_stack_trace = capture_stack_trace_available() && safe_to_get_stack_trace && stack_trace_is_empty(&session_status);
1490 + bool safe_to_get_stack_trace = reason != EXIT_REASON_SIGABRT || stacktrace_capture_is_async_signal_safe();
1491 + bool get_stack_trace = stacktrace_available() && safe_to_get_stack_trace && stack_trace_is_empty(&session_status);
1492
1493 // save it
1494 if(get_stack_trace)
1495 daemon_status_file_save_twice_if_we_can_get_stack_trace(static_save_buffer, &session_status, true);
1496 else {
1497 - if (!capture_stack_trace_available())
1497 + if (!stacktrace_available())
1498 set_stack_trace_message_if_empty(&session_status, STACK_TRACE_INFO_PREFIX "no stack trace backend available");
1499 else
1500 set_stack_trace_message_if_empty(&session_status, STACK_TRACE_INFO_PREFIX "not safe to get a stack trace for this signal using this backend");
src/daemon/unit_test.h
+1
@@ -13,6 +13,7 @@ int unit_test_buffer(void);
13 int unit_test_static_threads(void);
14 int test_sqlite(void);
15 int unit_test_bitmaps(void);
16 +int stacktrace_unittest(void);
17 #ifdef ENABLE_DBENGINE
18 int test_dbengine(void);
19 void generate_dbengine_dataset(unsigned history_seconds);
src/database/contexts/api_v2_contexts_alerts.c
+9 -3
@@ -193,9 +193,15 @@ static void alerts_v2_add(struct alert_v2_entry *t, RRDCALC *rc) {
193 dictionary_set(t->configs, key, NULL, 0);
194 }
195
196 +static STRING *silent_string = NULL;
197 +void alerts_by_x_cleanup(void) {
198 + string_freez(silent_string);
199 + silent_string = NULL;
200 +}
201 +
202 static void alerts_by_x_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data) {
197 - static STRING *silent = NULL;
198 - if(unlikely(!silent)) silent = string_strdupz("silent");
203 + if(unlikely(!silent_string))
204 + silent_string = string_strdupz("silent");
205
206 struct alert_by_x_entry *b = value;
207 RRDCALC *rc = data;
@@ -208,7 +214,7 @@ static void alerts_by_x_insert_callback(const DICTIONARY_ITEM *item __maybe_unus
214
215 b->running.total++;
216
211 - if (rc->config.recipient == silent)
217 + if (rc->config.recipient == silent_string)
218 b->running.silent++;
219 }
220 }
src/exporting/init_connectors.c
+7 -1
@@ -169,10 +169,11 @@ void simple_connector_init(struct instance *instance)
169 if (connector_specific_data->first_buffer)
170 return;
171
172 + // Initialize the active buffers that will be used for sending data
173 connector_specific_data->header = buffer_create(0, &netdata_buffers_statistics.buffers_exporters);
174 connector_specific_data->buffer = buffer_create(0, &netdata_buffers_statistics.buffers_exporters);
175
175 - // create a ring buffer
176 + // create a ring buffer with all buffers initialized
177 struct simple_connector_buffer *first_buffer = NULL;
178
179 if (instance->config.buffer_on_failures < 1)
@@ -180,6 +181,11 @@ void simple_connector_init(struct instance *instance)
181
182 for (int i = 0; i < instance->config.buffer_on_failures; i++) {
183 struct simple_connector_buffer *current_buffer = callocz(1, sizeof(struct simple_connector_buffer));
184 +
185 + // Initialize both header and buffer for each ring buffer entry
186 + // This ensures we never have NULL pointers during buffer swapping
187 + current_buffer->header = buffer_create(0, &netdata_buffers_statistics.buffers_exporters);
188 + current_buffer->buffer = buffer_create(0, &netdata_buffers_statistics.buffers_exporters);
189
190 if (!connector_specific_data->first_buffer)
191 first_buffer = current_buffer;
src/exporting/send_data.c
+9
@@ -129,6 +129,15 @@ void simple_connector_send_buffer(
129 flags += MSG_NOSIGNAL;
130 #endif
131
132 + // Safety check to prevent NULL pointer crashes, but don't allocate new memory
133 + if (unlikely(!buffer || !header)) {
134 + netdata_log_error("EXPORTING: NULL %s passed to simple_connector_send_buffer for instance %s",
135 + (!buffer && !header) ? "buffer and header" : (!buffer ? "buffer" : "header"),
136 + instance->config.name ? instance->config.name : "unknown");
137 + (*failures)++;
138 + return;
139 + }
140 +
141 uint32_t options = (uint32_t)instance->config.options;
142 struct simple_connector_data *connector_specific_data = instance->connector_specific_data;
143
src/health/health.c
+16
@@ -178,6 +178,22 @@ void health_plugin_destroy(void) {
178 health_globals.prototypes.dict = NULL;
179 }
180
181 + // Free allocated strings
182 + string_freez(health_globals.config.default_exec);
183 + string_freez(health_globals.config.default_recipient);
184 + string_freez(health_globals.config.silencers_filename);
185 +
186 + // Free the enabled_alerts pattern
187 + simple_pattern_free(health_globals.config.enabled_alerts);
188 +
189 + // Reset pointers to NULL
190 + health_globals.config.default_exec = NULL;
191 + health_globals.config.default_recipient = NULL;
192 + health_globals.config.silencers_filename = NULL;
193 + health_globals.config.enabled_alerts = NULL;
194 +
195 + alert_variable_lookup_cleanup();
196 +
197 health_globals.initialization.done = false;
198
199 spinlock_unlock(&health_globals.initialization.spinlock);
src/health/health.h
+2
@@ -111,4 +111,6 @@ uint64_t rrdhost_health_evloop_last_iteration(RRDHOST *host);
111
112 void health_load_config_defaults(void);
113
114 +void alert_variable_lookup_cleanup(void);
115 +
116 #endif //NETDATA_HEALTH_H
src/health/health_variable.c
+43 -15
@@ -153,22 +153,50 @@ bool alert_variable_from_running_alerts(struct variable_lookup_job *vbd) {
153 return found;
154 }
155
156 -bool alert_variable_lookup_internal(STRING *variable, void *data, NETDATA_DOUBLE *result, BUFFER *wb) {
157 - static STRING *this_string = NULL,
158 - *now_string = NULL,
159 - *after_string = NULL,
160 - *before_string = NULL,
161 - *status_string = NULL,
162 - *removed_string = NULL,
163 - *uninitialized_string = NULL,
164 - *undefined_string = NULL,
165 - *clear_string = NULL,
166 - *warning_string = NULL,
167 - *critical_string = NULL,
168 - *last_collected_t_string = NULL,
169 - *update_every_string = NULL;
170 -
156 +static STRING *this_string = NULL,
157 + *now_string = NULL,
158 + *after_string = NULL,
159 + *before_string = NULL,
160 + *status_string = NULL,
161 + *removed_string = NULL,
162 + *uninitialized_string = NULL,
163 + *undefined_string = NULL,
164 + *clear_string = NULL,
165 + *warning_string = NULL,
166 + *critical_string = NULL,
167 + *last_collected_t_string = NULL,
168 + *update_every_string = NULL;
169 +
170 +void alert_variable_lookup_cleanup(void) {
171 + string_freez(this_string);
172 + string_freez(now_string);
173 + string_freez(after_string);
174 + string_freez(before_string);
175 + string_freez(status_string);
176 + string_freez(removed_string);
177 + string_freez(uninitialized_string);
178 + string_freez(undefined_string);
179 + string_freez(clear_string);
180 + string_freez(warning_string);
181 + string_freez(critical_string);
182 + string_freez(last_collected_t_string);
183 + string_freez(update_every_string);
184 + this_string = NULL;
185 + now_string = NULL;
186 + after_string = NULL;
187 + before_string = NULL;
188 + status_string = NULL;
189 + removed_string = NULL;
190 + uninitialized_string = NULL;
191 + undefined_string = NULL;
192 + clear_string = NULL;
193 + warning_string = NULL;
194 + critical_string = NULL;
195 + last_collected_t_string = NULL;
196 + update_every_string = NULL;
197 +}
198
199 +bool alert_variable_lookup_internal(STRING *variable, void *data, NETDATA_DOUBLE *result, BUFFER *wb) {
200 struct variable_lookup_job vbd = { 0 };
201
202 // const char *v_name = string2str(variable);
src/libnetdata/dictionary/dictionary-callbacks.h
+12 -24
@@ -15,12 +15,9 @@ static inline void dictionary_execute_insert_callback(DICTIONARY *dict, DICTIONA
15 if(unlikely(is_view_dictionary(dict)))
16 fatal("DICTIONARY: called %s() on a view.", __FUNCTION__ );
17
18 - internal_error(false,
19 - "DICTIONARY: Running insert callback on item '%s' of dictionary created from %s() %zu@%s.",
20 - item_get_name(item),
21 - dict->creation_function,
22 - dict->creation_line,
23 - dict->creation_file);
18 + dictionary_internal_error(false, dict,
19 + "DICTIONARY: Running insert callback on item '%s' of dictionary",
20 + item_get_name(item));
21
22 dict->hooks->insert_callback(item, item->shared->value, constructor_data?constructor_data:dict->hooks->insert_callback_data);
23 DICTIONARY_STATS_CALLBACK_INSERTS_PLUS1(dict);
@@ -33,12 +30,9 @@ static inline bool dictionary_execute_conflict_callback(DICTIONARY *dict, DICTIO
30 if(unlikely(is_view_dictionary(dict)))
31 fatal("DICTIONARY: called %s() on a view.", __FUNCTION__ );
32
36 - internal_error(false,
37 - "DICTIONARY: Running conflict callback on item '%s' of dictionary created from %s() %zu@%s.",
38 - item_get_name(item),
39 - dict->creation_function,
40 - dict->creation_line,
41 - dict->creation_file);
33 + dictionary_internal_error(false, dict,
34 + "DICTIONARY: Running conflict callback on item '%s' of dictionary",
35 + item_get_name(item));
36
37 bool ret = dict->hooks->conflict_callback(
38 item, item->shared->value, new_value,
@@ -56,12 +50,9 @@ static inline void dictionary_execute_react_callback(DICTIONARY *dict, DICTIONAR
50 if(unlikely(is_view_dictionary(dict)))
51 fatal("DICTIONARY: called %s() on a view.", __FUNCTION__ );
52
59 - internal_error(false,
60 - "DICTIONARY: Running react callback on item '%s' of dictionary created from %s() %zu@%s.",
61 - item_get_name(item),
62 - dict->creation_function,
63 - dict->creation_line,
64 - dict->creation_file);
53 + dictionary_internal_error(false, dict,
54 + "DICTIONARY: Running react callback on item '%s' of dictionary",
55 + item_get_name(item));
56
57 dict->hooks->react_callback(item, item->shared->value,
58 constructor_data?constructor_data:dict->hooks->react_callback_data);
@@ -77,12 +68,9 @@ static inline void dictionary_execute_delete_callback(DICTIONARY *dict, DICTIONA
68 // because we may have references to it, after the master is gone
69 // so, the shared structure will remain until the last reference is released.
70
80 - internal_error(false,
81 - "DICTIONARY: Running delete callback on item '%s' of dictionary created from %s() %zu@%s.",
82 - item_get_name(item),
83 - dict->creation_function,
84 - dict->creation_line,
85 - dict->creation_file);
71 + dictionary_internal_error(false, dict,
72 + "DICTIONARY: Running delete callback on item '%s' of dictionary",
73 + item_get_name(item));
74
75 dict->hooks->delete_callback(item, item->shared->value, dict->hooks->delelte_callback_data);
76
src/libnetdata/dictionary/dictionary-debug.c new
+378
@@ -0,0 +1,378 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "dictionary-internals.h"
4 +
5 +#if defined(FSANITIZE_ADDRESS)
6 +
7 +// Judyl for tracking all dictionaries ever created
8 +static SPINLOCK all_dictionaries_spinlock = SPINLOCK_INITIALIZER;
9 +static Pvoid_t all_dictionaries = NULL;
10 +static bool all_dictionaries_initialized = false;
11 +
12 +// Initialize the tracking system
13 +static void all_dictionaries_initialize(void) {
14 + // Only do this once
15 + if (all_dictionaries_initialized)
16 + return;
17 +
18 + spinlock_lock(&all_dictionaries_spinlock);
19 + if (!all_dictionaries_initialized) {
20 + all_dictionaries = NULL;
21 + all_dictionaries_initialized = true;
22 + }
23 + spinlock_unlock(&all_dictionaries_spinlock);
24 +}
25 +
26 +// Comparison function for sorting stacktraces by count (descending)
27 +static int stacktrace_count_compare(const void *a, const void *b) {
28 + typedef struct {
29 + STACKTRACE stacktrace;
30 + Word_t count;
31 + } StacktraceInfo;
32 +
33 + const StacktraceInfo *sta = (const StacktraceInfo *)a;
34 + const StacktraceInfo *stb = (const StacktraceInfo *)b;
35 +
36 + if (stb->count > sta->count) return 1;
37 + if (stb->count < sta->count) return -1;
38 + return 0;
39 +}
40 +
41 +// Report all allocated dictionaries that are not part of the destroyed list
42 +static size_t report_allocated_dictionaries(void) {
43 + // Ensure initialization
44 + if (!all_dictionaries_initialized)
45 + all_dictionaries_initialize();
46 +
47 + spinlock_lock(&all_dictionaries_spinlock);
48 +
49 + Word_t index = 0;
50 + Pvoid_t PValue;
51 + Word_t count = 0;
52 +
53 + // First count dictionaries
54 + PValue = JudyLFirst(all_dictionaries, &index, PJE0);
55 + while (PValue != NULL) {
56 + count++;
57 + PValue = JudyLNext(all_dictionaries, &index, PJE0);
58 + }
59 +
60 + if (count > 0) {
61 + fprintf(stderr, "\nASAN: ===== DICTIONARY TRACKING: Detected %lu dictionaries that are still allocated =====\n", count);
62 + fflush(stderr);
63 +
64 + // First, group by stacktrace
65 + // Key: stacktrace pointer, Value: count
66 + Pvoid_t stacktrace_counts = NULL;
67 + // Key: stacktrace pointer, Value: array of dictionary pointers
68 + Pvoid_t stacktrace_dictionaries = NULL;
69 +
70 + // Group dictionaries by stacktrace
71 + index = 0;
72 + PValue = JudyLFirst(all_dictionaries, &index, PJE0);
73 + while (PValue != NULL) {
74 + DICTIONARY *dict = (DICTIONARY *)index;
75 +
76 + // Process each stacktrace in the array
77 + for (int st_idx = 0; st_idx < dict->stacktraces.num_stacktraces; st_idx++) {
78 + STACKTRACE st = dict->stacktraces.stacktraces[st_idx];
79 +
80 + if (st) {
81 + Word_t st_key = (Word_t)st;
82 +
83 + // Update count for this stacktrace
84 + Pvoid_t PCount;
85 + PCount = JudyLIns(&stacktrace_counts, st_key, PJE0);
86 + if (PCount) {
87 + (*(Word_t*)PCount)++;
88 + } else {
89 + *(Word_t*)PCount = 1;
90 + }
91 +
92 + // Add dictionary to the list for this stacktrace
93 + Pvoid_t PDictList;
94 + PDictList = JudyLGet(stacktrace_dictionaries, st_key, PJE0);
95 + if (!PDictList) {
96 + // Create a new array (starting with size 16)
97 + DICTIONARY **dict_list = (DICTIONARY **)calloc(16, sizeof(DICTIONARY*));
98 + if (dict_list) {
99 + dict_list[0] = dict;
100 + JudyLIns(&stacktrace_dictionaries, st_key, PJE0);
101 + PDictList = JudyLGet(stacktrace_dictionaries, st_key, PJE0);
102 + if (PDictList) {
103 + *(void**)PDictList = dict_list;
104 + }
105 + }
106 + } else {
107 + // Add to existing array if not already present
108 + DICTIONARY **dict_list = *(DICTIONARY***)PDictList;
109 + bool already_added = false;
110 + size_t i = 0;
111 +
112 + // Check if dictionary is already in this list
113 + while (dict_list[i] && i < 1024) {
114 + if (dict_list[i] == dict) {
115 + already_added = true;
116 + break;
117 + }
118 + i++;
119 + }
120 +
121 + // Add if not already present
122 + if (!already_added && i < 1024) {
123 + dict_list[i] = dict;
124 + }
125 + }
126 + }
127 + }
128 +
129 + PValue = JudyLNext(all_dictionaries, &index, PJE0);
130 + }
131 +
132 + // Now build a sorted array of stacktraces
133 + // Use the StacktraceInfo type defined for the comparator
134 + typedef struct {
135 + STACKTRACE stacktrace;
136 + Word_t count;
137 + } StacktraceInfo;
138 +
139 + Word_t num_stacktraces = 0;
140 +
141 + // Count stacktraces
142 + index = 0;
143 + PValue = JudyLFirst(stacktrace_counts, &index, PJE0);
144 + while (PValue != NULL) {
145 + num_stacktraces++;
146 + PValue = JudyLNext(stacktrace_counts, &index, PJE0);
147 + }
148 +
149 + if (num_stacktraces > 0) {
150 + StacktraceInfo *stacktraces = (StacktraceInfo *)calloc(num_stacktraces, sizeof(StacktraceInfo));
151 + if (stacktraces) {
152 + // Fill array
153 + Word_t i = 0;
154 + index = 0;
155 + PValue = JudyLFirst(stacktrace_counts, &index, PJE0);
156 + while (PValue != NULL) {
157 + stacktraces[i].stacktrace = (STACKTRACE)index;
158 + stacktraces[i].count = *(Word_t*)PValue;
159 + i++;
160 + PValue = JudyLNext(stacktrace_counts, &index, PJE0);
161 + }
162 +
163 + // Sort by count (descending)
164 + qsort(stacktraces, num_stacktraces, sizeof(StacktraceInfo), stacktrace_count_compare);
165 +
166 + // Print stacktraces in order
167 + for (i = 0; i < num_stacktraces; i++) {
168 + STACKTRACE st = stacktraces[i].stacktrace;
169 + Word_t st_count = stacktraces[i].count;
170 +
171 + fprintf(stderr, "\n > DICTIONARY STACKTRACE GROUP %lu/%lu (count: %lu):\n",
172 + i+1, num_stacktraces, st_count);
173 + fflush(stderr);
174 +
175 + // Print stacktrace
176 + BUFFER *wb = buffer_create(16384, NULL);
177 + stacktrace_to_buffer(st, wb);
178 + fprintf(stderr, "%s\n", buffer_tostring(wb));
179 + buffer_free(wb);
180 + fflush(stderr);
181 +
182 + // Print dictionary pointers
183 + Pvoid_t PDictList = JudyLGet(stacktrace_dictionaries, (Word_t)st, PJE0);
184 + if (PDictList) {
185 + DICTIONARY **dict_list = *(DICTIONARY***)PDictList;
186 + fprintf(stderr, " Dictionary pointers:");
187 + int displayed = 0;
188 +
189 + for (int j = 0; j < 1024 && dict_list[j] && displayed < 10; j++) {
190 + fprintf(stderr, " %p", dict_list[j]);
191 + displayed++;
192 + }
193 +
194 + if (st_count > 10) {
195 + fprintf(stderr, " ... (plus %lu more)", st_count - 10);
196 + }
197 + fprintf(stderr, "\n");
198 + fflush(stderr);
199 + }
200 + }
201 +
202 + free(stacktraces);
203 + }
204 +
205 + // Clean up
206 + index = 0;
207 + PValue = JudyLFirst(stacktrace_dictionaries, &index, PJE0);
208 + while (PValue != NULL) {
209 + free(*(void**)PValue);
210 + PValue = JudyLNext(stacktrace_dictionaries, &index, PJE0);
211 + }
212 +
213 + JudyLFreeArray(&stacktrace_dictionaries, PJE0);
214 + JudyLFreeArray(&stacktrace_counts, PJE0);
215 + }
216 + }
217 + else {
218 + fprintf(stderr, "\nASAN: ===== DICTIONARY TRACKING: No allocated dictionaries found =====\n");
219 + fflush(stderr);
220 + }
221 +
222 + spinlock_unlock(&all_dictionaries_spinlock);
223 +
224 + return count;
225 +}
226 +
227 +/**
228 + * @brief Print information about dictionaries that have delayed destruction
229 + *
230 + * This function is called during shutdown to print information about dictionaries that
231 + * could not be destroyed because they have referenced items
232 + */
233 +void dictionary_debug_print_delayed_dictionaries(size_t destroyed_dicts) {
234 + if (destroyed_dicts > 0) {
235 + fprintf(stderr, "\nASAN: ===== DICTIONARY TRACKING: %zu dictionaries with references couldn't be destroyed =====\n",
236 + destroyed_dicts);
237 + fflush(stderr);
238 + }
239 +}
240 +
241 +// Public API functions
242 +void dictionary_debug_init(void) {
243 + all_dictionaries_initialize();
244 +}
245 +
246 +void dictionary_debug_track_dict(DICTIONARY *dict) {
247 + // Make sure we're initialized
248 + if (!all_dictionaries_initialized)
249 + all_dictionaries_initialize();
250 +
251 + spinlock_lock(&all_dictionaries_spinlock);
252 + // No need to capture return value
253 + JudyLIns(&all_dictionaries, (Word_t)dict, PJE0);
254 + spinlock_unlock(&all_dictionaries_spinlock);
255 +}
256 +
257 +void dictionary_debug_untrack_dict(DICTIONARY *dict) {
258 + // Only attempt to untrack if we've initialized the tracking
259 + if (!all_dictionaries_initialized)
260 + return;
261 +
262 + spinlock_lock(&all_dictionaries_spinlock);
263 + // No need to capture return value
264 + JudyLDel(&all_dictionaries, (Word_t)dict, PJE0);
265 + spinlock_unlock(&all_dictionaries_spinlock);
266 +}
267 +
268 +void dictionary_print_still_allocated_stacktraces(void) {
269 + size_t allocated = report_allocated_dictionaries();
270 + if (allocated > 0) {
271 + fprintf(stderr, "\nASAN: ===== DICTIONARY TRACKING: Found %zu dictionaries that are still allocated but not in the destroyed list =====\n",
272 + allocated);
273 + fflush(stderr);
274 + }
275 +}
276 +
277 +void dictionary_debug_shutdown(void) {
278 + if (!all_dictionaries_initialized)
279 + return;
280 +
281 + spinlock_lock(&all_dictionaries_spinlock);
282 + JudyLFreeArray(&all_dictionaries, PJE0);
283 + spinlock_unlock(&all_dictionaries_spinlock);
284 +}
285 +
286 +void dictionary_debug_internal_check_with_trace(DICTIONARY *dict, DICTIONARY_ITEM *item, const char *function, bool allow_null_dict, bool allow_null_item) {
287 + if(!allow_null_dict && !dict) {
288 + // Create a buffer for the item's dict stacktrace
289 + BUFFER *wb = buffer_create(1024, NULL);
290 + if (item && item->dict && item->dict->stacktraces.num_stacktraces > 0) {
291 + buffer_strcat(wb, "\nItem's dictionary stacktraces:\n");
292 + for (int i = 0; i < item->dict->stacktraces.num_stacktraces && i < 3; i++) {
293 + if (item->dict->stacktraces.stacktraces[i]) {
294 + buffer_sprintf(wb, "Stacktrace #%d:\n", i+1);
295 + stacktrace_to_buffer(item->dict->stacktraces.stacktraces[i], wb);
296 + buffer_strcat(wb, "\n");
297 + }
298 + }
299 + if (item->dict->stacktraces.num_stacktraces > 3)
300 + buffer_sprintf(wb, "...and %d more stacktraces\n", item->dict->stacktraces.num_stacktraces - 3);
301 + } else {
302 + buffer_strcat(wb, "\nItem's dictionary stacktrace not available");
303 + }
304 +
305 + internal_error(
306 + item,
307 + "DICTIONARY: attempted to %s() with a NULL dictionary, passing an item. %s",
308 + function,
309 + buffer_tostring(wb));
310 +
311 + buffer_free(wb);
312 + fatal("DICTIONARY: attempted to %s() but dict is NULL", function);
313 + }
314 +
315 + if(!allow_null_item && !item) {
316 + dictionary_internal_error(true, dict,
317 + "DICTIONARY: attempted to %s() without an item on a dictionary",
318 + function);
319 + fatal("DICTIONARY: attempted to %s() but item is NULL", function);
320 + }
321 +
322 + if(dict && item && dict != item->dict) {
323 + // Create buffer for both dictionaries' stacktraces
324 + BUFFER *wb = buffer_create(1024, NULL);
325 +
326 + if (dict->stacktraces.num_stacktraces > 0) {
327 + buffer_strcat(wb, "\nDictionary stacktraces:\n");
328 + for (int i = 0; i < dict->stacktraces.num_stacktraces && i < 3; i++) {
329 + if (dict->stacktraces.stacktraces[i]) {
330 + buffer_sprintf(wb, "Stacktrace #%d:\n", i+1);
331 + stacktrace_to_buffer(dict->stacktraces.stacktraces[i], wb);
332 + buffer_strcat(wb, "\n");
333 + }
334 + }
335 + if (dict->stacktraces.num_stacktraces > 3)
336 + buffer_sprintf(wb, "...and %d more stacktraces\n", dict->stacktraces.num_stacktraces - 3);
337 + } else {
338 + buffer_strcat(wb, "\nDictionary stacktrace not available");
339 + }
340 +
341 + if (item->dict && item->dict->stacktraces.num_stacktraces > 0) {
342 + buffer_strcat(wb, "\nItem's dictionary stacktraces:\n");
343 + for (int i = 0; i < item->dict->stacktraces.num_stacktraces && i < 3; i++) {
344 + if (item->dict->stacktraces.stacktraces[i]) {
345 + buffer_sprintf(wb, "Stacktrace #%d:\n", i+1);
346 + stacktrace_to_buffer(item->dict->stacktraces.stacktraces[i], wb);
347 + buffer_strcat(wb, "\n");
348 + }
349 + }
350 + if (item->dict->stacktraces.num_stacktraces > 3)
351 + buffer_sprintf(wb, "...and %d more stacktraces\n", item->dict->stacktraces.num_stacktraces - 3);
352 + } else {
353 + buffer_strcat(wb, "\nItem's dictionary stacktrace not available");
354 + }
355 +
356 + internal_error(
357 + true,
358 + "DICTIONARY: attempted to %s() an item on a dictionary different from the item's dictionary. %s",
359 + function,
360 + buffer_tostring(wb));
361 +
362 + buffer_free(wb);
363 + fatal("DICTIONARY: %s(): item does not belong to this dictionary.", function);
364 + }
365 +
366 + if(item) {
367 + REFCOUNT refcount = DICTIONARY_ITEM_REFCOUNT_GET(dict, item);
368 + if (unlikely(refcount <= 0)) {
369 + dictionary_internal_error(true, item->dict,
370 + "DICTIONARY: attempted to %s() of an item with reference counter = %d on a dictionary",
371 + function,
372 + refcount);
373 + fatal("DICTIONARY: attempted to %s but item is having refcount = %d", function, refcount);
374 + }
375 + }
376 +}
377 +
378 +#endif // FSANITIZE_ADDRESS
src/libnetdata/dictionary/dictionary-debug.h new
+94
@@ -0,0 +1,94 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_DICTIONARY_DEBUG_H
4 +#define NETDATA_DICTIONARY_DEBUG_H
5 +
6 +#include "dictionary.h"
7 +
8 +#ifdef FSANITIZE_ADDRESS
9 +
10 +// Initialize dictionary debugging
11 +void dictionary_debug_init(void);
12 +
13 +// Register a dictionary for tracking (when FSANITIZE_ADDRESS is defined)
14 +void dictionary_debug_track_dict(DICTIONARY *dict);
15 +
16 +// Unregister a dictionary from tracking
17 +void dictionary_debug_untrack_dict(DICTIONARY *dict);
18 +
19 +// Print information about dictionaries that are still allocated
20 +void dictionary_print_still_allocated_stacktraces(void);
21 +
22 +// Print information about dictionaries that have delayed destruction due to referenced items
23 +void dictionary_debug_print_delayed_dictionaries(size_t destroyed_dicts);
24 +
25 +// Clean up resources used for tracking
26 +void dictionary_debug_shutdown(void);
27 +
28 +void dictionary_debug_internal_check_with_trace(DICTIONARY *dict, DICTIONARY_ITEM *item, const char *function, bool allow_null_dict, bool allow_null_item);
29 +#define dictionary_debug_internal_check(dict, item, allow_null_dict, allow_null_item) \
30 + dictionary_debug_internal_check_with_trace(dict, item, __FUNCTION__, allow_null_dict, allow_null_item)
31 +
32 +// For internal_error with dictionary stacktrace
33 +#define dictionary_internal_error(condition, dict, fmt, args...) do { \
34 + if(unlikely(condition)) { \
35 + BUFFER *__wb = buffer_create(1024, NULL); \
36 + if ((dict) && ((dict)->stacktraces.num_stacktraces > 0)) { \
37 + buffer_sprintf(__wb, fmt " Dictionary accessed at:\n", ##args); \
38 + for (int i = 0; i < (dict)->stacktraces.num_stacktraces && i < 3; i++) { \
39 + if ((dict)->stacktraces.stacktraces[i]) { \
40 + buffer_sprintf(__wb, "Stacktrace #%d:\n", i+1); \
41 + stacktrace_to_buffer((dict)->stacktraces.stacktraces[i], __wb); \
42 + buffer_strcat(__wb, "\n"); \
43 + } \
44 + } \
45 + if ((dict)->stacktraces.num_stacktraces > 3) \
46 + buffer_sprintf(__wb, "...and %d more stacktraces\n", (dict)->stacktraces.num_stacktraces - 3); \
47 + } else { \
48 + buffer_sprintf(__wb, fmt " Dictionary stacktrace not available", ##args); \
49 + } \
50 + netdata_logger(NDLS_DAEMON, NDLP_DEBUG, __FILE__, __FUNCTION__, __LINE__, "BEGIN --\n%s\n-- END", buffer_tostring(__wb)); \
51 + buffer_free(__wb); \
52 + } \
53 +} while(0)
54 +
55 +// For internal_fatal with dictionary stacktrace
56 +#define dictionary_internal_fatal(condition, dict, fmt, args...) do { \
57 + if(unlikely(condition)) { \
58 + BUFFER *__wb = buffer_create(1024, NULL); \
59 + if ((dict) && ((dict)->stacktraces.num_stacktraces > 0)) { \
60 + buffer_sprintf(__wb, fmt " Dictionary accessed at:\n", ##args); \
61 + for (int i = 0; i < (dict)->stacktraces.num_stacktraces && i < 3; i++) { \
62 + if ((dict)->stacktraces.stacktraces[i]) { \
63 + buffer_sprintf(__wb, "Stacktrace #%d:\n", i+1); \
64 + stacktrace_to_buffer((dict)->stacktraces.stacktraces[i], __wb); \
65 + buffer_strcat(__wb, "\n"); \
66 + } \
67 + } \
68 + if ((dict)->stacktraces.num_stacktraces > 3) \
69 + buffer_sprintf(__wb, "...and %d more stacktraces\n", (dict)->stacktraces.num_stacktraces - 3); \
70 + } else { \
71 + buffer_sprintf(__wb, fmt " Dictionary stacktrace not available", ##args); \
72 + } \
73 + netdata_logger_fatal(__FILE__, __FUNCTION__, __LINE__, "BEGIN --\n%s\n--END", buffer_tostring(__wb)); \
74 + buffer_free(__wb); \
75 + } \
76 +} while(0)
77 +
78 +#else
79 +
80 +#define dictionary_debug_init(void) debug_dummy()
81 +#define dictionary_debug_track_dict(dict) debug_dummy()
82 +#define dictionary_debug_untrack_dict(dict) debug_dummy()
83 +#define dictionary_print_still_allocated_stacktraces(void) debug_dummy()
84 +#define dictionary_debug_print_delayed_dictionaries(destroyed_dicts) debug_dummy()
85 +#define dictionary_debug_shutdown(void) debug_dummy()
86 +
87 +#define dictionary_debug_internal_check(dict, item, allow_null_dict, allow_null_item) debug_dummy()
88 +#define dictionary_debug_internal_check_with_trace(dict, item, function, allow_null_dict, allow_null_item) debug_dummy()
89 +
90 +#define dictionary_internal_error(condition, dict, fmt, args...) debug_dummy()
91 +#define dictionary_internal_fatal(condition, dict, fmt, args...) debug_dummy()
92 +#endif
93 +
94 +#endif // NETDATA_DICTIONARY_DEBUG_H
src/libnetdata/dictionary/dictionary-internals.h
+6 -5
@@ -75,6 +75,9 @@ typedef struct dictionary_item_shared {
75 } DICTIONARY_ITEM_SHARED;
76
77 struct dictionary_item {
78 +#ifdef FSANITIZE_ADDRESS
79 + STACKTRACE_ARRAY stacktraces; // stack traces from all acquisition points
80 +#endif
81 #ifdef NETDATA_INTERNAL_CHECKS
82 DICTIONARY *dict;
83 pid_t creator_pid;
@@ -127,11 +130,8 @@ struct dictionary_hooks {
130 };
131
132 struct dictionary {
130 -#ifdef NETDATA_INTERNAL_CHECKS
131 - const char *creation_function;
132 - const char *creation_file;
133 - size_t creation_line;
134 - pid_t creation_tid;
133 +#ifdef FSANITIZE_ADDRESS
134 + STACKTRACE_ARRAY stacktraces; // stack traces from all acquisition points
135 #endif
136
137 usec_t last_gc_run_us;
@@ -245,6 +245,7 @@ static inline void pointer_del(DICTIONARY *dict __maybe_unused, DICTIONARY_ITEM
245 extern ARAL *dict_items_aral;
246 extern ARAL *dict_shared_items_aral;
247
248 +#include "dictionary-debug.h"
249 #include "dictionary-statistics.h"
250 #include "dictionary-locks.h"
251 #include "dictionary-refcount.h"
src/libnetdata/dictionary/dictionary-item.h
+13 -14
@@ -55,6 +55,13 @@ static inline DICTIONARY_ITEM *dict_item_create(DICTIONARY *dict __maybe_unused,
55 #ifdef NETDATA_INTERNAL_CHECKS
56 item->creator_pid = gettid_cached();
57 #endif
58 +#ifdef FSANITIZE_ADDRESS
59 + // Initialize stacktrace tracking
60 + stacktrace_array_init(&item->stacktraces);
61 +
62 + // Add the first stack trace at creation time
63 + stacktrace_array_add(&item->stacktraces, 1);
64 +#endif
65
66 item->refcount = 1;
67 item->flags = ITEM_FLAG_BEING_CREATED;
@@ -404,13 +411,9 @@ static inline bool dict_item_del(DICTIONARY *dict, const char *name, ssize_t nam
411
412 static inline DICTIONARY_ITEM *dict_item_add_or_reset_value_and_acquire(DICTIONARY *dict, const char *name, ssize_t name_len, void *value, size_t value_len, void *constructor_data, DICTIONARY_ITEM *master_item) {
413 if(unlikely(!name || !*name)) {
407 - internal_error(
408 - true,
409 - "DICTIONARY: attempted to %s() without a name on a dictionary created from %s() %zu@%s.",
410 - __FUNCTION__,
411 - dict->creation_function,
412 - dict->creation_line,
413 - dict->creation_file);
414 + dictionary_internal_error(true, dict,
415 + "DICTIONARY: attempted to %s() without a name on a dictionary.",
416 + __FUNCTION__);
417 return NULL;
418 }
419
@@ -520,13 +523,9 @@ static inline DICTIONARY_ITEM *dict_item_add_or_reset_value_and_acquire(DICTIONA
523
524 static inline DICTIONARY_ITEM *dict_item_find_and_acquire(DICTIONARY *dict, const char *name, ssize_t name_len) {
525 if(unlikely(!name || !*name)) {
523 - internal_error(
524 - true,
525 - "DICTIONARY: attempted to %s() without a name on a dictionary created from %s() %zu@%s.",
526 - __FUNCTION__,
527 - dict->creation_function,
528 - dict->creation_line,
529 - dict->creation_file);
526 + dictionary_internal_error(true, dict,
527 + "DICTIONARY: attempted to %s() without a name on a dictionary.",
528 + __FUNCTION__);
529 return NULL;
530 }
531
src/libnetdata/dictionary/dictionary-refcount.h
+16 -16
@@ -32,15 +32,10 @@ static inline void item_acquire(DICTIONARY *dict, DICTIONARY_ITEM *item) {
32
33
34 if(refcount <= 0) {
35 - internal_error(
36 - true,
37 - "DICTIONARY: attempted to acquire item which is deleted (refcount = %d): "
38 - "'%s' on dictionary created by %s() (%zu@%s)",
35 + dictionary_internal_error(true, dict,
36 + "DICTIONARY: attempted to acquire item which is deleted (refcount = %d): '%s'",
37 refcount - 1,
40 - item_get_name(item),
41 - dict->creation_function,
42 - dict->creation_line,
43 - dict->creation_file);
38 + item_get_name(item));
39
40 fatal(
41 "DICTIONARY: request to acquire item '%s', which is deleted (refcount = %d)!",
@@ -58,6 +53,11 @@ static inline void item_acquire(DICTIONARY *dict, DICTIONARY_ITEM *item) {
53 if(item_flag_check(item, ITEM_FLAG_DELETED))
54 DICTIONARY_PENDING_DELETES_MINUS1(dict);
55 }
56 +
57 +#ifdef FSANITIZE_ADDRESS
58 + // Add a stacktrace for this acquisition point
59 + stacktrace_array_add(&item->stacktraces, 1);
60 +#endif
61 }
62
63 static inline void item_release(DICTIONARY *dict, DICTIONARY_ITEM *item) {
@@ -81,15 +81,10 @@ static inline void item_release(DICTIONARY *dict, DICTIONARY_ITEM *item) {
81 }
82
83 if(refcount < 0) {
84 - internal_error(
85 - true,
86 - "DICTIONARY: attempted to release item without references (refcount = %d): "
87 - "'%s' on dictionary created by %s() (%zu@%s)",
84 + dictionary_internal_error(true, dict,
85 + "DICTIONARY: attempted to release item without references (refcount = %d): '%s'",
86 refcount + 1,
89 - item_get_name(item),
90 - dict->creation_function,
91 - dict->creation_line,
92 - dict->creation_file);
87 + item_get_name(item));
88
89 fatal(
90 "DICTIONARY: attempted to release item '%s' without references (refcount = %d)",
@@ -175,6 +170,11 @@ static inline int item_check_and_acquire_advanced(DICTIONARY *dict, DICTIONARY_I
170
171 if(desired == 1)
172 DICTIONARY_REFERENCED_ITEMS_PLUS1(dict);
173 +
174 +#ifdef FSANITIZE_ADDRESS
175 + // Add a stacktrace for this acquisition point
176 + stacktrace_array_add(&item->stacktraces, 1);
177 +#endif
178 }
179
180 if(unlikely(spins > 1))
src/libnetdata/dictionary/dictionary-statistics.h
+5 -11
@@ -78,11 +78,8 @@ static inline void DICTIONARY_ENTRIES_MINUS1(DICTIONARY *dict) {
78 entries = __atomic_fetch_sub(&dict->entries, 1, __ATOMIC_RELAXED);
79 }
80
81 - internal_fatal(entries == 0,
82 - "DICT: negative number of entries in dictionary created from %s() (%zu@%s)",
83 - dict->creation_function,
84 - dict->creation_line,
85 - dict->creation_file);
81 + dictionary_internal_fatal(entries == 0, dict,
82 + "DICT: negative number of entries in dictionary");
83 }
84
85 static inline void DICTIONARY_VALUE_RESETS_PLUS1(DICTIONARY *dict) {
@@ -188,12 +185,9 @@ static inline void DICTIONARY_REFERENCED_ITEMS_MINUS1(DICTIONARY *dict) {
185 else
186 referenced_items = __atomic_sub_fetch(&dict->referenced_items, 1, __ATOMIC_SEQ_CST);
187
191 - internal_fatal(referenced_items < 0,
192 - "DICT: negative number of referenced items (%ld) in dictionary created from %s() (%zu@%s)",
193 - referenced_items,
194 - dict->creation_function,
195 - dict->creation_line,
196 - dict->creation_file);
188 + dictionary_internal_fatal(referenced_items < 0, dict,
189 + "DICT: negative number of referenced items (%ld) in dictionary",
190 + referenced_items);
191 }
192
193 static inline void DICTIONARY_PENDING_DELETES_PLUS1(DICTIONARY *dict) {
src/libnetdata/dictionary/dictionary-unittest.c
+1 -1
@@ -1183,7 +1183,7 @@ int dictionary_unittest(size_t entries) {
1183 errors += dictionary_unittest_threads();
1184 errors += dictionary_unittest_view_threads();
1185
1186 - cleanup_destroyed_dictionaries();
1186 + cleanup_destroyed_dictionaries(false);
1187
1188 fprintf(stderr, "\n%zu errors found\n", errors);
1189 return errors ? 1 : 0;
src/libnetdata/dictionary/dictionary.c
+171 -143
@@ -149,6 +149,11 @@ void dictionary_version_increment(DICTIONARY *dict) {
149 __atomic_fetch_add(&dict->version, 1, __ATOMIC_RELAXED);
150 }
151
152 +// ----------------------------------------------------------------------------
153 +// tracking allocated dictionaries
154 +
155 +#include "dictionary-debug.h"
156 +
157 // ----------------------------------------------------------------------------
158 // items garbage collector
159
@@ -212,9 +217,8 @@ void garbage_collect_pending_deletes(DICTIONARY *dict) {
217 (void)deleted;
218 (void)examined;
219
215 - internal_error(false, "DICTIONARY: garbage collected dictionary created by %s (%zu@%s), "
220 + dictionary_internal_error(false, dict, "DICTIONARY: garbage collected dictionary, "
221 "examined %zu items, deleted %zu items, still pending %zu items",
217 - dict->creation_function, dict->creation_line, dict->creation_file,
222 examined, deleted, pending);
223 }
224
@@ -258,9 +262,6 @@ static bool dictionary_free_all_resources(DICTIONARY *dict, size_t *mem, bool fo
262 long int entries = dict->entries;
263 long int referenced_items = dict->referenced_items;
264 long int pending_deletion_items = dict->pending_deletion_items;
261 - const char *creation_function = dict->creation_function;
262 - const char *creation_file = dict->creation_file;
263 - size_t creation_line = dict->creation_line;
265 #endif
266
267 // destroy the index
@@ -295,14 +296,12 @@ static bool dictionary_free_all_resources(DICTIONARY *dict, size_t *mem, bool fo
296 if(dict->value_aral)
297 aral_by_size_release(dict->value_aral);
298
299 + dictionary_debug_untrack_dict(dict);
300 aral_freez(ar_dict, dict);
301
302 internal_error(
303 false,
302 - "DICTIONARY: Freed dictionary created from %s() %zu@%s, having %ld (counted %zu) entries, %ld referenced, %ld pending deletion, total freed memory: %zu bytes (sizeof(dict) = %zu, sizeof(item) = %zu).",
303 - creation_function,
304 - creation_line,
305 - creation_file,
304 + "DICTIONARY: Freed dictionary having %ld (counted %zu) entries, %ld referenced, %ld pending deletion, total freed memory: %zu bytes (sizeof(dict) = %zu, sizeof(item) = %zu).",
305 entries, counted_items, referenced_items, pending_deletion_items,
306 dict_size + item_size, sizeof(DICTIONARY), sizeof(DICTIONARY_ITEM) + sizeof(DICTIONARY_ITEM_SHARED));
307
@@ -315,6 +314,10 @@ static bool dictionary_free_all_resources(DICTIONARY *dict, size_t *mem, bool fo
314 netdata_mutex_t dictionaries_waiting_to_be_destroyed_mutex = NETDATA_MUTEX_INITIALIZER;
315 static DICTIONARY *dictionaries_waiting_to_be_destroyed = NULL;
316
317 +#ifdef FSANITIZE_ADDRESS
318 +DEFINE_JUDYL_TYPED(STACKTRACE, size_t);
319 +#endif
320 +
321 static void dictionary_queue_for_destruction(DICTIONARY *dict) {
322 if(is_dictionary_destroyed(dict))
323 return;
@@ -330,7 +333,7 @@ static void dictionary_queue_for_destruction(DICTIONARY *dict) {
333 netdata_mutex_unlock(&dictionaries_waiting_to_be_destroyed_mutex);
334 }
335
333 -size_t cleanup_destroyed_dictionaries(void) {
336 +size_t cleanup_destroyed_dictionaries(bool shutdown __maybe_unused) {
337 netdata_mutex_lock(&dictionaries_waiting_to_be_destroyed_mutex);
338 if (!dictionaries_waiting_to_be_destroyed) {
339 netdata_mutex_unlock(&dictionaries_waiting_to_be_destroyed_mutex);
@@ -339,35 +342,49 @@ size_t cleanup_destroyed_dictionaries(void) {
342
343 size_t remaining = 0;
344
345 +#ifdef FSANITIZE_ADDRESS
346 + // Create Judy arrays for tracking stats by stacktrace
347 + STACKTRACE_JudyLSet dict_counts = { 0 }; // Count of dictionaries per stacktrace
348 + STACKTRACE_JudyLSet item_counts = { 0 }; // Count of items per stacktrace
349 + STACKTRACE_JudyLSet item_stacktrace_counts = { 0 }; // Count of items by their creation stacktrace
350 +
351 + STACKTRACE_INIT(&dict_counts);
352 + STACKTRACE_INIT(&item_counts);
353 + STACKTRACE_INIT(&item_stacktrace_counts);
354 +#endif
355 +
356 DICTIONARY *dict, *last = NULL, *next = NULL;
357 for(dict = dictionaries_waiting_to_be_destroyed; dict ; dict = next) {
358 next = dict->next;
359
346 -#ifdef NETDATA_INTERNAL_CHECKS
347 - size_t line = dict->creation_line;
348 - const char *file = dict->creation_file;
349 - const char *function = dict->creation_function;
350 - pid_t pid = dict->creation_tid;
351 -#endif
352 -
360 DICTIONARY_STATS_DICT_DESTROY_QUEUED_MINUS1(dict);
361 if(dictionary_free_all_resources(dict, NULL, false)) {
355 -
356 - internal_error(
357 - true,
358 - "DICTIONARY DELAYED: freed dict created from %s() %zu@%s pid %d.",
359 - function, line, file, pid);
360 -
362 if(last) last->next = next;
363 else dictionaries_waiting_to_be_destroyed = next;
364 }
365 else {
365 -
366 - internal_error(
367 - true,
368 - "DICTIONARY DELAYED %zu: %zu referenced in dict created from %s() %zu@%s pid %d.",
369 - remaining + 1, dictionary_referenced_items(dict),
370 - function, line, file, pid);
366 + size_t ref_items = dictionary_referenced_items(dict);
367 +
368 +#ifdef FSANITIZE_ADDRESS
369 + // Track this dictionary for deduplication reporting
370 + if (shutdown) {
371 + // Process all stacktraces from this dictionary
372 + for (int i = 0; i < dict->stacktraces.num_stacktraces; i++) {
373 + if (dict->stacktraces.stacktraces[i]) {
374 + // Update dictionary count
375 + uintptr_t key = (uintptr_t)dict->stacktraces.stacktraces[i];
376 + size_t dict_count = STACKTRACE_GET(&dict_counts, key);
377 + dict_count++;
378 + STACKTRACE_SET(&dict_counts, key, dict_count);
379 +
380 + // Update item count
381 + size_t items_count = STACKTRACE_GET(&item_counts, key);
382 + items_count += ref_items;
383 + STACKTRACE_SET(&item_counts, key, items_count);
384 + }
385 + }
386 + }
387 +#endif
388
389 DICTIONARY_STATS_DICT_DESTROY_QUEUED_PLUS1(dict);
390 last = dict;
@@ -375,120 +392,133 @@ size_t cleanup_destroyed_dictionaries(void) {
392 }
393 }
394
378 - netdata_mutex_unlock(&dictionaries_waiting_to_be_destroyed_mutex);
395 +#ifdef FSANITIZE_ADDRESS
396 + if (remaining > 0 && shutdown) {
397 + // Print deduplicated report
398 + fprintf(stderr, "WARNING: There are %zu dictionaries with references in them, that cannot be destroyed.\n",
399 + remaining);
400 +
401 + // Buffer for formatting stack traces
402 + BUFFER *wb = buffer_create(16384, NULL);
403 +
404 + // Print each unique stacktrace group
405 + Word_t stacktrace_idx = 0;
406 + size_t i = 0;
407 +
408 + // Get the first key from dict_counts (both Judy arrays have the same keys)
409 + for(size_t key_index = STACKTRACE_FIRST(&dict_counts, &stacktrace_idx);
410 + key_index;
411 + key_index = STACKTRACE_NEXT(&dict_counts, &stacktrace_idx)) {
412 +
413 + i++;
414 + STACKTRACE st = (STACKTRACE)stacktrace_idx;
415 + size_t dict_count = STACKTRACE_GET(&dict_counts, stacktrace_idx);
416 + size_t item_count = STACKTRACE_GET(&item_counts, stacktrace_idx);
417 +
418 + // Format stacktrace to buffer
419 + buffer_flush(wb);
420 + stacktrace_to_buffer(st, wb);
421 +
422 + fprintf(stderr, "\n > DICTIONARY DELAYED %zu: %zu items in %zu dictionaries accessed from:\n%s\n\n",
423 + i, item_count, dict_count, buffer_tostring(wb));
424 + }
425 +
426 + // Clean up
427 + // Now collect and report information about dictionary items grouped by their creation stacktrace
428 + fprintf(stderr, "\n========= DICTIONARY ITEMS GROUPED BY CREATION STACKTRACE =========\n");
429 +
430 + // Loop through dictionaries to collect item stacktraces
431 + for(dict = dictionaries_waiting_to_be_destroyed; dict; dict = dict->next) {
432 + // Iterate through all items and count by stacktrace
433 + DICTIONARY_ITEM *item;
434 + for(item = dict->items.list; item; item = item->next) {
435 + // Count each unique stacktrace in the item's array
436 + for (int i = 0; i < item->stacktraces.num_stacktraces; i++) {
437 + if (item->stacktraces.stacktraces[i]) {
438 + uintptr_t item_key = (uintptr_t)item->stacktraces.stacktraces[i];
439 + size_t count = STACKTRACE_GET(&item_stacktrace_counts, item_key);
440 + count++;
441 + STACKTRACE_SET(&item_stacktrace_counts, item_key, count);
442 + }
443 + }
444 + }
445 + }
446
380 - return remaining;
381 -}
447 + // Print report of items by stacktrace
448 + Word_t item_st_idx = 0;
449 + size_t j = 0;
450
383 -// ----------------------------------------------------------------------------
384 -// API internal checks
451 + for(size_t key_index = STACKTRACE_FIRST(&item_stacktrace_counts, &item_st_idx);
452 + key_index;
453 + key_index = STACKTRACE_NEXT(&item_stacktrace_counts, &item_st_idx)) {
454
386 -#ifdef NETDATA_INTERNAL_CHECKS
387 -#define api_internal_check(dict, item, allow_null_dict, allow_null_item) api_internal_check_with_trace(dict, item, __FUNCTION__, allow_null_dict, allow_null_item)
388 -static inline void api_internal_check_with_trace(DICTIONARY *dict, DICTIONARY_ITEM *item, const char *function, bool allow_null_dict, bool allow_null_item) {
389 - if(!allow_null_dict && !dict) {
390 - internal_error(
391 - item,
392 - "DICTIONARY: attempted to %s() with a NULL dictionary, passing an item created from %s() %zu@%s.",
393 - function,
394 - item->dict->creation_function,
395 - item->dict->creation_line,
396 - item->dict->creation_file);
397 - fatal("DICTIONARY: attempted to %s() but dict is NULL", function);
398 - }
455 + j++;
456 + STACKTRACE st = (STACKTRACE)item_st_idx;
457 + size_t count = STACKTRACE_GET(&item_stacktrace_counts, item_st_idx);
458
400 - if(!allow_null_item && !item) {
401 - internal_error(
402 - true,
403 - "DICTIONARY: attempted to %s() without an item on a dictionary created from %s() %zu@%s.",
404 - function,
405 - dict?dict->creation_function:"unknown",
406 - dict?dict->creation_line:0,
407 - dict?dict->creation_file:"unknown");
408 - fatal("DICTIONARY: attempted to %s() but item is NULL", function);
409 - }
459 + // Format stacktrace to buffer
460 + buffer_flush(wb);
461 + stacktrace_to_buffer(st, wb);
462
411 - if(dict && item && dict != item->dict) {
412 - internal_error(
413 - true,
414 - "DICTIONARY: attempted to %s() an item on a dictionary created from %s() %zu@%s, but the item belongs to the dictionary created from %s() %zu@%s.",
415 - function,
416 - dict->creation_function,
417 - dict->creation_line,
418 - dict->creation_file,
419 - item->dict->creation_function,
420 - item->dict->creation_line,
421 - item->dict->creation_file
422 - );
423 - fatal("DICTIONARY: %s(): item does not belong to this dictionary.", function);
424 - }
425 -
426 - if(item) {
427 - REFCOUNT refcount = DICTIONARY_ITEM_REFCOUNT_GET(dict, item);
428 - if (unlikely(refcount <= 0)) {
429 - internal_error(
430 - true,
431 - "DICTIONARY: attempted to %s() of an item with reference counter = %d on a dictionary created from %s() %zu@%s",
432 - function,
433 - refcount,
434 - item->dict->creation_function,
435 - item->dict->creation_line,
436 - item->dict->creation_file);
437 - fatal("DICTIONARY: attempted to %s but item is having refcount = %d", function, refcount);
463 + fprintf(stderr, "\n > DICTIONARY ITEMS DELAYED %zu: %zu items accessed from:\n%s\n\n",
464 + j, count, buffer_tostring(wb));
465 }
466 +
467 + fprintf(stderr, "Total: %zu dictionaries\n", j);
468 + fprintf(stderr, "==================================================================\n");
469 +
470 + STACKTRACE_FREE(&dict_counts, NULL, NULL);
471 + STACKTRACE_FREE(&item_counts, NULL, NULL);
472 + STACKTRACE_FREE(&item_stacktrace_counts, NULL, NULL);
473 +
474 + buffer_free(wb);
475 }
440 -}
441 -#else
442 -#define api_internal_check(dict, item, allow_null_dict, allow_null_item) debug_dummy()
476 #endif
477
478 + netdata_mutex_unlock(&dictionaries_waiting_to_be_destroyed_mutex);
479 +
480 + return remaining;
481 +}
482 +
483 +// ----------------------------------------------------------------------------
484 +// API internal checks
485 +
486 +// Use the debug version from dictionary-debug.h
487 +#define api_internal_check(dict, item, allow_null_dict, allow_null_item) dictionary_debug_internal_check(dict, item, allow_null_dict, allow_null_item)
488 +
489 #define api_is_name_good(dict, name, name_len) api_is_name_good_with_trace(dict, name, name_len, __FUNCTION__)
490 static bool api_is_name_good_with_trace(DICTIONARY *dict __maybe_unused, const char *name, ssize_t name_len __maybe_unused, const char *function __maybe_unused) {
491 if(unlikely(!name)) {
448 - internal_error(
449 - true,
450 - "DICTIONARY: attempted to %s() with name = NULL on a dictionary created from %s() %zu@%s.",
451 - function,
452 - dict?dict->creation_function:"unknown",
453 - dict?dict->creation_line:0,
454 - dict?dict->creation_file:"unknown");
492 + dictionary_internal_error(true, dict,
493 + "DICTIONARY: attempted to %s() with name = NULL on a dictionary",
494 + function);
495 return false;
496 }
497
498 if(unlikely(!*name)) {
459 - internal_error(
460 - true,
461 - "DICTIONARY: attempted to %s() with empty name on a dictionary created from %s() %zu@%s.",
462 - function,
463 - dict?dict->creation_function:"unknown",
464 - dict?dict->creation_line:0,
465 - dict?dict->creation_file:"unknown");
499 + dictionary_internal_error(true, dict,
500 + "DICTIONARY: attempted to %s() with empty name on a dictionary",
501 + function);
502 return false;
503 }
504
469 - internal_error(
470 - name_len > 0 && name_len != (ssize_t)strlen(name),
505 + dictionary_internal_error(
506 + name_len > 0 && name_len != (ssize_t)strlen(name), dict,
507 "DICTIONARY: attempted to %s() with a name of '%s', having length of %zu, "
472 - "but the supplied name_len = %ld, on a dictionary created from %s() %zu@%s.",
508 + "but the supplied name_len = %ld",
509 function,
510 name,
511 strlen(name),
476 - (long int) name_len,
477 - dict?dict->creation_function:"unknown",
478 - dict?dict->creation_line:0,
479 - dict?dict->creation_file:"unknown");
512 + (long int) name_len);
513
481 - internal_error(
482 - name_len <= 0 && name_len != -1,
514 + dictionary_internal_error(
515 + name_len <= 0 && name_len != -1, dict,
516 "DICTIONARY: attempted to %s() with a name of '%s', having length of %zu, "
484 - "but the supplied name_len = %ld, on a dictionary created from %s() %zu@%s.",
517 + "but the supplied name_len = %ld",
518 function,
519 name,
520 strlen(name),
488 - (long int) name_len,
489 - dict?dict->creation_function:"unknown",
490 - dict?dict->creation_line:0,
491 - dict?dict->creation_file:"unknown");
521 + (long int) name_len);
522
523 return true;
524 }
@@ -498,7 +528,7 @@ static bool api_is_name_good_with_trace(DICTIONARY *dict __maybe_unused, const c
528
529 static DICTIONARY *dictionary_create_internal(DICT_OPTIONS options, struct dictionary_stats *stats, size_t fixed_size) {
530 dictionary_init_aral();
501 - cleanup_destroyed_dictionaries();
531 + cleanup_destroyed_dictionaries(false);
532
533 DICTIONARY *dict = aral_callocz(ar_dict);
534 dict->options = options;
@@ -535,30 +565,30 @@ static DICTIONARY *dictionary_create_internal(DICT_OPTIONS options, struct dicti
565 return dict;
566 }
567
538 -#ifdef NETDATA_INTERNAL_CHECKS
539 -DICTIONARY *dictionary_create_advanced_with_trace(DICT_OPTIONS options, struct dictionary_stats *stats, size_t fixed_size, const char *function, size_t line, const char *file) {
540 -#else
541 -DICTIONARY *dictionary_create_advanced(DICT_OPTIONS options, struct dictionary_stats *stats, size_t fixed_size) {
568 +// Helper function to add a stacktrace to a dictionary
569 +#ifdef FSANITIZE_ADDRESS
570 +static inline void dict_add_stacktrace(DICTIONARY *dict) {
571 + stacktrace_array_add(&dict->stacktraces, 1);
572 +}
573 #endif
574
575 +DICTIONARY *dictionary_create_advanced(DICT_OPTIONS options, struct dictionary_stats *stats, size_t fixed_size) {
576 DICTIONARY *dict = dictionary_create_internal(options, stats?stats:&dictionary_stats_category_other, fixed_size);
577
546 -#ifdef NETDATA_INTERNAL_CHECKS
547 - dict->creation_function = function;
548 - dict->creation_file = file;
549 - dict->creation_line = line;
578 +#ifdef FSANITIZE_ADDRESS
579 + // Initialize stacktrace tracking
580 + stacktrace_array_init(&dict->stacktraces);
581 +
582 + // Add the first stack trace at creation time
583 + dict_add_stacktrace(dict);
584 #endif
585
586 DICTIONARY_STATS_DICT_CREATIONS_PLUS1(dict);
587 + dictionary_debug_track_dict(dict);
588 return dict;
589 }
590
556 -#ifdef NETDATA_INTERNAL_CHECKS
557 -DICTIONARY *dictionary_create_view_with_trace(DICTIONARY *master, const char *function, size_t line, const char *file) {
558 -#else
591 DICTIONARY *dictionary_create_view(DICTIONARY *master) {
560 -#endif
561 -
592 DICTIONARY *dict = dictionary_create_internal(master->options, master->stats,
593 master->value_aral ? aral_requested_element_size(master->value_aral) : 0);
594
@@ -572,14 +602,16 @@ DICTIONARY *dictionary_create_view(DICTIONARY *master) {
602 dict->hooks = master->hooks;
603 __atomic_add_fetch(&master->hooks->links, 1, __ATOMIC_ACQUIRE);
604
575 -#ifdef NETDATA_INTERNAL_CHECKS
576 - dict->creation_function = function;
577 - dict->creation_file = file;
578 - dict->creation_line = line;
579 - dict->creation_tid = gettid_cached();
605 +#ifdef FSANITIZE_ADDRESS
606 + // Initialize stacktrace tracking
607 + stacktrace_array_init(&dict->stacktraces);
608 +
609 + // Add the first stack trace at creation time
610 + dict_add_stacktrace(dict);
611 #endif
612
613 DICTIONARY_STATS_DICT_CREATIONS_PLUS1(dict);
614 + dictionary_debug_track_dict(dict);
615 return dict;
616 }
617
@@ -603,7 +635,7 @@ void dictionary_flush(DICTIONARY *dict) {
635 }
636
637 size_t dictionary_destroy(DICTIONARY *dict) {
606 - cleanup_destroyed_dictionaries();
638 + cleanup_destroyed_dictionaries(false);
639
640 if(!dict) return 0;
641
@@ -616,14 +648,10 @@ size_t dictionary_destroy(DICTIONARY *dict) {
648 dictionary_flush(dict);
649 dictionary_queue_for_destruction(dict);
650
619 - internal_error(
620 - true,
621 - "DICTIONARY: delaying destruction of dictionary created from %s() %zu@%s, because it has %d referenced items in it (%d total).",
622 - dict->creation_function,
623 - dict->creation_line,
624 - dict->creation_file,
625 - dict->referenced_items,
626 - dict->entries);
651 + dictionary_internal_error(
652 + true, dict,
653 + "DICTIONARY: delaying destruction of dictionary, because it has %d referenced items in it (%d total).",
654 + dict->referenced_items, dict->entries);
655
656 ll_recursive_unlock(dict, DICTIONARY_LOCK_WRITE);
657 return 0;
src/libnetdata/dictionary/dictionary.h
+4 -12
@@ -113,22 +113,11 @@ struct dictionary_stats {
113 };
114
115 // Create a dictionary
116 -#ifdef NETDATA_INTERNAL_CHECKS
117 -#define dictionary_create(options) dictionary_create_advanced_with_trace(options, NULL, 0, __FUNCTION__, __LINE__, __FILE__)
118 -#define dictionary_create_advanced(options, stats, fixed_size) dictionary_create_advanced_with_trace(options, stats, fixed_size, __FUNCTION__, __LINE__, __FILE__)
119 -DICTIONARY *dictionary_create_advanced_with_trace(DICT_OPTIONS options, struct dictionary_stats *stats, size_t fixed_size, const char *function, size_t line, const char *file);
120 -#else
116 #define dictionary_create(options) dictionary_create_advanced(options, NULL, 0)
117 DICTIONARY *dictionary_create_advanced(DICT_OPTIONS options, struct dictionary_stats *stats, size_t fixed_size);
123 -#endif
118
119 // Create a view on a dictionary
126 -#ifdef NETDATA_INTERNAL_CHECKS
127 -#define dictionary_create_view(master) dictionary_create_view_with_trace(master, __FUNCTION__, __LINE__, __FILE__)
128 -DICTIONARY *dictionary_create_view_with_trace(DICTIONARY *master, const char *function, size_t line, const char *file);
129 -#else
120 DICTIONARY *dictionary_create_view(DICTIONARY *master);
131 -#endif
121
122 // an insert callback to be called just after an item is added to the dictionary
123 // this callback is called while the dictionary is write locked!
@@ -168,7 +157,10 @@ void dictionary_version_increment(DICTIONARY *dict);
157
158 void dictionary_garbage_collect(DICTIONARY *dict);
159
171 -size_t cleanup_destroyed_dictionaries(void);
160 +size_t cleanup_destroyed_dictionaries(bool shutdown);
161 +
162 +// Report on allocated dictionaries - used during Address Sanitizer builds
163 +void dictionary_print_still_allocated_stacktraces(void);
164
165 // ----------------------------------------------------------------------------
166 // Set an item in the dictionary
src/libnetdata/libjudy/judy-malloc.c
+2 -2
@@ -74,8 +74,8 @@ ALWAYS_INLINE int64_t JudyAllocThreadPulseGetAndReset(void) {
74 // --------------------------------------------------------------------------------------------------------------------
75 // Judy dedicated jemalloc arena
76
77 -static unsigned jemalloc_arena_index = 0;
78 -static bool jemalloc_initialized = false;
77 +static unsigned jemalloc_arena_index __maybe_unused = 0;
78 +static bool jemalloc_initialized __maybe_unused = false;
79
80 #ifdef HAVE_JEMALLOC_ARENA_API
81 #include <jemalloc/jemalloc.h>
src/libnetdata/libnetdata.h
+1
@@ -134,6 +134,7 @@ extern const char *netdata_configured_host_prefix;
134 #include "facets/facets.h"
135 #include "functions_evloop/functions_evloop.h"
136 #include "query_progress/progress.h"
137 +#include "stacktrace/stacktrace.h"
138
139 static ALWAYS_INLINE PPvoid_t JudyLFirstThenNext(Pcvoid_t PArray, Word_t * PIndex, bool *first) {
140 if(unlikely(*first)) {
src/libnetdata/log/nd_log-init.c
+4 -4
@@ -136,7 +136,7 @@ void nd_log_initialize_for_external_plugins(const char *name) {
136
137 // nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "FINAL_LOG_METHOD: %s", nd_log_id2method(method));
138
139 - capture_stack_trace_init();
139 + stacktrace_init();
140 }
141
142 // --------------------------------------------------------------------------------------------------------------------
@@ -266,7 +266,7 @@ void nd_log_initialize(void) {
266 for(size_t i = 0 ; i < _NDLS_MAX ; i++)
267 nd_log_open(&nd_log.sources[i], i);
268
269 - capture_stack_trace_init();
269 + stacktrace_init();
270 }
271
272 void nd_log_reopen_log_files(bool log) {
@@ -284,12 +284,12 @@ int nd_log_systemd_journal_fd(void) {
284 }
285
286 void nd_log_reopen_log_files_for_spawn_server(const char *name) {
287 - nd_log_forked = true;
287 nd_log.fatal_hook_cb = NULL;
288 nd_log.fatal_final_cb = NULL;
289
290 gettid_uncached();
292 - capture_stack_trace_flush();
291 + stacktrace_flush();
292 + stacktrace_forked();
293
294 if(nd_log.syslog.initialized) {
295 closelog();
src/libnetdata/log/nd_log-internals.h
-3
@@ -201,9 +201,6 @@ const char *errno_annotator(struct log_field *lf);
201 const char *priority_annotator(struct log_field *lf);
202 const char *timestamp_usec_annotator(struct log_field *lf);
203
204 -extern bool nd_log_forked;
205 -bool stack_trace_formatter(BUFFER *wb, void *data);
206 -
204 #if defined(OS_WINDOWS)
205 const char *winerror_annotator(struct log_field *lf);
206 #endif
src/libnetdata/log/nd_log-stacktrace.c deleted
-560
@@ -1,560 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "nd_log-internals.h"
4 -
5 -bool nd_log_forked = false;
6 -
7 -#define NO_STACK_TRACE_PREFIX STACK_TRACE_INFO_PREFIX "stack trace is not available, "
8 -
9 -// The signal handler function name to filter out in stack traces
10 -static const char *signal_handler_function = "nd_signal_handler";
11 -
12 -// List of auxiliary functions that should not be reported as root cause
13 -static const char *auxiliary_functions[] = {
14 - "nd_uuid_copy",
15 - "out_of_memory",
16 - "shutdown_timed_out",
17 - NULL // Terminator
18 -};
19 -
20 -// List of logging functions to filter out
21 -static const char *logging_functions[] = {
22 - "netdata_logger",
23 - "netdata_logger_with_limit",
24 - "netdata_logger_fatal",
25 - NULL // Terminator
26 -};
27 -
28 -// Set the signal handler function name to filter out in stack traces
29 -void capture_stack_trace_set_signal_handler_function(const char *function_name) {
30 - signal_handler_function = function_name;
31 -}
32 -
33 -// Exact match check if a function is in the auxiliary list (strcmp)
34 -static inline bool is_auxiliary_function(const char *function) {
35 - if (!function || !*function)
36 - return false;
37 -
38 - for (int i = 0; auxiliary_functions[i]; i++) {
39 - if (strcmp(function, auxiliary_functions[i]) == 0)
40 - return true;
41 - }
42 -
43 - return false;
44 -}
45 -
46 -// Exact match check if a function is a logging function (strcmp)
47 -static inline bool is_logging_function(const char *function) {
48 - if (!function || !*function)
49 - return false;
50 -
51 - for (int i = 0; logging_functions[i]; i++) {
52 - if (strcmp(function, logging_functions[i]) == 0)
53 - return true;
54 - }
55 -
56 - return false;
57 -}
58 -
59 -// Substring check if a function contains a logging function name (strstr)
60 -static inline bool contains_logging_function(const char *text) {
61 - if (!text || !*text)
62 - return false;
63 -
64 - for (int i = 0; logging_functions[i]; i++) {
65 - if (strstr(text, logging_functions[i]) != NULL)
66 - return true;
67 - }
68 -
69 - return false;
70 -}
71 -
72 -static inline bool is_netdata_function(const char *function, const char *filename) {
73 - return function && *function && filename && *filename &&
74 - strstr(filename, "/src/") &&
75 - !strstr(filename, "/vendored/") &&
76 - !contains_logging_function(function);
77 -}
78 -
79 -// Exact match check if a function is the signal handler (strcmp)
80 -static inline bool is_signal_handler_function(const char *function) {
81 - return function && *function &&
82 - signal_handler_function && *signal_handler_function &&
83 - strcmp(function, signal_handler_function) == 0;
84 -}
85 -
86 -// Substring check if a function contains the signal handler name (strstr)
87 -static inline bool contains_signal_handler_function(const char *text) {
88 - return text && *text &&
89 - signal_handler_function && *signal_handler_function &&
90 - strstr(text, signal_handler_function) != NULL;
91 -}
92 -
93 -// Thread-local buffer to store the first netdata function encountered in a stack trace
94 -static __thread char root_cause_function[48];
95 -
96 -// Returns the first netdata function found in the stack trace
97 -const char *capture_stack_trace_root_cause_function(void) {
98 - return root_cause_function[0] ? root_cause_function : NULL;
99 -}
100 -
101 -// Store a function name as the first netdata function found
102 -static inline void keep_first_root_cause_function(const char *function) {
103 - if (!function || !*function || root_cause_function[0])
104 - return; // Already have a function or null input
105 -
106 - // Skip auxiliary functions and logging functions
107 - if (is_auxiliary_function(function) || is_logging_function(function))
108 - return;
109 -
110 - strncpyz(root_cause_function, function, sizeof(root_cause_function) - 1);
111 -}
112 -
113 -#if defined(HAVE_LIBBACKTRACE)
114 -#include "backtrace-supported.h"
115 -#endif
116 -
117 -#if defined(HAVE_LIBBACKTRACE) && BACKTRACE_SUPPORTED == 1 /* && BACKTRACE_SUPPORTS_THREADS == 1 */
118 -#include "backtrace.h"
119 -
120 -static struct backtrace_state *backtrace_state = NULL;
121 -
122 -typedef struct {
123 - BUFFER *wb; // Buffer to write to
124 - size_t frame_count; // Number of frames processed
125 - bool first_frame; // Is this the first frame?
126 - bool found_signal_handler; // Have we found the signal handler frame?
127 -} backtrace_data_t;
128 -
129 -// Common function to format and add a stack frame to the buffer
130 -static void add_stack_frame(backtrace_data_t *bt_data, uintptr_t pc, const char *function,
131 - const char *filename, int lineno) {
132 - BUFFER *wb = bt_data->wb;
133 -
134 - if (!wb)
135 - return;
136 -
137 - // Check if we found the signal handler frame
138 - if (!bt_data->found_signal_handler && is_signal_handler_function(function)) {
139 - // We found the signal handler, reset the buffer and clear function name
140 - buffer_flush(wb);
141 - bt_data->frame_count = 0;
142 - bt_data->first_frame = true;
143 - bt_data->found_signal_handler = true;
144 - root_cause_function[0] = '\0';
145 - return; // Skip adding the signal handler itself
146 - }
147 -
148 - // Check for logging functions, but only if we haven't found a signal handler yet
149 - // This prevents double resets when crashing inside logging code
150 - if (!bt_data->found_signal_handler && is_logging_function(function)) {
151 - // Found a logging function, reset the buffer and clear function name
152 - buffer_flush(wb);
153 - bt_data->frame_count = 0;
154 - bt_data->first_frame = true;
155 - root_cause_function[0] = '\0';
156 - // continue to add the function to the stack trace
157 - }
158 -
159 - // Check if this is a netdata source file and store the function name if it is
160 - // (but only if we haven't already stored one)
161 - if (!root_cause_function[0] && is_netdata_function(function, filename))
162 - keep_first_root_cause_function(function);
163 -
164 - // Add a newline between frames
165 - if (!bt_data->first_frame)
166 - buffer_putc(wb, '\n');
167 - else
168 - bt_data->first_frame = false;
169 -
170 - // Format: #ID function (filename.c:NNN)
171 - buffer_putc(wb, '#');
172 - buffer_print_uint64(wb, bt_data->frame_count);
173 - buffer_putc(wb, ' ');
174 -
175 - if (function && *function)
176 - buffer_strcat(wb, function);
177 - else
178 - buffer_strcat(wb, "<unknown>");
179 -
180 - if(pc) {
181 - buffer_strcat(wb, " [");
182 - buffer_print_uint64_hex(wb, pc);
183 - buffer_putc(wb, ']');
184 - }
185 -
186 - if (filename && *filename) {
187 - buffer_strcat(wb, " (");
188 -
189 - const char *f = strstr(filename, "/src/");
190 - if (f) {
191 - const char *f2 = strstr(f + 1, "/src/");
192 - if(f2) f = f2;
193 - }
194 - if(!f) f = filename;
195 -
196 - buffer_strcat(wb, f);
197 -
198 - if (lineno > 0) {
199 - buffer_strcat(wb, ":");
200 - buffer_print_uint64(wb, (uint64_t)lineno);
201 - }
202 -
203 - buffer_putc(wb, ')');
204 - }
205 -
206 - bt_data->frame_count++;
207 -}
208 -
209 -// Error callback for libbacktrace
210 -static void bt_error_handler(void *data, const char *msg, int errnum) {
211 - backtrace_data_t *bt_data = (backtrace_data_t *)data;
212 -
213 - if (!bt_data || !bt_data->wb)
214 - return;
215 -
216 - // Use <unknown> for function name in error cases
217 - const char *function = "<unknown>";
218 -
219 - // Format the error message as the filename
220 - char error_buf[512] = "error: ";
221 - size_t len = 7; // Length of "error: "
222 -
223 - // Add the error message
224 - if (msg)
225 - len = strcatz(error_buf, len, msg, sizeof(error_buf));
226 -
227 - // Add the error number description if available
228 - if (errnum > 0) {
229 - if (msg) {
230 - len = strcatz(error_buf, len, ": ", sizeof(error_buf));
231 - }
232 - len = strcatz(error_buf, len, strerror(errnum), sizeof(error_buf));
233 - }
234 -
235 - add_stack_frame(bt_data, 0, function, error_buf, 0);
236 -}
237 -
238 -// Full callback for libbacktrace
239 -static int bt_full_handler(void *data, uintptr_t pc,
240 - const char *filename, int lineno,
241 - const char *function) {
242 - backtrace_data_t *bt_data = (backtrace_data_t *)data;
243 - if (!bt_data)
244 - return 0;
245 -
246 - add_stack_frame(bt_data, pc, function, filename, lineno);
247 -
248 - return 0; // Continue backtrace
249 -}
250 -
251 -const char *capture_stack_trace_backend(void) {
252 -#if BACKTRACE_SUPPORTS_DATA
253 -#define BACKTRACE_DATA "data"
254 -#else
255 -#define BACKTRACE_DATA "no-data"
256 -#endif
257 -
258 -#if BACKTRACE_USES_MALLOC
259 -#define BACKTRACE_MEMORY "malloc"
260 -#else
261 -#define BACKTRACE_MEMORY "mmap"
262 -#endif
263 -
264 -#if BACKTRACE_SUPPORTS_THREADS
265 -#define BACKTRACE_THREADS "threads"
266 -#else
267 -#define BACKTRACE_THREADS "no-threads"
268 -#endif
269 -
270 - return "libbacktrace (" BACKTRACE_MEMORY ", " BACKTRACE_THREADS ", " BACKTRACE_DATA ")";
271 -}
272 -
273 -void capture_stack_trace_init(void) {
274 - if (!backtrace_state) {
275 - backtrace_state = backtrace_create_state(NULL, BACKTRACE_SUPPORTS_THREADS,
276 - NULL, // We'll handle errors in bt_full_handler
277 - NULL);
278 - }
279 -}
280 -
281 -void capture_stack_trace_flush(void) {
282 - // Nothing to flush with libbacktrace
283 -}
284 -
285 -bool capture_stack_trace_is_async_signal_safe(void) {
286 -// libbacktrace may use malloc depending on configuration
287 -// Check the BACKTRACE_USES_MALLOC define
288 -#if BACKTRACE_USES_MALLOC
289 - return false;
290 -#else
291 - return true;
292 -#endif
293 -}
294 -
295 -bool capture_stack_trace_available(void) {
296 - return backtrace_state != NULL && BACKTRACE_SUPPORTED;
297 -}
298 -
299 -NEVER_INLINE
300 -void capture_stack_trace(BUFFER *wb) {
301 - root_cause_function[0] = '\0';
302 -
303 - if (!backtrace_state) {
304 - buffer_strcat(wb, NO_STACK_TRACE_PREFIX "libbacktrace not initialized");
305 - return;
306 - }
307 -
308 - backtrace_data_t bt_data = {
309 - .wb = wb,
310 - .frame_count = 0,
311 - .first_frame = true,
312 - .found_signal_handler = false
313 - };
314 -
315 - // Skip one frame to hide capture_stack_trace() itself
316 - backtrace_full(backtrace_state, 0, bt_full_handler,
317 - bt_error_handler, &bt_data);
318 -
319 - // If no frames were reported
320 - if (bt_data.frame_count == 0) {
321 - buffer_strcat(wb, NO_STACK_TRACE_PREFIX "libbacktrace reports no frames");
322 - }
323 -}
324 -
325 -#elif defined(HAVE_LIBUNWIND)
326 -#if !defined(STATIC_BUILD)
327 -#define UNW_LOCAL_ONLY
328 -#endif
329 -#include <libunwind.h>
330 -
331 -const char *capture_stack_trace_backend(void) {
332 - return "libunwind";
333 -}
334 -
335 -void capture_stack_trace_init(void) {
336 - unw_set_caching_policy(unw_local_addr_space, UNW_CACHE_NONE);
337 -}
338 -
339 -void capture_stack_trace_flush(void) {
340 - unw_flush_cache(unw_local_addr_space, 0, 0);
341 -}
342 -
343 -bool capture_stack_trace_is_async_signal_safe(void) {
344 -#if defined(STATIC_BUILD)
345 - return false;
346 -#else
347 - return true;
348 -#endif
349 -}
350 -
351 -bool capture_stack_trace_available(void) {
352 - return true;
353 -}
354 -
355 -NEVER_INLINE
356 -void capture_stack_trace(BUFFER *wb) {
357 - // this function is async-signal-safe, if the buffer has enough space to hold the stack trace
358 -
359 - root_cause_function[0] = '\0';
360 -
361 - unw_cursor_t cursor;
362 - unw_context_t context;
363 - size_t frames = 0;
364 -
365 - // Initialize context for current thread
366 - unw_getcontext(&context);
367 - unw_init_local(&cursor, &context);
368 -
369 - size_t added = 0;
370 - bool found_signal_handler = false;
371 -
372 - while (unw_step(&cursor) > 0) {
373 - unw_word_t offset, pc;
374 - char sym[256];
375 -
376 - unw_get_reg(&cursor, UNW_REG_IP, &pc);
377 - if (!pc)
378 - break;
379 -
380 - const char *name = sym;
381 - if (unw_get_proc_name(&cursor, sym, sizeof(sym), &offset) != 0) {
382 - name = "<unknown>";
383 - offset = 0;
384 - }
385 -
386 - // Check if we found the signal handler frame
387 - if (!found_signal_handler && is_signal_handler_function(name)) {
388 - // We found the signal handler, reset the buffer
389 - buffer_flush(wb);
390 - added = 0;
391 - frames = 0;
392 - found_signal_handler = true;
393 - continue; // Skip adding the signal handler itself
394 - }
395 -
396 - // Check for logging functions, but only if we haven't found a signal handler yet
397 - if (!found_signal_handler && is_logging_function(name)) {
398 - // Found a logging function, reset the buffer
399 - buffer_flush(wb);
400 - added = 0;
401 - frames = 0;
402 - // continue to add the function to the stack trace
403 - }
404 -
405 - if (frames++)
406 - buffer_putc(wb, '\n');
407 -
408 - buffer_putc(wb, '#');
409 - buffer_print_uint64(wb, added);
410 - buffer_putc(wb, ' ');
411 - buffer_strcat(wb, name);
412 -
413 - if(offset) {
414 - buffer_putc(wb, '+');
415 - buffer_print_uint64_hex(wb, offset);
416 - }
417 -
418 - added++;
419 - }
420 -
421 - if (!added)
422 - buffer_strcat(wb, NO_STACK_TRACE_PREFIX "libunwind reports no frames");
423 -}
424 -
425 -#elif defined(HAVE_BACKTRACE)
426 -
427 -const char *capture_stack_trace_backend(void) {
428 - return "backtrace";
429 -}
430 -
431 -bool capture_stack_trace_available(void) {
432 - return true;
433 -}
434 -
435 -void capture_stack_trace_init(void) {
436 - ;
437 -}
438 -
439 -void capture_stack_trace_flush(void) {
440 - ;
441 -}
442 -
443 -bool capture_stack_trace_is_async_signal_safe(void) {
444 - return false;
445 -}
446 -
447 -NEVER_INLINE
448 -void capture_stack_trace(BUFFER *wb) {
449 - void *array[50];
450 - char **messages;
451 - int size, i;
452 -
453 - root_cause_function[0] = '\0';
454 -
455 - size = backtrace(array, _countof(array));
456 - messages = backtrace_symbols(array, size);
457 -
458 - if (!messages) {
459 - buffer_strcat(wb, NO_STACK_TRACE_PREFIX "backtrace() reports no symbols");
460 - return;
461 - }
462 -
463 - size_t added = 0;
464 - bool found_signal_handler = false;
465 -
466 - // Format the stack trace (removing the address part)
467 - for (i = 0; i < size; i++) {
468 - if(messages[i] && *messages[i]) {
469 - // Check if we found the signal handler frame
470 - if (!found_signal_handler && contains_signal_handler_function(messages[i])) {
471 - // We found the signal handler, reset the buffer
472 - buffer_flush(wb);
473 - added = 0;
474 - found_signal_handler = true;
475 - continue; // Skip adding the signal handler itself
476 - }
477 -
478 - // Check for logging functions, but only if we haven't found a signal handler yet
479 - if (!found_signal_handler && contains_logging_function(messages[i])) {
480 - // Found a logging function, reset the buffer
481 - buffer_flush(wb);
482 - added = 0;
483 - // continue to add the function to the stack trace
484 - }
485 -
486 - if(added)
487 - buffer_putc(wb, '\n');
488 -
489 - buffer_putc(wb, '#');
490 - buffer_print_uint64(wb, added);
491 - buffer_putc(wb, ' ');
492 - buffer_strcat(wb, messages[i]);
493 - added++;
494 - }
495 - }
496 -
497 - if(!added)
498 - buffer_strcat(wb, NO_STACK_TRACE_PREFIX "backtrace() reports no frames");
499 -
500 - free(messages);
501 -}
502 -
503 -#else
504 -
505 -const char *capture_stack_trace_backend(void) {
506 - return "none";
507 -}
508 -
509 -bool capture_stack_trace_available(void) {
510 - return false;
511 -}
512 -
513 -void capture_stack_trace_init(void) {
514 - ;
515 -}
516 -
517 -void capture_stack_trace_flush(void) {
518 - ;
519 -}
520 -
521 -bool capture_stack_trace_is_async_signal_safe(void) {
522 - return false;
523 -}
524 -
525 -NEVER_INLINE
526 -void capture_stack_trace(BUFFER *wb) {
527 - root_cause_function[0] = '\0';
528 -
529 - buffer_strcat(wb, NO_STACK_TRACE_PREFIX "no back-end available");
530 -
531 - // probably we can have something like this?
532 - // https://maskray.me/blog/2022-04-09-unwinding-through-signal-handler
533 - // (at the end - but it needs the frame pointer)
534 -}
535 -
536 -#endif
537 -
538 -NEVER_INLINE
539 -bool stack_trace_formatter(BUFFER *wb, void *data __maybe_unused) {
540 - static __thread bool in_stack_trace = false;
541 -
542 - if (nd_log_forked) {
543 - // libunwind freezes in forked children
544 - buffer_strcat(wb, NO_STACK_TRACE_PREFIX "stack trace after fork is disabled");
545 - return true;
546 - }
547 -
548 - if (in_stack_trace) {
549 - // Prevent recursion
550 - buffer_strcat(wb, NO_STACK_TRACE_PREFIX "stack trace recursion detected");
551 - return true;
552 - }
553 -
554 - in_stack_trace = true;
555 -
556 - capture_stack_trace(wb);
557 -
558 - in_stack_trace = false; // Ensure the flag is reset
559 - return true;
560 -}
src/libnetdata/log/nd_log.c
+1
@@ -6,6 +6,7 @@
6
7 #include "../libnetdata.h"
8 #include "nd_log-internals.h"
9 +#include "../stacktrace/stacktrace.h"
10
11 const char *program_name = "";
12 uint64_t debug_flags = 0;
src/libnetdata/log/nd_log.h
-11
@@ -33,17 +33,6 @@ const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority);
33 const char *nd_log_method_for_external_plugins(const char *s);
34 ND_UUID nd_log_get_invocation_id(void);
35
36 -#define STACK_TRACE_INFO_PREFIX "info: "
37 -void capture_stack_trace(BUFFER *wb);
38 -
39 -void capture_stack_trace_init(void);
40 -void capture_stack_trace_flush(void);
41 -bool capture_stack_trace_available(void);
42 -bool capture_stack_trace_is_async_signal_safe(void);
43 -const char *capture_stack_trace_backend(void);
44 -void capture_stack_trace_set_signal_handler_function(const char *function_name);
45 -const char *capture_stack_trace_root_cause_function(void);
46 -
36 typedef void (*log_event_t)(const char *filename, const char *function, const char *message, const char *errno_str, const char *stack_trace, long line);
37 void nd_log_register_fatal_hook_cb(log_event_t cb);
38
src/libnetdata/stacktrace/stacktrace-array.c new
+91
@@ -0,0 +1,91 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "stacktrace-array.h"
4 +#include "stacktrace-common.h"
5 +
6 +// Initialize a stacktrace array
7 +void stacktrace_array_init(STACKTRACE_ARRAY *array) {
8 + if (unlikely(!array))
9 + return;
10 +
11 + spinlock_init(&array->spinlock);
12 + array->num_stacktraces = 0;
13 + memset(array->stacktraces, 0, sizeof(array->stacktraces));
14 +}
15 +
16 +// Add a stacktrace to an array (captures current stacktrace)
17 +NEVER_INLINE
18 +bool stacktrace_array_add(STACKTRACE_ARRAY *array, int skip_frames) {
19 + if (unlikely(!array))
20 + return false;
21 +
22 + // Get current stacktrace
23 + STACKTRACE current = stacktrace_get(skip_frames + 1); // +1 to skip this function
24 + if (!current)
25 + return false;
26 +
27 + bool added = false;
28 +
29 + // Protect the stacktraces array with a spinlock
30 + spinlock_lock(&array->spinlock);
31 +
32 + // Check if this stacktrace already exists in the array
33 + bool found = false;
34 + for (int i = 0; i < array->num_stacktraces; i++) {
35 + if (array->stacktraces[i] == current) {
36 + found = true;
37 + break;
38 + }
39 + }
40 +
41 + // Add the stacktrace if it's unique and there's room
42 + if (!found && array->num_stacktraces < STACKTRACE_ARRAY_MAX_TRACES) {
43 + array->stacktraces[array->num_stacktraces++] = current;
44 + added = true;
45 + }
46 +
47 + spinlock_unlock(&array->spinlock);
48 +
49 + return added;
50 +}
51 +
52 +// Report stacktraces to a buffer
53 +size_t stacktrace_array_to_buffer(STACKTRACE_ARRAY *array, BUFFER *wb, size_t *total_count, const char *prefix, bool brief_output) {
54 + if (unlikely(!array || !wb))
55 + return 0;
56 +
57 + if (!prefix)
58 + prefix = "STACKTRACE";
59 +
60 + size_t reported = 0;
61 +
62 + // Lock the array while we're operating on it
63 + spinlock_lock(&array->spinlock);
64 +
65 + // Update total count if requested
66 + if (total_count)
67 + *total_count = array->num_stacktraces;
68 +
69 + // If brief output is requested, just report the number of stacktraces
70 + if (brief_output) {
71 + buffer_sprintf(wb, "%s: %d stacktraces captured\n", prefix, array->num_stacktraces);
72 + spinlock_unlock(&array->spinlock);
73 + return array->num_stacktraces;
74 + }
75 +
76 + // Report each stacktrace in the array
77 + for (int i = 0; i < array->num_stacktraces; i++) {
78 + if (array->stacktraces[i]) {
79 + if (i > 0)
80 + buffer_strcat(wb, "\n");
81 +
82 + buffer_sprintf(wb, "%s #%d:\n", prefix, i+1);
83 + stacktrace_to_buffer(array->stacktraces[i], wb);
84 + reported++;
85 + }
86 + }
87 +
88 + spinlock_unlock(&array->spinlock);
89 +
90 + return reported;
91 +}
\ No newline at end of file
src/libnetdata/stacktrace/stacktrace-array.h new
+35
@@ -0,0 +1,35 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_STACKTRACE_ARRAY_H
4 +#define NETDATA_STACKTRACE_ARRAY_H 1
5 +
6 +#include "libnetdata/libnetdata.h"
7 +#include "stacktrace.h"
8 +
9 +// Default maximum number of stacktraces to track per array
10 +#ifndef STACKTRACE_ARRAY_MAX_TRACES
11 +#define STACKTRACE_ARRAY_MAX_TRACES 20
12 +#endif
13 +
14 +// Structure to track multiple stacktraces
15 +typedef struct stacktrace_array {
16 + SPINLOCK spinlock; // spinlock to protect the stacktraces array
17 + int num_stacktraces; // number of stored stacktraces (0 to STACKTRACE_ARRAY_MAX_TRACES)
18 + STACKTRACE stacktraces[STACKTRACE_ARRAY_MAX_TRACES]; // array of stacktraces from different acquisition points
19 +} STACKTRACE_ARRAY;
20 +
21 +// Initialize a stacktrace array
22 +void stacktrace_array_init(STACKTRACE_ARRAY *array);
23 +
24 +// Add a stacktrace to an array (captures current stacktrace)
25 +// Only adds if it's not already present
26 +// Returns true if the stacktrace was added, false if it was already there or array is full
27 +bool stacktrace_array_add(STACKTRACE_ARRAY *array, int skip_frames);
28 +
29 +// Report stacktraces to a buffer
30 +// If total_count is not NULL, it will be updated with the total number of array elements
31 +// If brief_output is true, only summary information is included
32 +// Returns the number of unique stacktraces reported
33 +size_t stacktrace_array_to_buffer(STACKTRACE_ARRAY *array, BUFFER *wb, size_t *total_count, const char *prefix, bool brief_output);
34 +
35 +#endif /* NETDATA_STACKTRACE_ARRAY_H */
\ No newline at end of file
src/libnetdata/stacktrace/stacktrace-backtrace.c new
+159
@@ -0,0 +1,159 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "stacktrace-common.h"
4 +
5 +#if defined(USE_BACKTRACE)
6 +#include <execinfo.h>
7 +
8 +const char *stacktrace_backend(void) {
9 + return "backtrace";
10 +}
11 +
12 +bool stacktrace_available(void) {
13 + return true;
14 +}
15 +
16 +void impl_stacktrace_init(void) {
17 + // Nothing to initialize for backtrace backend
18 +}
19 +
20 +void stacktrace_flush(void) {
21 + // Nothing to flush
22 +}
23 +
24 +bool stacktrace_capture_is_async_signal_safe(void) {
25 + return false;
26 +}
27 +
28 +NEVER_INLINE
29 +void stacktrace_capture(BUFFER *wb) {
30 + void *array[50];
31 + char **messages;
32 + int size, i;
33 +
34 + root_cause_function[0] = '\0';
35 +
36 + size = backtrace(array, _countof(array));
37 + messages = backtrace_symbols(array, size);
38 +
39 + if (!messages) {
40 + buffer_strcat(wb, NO_STACK_TRACE_PREFIX "backtrace() reports no symbols");
41 + return;
42 + }
43 +
44 + size_t added = 0;
45 + bool found_signal_handler = false;
46 +
47 + // Format the stack trace (removing the address part)
48 + // Skip the first frame (stacktrace_capture itself)
49 + for (i = 1; i < size; i++) {
50 + if(messages[i] && *messages[i]) {
51 + // Check if we found the signal handler frame
52 + if (!found_signal_handler && stacktrace_contains_signal_handler_function(messages[i])) {
53 + // We found the signal handler, reset the buffer
54 + buffer_flush(wb);
55 + added = 0;
56 + found_signal_handler = true;
57 + continue; // Skip adding the signal handler itself
58 + }
59 +
60 + // Check for logging functions, but only if we haven't found a signal handler yet
61 + if (!found_signal_handler && stacktrace_contains_logging_function(messages[i])) {
62 + // Found a logging function, reset the buffer
63 + buffer_flush(wb);
64 + added = 0;
65 + // continue to add the function to the stack trace
66 + }
67 +
68 + if(added)
69 + buffer_putc(wb, '\n');
70 +
71 + buffer_putc(wb, '#');
72 + buffer_print_uint64(wb, added);
73 + buffer_putc(wb, ' ');
74 + buffer_strcat(wb, messages[i]);
75 + added++;
76 + }
77 + }
78 +
79 + if(!added)
80 + buffer_strcat(wb, NO_STACK_TRACE_PREFIX "backtrace() reports no frames");
81 +
82 + free(messages);
83 +}
84 +
85 +// Implementation-specific function to collect stack trace frames
86 +int impl_stacktrace_get_frames(void **frames, int max_frames, int skip_frames) {
87 + if (!frames || max_frames <= 0)
88 + return 0;
89 +
90 + // Add 1 to skip_frames to also skip this function itself
91 + skip_frames += 1;
92 +
93 + // Collect all stack frames without skipping at this level
94 + void *array[100 + 50]; // Use a larger array to account for skipped frames (100) plus the max we need (50)
95 + int size = backtrace(array, _countof(array));
96 +
97 + if (size <= skip_frames) // Not enough frames after skipping
98 + return 0;
99 +
100 + // Apply skip_frames here, in the aftermath of backtrace, not during capture
101 + // This ensures we're not skipping inlined functions
102 + int available_frames = size - skip_frames;
103 + int frames_to_copy = available_frames < max_frames ? available_frames : max_frames;
104 +
105 + // Copy the frames, skipping the requested number of frames
106 + memcpy(frames, array + skip_frames, frames_to_copy * sizeof(void *));
107 +
108 + return frames_to_copy;
109 +}
110 +
111 +// Implementation-specific function to convert a stacktrace to a buffer
112 +void impl_stacktrace_to_buffer(STACKTRACE trace, BUFFER *wb) {
113 + struct stacktrace *st = (struct stacktrace *)trace;
114 +
115 + // Convert to text representation
116 + char **messages = backtrace_symbols(st->frames, st->frame_count);
117 +
118 + if (!messages) {
119 + buffer_strcat(wb, NO_STACK_TRACE_PREFIX "backtrace_symbols() failed");
120 + return;
121 + }
122 +
123 + bool found_signal_handler = false;
124 + int added = 0;
125 +
126 + for (int i = 0; i < st->frame_count; i++) {
127 + if (!messages[i] || !*messages[i])
128 + continue;
129 +
130 + // Handle filtering
131 + if (!found_signal_handler && stacktrace_contains_signal_handler_function(messages[i])) {
132 + buffer_flush(wb);
133 + added = 0;
134 + found_signal_handler = true;
135 + continue;
136 + }
137 +
138 + if (!found_signal_handler && stacktrace_contains_logging_function(messages[i])) {
139 + buffer_flush(wb);
140 + added = 0;
141 + }
142 +
143 + if (added > 0)
144 + buffer_putc(wb, '\n');
145 +
146 + buffer_putc(wb, '#');
147 + buffer_print_uint64(wb, added);
148 + buffer_putc(wb, ' ');
149 + buffer_strcat(wb, messages[i]);
150 + added++;
151 + }
152 +
153 + free(messages);
154 +
155 + if (added == 0)
156 + buffer_strcat(wb, NO_STACK_TRACE_PREFIX "no valid frames");
157 +}
158 +
159 +#endif // USE_BACKTRACE
\ No newline at end of file
src/libnetdata/stacktrace/stacktrace-common.c new
+227
@@ -0,0 +1,227 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "stacktrace-common.h"
4 +
5 +// Stacktrace cache
6 +STACKTRACE_JudyLSet stacktrace_cache;
7 +SPINLOCK stacktrace_lock = SPINLOCK_INITIALIZER;
8 +bool cache_initialized = false;
9 +
10 +// The signal handler function name to filter out in stack traces
11 +const char *signal_handler_function = "nd_signal_handler";
12 +
13 +// List of auxiliary functions that should not be reported as root cause
14 +const char *auxiliary_functions[] = {
15 + "nd_uuid_copy",
16 + "out_of_memory",
17 + "shutdown_timed_out",
18 + NULL // Terminator
19 +};
20 +
21 +// List of logging functions to filter out
22 +const char *logging_functions[] = {
23 + "netdata_logger",
24 + "netdata_logger_with_limit",
25 + "netdata_logger_fatal",
26 + NULL // Terminator
27 +};
28 +
29 +// Thread-local buffer to store the first netdata function encountered in a stack trace
30 +__thread char root_cause_function[48];
31 +
32 +// Set the signal handler function name to filter out in stack traces
33 +void stacktrace_set_signal_handler_function(const char *function_name) {
34 + signal_handler_function = function_name;
35 +}
36 +
37 +// Returns the first netdata function found in the stack trace
38 +const char *stacktrace_root_cause_function(void) {
39 + return root_cause_function[0] ? root_cause_function : NULL;
40 +}
41 +
42 +// Initialize the stacktrace cache
43 +void stacktrace_cache_init(void) {
44 + if (cache_initialized)
45 + return;
46 +
47 + spinlock_lock(&stacktrace_lock);
48 + if (!cache_initialized) {
49 + STACKTRACE_INIT(&stacktrace_cache);
50 + cache_initialized = true;
51 + }
52 + spinlock_unlock(&stacktrace_lock);
53 +}
54 +
55 +// Main initialization function - this ensures the cache is always initialized
56 +void stacktrace_init(void) {
57 + // Always initialize the cache
58 + stacktrace_cache_init();
59 +
60 + // Call the backend-specific initialization
61 + impl_stacktrace_init();
62 +}
63 +
64 +// Allocate a new stacktrace structure with the given number of frames
65 +struct stacktrace *stacktrace_create(int num_frames) {
66 + size_t size = sizeof(struct stacktrace) + ((num_frames - 1) * sizeof(void *));
67 + struct stacktrace *trace = callocz(1, size);
68 + trace->frame_count = num_frames;
69 + return trace;
70 +}
71 +
72 +// Exact match check if a function is in the auxiliary list (strcmp)
73 +bool stacktrace_is_auxiliary_function(const char *function) {
74 + if (!function || !*function)
75 + return false;
76 +
77 + for (int i = 0; auxiliary_functions[i]; i++) {
78 + if (strcmp(function, auxiliary_functions[i]) == 0)
79 + return true;
80 + }
81 +
82 + return false;
83 +}
84 +
85 +// Exact match check if a function is a logging function (strcmp)
86 +bool stacktrace_is_logging_function(const char *function) {
87 + if (!function || !*function)
88 + return false;
89 +
90 + for (int i = 0; logging_functions[i]; i++) {
91 + if (strcmp(function, logging_functions[i]) == 0)
92 + return true;
93 + }
94 +
95 + return false;
96 +}
97 +
98 +// Substring check if a function contains a logging function name (strstr)
99 +bool stacktrace_contains_logging_function(const char *text) {
100 + if (!text || !*text)
101 + return false;
102 +
103 + for (int i = 0; logging_functions[i]; i++) {
104 + if (strstr(text, logging_functions[i]) != NULL)
105 + return true;
106 + }
107 +
108 + return false;
109 +}
110 +
111 +bool stacktrace_is_netdata_function(const char *function, const char *filename) {
112 + return function && *function && filename && *filename &&
113 + strstr(filename, "/src/") &&
114 + !strstr(filename, "/vendored/") &&
115 + !stacktrace_contains_logging_function(function);
116 +}
117 +
118 +// Exact match check if a function is the signal handler (strcmp)
119 +bool stacktrace_is_signal_handler_function(const char *function) {
120 + return function && *function &&
121 + signal_handler_function && *signal_handler_function &&
122 + strcmp(function, signal_handler_function) == 0;
123 +}
124 +
125 +// Substring check if a function contains the signal handler name (strstr)
126 +bool stacktrace_contains_signal_handler_function(const char *text) {
127 + return text && *text &&
128 + signal_handler_function && *signal_handler_function &&
129 + strstr(text, signal_handler_function) != NULL;
130 +}
131 +
132 +// Store a function name as the first netdata function found
133 +void stacktrace_keep_first_root_cause_function(const char *function) {
134 + if (!function || !*function || root_cause_function[0])
135 + return; // Already have a function or null input
136 +
137 + // Skip auxiliary functions and logging functions
138 + if (stacktrace_is_auxiliary_function(function) || stacktrace_is_logging_function(function))
139 + return;
140 +
141 + strncpyz(root_cause_function, function, sizeof(root_cause_function) - 1);
142 +}
143 +
144 +// Get the current stacktrace - public API
145 +NEVER_INLINE
146 +STACKTRACE stacktrace_get(int skip_frames) {
147 + // Make sure cache is initialized
148 + stacktrace_cache_init();
149 +
150 + // Get the frames from the implementation
151 + void *frames[50] = {0};
152 + // Add 1 to skip_frames to also skip stacktrace_get() itself
153 + int num_frames = impl_stacktrace_get_frames(frames, 50, skip_frames + 1);
154 +
155 + if (num_frames <= 0)
156 + return NULL;
157 +
158 + // Calculate hash
159 + uint64_t hash = XXH3_64bits(frames, num_frames * sizeof(void *));
160 +
161 + // Look up in cache first
162 + spinlock_lock(&stacktrace_lock);
163 +
164 + struct stacktrace *trace = STACKTRACE_GET(&stacktrace_cache, hash);
165 +
166 + // If existing trace found, verify it's the same frames
167 + // This handles hash collisions
168 + if (trace) {
169 + if (trace->frame_count != num_frames ||
170 + memcmp(trace->frames, frames, num_frames * sizeof(void *)) != 0) {
171 + // Hash collision - use linear probing
172 + int i = 1;
173 + uint64_t new_hash = hash;
174 + do {
175 + new_hash = hash + i;
176 + trace = STACKTRACE_GET(&stacktrace_cache, new_hash);
177 +
178 + if (!trace ||
179 + (trace->frame_count == num_frames &&
180 + memcmp(trace->frames, frames, num_frames * sizeof(void *)) == 0)) {
181 + break; // Either found a match or empty slot
182 + }
183 + i++;
184 + } while (i < 10); // Limit search to avoid infinite loops
185 +
186 + hash = new_hash;
187 + }
188 + }
189 +
190 + // If not found or hash collision, create new entry
191 + if (!trace) {
192 + trace = stacktrace_create(num_frames);
193 + trace->hash = hash;
194 + memcpy(trace->frames, frames, num_frames * sizeof(void *));
195 + STACKTRACE_SET(&stacktrace_cache, hash, trace);
196 + }
197 +
198 + spinlock_unlock(&stacktrace_lock);
199 +
200 + return trace;
201 +}
202 +
203 +// Convert a stacktrace to a buffer - public API
204 +void stacktrace_to_buffer(STACKTRACE trace, BUFFER *wb) {
205 + if (!trace || !wb) {
206 + if (wb)
207 + buffer_strcat(wb, NO_STACK_TRACE_PREFIX "invalid stacktrace");
208 + return;
209 + }
210 +
211 + struct stacktrace *st = (struct stacktrace *)trace;
212 +
213 + // If we already have cached text representation, use it
214 + if (st->text) {
215 + buffer_strcat(wb, st->text);
216 + return;
217 + }
218 +
219 + // Use the implementation-specific function for conversion
220 + impl_stacktrace_to_buffer(trace, wb);
221 +
222 + // Cache the text representation
223 + spinlock_lock(&stacktrace_lock);
224 + if (!st->text)
225 + st->text = strdupz(buffer_tostring(wb));
226 + spinlock_unlock(&stacktrace_lock);
227 +}
src/libnetdata/stacktrace/stacktrace-common.h new
+69
@@ -0,0 +1,69 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_STACKTRACE_COMMON_H
4 +#define NETDATA_STACKTRACE_COMMON_H 1
5 +
6 +#include "libnetdata/libnetdata.h"
7 +#include "stacktrace.h"
8 +#include "../libjudy/judyl-typed.h"
9 +
10 +#define NO_STACK_TRACE_PREFIX STACK_TRACE_INFO_PREFIX "stack trace is not available, "
11 +
12 +#if defined(HAVE_LIBBACKTRACE)
13 +#include "backtrace-supported.h"
14 +#if BACKTRACE_SUPPORTED == 1
15 +#define USE_LIBBACKTRACE 1
16 +#endif
17 +#endif
18 +
19 +#if !defined(USE_LIBBACKTRACE) && defined(HAVE_LIBUNWIND)
20 +#define USE_LIBUNWIND 1
21 +#endif
22 +
23 +#if !defined(USE_LIBBACKTRACE) && !defined(USE_LIBUNWIND) && defined(HAVE_BACKTRACE)
24 +#define USE_BACKTRACE 1
25 +#endif
26 +
27 +#if !defined(USE_LIBBACKTRACE) && !defined(USE_LIBUNWIND) && !defined(USE_BACKTRACE) && defined(HAVE_BACKTRACE)
28 +#define USE_NOTRACE 1
29 +#endif
30 +
31 +// The structure for stacktrace storage
32 +struct stacktrace {
33 + uint64_t hash; // Hash of the stack trace
34 + char *text; // Text representation (cached, lazy-initialized)
35 + int frame_count; // Number of frames
36 + void *frames[1]; // Variable-length array of frame pointers
37 +};
38 +
39 +// Cache for storing stack traces
40 +DEFINE_JUDYL_TYPED(STACKTRACE, struct stacktrace *);
41 +extern STACKTRACE_JudyLSet stacktrace_cache;
42 +extern SPINLOCK stacktrace_lock;
43 +extern bool cache_initialized;
44 +
45 +// Filter names
46 +extern const char *signal_handler_function;
47 +extern const char *auxiliary_functions[];
48 +extern const char *logging_functions[];
49 +
50 +// Thread-local storage for root cause
51 +extern __thread char root_cause_function[48];
52 +
53 +// Common helper functions
54 +void stacktrace_cache_init(void);
55 +struct stacktrace *stacktrace_create(int num_frames);
56 +bool stacktrace_is_auxiliary_function(const char *function);
57 +bool stacktrace_is_logging_function(const char *function);
58 +bool stacktrace_contains_logging_function(const char *text);
59 +bool stacktrace_is_netdata_function(const char *function, const char *filename);
60 +bool stacktrace_is_signal_handler_function(const char *function);
61 +bool stacktrace_contains_signal_handler_function(const char *text);
62 +void stacktrace_keep_first_root_cause_function(const char *function);
63 +
64 +// Implementation-specific declarations
65 +void impl_stacktrace_init(void);
66 +int impl_stacktrace_get_frames(void **frames, int max_frames, int skip_frames);
67 +void impl_stacktrace_to_buffer(STACKTRACE trace, BUFFER *wb);
68 +
69 +#endif /* NETDATA_STACKTRACE_COMMON_H */
\ No newline at end of file
src/libnetdata/stacktrace/stacktrace-libbacktrace.c new
+294
@@ -0,0 +1,294 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "stacktrace-common.h"
4 +
5 +#if defined(USE_LIBBACKTRACE)
6 +#include "backtrace.h"
7 +
8 +static struct backtrace_state *backtrace_state = NULL;
9 +
10 +typedef struct {
11 + BUFFER *wb; // Buffer to write to
12 + size_t frame_count; // Number of frames processed
13 + bool first_frame; // Is this the first frame?
14 + bool found_signal_handler; // Have we found the signal handler frame?
15 +} backtrace_data_t;
16 +
17 +// For collecting raw frames
18 +typedef struct {
19 + void **frames;
20 + int max_frames;
21 + int num_frames;
22 + int skip_frames;
23 +} collect_frames_data_t;
24 +
25 +// Simple callback for collecting PC addresses
26 +static int bt_collect_frames_callback(void *data, uintptr_t pc) {
27 + collect_frames_data_t *cf_data = (collect_frames_data_t *)data;
28 +
29 + // Skip frames at the top of the stack
30 + if (cf_data->skip_frames > 0) {
31 + cf_data->skip_frames--;
32 + return 0;
33 + }
34 +
35 + if (cf_data->num_frames < cf_data->max_frames) {
36 + cf_data->frames[cf_data->num_frames++] = (void *)pc;
37 + }
38 +
39 + return 0;
40 +}
41 +
42 +// Common function to format and add a stack frame to the buffer
43 +static void add_stack_frame(backtrace_data_t *bt_data, uintptr_t pc, const char *function,
44 + const char *filename, int lineno) {
45 + BUFFER *wb = bt_data->wb;
46 +
47 + if (!wb)
48 + return;
49 +
50 + // Check if we found the signal handler frame
51 + if (!bt_data->found_signal_handler && stacktrace_is_signal_handler_function(function)) {
52 + // We found the signal handler, reset the buffer and clear function name
53 + buffer_flush(wb);
54 + bt_data->frame_count = 0;
55 + bt_data->first_frame = true;
56 + bt_data->found_signal_handler = true;
57 + root_cause_function[0] = '\0';
58 + return; // Skip adding the signal handler itself
59 + }
60 +
61 + // Check for logging functions, but only if we haven't found a signal handler yet
62 + // This prevents double resets when crashing inside logging code
63 + if (!bt_data->found_signal_handler && stacktrace_is_logging_function(function)) {
64 + // Found a logging function, reset the buffer and clear function name
65 + buffer_flush(wb);
66 + bt_data->frame_count = 0;
67 + bt_data->first_frame = true;
68 + root_cause_function[0] = '\0';
69 + // continue to add the function to the stack trace
70 + }
71 +
72 + // Check if this is a netdata source file and store the function name if it is
73 + // (but only if we haven't already stored one)
74 + if (!root_cause_function[0] && stacktrace_is_netdata_function(function, filename))
75 + stacktrace_keep_first_root_cause_function(function);
76 +
77 + // Add a newline between frames
78 + if (!bt_data->first_frame)
79 + buffer_putc(wb, '\n');
80 + else
81 + bt_data->first_frame = false;
82 +
83 + // Format: #ID function (filename.c:NNN)
84 + buffer_putc(wb, '#');
85 + buffer_print_uint64(wb, bt_data->frame_count);
86 + buffer_putc(wb, ' ');
87 +
88 + if (function && *function)
89 + buffer_strcat(wb, function);
90 + else
91 + buffer_strcat(wb, "<unknown>");
92 +
93 + if(pc) {
94 + buffer_strcat(wb, " [");
95 + buffer_print_uint64_hex(wb, pc);
96 + buffer_putc(wb, ']');
97 + }
98 +
99 + if (filename && *filename) {
100 + buffer_strcat(wb, " (");
101 +
102 + const char *f = strstr(filename, "/src/");
103 + if (f) {
104 + const char *f2 = strstr(f + 1, "/src/");
105 + if(f2) f = f2;
106 + }
107 + if(!f) f = filename;
108 +
109 + buffer_strcat(wb, f);
110 +
111 + if (lineno > 0) {
112 + buffer_strcat(wb, ":");
113 + buffer_print_uint64(wb, (uint64_t)lineno);
114 + }
115 +
116 + buffer_putc(wb, ')');
117 + }
118 +
119 + bt_data->frame_count++;
120 +}
121 +
122 +// Error callback for libbacktrace
123 +static void bt_error_handler(void *data, const char *msg, int errnum) {
124 + backtrace_data_t *bt_data = (backtrace_data_t *)data;
125 +
126 + if (!bt_data || !bt_data->wb)
127 + return;
128 +
129 + // Use <unknown> for function name in error cases
130 + const char *function = "<unknown>";
131 +
132 + // Format the error message as the filename
133 + char error_buf[512] = "error: ";
134 + size_t len = 7; // Length of "error: "
135 +
136 + // Add the error message
137 + if (msg)
138 + len = strcatz(error_buf, len, msg, sizeof(error_buf));
139 +
140 + // Add the error number description if available
141 + if (errnum > 0) {
142 + if (msg) {
143 + len = strcatz(error_buf, len, ": ", sizeof(error_buf));
144 + }
145 + len = strcatz(error_buf, len, strerror(errnum), sizeof(error_buf));
146 + }
147 +
148 + add_stack_frame(bt_data, 0, function, error_buf, 0);
149 +}
150 +
151 +// Full callback for libbacktrace
152 +static int bt_full_handler(void *data, uintptr_t pc,
153 + const char *filename, int lineno,
154 + const char *function) {
155 + backtrace_data_t *bt_data = (backtrace_data_t *)data;
156 + if (!bt_data)
157 + return 0;
158 +
159 + add_stack_frame(bt_data, pc, function, filename, lineno);
160 +
161 + return 0; // Continue backtrace
162 +}
163 +
164 +const char *stacktrace_backend(void) {
165 +#if BACKTRACE_SUPPORTS_DATA
166 +#define BACKTRACE_DATA "data"
167 +#else
168 +#define BACKTRACE_DATA "no-data"
169 +#endif
170 +
171 +#if BACKTRACE_USES_MALLOC
172 +#define BACKTRACE_MEMORY "malloc"
173 +#else
174 +#define BACKTRACE_MEMORY "mmap"
175 +#endif
176 +
177 +#if BACKTRACE_SUPPORTS_THREADS
178 +#define BACKTRACE_THREADS "threads"
179 +#else
180 +#define BACKTRACE_THREADS "no-threads"
181 +#endif
182 +
183 + return "libbacktrace (" BACKTRACE_MEMORY ", " BACKTRACE_THREADS ", " BACKTRACE_DATA ")";
184 +}
185 +
186 +void impl_stacktrace_init(void) {
187 + if (!backtrace_state) {
188 + backtrace_state = backtrace_create_state(NULL, BACKTRACE_SUPPORTS_THREADS,
189 + bt_error_handler, NULL);
190 + }
191 +}
192 +
193 +void stacktrace_flush(void) {
194 + // Nothing to flush with libbacktrace
195 +}
196 +
197 +bool stacktrace_capture_is_async_signal_safe(void) {
198 +// libbacktrace may use malloc depending on configuration
199 +// Check the BACKTRACE_USES_MALLOC define
200 +#if BACKTRACE_USES_MALLOC
201 + return false;
202 +#else
203 + return true;
204 +#endif
205 +}
206 +
207 +bool stacktrace_available(void) {
208 + return backtrace_state != NULL;
209 +}
210 +
211 +NEVER_INLINE
212 +void stacktrace_capture(BUFFER *wb) {
213 + root_cause_function[0] = '\0';
214 +
215 + if (!backtrace_state) {
216 + buffer_strcat(wb, NO_STACK_TRACE_PREFIX "libbacktrace not initialized");
217 + return;
218 + }
219 +
220 + backtrace_data_t bt_data = {
221 + .wb = wb,
222 + .frame_count = 0,
223 + .first_frame = true,
224 + .found_signal_handler = false
225 + };
226 +
227 + // Skip one frame to hide stacktrace_capture() itself
228 + backtrace_full(backtrace_state, 1, bt_full_handler,
229 + bt_error_handler, &bt_data);
230 +
231 + // If no frames were reported
232 + if (bt_data.frame_count == 0) {
233 + buffer_strcat(wb, NO_STACK_TRACE_PREFIX "libbacktrace reports no frames");
234 + }
235 +}
236 +
237 +// Implementation-specific function to collect stack trace frames
238 +NEVER_INLINE
239 +int impl_stacktrace_get_frames(void **frames, int max_frames, int skip_frames) {
240 + if (!backtrace_state || !frames || max_frames <= 0)
241 + return 0;
242 +
243 + // Collect frames
244 + collect_frames_data_t data = {
245 + .frames = frames,
246 + .max_frames = max_frames,
247 + .num_frames = 0,
248 + .skip_frames = skip_frames + 1 // +1 to also skip this function itself
249 + };
250 +
251 + // Pass 0 as skip_frames to backtrace_simple and let the callback handle skipping
252 + backtrace_simple(backtrace_state, 0, bt_collect_frames_callback, bt_error_handler, &data);
253 +
254 + return data.num_frames;
255 +}
256 +
257 +// Implementation-specific function to convert a stacktrace to a buffer
258 +void impl_stacktrace_to_buffer(STACKTRACE trace, BUFFER *wb) {
259 + struct stacktrace *st = (struct stacktrace *)trace;
260 +
261 + // Resolve each frame
262 + backtrace_data_t bt_data = {
263 + .wb = wb,
264 + .frame_count = 0,
265 + .first_frame = true,
266 + .found_signal_handler = false
267 + };
268 +
269 + for (int i = 0; i < st->frame_count; i++) {
270 + backtrace_pcinfo(
271 + backtrace_state,
272 + (uintptr_t)st->frames[i],
273 + bt_full_handler,
274 + bt_error_handler,
275 + &bt_data
276 + );
277 + }
278 +
279 + // If we couldn't resolve any frames, use addresses
280 + if (bt_data.frame_count == 0) {
281 + for (int i = 0; i < st->frame_count; i++) {
282 + if (i > 0)
283 + buffer_putc(wb, '\n');
284 +
285 + buffer_putc(wb, '#');
286 + buffer_print_uint64(wb, i);
287 + buffer_strcat(wb, " <unknown> [");
288 + buffer_print_uint64_hex(wb, (uint64_t)st->frames[i]);
289 + buffer_putc(wb, ']');
290 + }
291 + }
292 +}
293 +
294 +#endif // USE_LIBBACKTRACE
src/libnetdata/stacktrace/stacktrace-libunwind.c new
+186
@@ -0,0 +1,186 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "stacktrace-common.h"
4 +
5 +#if defined(USE_LIBUNWIND)
6 +#if !defined(STATIC_BUILD)
7 +#define UNW_LOCAL_ONLY
8 +#endif
9 +#include <libunwind.h>
10 +
11 +const char *stacktrace_capture_backend(void) {
12 + return "libunwind";
13 +}
14 +
15 +void impl_stacktrace_init(void) {
16 + unw_set_caching_policy(unw_local_addr_space, UNW_CACHE_NONE);
17 +}
18 +
19 +void stacktrace_flush(void) {
20 + unw_flush_cache(unw_local_addr_space, 0, 0);
21 +}
22 +
23 +bool stacktrace_capture_is_async_signal_safe(void) {
24 +#if defined(STATIC_BUILD)
25 + return false;
26 +#else
27 + return true;
28 +#endif
29 +}
30 +
31 +bool stacktrace_available(void) {
32 + return true;
33 +}
34 +
35 +NEVER_INLINE
36 +void stack_trace_capture(BUFFER *wb) {
37 + // this function is async-signal-safe, if the buffer has enough space to hold the stack trace
38 +
39 + root_cause_function[0] = '\0';
40 +
41 + unw_cursor_t cursor;
42 + unw_context_t context;
43 + size_t frames = 0;
44 +
45 + // Initialize context for current thread
46 + unw_getcontext(&context);
47 + unw_init_local(&cursor, &context);
48 +
49 + // Skip one frame to hide stacktrace_capture() itself
50 + unw_step(&cursor);
51 +
52 + size_t added = 0;
53 + bool found_signal_handler = false;
54 +
55 + while (unw_step(&cursor) > 0) {
56 + unw_word_t offset, pc;
57 + char sym[256];
58 +
59 + unw_get_reg(&cursor, UNW_REG_IP, &pc);
60 + if (!pc)
61 + break;
62 +
63 + const char *name = sym;
64 + if (unw_get_proc_name(&cursor, sym, sizeof(sym), &offset) != 0) {
65 + name = "<unknown>";
66 + offset = 0;
67 + }
68 +
69 + // Check if we found the signal handler frame
70 + if (!found_signal_handler && stacktrace_is_signal_handler_function(name)) {
71 + // We found the signal handler, reset the buffer
72 + buffer_flush(wb);
73 + added = 0;
74 + frames = 0;
75 + found_signal_handler = true;
76 + continue; // Skip adding the signal handler itself
77 + }
78 +
79 + // Check for logging functions, but only if we haven't found a signal handler yet
80 + if (!found_signal_handler && stacktrace_is_logging_function(name)) {
81 + // Found a logging function, reset the buffer
82 + buffer_flush(wb);
83 + added = 0;
84 + frames = 0;
85 + // continue to add the function to the stack trace
86 + }
87 +
88 + if (frames++)
89 + buffer_putc(wb, '\n');
90 +
91 + buffer_putc(wb, '#');
92 + buffer_print_uint64(wb, added);
93 + buffer_putc(wb, ' ');
94 + buffer_strcat(wb, name);
95 +
96 + if(offset) {
97 + buffer_putc(wb, '+');
98 + buffer_print_uint64_hex(wb, offset);
99 + }
100 +
101 + added++;
102 + }
103 +
104 + if (!added)
105 + buffer_strcat(wb, NO_STACK_TRACE_PREFIX "libunwind reports no frames");
106 +}
107 +
108 +// Collect frames using libunwind
109 +static int collect_frames_libunwind(void **frames, int max_frames, int skip) {
110 + unw_cursor_t cursor;
111 + unw_context_t context;
112 +
113 + unw_getcontext(&context);
114 + unw_init_local(&cursor, &context);
115 +
116 + // Collect all frames first, including the ones we'll skip
117 + // This way we ensure we capture all frames, including inlined ones
118 + void *all_frames[150]; // Allocate a larger buffer to hold all frames
119 + int total_frames = 0;
120 +
121 + // Collect as many frames as possible
122 + while (total_frames < 150 && unw_step(&cursor) > 0) {
123 + unw_word_t pc;
124 + unw_get_reg(&cursor, UNW_REG_IP, &pc);
125 + if (!pc)
126 + break;
127 +
128 + all_frames[total_frames++] = (void *)pc;
129 + }
130 +
131 + // Now copy the frames we want, skipping the requested number
132 + int frame_count = 0;
133 + for (int i = skip; i < total_frames && frame_count < max_frames; i++) {
134 + frames[frame_count++] = all_frames[i];
135 + }
136 +
137 + return frame_count;
138 +}
139 +
140 +// Implementation-specific function to collect stack trace frames
141 +int impl_stacktrace_get_frames(void **frames, int max_frames, int skip_frames) {
142 + if (!frames || max_frames <= 0)
143 + return 0;
144 +
145 + // Add 1 to skip_frames to also skip this function itself
146 + return collect_frames_libunwind(frames, max_frames, skip_frames + 1);
147 +}
148 +
149 +// Implementation-specific function to convert a stacktrace to a buffer
150 +void impl_stacktrace_to_buffer(STACKTRACE trace, BUFFER *wb) {
151 + struct stacktrace *st = (struct stacktrace *)trace;
152 +
153 + // Format each frame
154 + for (int i = 0; i < st->frame_count; i++) {
155 + if (i > 0)
156 + buffer_putc(wb, '\n');
157 +
158 + buffer_putc(wb, '#');
159 + buffer_print_uint64(wb, i);
160 + buffer_putc(wb, ' ');
161 +
162 + // Try to resolve symbol name
163 + unw_cursor_t cursor;
164 + unw_context_t context;
165 + char sym[256] = "<unknown>";
166 + unw_word_t offset = 0;
167 +
168 + // We don't have the context anymore, so do the best we can
169 + // Try to resolve the address to a symbol name
170 + if (dladdr(st->frames[i], (Dl_info *)sym) == 0) {
171 + buffer_strcat(wb, "<unknown>");
172 + } else {
173 + Dl_info *info = (Dl_info *)sym;
174 + if (info->dli_sname)
175 + buffer_strcat(wb, info->dli_sname);
176 + else
177 + buffer_strcat(wb, "<unknown>");
178 + }
179 +
180 + buffer_strcat(wb, " [");
181 + buffer_print_uint64_hex(wb, (uint64_t)st->frames[i]);
182 + buffer_putc(wb, ']');
183 + }
184 +}
185 +
186 +#endif // USE_LIBUNWIND
src/libnetdata/stacktrace/stacktrace-log.c new
+38
@@ -0,0 +1,38 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "libnetdata/libnetdata.h"
4 +#include "stacktrace.h"
5 +
6 +// External variable needed by other modules
7 +static bool nd_log_forked = false;
8 +
9 +void stacktrace_forked(void) {
10 + nd_log_forked = true;
11 +}
12 +
13 +// The stack trace formatter called by logger
14 +NEVER_INLINE
15 +bool stack_trace_formatter(BUFFER *wb, void *data __maybe_unused) {
16 + static __thread bool in_stack_trace = false;
17 +
18 + extern bool nd_log_forked;
19 + if (nd_log_forked) {
20 + // libunwind freezes in forked children
21 + buffer_strcat(wb, STACK_TRACE_INFO_PREFIX "stack trace is not available, stack trace after fork is disabled");
22 + return true;
23 + }
24 +
25 + if (in_stack_trace) {
26 + // Prevent recursion
27 + buffer_strcat(wb, STACK_TRACE_INFO_PREFIX "stack trace is not available, stack trace recursion detected");
28 + return true;
29 + }
30 +
31 + in_stack_trace = true;
32 +
33 + // Use the existing stacktrace_capture
34 + stacktrace_capture(wb);
35 +
36 + in_stack_trace = false; // Ensure the flag is reset
37 + return true;
38 +}
\ No newline at end of file
src/libnetdata/stacktrace/stacktrace-none.c new
+65
@@ -0,0 +1,65 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "stacktrace-common.h"
4 +
5 +// This implementation is used when no stacktrace backend is available
6 +#if defined(USE_NOTRACE)
7 +
8 +const char *stacktrace_backend(void) {
9 + return "none";
10 +}
11 +
12 +bool stacktrace_available(void) {
13 + return false;
14 +}
15 +
16 +void impl_stacktrace_init(void) {
17 + // Nothing to initialize for null backend
18 +}
19 +
20 +void stacktrace_flush(void) {
21 + // Nothing to flush
22 +}
23 +
24 +bool stacktrace_capture_is_async_signal_safe(void) {
25 + return false;
26 +}
27 +
28 +NEVER_INLINE
29 +void stacktrace_capture(BUFFER *wb) {
30 + root_cause_function[0] = '\0';
31 +
32 + buffer_strcat(wb, NO_STACK_TRACE_PREFIX "no back-end available");
33 +
34 + // probably we can have something like this?
35 + // https://maskray.me/blog/2022-04-09-unwinding-through-signal-handler
36 + // (at the end - but it needs the frame pointer)
37 +}
38 +
39 +// Simple dummy implementation for platforms without stack trace support
40 +int impl_stacktrace_get_frames(void **frames, int max_frames, int skip_frames) {
41 + if (!frames || max_frames <= 0)
42 + return 0;
43 +
44 + // No need to adjust skip_frames here as we're just creating a dummy frame
45 + // but we include the comment for consistency with other implementations
46 + // skip_frames += 1; // Skip this function itself
47 +
48 + // Just use a counter to create a unique "frame"
49 + static uint64_t counter = 0;
50 + uint64_t id = __atomic_fetch_add(&counter, 1, __ATOMIC_SEQ_CST);
51 +
52 + // Store one dummy frame
53 + frames[0] = (void *)(uintptr_t)id;
54 +
55 + return 1;
56 +}
57 +
58 +void impl_stacktrace_to_buffer(STACKTRACE trace, BUFFER *wb) {
59 + struct stacktrace *st = (struct stacktrace *)trace;
60 +
61 + // Simple representation for platforms without stack trace support
62 + buffer_sprintf(wb, NO_STACK_TRACE_PREFIX "no back-end available (id: %" PRIu64 ")", st->hash);
63 +}
64 +
65 +#endif // USE_NOTRACE
\ No newline at end of file
src/libnetdata/stacktrace/stacktrace-unittest.c new
+141
@@ -0,0 +1,141 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "libnetdata/libnetdata.h"
4 +#include "stacktrace.h"
5 +
6 +// Structure to hold all test data
7 +typedef struct {
8 + BUFFER *direct_trace; // Buffer for direct stack trace
9 + BUFFER *indirect_trace; // Buffer for indirect stack trace
10 + BUFFER *direct_root_cause; // Root cause function from direct capture
11 + BUFFER *indirect_root_cause; // Root cause function from indirect capture
12 + const char *never_inline_fn; // Name of the never-inline function
13 + const char *always_inline_fn; // Name of the always-inline function
14 +} stacktrace_test_data_t;
15 +
16 +// Function to analyze a stack trace
17 +static bool analyze_stack_trace(
18 + const char *stack_trace,
19 + const char *never_inline_fn,
20 + const char *always_inline_fn,
21 + const char *unittest_fn,
22 + const char *root_cause)
23 +{
24 + fprintf(stderr, "--------------------------------------------------------------------------------\n");
25 + fprintf(stderr, "%s\n", stack_trace);
26 + fprintf(stderr, "--------------------------------------------------------------------------------\n");
27 +
28 + if (!stack_trace || !*stack_trace) {
29 + fprintf(stderr, " - empty stack trace\n");
30 + return false;
31 + }
32 +
33 + // Report presence of each function
34 + bool never_inline_found = strstr(stack_trace, never_inline_fn) != NULL;
35 + bool always_inline_found = strstr(stack_trace, always_inline_fn) != NULL;
36 + bool unittest_found = strstr(stack_trace, unittest_fn) != NULL;
37 +
38 + fprintf(stderr, " - %50.50s: %s\n",
39 + never_inline_fn, never_inline_found ? "FOUND" : "NOT FOUND");
40 + fprintf(stderr, " - %50.50s: %s\n",
41 + always_inline_fn, always_inline_found ? "FOUND" : "NOT FOUND");
42 + fprintf(stderr, " - %50.50s: %s\n",
43 + unittest_fn, unittest_found ? "FOUND" : "NOT FOUND");
44 + fprintf(stderr, " - %50.50s: %s\n",
45 + "root cause function",
46 + root_cause && *root_cause ? root_cause : "NOT FOUND");
47 +
48 + // We only require the unittest function to be present for the test to pass
49 + return unittest_found;
50 +}
51 +
52 +// This function will never be inlined
53 +NEVER_INLINE
54 +static void never_inline_function_to_capture_stack_trace(stacktrace_test_data_t *test_data) {
55 + test_data->never_inline_fn = __FUNCTION__;
56 +
57 + BUFFER *wb = buffer_create(4096, NULL);
58 +
59 + // Test 1: Direct capture
60 + stacktrace_capture(wb);
61 + buffer_strcat(test_data->direct_trace, buffer_tostring(wb));
62 +
63 + // Get root cause (first time)
64 + buffer_flush(test_data->direct_root_cause);
65 + buffer_strcat(test_data->direct_root_cause, stacktrace_root_cause_function());
66 +
67 + // Test 2: Indirect capture (get + to_buffer)
68 + buffer_flush(wb);
69 + STACKTRACE trace = stacktrace_get(0);
70 + if (trace) {
71 + stacktrace_to_buffer(trace, wb);
72 + buffer_strcat(test_data->indirect_trace, buffer_tostring(wb));
73 +
74 + // Get root cause (second time)
75 + buffer_flush(test_data->indirect_root_cause);
76 + buffer_strcat(test_data->indirect_root_cause, stacktrace_root_cause_function());
77 + }
78 +
79 + buffer_free(wb);
80 +}
81 +
82 +// This function will be inlined in the caller
83 +ALWAYS_INLINE
84 +static void inline_function_to_capture_stack_trace(stacktrace_test_data_t *test_data) {
85 + test_data->always_inline_fn = __FUNCTION__;
86 +
87 + // Call the non-inlined function
88 + never_inline_function_to_capture_stack_trace(test_data);
89 +}
90 +
91 +// Run the stacktrace unittest
92 +int stacktrace_unittest(void) {
93 + // Initialize stacktrace subsystem
94 + stacktrace_init();
95 +
96 + // Setup test data structure
97 + stacktrace_test_data_t test_data = {
98 + .direct_trace = buffer_create(4096, NULL),
99 + .indirect_trace = buffer_create(4096, NULL),
100 + .direct_root_cause = buffer_create(4096, NULL),
101 + .indirect_root_cause = buffer_create(4096, NULL),
102 + .never_inline_fn = NULL,
103 + .always_inline_fn = NULL,
104 + };
105 +
106 + // Run the test function to gather stack traces
107 + inline_function_to_capture_stack_trace(&test_data);
108 +
109 + // Print basic test information
110 + fprintf(stderr, "\nSTACKTRACE TEST: Backend: %s\n", stacktrace_backend());
111 +
112 + // Analyze both stack traces
113 + fprintf(stderr, "\nDIRECT STACK TRACE\n");
114 + bool direct_analysis = analyze_stack_trace(
115 + buffer_tostring(test_data.direct_trace),
116 + test_data.never_inline_fn,
117 + test_data.always_inline_fn,
118 + "stacktrace_unittest",
119 + buffer_tostring(test_data.direct_root_cause));
120 +
121 + fprintf(stderr, "\nINDIRECT STACK TRACE\n");
122 + bool indirect_analysis = analyze_stack_trace(
123 + buffer_tostring(test_data.indirect_trace),
124 + test_data.never_inline_fn,
125 + test_data.always_inline_fn,
126 + "stacktrace_unittest",
127 + buffer_tostring(test_data.indirect_root_cause));
128 +
129 + // Free resources
130 + buffer_free(test_data.direct_trace);
131 + buffer_free(test_data.indirect_trace);
132 + buffer_free(test_data.direct_root_cause);
133 + buffer_free(test_data.indirect_root_cause);
134 +
135 + // Report overall test status - success if both analyses succeed
136 + bool test_success = direct_analysis && indirect_analysis;
137 + fprintf(stderr, "\nSTACKTRACE TEST: Overall result: %s\n",
138 + test_success ? "SUCCESS" : "FAILURE");
139 +
140 + return test_success ? 0 : 1;
141 +}
\ No newline at end of file
src/libnetdata/stacktrace/stacktrace.h new
+52
@@ -0,0 +1,52 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_STACKTRACE_H
4 +#define NETDATA_STACKTRACE_H 1
5 +
6 +#include "libnetdata/common.h"
7 +
8 +#define STACK_TRACE_INFO_PREFIX "info: "
9 +
10 +// Opaque pointer to a stack trace
11 +typedef struct stacktrace *STACKTRACE;
12 +
13 +// Set the signal handler function name to filter out in stack traces
14 +void stacktrace_set_signal_handler_function(const char *function_name);
15 +
16 +// Returns the first netdata function found in the stack trace
17 +const char *stacktrace_root_cause_function(void);
18 +
19 +// Initialize the stacktrace capture mechanism
20 +void stacktrace_init(void);
21 +
22 +// Free any resources used by the stacktrace mechanism
23 +void stacktrace_flush(void);
24 +
25 +// Return true if the stacktrace mechanism can be safely used in signal handlers
26 +bool stacktrace_capture_is_async_signal_safe(void);
27 +
28 +// Return true if stacktrace capture is available on this platform
29 +bool stacktrace_available(void);
30 +
31 +// Capture a stacktrace to a buffer
32 +struct web_buffer;
33 +void stacktrace_capture(BUFFER *wb);
34 +
35 +// Return a string describing the backend used for capturing stacktraces
36 +const char *stacktrace_backend(void);
37 +
38 +// Get the current stacktrace, hash it, and store it in a cache
39 +STACKTRACE stacktrace_get(int skip_frames);
40 +
41 +// Convert a stacktrace to a buffer
42 +void stacktrace_to_buffer(STACKTRACE trace, struct web_buffer *wb);
43 +
44 +void stacktrace_forked(void);
45 +bool stack_trace_formatter(struct web_buffer *wb, void *data);
46 +
47 +// Unit testing
48 +int stacktrace_unittest(void);
49 +
50 +#include "stacktrace-array.h"
51 +
52 +#endif /* NETDATA_STACKTRACE_H */
src/libnetdata/string/string.c
+183 -14
@@ -17,6 +17,10 @@ struct netdata_string {
17 REFCOUNT refcount; // how many times this string is used
18 // We use a signed number to be able to detect duplicate frees of a string.
19 // If at any point this goes below zero, we have a duplicate free.
20 +
21 +#ifdef FSANITIZE_ADDRESS
22 + STACKTRACE_ARRAY stacktraces; // stack traces from all acquisition points
23 +#endif
24
25 const char str[]; // the string itself, is appended to this structure
26 };
@@ -33,9 +37,12 @@ static struct string_partition {
37 long int memory; // the memory used
38 long int memory_index; // JudyHS (accurate)
39
40 +#ifdef FSANITIZE_ADDRESS
41 + Pvoid_t JudyLPointers; // JudyL array to keep track of all string pointers for traversal
42 +#endif
43 +
44 #ifdef NETDATA_INTERNAL_CHECKS
45 // internal statistics
38 -
46 struct {
47 size_t searches; // the number of successful searches in the index
48 size_t releases; // when a string is unreferenced
@@ -114,6 +121,10 @@ STRING *string_dup(STRING *string) {
121 uint8_t partition = string_partition(string);
122 #endif
123
124 +#ifdef FSANITIZE_ADDRESS
125 + stacktrace_array_add(&string->stacktraces, 0);
126 +#endif
127 +
128 // statistics
129 string_stats_atomic_increment(partition, active_references);
130 string_stats_atomic_increment(partition, duplications);
@@ -201,6 +212,18 @@ static inline STRING *string_index_insert(const char *str, size_t length) {
212 strcpy((char *)string->str, str);
213 string->length = length;
214 string->refcount = 1;
215 +
216 +#ifdef FSANITIZE_ADDRESS
217 + // Initialize stacktrace tracking
218 + stacktrace_array_init(&string->stacktraces);
219 +
220 + // Add to JudyL array for tracking strings by pointer
221 + Pvoid_t *PValue;
222 + PValue = JudyLIns(&string_base[partition].JudyLPointers, (Word_t)string, PJE0);
223 + if (PValue != PJERR)
224 + *PValue = (void *)1; // Use a simple value of 1 for now
225 +#endif
226 +
227 *ptr = string;
228 string_base[partition].inserts++;
229 string_base[partition].entries++;
@@ -265,6 +288,13 @@ static inline void string_index_delete(STRING *string) {
288 string_base[partition].entries--;
289 string_base[partition].memory -= mem_size;
290 string_base[partition].memory_index += judy_mem;
291 +
292 +#ifdef FSANITIZE_ADDRESS
293 + // Remove from the JudyL array if it exists
294 + if (string_base[partition].JudyLPointers)
295 + JudyLDel(&string_base[partition].JudyLPointers, (Word_t)string, PJE0);
296 +#endif
297 +
298 freez(string);
299 }
300
@@ -292,6 +322,11 @@ STRING *string_strdupz(const char *str) {
322 // statistics
323 string_stats_atomic_increment(partition, active_references);
324
325 +#ifdef FSANITIZE_ADDRESS
326 + // Add a stacktrace for this acquisition point too
327 + stacktrace_array_add(&string->stacktraces, 0);
328 +#endif
329 +
330 return string;
331 }
332
@@ -311,6 +346,12 @@ STRING *string_strndupz(const char *str, size_t len) {
346 string = string_index_insert(buf, len + 1);
347
348 string_stats_atomic_increment(partition, active_references);
349 +
350 +#ifdef FSANITIZE_ADDRESS
351 + // Add a stacktrace for this acquisition point too
352 + stacktrace_array_add(&string->stacktraces, 0);
353 +#endif
354 +
355 return string;
356 }
357
@@ -357,22 +398,22 @@ bool string_starts_with_string(const STRING *whole, const STRING *end) {
398 return strncmp(string2str(whole), string2str(end), string_strlen(end)) == 0;
399 }
400
360 -STRING *string_2way_merge(STRING *a, STRING *b) {
361 - static STRING *X = NULL;
401 +// Static X used by string_2way_merge
402 +static STRING *string_2way_merge_X = NULL;
403
363 - if(unlikely(!X)) {
364 - X = string_strdupz("[x]");
365 - }
404 +STRING *string_2way_merge(STRING *a, STRING *b) {
405 + if(unlikely(!string_2way_merge_X))
406 + string_2way_merge_X = string_strdupz("[x]");
407
408 if(unlikely(a == b)) return string_dup(a);
368 - if(unlikely(a == X)) return string_dup(a);
369 - if(unlikely(b == X)) return string_dup(b);
370 - if(unlikely(!a)) return string_dup(X);
371 - if(unlikely(!b)) return string_dup(X);
409 + if(unlikely(a == string_2way_merge_X)) return string_dup(a);
410 + if(unlikely(b == string_2way_merge_X)) return string_dup(b);
411 + if(unlikely(!a)) return string_dup(string_2way_merge_X);
412 + if(unlikely(!b)) return string_dup(string_2way_merge_X);
413
414 size_t alen = string_strlen(a);
415 size_t blen = string_strlen(b);
375 - size_t length = alen + blen + string_strlen(X) + 1;
416 + size_t length = alen + blen + string_strlen(string_2way_merge_X) + 1;
417 char buf1[length + 1], buf2[length + 1], *dst1;
418 const char *s1, *s2;
419
@@ -460,15 +501,65 @@ static long unittest_string_entries(void) {
501 size_t string_destroy(void) {
502 size_t referenced = 0;
503
504 + // Free the static X string used by string_2way_merge
505 + string_freez(string_2way_merge_X);
506 + string_2way_merge_X = NULL;
507 +
508 +#ifdef FSANITIZE_ADDRESS
509 + // Create JudyL array for tracking stats by stacktrace
510 + Pvoid_t string_counts = NULL; // JudyL array to count strings per stacktrace
511 +
512 + BUFFER *wb = buffer_create(16384, NULL);
513 +
514 + fprintf(stderr, "\n========= STRINGS GROUPED BY CREATION STACKTRACE =========\n");
515 +#endif
516 +
517 // Traverse all partitions
518 for (size_t partition = 0; partition < STRING_PARTITIONS; partition++) {
519 // Lock the partition to prevent new entries while we're cleaning up
520 rw_spinlock_write_lock(&string_base[partition].spinlock);
521
522 +#ifdef FSANITIZE_ADDRESS
523 + // First, collect statistics about remaining strings
524 + if (string_base[partition].JudyLPointers) {
525 + // Traverse the JudyL array to count strings by stacktrace
526 + Word_t string_idx = 0;
527 + Pvoid_t *PValue;
528 +
529 + PValue = JudyLFirst(string_base[partition].JudyLPointers, &string_idx, PJE0);
530 + while (PValue) {
531 + STRING *string = (STRING *)string_idx;
532 + if(string) {
533 + for (int i = 0; i < string->stacktraces.num_stacktraces; i++) {
534 + if (string->stacktraces.stacktraces[i]) {
535 + Word_t key = (Word_t)string->stacktraces.stacktraces[i];
536 + PValue = JudyLGet(string_counts, key, PJE0);
537 + if (PValue) {
538 + // Increment existing count
539 + size_t count = (size_t)(uintptr_t)*PValue;
540 + count++;
541 + *PValue = (Pvoid_t)(uintptr_t)count;
542 + } else {
543 + // Insert new count
544 + PValue = JudyLIns(&string_counts, key, PJE0);
545 + if (PValue != PJERR)
546 + *PValue = (Pvoid_t)(uintptr_t)1;
547 + }
548 + }
549 + }
550 + }
551 +
552 + PValue = JudyLNext(string_base[partition].JudyLPointers, &string_idx, PJE0);
553 + }
554 +
555 + // Free the JudyL pointers array
556 + JudyLFreeArray(&string_base[partition].JudyLPointers, PJE0);
557 + string_base[partition].JudyLPointers = NULL;
558 + }
559 +#endif
560 +
561 // Since JudyHS doesn't have simple traversal functions,
562 // we'll free the entire array at once.
470 - // This is a bit inefficient because we won't be able to
471 - // determine exactly how many strings were referenced.
563 if (string_base[partition].JudyHSArray) {
564 // We'll count all entries as "referenced" since we can't check them individually
565 referenced += string_base[partition].entries;
@@ -500,6 +591,78 @@ size_t string_destroy(void) {
591 rw_spinlock_write_unlock(&string_base[partition].spinlock);
592 }
593
594 +#ifdef FSANITIZE_ADDRESS
595 + // Collect stacktraces into an array for sorting
596 + typedef struct {
597 + STACKTRACE st;
598 + size_t count;
599 + } StacktraceEntry;
600 +
601 + // First, count the number of unique stacktraces
602 + Word_t Index = 0;
603 + Pvoid_t *PValue;
604 + size_t unique_stacktraces = 0;
605 +
606 + if (string_counts) {
607 + PValue = JudyLFirst(string_counts, &Index, PJE0);
608 + while (PValue) {
609 + unique_stacktraces++;
610 + PValue = JudyLNext(string_counts, &Index, PJE0);
611 + }
612 + }
613 +
614 + // Allocate an array for sorting
615 + StacktraceEntry *entries = mallocz(sizeof(StacktraceEntry) * unique_stacktraces);
616 + size_t entry_count = 0;
617 +
618 + // Populate the array with stacktraces and counts
619 + if (string_counts && unique_stacktraces > 0) {
620 + Index = 0;
621 + PValue = JudyLFirst(string_counts, &Index, PJE0);
622 + while (PValue) {
623 + entries[entry_count].st = (STACKTRACE)Index;
624 + entries[entry_count].count = (size_t)(uintptr_t)*PValue;
625 + entry_count++;
626 + PValue = JudyLNext(string_counts, &Index, PJE0);
627 + }
628 + }
629 +
630 + // Sort by count in descending order
631 + // Simple insertion sort is sufficient for a small number of entries
632 + for (size_t i = 1; i < entry_count; i++) {
633 + StacktraceEntry key = entries[i];
634 + ssize_t j = i - 1;
635 +
636 + // Move elements that are greater than key to one position ahead of their current position
637 + while (j >= 0 && entries[j].count < key.count) {
638 + entries[j + 1] = entries[j];
639 + j--;
640 + }
641 + entries[j + 1] = key;
642 + }
643 +
644 + // Print sorted stacktraces
645 + fprintf(stderr, "\nTop string creation stacktraces by count:\n");
646 +
647 + for (size_t i = 0; i < entry_count; i++) {
648 + // Format stacktrace to buffer
649 + buffer_flush(wb);
650 + stacktrace_to_buffer(entries[i].st, wb);
651 +
652 + fprintf(stderr, "\n > STRINGS REMAINING %zu: %zu strings created from:\n%s\n",
653 + i + 1, entries[i].count, buffer_tostring(wb));
654 + }
655 +
656 + fprintf(stderr, "==================================================================\n\n");
657 +
658 + // Clean up
659 + freez(entries);
660 + if (string_counts)
661 + JudyLFreeArray(&string_counts, PJE0);
662 + buffer_free(wb);
663 +#endif
664 +
665 + memset(&string_base, 0, sizeof(string_base));
666 return referenced;
667 }
668
@@ -751,6 +914,12 @@ int string_unittest(size_t entries) {
914 }
915
916 void string_init(void) {
754 - for (size_t i = 0; i != STRING_PARTITIONS; i++)
917 + for (size_t i = 0; i != STRING_PARTITIONS; i++) {
918 rw_spinlock_init(&string_base[i].spinlock);
919 +
920 +#ifdef FSANITIZE_ADDRESS
921 + // Initialize the JudyL pointers array to NULL
922 + string_base[i].JudyLPointers = NULL;
923 +#endif
924 + }
925 }
src/libnetdata/worker_utilization/worker_utilization.c
+250 -11
@@ -87,11 +87,23 @@ static struct workers_globals {
87 SPINLOCK spinlock;
88 Pvoid_t worknames_JudyHS;
89 size_t memory;
90 +
91 +#ifdef FSANITIZE_ADDRESS
92 + // For tracking all registered worker items during ASAN builds
93 + Pvoid_t workers_JudyL; // JudyL array of all worker structs
94 + Pvoid_t worknames_JudyL; // JudyL array of all workname structs
95 + Pvoid_t worker_strings_JudyL; // JudyL array of all STRING objects in workers
96 +#endif
97
98 } workers_globals = { // workers globals, the base of all worknames
99 .enabled = false,
100 .spinlock = SPINLOCK_INITIALIZER, // a lock for the worknames index
101 .worknames_JudyHS = NULL, // the worknames index
102 +#ifdef FSANITIZE_ADDRESS
103 + .workers_JudyL = NULL,
104 + .worknames_JudyL = NULL,
105 + .worker_strings_JudyL = NULL,
106 +#endif
107 };
108
109 static __thread struct worker *worker = NULL; // the current thread worker
@@ -113,15 +125,6 @@ void workers_utilization_enable(void) {
125 workers_globals.enabled = true;
126 }
127
116 -void worker_utilization_cleanup(void) {
117 - if(!workers_globals.enabled)
118 - return;
119 -
120 - spinlock_lock(&workers_globals.spinlock);
121 - JudyHSFreeArray(&workers_globals.worknames_JudyHS, PJE0);
122 - spinlock_unlock(&workers_globals.spinlock);
123 -}
124 -
128 size_t workers_allocated_memory(void) {
129 if(!workers_globals.enabled)
130 return 0;
@@ -152,6 +155,13 @@ void worker_register(const char *name) {
155
156 workers_globals.memory += sizeof(struct worker) + strlen(worker->tag) + 1 + strlen(worker->workname) + 1;
157
158 +#ifdef FSANITIZE_ADDRESS
159 + // Track the worker struct in our JudyL array for ASAN builds
160 + Pvoid_t *WValue = JudyLIns(&workers_globals.workers_JudyL, (Word_t)worker, PJE0);
161 + if (WValue != PJERR)
162 + *WValue = (void *)1;
163 +#endif
164 +
165 JudyAllocThreadPulseReset();
166 Pvoid_t *PValue = JudyHSIns(&workers_globals.worknames_JudyHS, (void *)name, name_size, PJE0);
167 int64_t judy_mem = JudyAllocThreadPulseGetAndReset();
@@ -164,6 +174,13 @@ void worker_register(const char *name) {
174 *PValue = workname;
175
176 workers_globals.memory = (int64_t)workers_globals.memory + (int64_t)sizeof(struct workers_workname) + judy_mem;
177 +
178 +#ifdef FSANITIZE_ADDRESS
179 + // Track the workname struct in our JudyL array for ASAN builds
180 + Pvoid_t *WValue = JudyLIns(&workers_globals.worknames_JudyL, (Word_t)workname, PJE0);
181 + if (WValue != PJERR)
182 + *WValue = (void *)1;
183 +#endif
184 }
185
186 spinlock_lock(&workname->spinlock);
@@ -190,9 +207,41 @@ void worker_register_job_custom_metric(size_t job_id, const char *name, const ch
207 return;
208 }
209
193 - worker->per_job_type[job_id].name = string_strdupz(name);
194 - worker->per_job_type[job_id].units = string_strdupz(units);
210 + STRING *name_str = string_strdupz(name);
211 + STRING *units_str = string_strdupz(units);
212 +
213 + worker->per_job_type[job_id].name = name_str;
214 + worker->per_job_type[job_id].units = units_str;
215 worker->per_job_type[job_id].type = type;
216 +
217 +#ifdef FSANITIZE_ADDRESS
218 + // Track the strings in our JudyL array for ASAN builds
219 + spinlock_lock(&workers_globals.spinlock);
220 +
221 + // Track the name STRING with reference counting
222 + if (name_str) {
223 + Pvoid_t *PValue = JudyLIns(&workers_globals.worker_strings_JudyL, (Word_t)name_str, PJE0);
224 + if (PValue != PJERR) {
225 + // Increment the reference count (or initialize to 1 if new)
226 + size_t count = (size_t)(uintptr_t)*PValue;
227 + count++;
228 + *PValue = (void *)(uintptr_t)count;
229 + }
230 + }
231 +
232 + // Track the units STRING with reference counting
233 + if (units_str) {
234 + Pvoid_t *PValue = JudyLIns(&workers_globals.worker_strings_JudyL, (Word_t)units_str, PJE0);
235 + if (PValue != PJERR) {
236 + // Increment the reference count (or initialize to 1 if new)
237 + size_t count = (size_t)(uintptr_t)*PValue;
238 + count++;
239 + *PValue = (void *)(uintptr_t)count;
240 + }
241 + }
242 +
243 + spinlock_unlock(&workers_globals.spinlock);
244 +#endif
245 }
246
247 void worker_register_job_name(size_t job_id, const char *name) {
@@ -220,10 +269,52 @@ void worker_unregister(void) {
269 }
270 }
271 workers_globals.memory -= sizeof(struct worker) + strlen(worker->tag) + 1 + strlen(worker->workname) + 1;
272 +
273 +#ifdef FSANITIZE_ADDRESS
274 + // Remove this worker from the tracking array
275 + JudyLDel(&workers_globals.workers_JudyL, (Word_t)worker, PJE0);
276 +#endif
277 +
278 spinlock_unlock(&workers_globals.spinlock);
279
280 // Free all thread-local resources associated with this worker
281 for(int i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES; i++) {
282 +#ifdef FSANITIZE_ADDRESS
283 + // Decrement reference count in tracking array before freeing
284 + if (worker->per_job_type[i].name) {
285 + spinlock_lock(&workers_globals.spinlock);
286 + Pvoid_t *PValue = JudyLGet(workers_globals.worker_strings_JudyL, (Word_t)worker->per_job_type[i].name, PJE0);
287 + if (PValue) {
288 + size_t count = (size_t)(uintptr_t)*PValue;
289 + if (count > 1) {
290 + // Decrement reference count
291 + count--;
292 + *PValue = (void *)(uintptr_t)count;
293 + } else {
294 + // Last reference, remove from tracking
295 + JudyLDel(&workers_globals.worker_strings_JudyL, (Word_t)worker->per_job_type[i].name, PJE0);
296 + }
297 + }
298 + spinlock_unlock(&workers_globals.spinlock);
299 + }
300 + if (worker->per_job_type[i].units) {
301 + spinlock_lock(&workers_globals.spinlock);
302 + Pvoid_t *PValue = JudyLGet(workers_globals.worker_strings_JudyL, (Word_t)worker->per_job_type[i].units, PJE0);
303 + if (PValue) {
304 + size_t count = (size_t)(uintptr_t)*PValue;
305 + if (count > 1) {
306 + // Decrement reference count
307 + count--;
308 + *PValue = (void *)(uintptr_t)count;
309 + } else {
310 + // Last reference, remove from tracking
311 + JudyLDel(&workers_globals.worker_strings_JudyL, (Word_t)worker->per_job_type[i].units, PJE0);
312 + }
313 + }
314 + spinlock_unlock(&workers_globals.spinlock);
315 + }
316 +#endif
317 + // Then free the strings
318 string_freez(worker->per_job_type[i].name);
319 string_freez(worker->per_job_type[i].units);
320 }
@@ -235,6 +326,154 @@ void worker_unregister(void) {
326 worker = NULL;
327 }
328
329 +// Cleanup all worker utilization resources
330 +void worker_utilization_cleanup(void) {
331 + if(!workers_globals.enabled)
332 + return;
333 +
334 + // Clean up the current thread's worker if it exists
335 + worker_unregister();
336 +
337 + spinlock_lock(&workers_globals.spinlock);
338 +
339 +#ifdef FSANITIZE_ADDRESS
340 + // Free any remaining strings in the tracking array according to their reference counts
341 + if (workers_globals.worker_strings_JudyL) {
342 + Word_t string_ptr = 0;
343 + Pvoid_t *PValue = JudyLFirst(workers_globals.worker_strings_JudyL, &string_ptr, PJE0);
344 + size_t total_strings = 0;
345 + size_t total_refs = 0;
346 +
347 + // First pass: count how many strings and references
348 + while (PValue) {
349 + total_strings++;
350 + size_t refs = (size_t)(uintptr_t)*PValue;
351 + total_refs += refs;
352 + PValue = JudyLNext(workers_globals.worker_strings_JudyL, &string_ptr, PJE0);
353 + }
354 +
355 + // If any strings remain, emit info about them
356 + if (total_strings > 0) {
357 + fprintf(stderr, "WORKERS UTILIZATION: Freeing %zu STRING objects with %zu total references\n",
358 + total_strings, total_refs);
359 +
360 + // Second pass: free each string the correct number of times
361 + string_ptr = 0;
362 + PValue = JudyLFirst(workers_globals.worker_strings_JudyL, &string_ptr, PJE0);
363 +
364 + while (PValue) {
365 + STRING *str = (STRING *)string_ptr;
366 + size_t refs = (size_t)(uintptr_t)*PValue;
367 +
368 + // Get the next one before we potentially delete this entry
369 + PValue = JudyLNext(workers_globals.worker_strings_JudyL, &string_ptr, PJE0);
370 +
371 + // Free the string exactly the number of times it was referenced
372 + for (size_t i = 0; i < refs; i++) {
373 + string_freez(str);
374 + }
375 + }
376 + }
377 +
378 + // Free the array itself
379 + JudyLFreeArray(&workers_globals.worker_strings_JudyL, PJE0);
380 + workers_globals.worker_strings_JudyL = NULL;
381 + }
382 +
383 + // We don't need to free worker or workname structs here as they should
384 + // be freed by worker_unregister, but we should free the tracking arrays
385 + if (workers_globals.workers_JudyL) {
386 + JudyLFreeArray(&workers_globals.workers_JudyL, PJE0);
387 + workers_globals.workers_JudyL = NULL;
388 + }
389 +
390 + if (workers_globals.worknames_JudyL) {
391 + JudyLFreeArray(&workers_globals.worknames_JudyL, PJE0);
392 + workers_globals.worknames_JudyL = NULL;
393 + }
394 +#endif
395 +
396 + // Free the JudyHS array that contains workname structs
397 + if (workers_globals.worknames_JudyHS) {
398 +#ifdef FSANITIZE_ADDRESS
399 + // For ASAN builds, we need to free any remaining workname structures
400 + // First, collect all workname pointers
401 + Pvoid_t worknames_to_free = NULL;
402 +
403 + // Cannot iterate JudyHS directly, but we can use worknames_JudyL which tracks all worknames
404 + if (workers_globals.worknames_JudyL) {
405 + Word_t workname_ptr = 0;
406 + Pvoid_t *PValue = JudyLFirst(workers_globals.worknames_JudyL, &workname_ptr, PJE0);
407 + size_t count = 0;
408 +
409 + while (PValue) {
410 + // Store this workname to free it later
411 + Pvoid_t *StoreValue = JudyLIns(&worknames_to_free, workname_ptr, PJE0);
412 + if (StoreValue != PJERR)
413 + *StoreValue = (void *)1;
414 +
415 + count++;
416 + PValue = JudyLNext(workers_globals.worknames_JudyL, &workname_ptr, PJE0);
417 + }
418 +
419 + if (count > 0) {
420 + fprintf(stderr, "WORKERS UTILIZATION: Freeing %zu workers_workname structures from tracking array\n", count);
421 + }
422 + }
423 +
424 + // Also try to get the LIBUV workname directly since it's the one leaking
425 + // This is a direct approach to ensure we don't miss any worknames
426 + const char *libuv_name = "LIBUV";
427 + size_t libuv_name_size = strlen(libuv_name) + 1;
428 + Pvoid_t *LiuvValue = JudyHSGet(workers_globals.worknames_JudyHS, (void *)libuv_name, libuv_name_size);
429 + if (LiuvValue && *LiuvValue) {
430 + struct workers_workname *libuv_workname = *LiuvValue;
431 + // Check if we've already got this workname in our tracking array
432 + bool already_tracked = false;
433 + if (worknames_to_free) {
434 + Pvoid_t *ExistingValue = JudyLGet(worknames_to_free, (Word_t)libuv_workname, PJE0);
435 + already_tracked = (ExistingValue != NULL);
436 + }
437 +
438 + if (!already_tracked) {
439 + // Add this workname to our list
440 + Pvoid_t *StoreValue = JudyLIns(&worknames_to_free, (Word_t)libuv_workname, PJE0);
441 + if (StoreValue != PJERR) {
442 + *StoreValue = (void *)1;
443 + fprintf(stderr, "WORKERS UTILIZATION: Found LIBUV workname not in tracking array\n");
444 + }
445 + }
446 + }
447 +#endif
448 +
449 + // Free the JudyHS array
450 + JudyHSFreeArray(&workers_globals.worknames_JudyHS, PJE0);
451 + workers_globals.worknames_JudyHS = NULL;
452 +
453 +#ifdef FSANITIZE_ADDRESS
454 + // Now free all the workname structures we collected
455 + if (worknames_to_free) {
456 + Word_t workname_ptr = 0;
457 + Pvoid_t *PValue = JudyLFirst(worknames_to_free, &workname_ptr, PJE0);
458 +
459 + while (PValue) {
460 + struct workers_workname *workname = (struct workers_workname *)workname_ptr;
461 + freez(workname);
462 +
463 + PValue = JudyLNext(worknames_to_free, &workname_ptr, PJE0);
464 + }
465 +
466 + JudyLFreeArray(&worknames_to_free, PJE0);
467 + }
468 +#endif
469 + }
470 +
471 + // Reset memory count
472 + workers_globals.memory = 0;
473 +
474 + spinlock_unlock(&workers_globals.spinlock);
475 +}
476 +
477 static void worker_is_idle_with_time(usec_t now) {
478 usec_t delta = now - worker->last_action_timestamp;
479 worker->busy_time += delta;
src/streaming/stream-conf.c
+15
@@ -14,7 +14,22 @@ static struct config stream_config = APPCONFIG_INITIALIZER;
14 * Called during shutdown to prevent memory leaks.
15 */
16 void stream_config_free(void) {
17 + // Free the configuration
18 inicfg_free(&stream_config);
19 +
20 + // Free the allocated strings in stream_send structure
21 + string_freez(stream_send.api_key);
22 + string_freez(stream_send.send_charts_matching);
23 + string_freez(stream_send.parents.destination);
24 + string_freez(stream_send.parents.ssl_ca_path);
25 + string_freez(stream_send.parents.ssl_ca_file);
26 +
27 + // Reset the pointers to NULL
28 + stream_send.api_key = NULL;
29 + stream_send.send_charts_matching = NULL;
30 + stream_send.parents.destination = NULL;
31 + stream_send.parents.ssl_ca_path = NULL;
32 + stream_send.parents.ssl_ca_file = NULL;
33 }
34
35 struct _stream_send stream_send = {
src/web/api/http_auth.c
+5
@@ -339,6 +339,11 @@ void bearer_tokens_init(void) {
339 bearer_tokens_load_from_disk();
340 }
341
342 +void bearer_tokens_destroy(void) {
343 + dictionary_destroy(netdata_authorized_bearers);
344 + netdata_authorized_bearers = NULL;
345 +}
346 +
347 bool extract_bearer_token_from_request(struct web_client *w, char *dst, size_t dst_len) {
348 if(!web_client_flag_check(w, WEB_CLIENT_FLAG_AUTH_BEARER) || dst_len != UUID_STR_LEN)
349 return false;