annotate logs with stack trace when libunwind is available (#19334)
* annotate logs with stack trace when libunwind is available * Update CMakeLists.txt Co-authored-by: Austin S. Hemmelgarn <ahferroin7@gmail.com> * fix error when libunwind is not available --------- Co-authored-by: Austin S. Hemmelgarn <ahferroin7@gmail.com>
Costa Tsaousis committed
Jan 8, 2025 at 21:32 UTC
989a1e270c4892ac8923d1c356079e0b8437b7e8
7 files changed
+96
-2
CMakeLists.txt
+20
@@ -957,6 +957,7 @@ set(LIBNETDATA_FILES
957
src/libnetdata/locks/benchmark.h
958
src/libnetdata/locks/benchmark-rw.c
959
src/libnetdata/locks/benchmark-rw.h
960
+ src/libnetdata/log/nd_log-libunwind.c
961
)
962
963
set(LIBH2O_FILES
@@ -2092,6 +2093,25 @@ netdata_add_jsonc_to_target(libnetdata)
2093
2094
netdata_add_libyaml_to_target(libnetdata)
2095
2096
+# libunwind
2097
+pkg_check_modules(LIBUNWIND libunwind IMPORTED_TARGET)
2098
+if(TARGET PkgConfig::LIBUNWIND)
2099
+ set(HAVE_LIBUNWIND On)
2100
+
2101
+ if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64)|(amd64)")
2102
+ target_link_libraries(libnetdata PUBLIC PkgConfig::LIBUNWIND -lunwind-x86_64)
2103
+ elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64")
2104
+ target_link_libraries(libnetdata PUBLIC PkgConfig::LIBUNWIND -lunwind-aarch64)
2105
+ elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "arm")
2106
+ target_link_libraries(libnetdata PUBLIC PkgConfig::LIBUNWIND -lunwind-arm)
2107
+ elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "powerpc|ppc")
2108
+ target_link_libraries(libnetdata PUBLIC PkgConfig::LIBUNWIND -lunwind-ppc64)
2109
+ else()
2110
+ message(WARNING "Unknown architecture ${CMAKE_SYSTEM_PROCESSOR} for libunwind. Stack traces may not work.")
2111
+ target_link_libraries(libnetdata PUBLIC PkgConfig::LIBUNWIND)
2112
+ endif()
2113
+endif()
2114
+
2115
# zlib
2116
if(OS_MACOS)
2117
find_package(ZLIB REQUIRED)
packaging/cmake/config.cmake.h.in
+1
@@ -75,6 +75,7 @@
75
#cmakedefine HAVE_GETRANDOM
76
#cmakedefine HAVE_SYSINFO
77
78
+#cmakedefine HAVE_LIBUNWIND
79
#cmakedefine HAVE_BACKTRACE
80
#cmakedefine HAVE_CLOSE_RANGE
81
#cmakedefine HAVE_SCHED_GETSCHEDULER
src/libnetdata/log/nd_log-common.h
+4
-2
@@ -119,11 +119,13 @@ typedef enum __attribute__((__packed__)) {
119
NDF_ALERT_NOTIFICATION_REALTIME_USEC = 62,
120
// NDF_ALERT_FLAGS,
121
122
+ NDF_STACK_TRACE = 63, // stack trace of the thread logging
123
+
124
// put new items here
125
// leave the request URL and the message last
126
125
- NDF_REQUEST = 63, // the request we are currently working on
126
- NDF_MESSAGE = 64, // the log message, if any
127
+ NDF_REQUEST = 64, // the request we are currently working on
128
+ NDF_MESSAGE = 65, // the log message, if any
129
130
// terminator
131
_NDF_MAX,
src/libnetdata/log/nd_log-internals.c
+6
@@ -707,6 +707,12 @@ __thread struct log_field thread_log_fields[_NDF_MAX] = {
707
.logfmt = "alert_notification_timestamp",
708
.annotator = timestamp_usec_annotator,
709
},
710
+ [NDF_STACK_TRACE] = {
711
+ .journal = "ND_STACK_TRACE",
712
+ .eventlog = "StackTrace",
713
+ .logfmt = NULL,
714
+ .annotator = stack_trace_annotator,
715
+ },
716
717
// put new items here
718
// leave the request URL and the message last
src/libnetdata/log/nd_log-internals.h
+1
@@ -195,6 +195,7 @@ struct log_field;
195
const char *errno_annotator(struct log_field *lf);
196
const char *priority_annotator(struct log_field *lf);
197
const char *timestamp_usec_annotator(struct log_field *lf);
198
+const char *stack_trace_annotator(struct log_field *lf);
199
200
#if defined(OS_WINDOWS)
201
const char *winerror_annotator(struct log_field *lf);
src/libnetdata/log/nd_log-libunwind.c
new
+61
@@ -0,0 +1,61 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "nd_log-internals.h"
4
+
5
+#ifdef HAVE_LIBUNWIND
6
+#include <libunwind.h>
7
+
8
+const char *stack_trace_annotator(struct log_field *lf __maybe_unused) {
9
+ static __thread char stack[4096];
10
+ static __thread bool in_stack_trace = false;
11
+
12
+ // prevent recursion
13
+ if(in_stack_trace)
14
+ return "stack trace recursion detected";
15
+
16
+ in_stack_trace = true;
17
+
18
+ unw_cursor_t cursor;
19
+ unw_context_t context;
20
+ char *d = stack;
21
+ size_t frames = 0;
22
+
23
+ // Initialize context for current thread
24
+ unw_getcontext(&context);
25
+ unw_init_local(&cursor, &context);
26
+
27
+ // Skip first 3 frames (our annotator and the logging infrastructure)
28
+ unw_step(&cursor);
29
+ unw_step(&cursor);
30
+ unw_step(&cursor);
31
+
32
+ while (unw_step(&cursor) > 0) {
33
+ unw_word_t offset, pc;
34
+ char sym[256];
35
+
36
+ unw_get_reg(&cursor, UNW_REG_IP, &pc);
37
+ if (pc == 0)
38
+ break;
39
+
40
+ const char *name = sym;
41
+ if (unw_get_proc_name(&cursor, sym, sizeof(sym), &offset) == 0) {
42
+ if(frames++)
43
+ d += snprintfz(d, sizeof(stack) - (d - stack), "\n");
44
+ d += snprintfz(d, sizeof(stack) - (d - stack), "%s+0x%lx", name, (unsigned long)offset);
45
+ }
46
+ else {
47
+ if(frames++)
48
+ d += snprintfz(d, sizeof(stack) - (d - stack), "\n");
49
+ d += snprintfz(d, sizeof(stack) - (d - stack), "<unknown>");
50
+ }
51
+ }
52
+
53
+ in_stack_trace = false;
54
+ return stack;
55
+}
56
+
57
+#else
58
+const char *stack_trace_annotator(struct log_field *lf __maybe_unused) {
59
+ return "libunwind not available";
60
+}
61
+#endif
src/libnetdata/log/nd_log.c
+3
@@ -232,6 +232,9 @@ static void nd_logger(const char *file, const char *function, const unsigned lon
232
233
// set the common fields that are automatically set by the logging subsystem
234
235
+ if(likely(!thread_log_fields[NDF_STACK_TRACE].entry.set))
236
+ thread_log_fields[NDF_STACK_TRACE].entry = ND_LOG_FIELD_U64(NDF_STACK_TRACE, 1);
237
+
238
if(likely(!thread_log_fields[NDF_INVOCATION_ID].entry.set))
239
thread_log_fields[NDF_INVOCATION_ID].entry = ND_LOG_FIELD_UUID(NDF_INVOCATION_ID, &nd_log.invocation_id);
240