added more annotations in spinlock deadlock detection (#20207)
Costa Tsaousis committed
Apr 29, 2025 at 14:24 UTC
0a00260947ff25af89e99ea41a34d74f458747c4
7 files changed
+16
-14
src/libnetdata/dictionary/dictionary.c
+2
-2
@@ -376,9 +376,9 @@ size_t cleanup_destroyed_dictionaries(bool shutdown __maybe_unused) {
376
else dictionaries_waiting_to_be_destroyed = next;
377
}
378
else {
379
- size_t ref_items = dictionary_referenced_items(dict);
380
-
379
#ifdef FSANITIZE_ADDRESS
380
+ size_t ref_items = dictionary_referenced_items(dict);
381
+
382
// Track this dictionary for deduplication reporting
383
if (shutdown) {
384
// Process all stacktraces from this dictionary
src/libnetdata/locks/rw-spinlock.c
+2
-2
@@ -56,7 +56,7 @@ ALWAYS_INLINE void rw_spinlock_read_lock_with_trace(RW_SPINLOCK *rw_spinlock, co
56
57
// Check for deadlock every SPINS_BEFORE_DEADLOCK_CHECK iterations
58
if ((spins % SPINS_BEFORE_DEADLOCK_CHECK) == 0) {
59
- spinlock_deadlock_detect(&deadlock_timestamp);
59
+ spinlock_deadlock_detect(&deadlock_timestamp, "rw-spinlock read lock", func);
60
}
61
62
microsleep(usec);
@@ -122,7 +122,7 @@ ALWAYS_INLINE void rw_spinlock_write_lock_with_trace(RW_SPINLOCK *rw_spinlock, c
122
123
// Check for deadlock every SPINS_BEFORE_DEADLOCK_CHECK iterations
124
if ((spins % SPINS_BEFORE_DEADLOCK_CHECK) == 0) {
125
- spinlock_deadlock_detect(&deadlock_timestamp);
125
+ spinlock_deadlock_detect(&deadlock_timestamp, "rw-spinlock write lock", func);
126
}
127
128
microsleep(usec);
src/libnetdata/locks/spinlock.c
+4
-4
@@ -6,7 +6,7 @@
6
7
// ----------------------------------------------------------------------------
8
// Deadlock detection function
9
-void spinlock_deadlock_detect(usec_t *timestamp) {
9
+void spinlock_deadlock_detect(usec_t *timestamp, const char *type, const char *func) {
10
if (!*timestamp) {
11
// First time checking - initialize the timestamp
12
*timestamp = now_monotonic_high_precision_usec();
@@ -17,8 +17,8 @@ void spinlock_deadlock_detect(usec_t *timestamp) {
17
usec_t now = now_monotonic_high_precision_usec();
18
if (now - *timestamp >= SPINLOCK_DEADLOCK_TIMEOUT_SEC * USEC_PER_SEC) {
19
// We've been spinning for too long - likely deadlock
20
- fatal("DEADLOCK DETECTED: spinlock could not be acquired for %d seconds",
21
- SPINLOCK_DEADLOCK_TIMEOUT_SEC);
20
+ fatal("DEADLOCK DETECTED: %s in function '%s' could not be acquired for %"PRIi64" seconds",
21
+ type, func, (int64_t)((now - *timestamp) / USEC_PER_SEC));
22
}
23
}
24
@@ -49,7 +49,7 @@ ALWAYS_INLINE void spinlock_lock_with_trace(SPINLOCK *spinlock, const char *func
49
50
// Check for deadlock every SPINS_BEFORE_DEADLOCK_CHECK iterations
51
if ((spins % SPINS_BEFORE_DEADLOCK_CHECK) == 0) {
52
- spinlock_deadlock_detect(&deadlock_timestamp);
52
+ spinlock_deadlock_detect(&deadlock_timestamp, "spinlock", func);
53
}
54
55
microsleep(usec);
src/libnetdata/locks/spinlock.h
+1
-1
@@ -39,7 +39,7 @@ typedef struct netdata_spinlock
39
#define SPINS_BEFORE_DEADLOCK_CHECK 20000 // Check for deadlock every 20000 spins (approx. once per second)
40
41
// Helper function to detect deadlocks
42
-void spinlock_deadlock_detect(usec_t *timestamp);
42
+void spinlock_deadlock_detect(usec_t *timestamp, const char *type, const char *func);
43
44
void spinlock_init_with_trace(SPINLOCK *spinlock, const char *func);
45
#define spinlock_init(spinlock) spinlock_init_with_trace(spinlock, __FUNCTION__)
src/libnetdata/locks/waitq.c
+1
-1
@@ -108,7 +108,7 @@ ALWAYS_INLINE void waitq_acquire_with_trace(WAITQ *waitq, WAITQ_PRIORITY priorit
108
109
// Check for deadlock every SPINS_BEFORE_DEADLOCK_CHECK iterations
110
if ((spins % SPINS_BEFORE_DEADLOCK_CHECK) == 0) {
111
- spinlock_deadlock_detect(&deadlock_timestamp);
111
+ spinlock_deadlock_detect(&deadlock_timestamp, "waitq", func);
112
}
113
114
microsleep(usec);
src/libnetdata/log/nd_log.c
+2
@@ -480,12 +480,14 @@ static void recursive_fatal_abort(void) {
480
_exit(1);
481
}
482
483
+#ifdef NETDATA_INTERNAL_CHECKS
484
NEVER_INLINE NORETURN
485
static void fatal_abort_internal_checks(void) {
486
// keep this as a separate function, to have it logged like this in sentry
487
abort();
488
_exit(1);
489
}
490
+#endif
491
492
NEVER_INLINE
493
void netdata_logger_fatal(const char *file, const char *function, const unsigned long line, const char *fmt, ... ) {
src/streaming/stream-circular-buffer.h
+4
-4
@@ -10,10 +10,10 @@
10
extern "C" {
11
#endif
12
13
-#define CBUFFER_INITIAL_SIZE (16 * 1024)
14
-#define CBUFFER_INITIAL_MAX_SIZE (10 * 1024 * 1024)
15
-#define HOST_THREAD_BUFFER_INITIAL_SIZE (256 * 1024)
16
-#define REPLICATION_THREAD_BUFFER_INITIAL_SIZE (512 * 1024)
13
+#define CBUFFER_INITIAL_SIZE (16ULL * 1024)
14
+#define CBUFFER_INITIAL_MAX_SIZE (10ULL * 1024 * 1024)
15
+#define HOST_THREAD_BUFFER_INITIAL_SIZE (256ULL * 1024)
16
+#define REPLICATION_THREAD_BUFFER_INITIAL_SIZE (512ULL * 1024)
17
18
#define STREAM_CIRCULAR_BUFFER_ADAPT_TO_TIMES_MAX_SIZE 3
19