improve pgc fatal errors (#20181)
* improve pgc fatal errors * fix threads joining corruption * add more info about thread linking
Costa Tsaousis committed
Apr 25, 2025 at 20:34 UTC
6a9ad6be02d62114e483dde587dc4bfdda7338a7
2 files changed
+55
-22
src/database/engine/cache.c
+18
-7
@@ -638,6 +638,9 @@ static ALWAYS_INLINE void pgc_queue_add(PGC *cache __maybe_unused, struct pgc_qu
638
int64_t mem_delta = 0;
639
640
Pvoid_t *section_pages_pptr = JudyLIns(&q->sections_judy, page->section, PJE0);
641
+ if(section_pages_pptr == NULL || section_pages_pptr == PJERR)
642
+ fatal("DBENGINE CACHE: JudyLIns(q->sections_judy, 0x%lx) failed, q->sections_judy = %p, result = %p",
643
+ (long unsigned)page->section, q->sections_judy, section_pages_pptr);
644
645
struct section_pages *sp = *section_pages_pptr;
646
if(!sp) {
@@ -714,7 +717,9 @@ static ALWAYS_INLINE void pgc_queue_del(PGC *cache __maybe_unused, struct pgc_qu
717
718
if(q->linked_list_in_sections_judy) {
719
Pvoid_t *section_pages_pptr = JudyLGet(q->sections_judy, page->section, PJE0);
717
- internal_fatal(!section_pages_pptr, "DBENGINE CACHE: page should be in Judy LL, but it is not");
720
+ if(section_pages_pptr == NULL || section_pages_pptr == PJERR)
721
+ fatal("DBENGINE CACHE: JudyLGet(q->sections_judy, 0x%lx) failed, q->sections_judy = %p",
722
+ (long unsigned)page->section, q->sections_judy);
723
724
struct section_pages *sp = *section_pages_pptr;
725
sp->entries--;
@@ -1418,15 +1423,18 @@ static PGC_PAGE *pgc_page_add(PGC *cache, PGC_ENTRY *entry, bool *added) {
1423
1424
Pvoid_t *metrics_judy_pptr = JudyLIns(&cache->index[partition].sections_judy, entry->section, PJE0);
1425
if(unlikely(!metrics_judy_pptr || metrics_judy_pptr == PJERR))
1421
- fatal("DBENGINE CACHE: corrupted sections judy array");
1426
+ fatal("DBENGINE CACHE: JudyLIns(sections_judy, 0x%lx) failed, sections_judy = %p, result = %p",
1427
+ (long unsigned)entry->section, cache->index[partition].sections_judy, metrics_judy_pptr);
1428
1429
Pvoid_t *pages_judy_pptr = JudyLIns(metrics_judy_pptr, entry->metric_id, PJE0);
1430
if(unlikely(!pages_judy_pptr || pages_judy_pptr == PJERR))
1425
- fatal("DBENGINE CACHE: corrupted pages judy array");
1431
+ fatal("DBENGINE CACHE: JudyLIns(metrics_judy, 0x%lx) failed, metrics_judy = %p, result = %p",
1432
+ (long unsigned)entry->metric_id, metrics_judy_pptr, pages_judy_pptr);
1433
1434
Pvoid_t *page_ptr = JudyLIns(pages_judy_pptr, entry->start_time_s, PJE0);
1435
if(unlikely(!page_ptr || page_ptr == PJERR))
1429
- fatal("DBENGINE CACHE: corrupted page in judy array");
1436
+ fatal("DBENGINE CACHE: JudyLIns(pages_judy, %ld) failed, pages_judy = %p, result = %p",
1437
+ (long)entry->start_time_s, pages_judy_pptr, page_ptr);
1438
1439
pgc_stats_index_judy_change(cache, JudyAllocThreadPulseGetAndReset());
1440
@@ -2471,7 +2479,8 @@ void pgc_open_cache_to_journal_v2(PGC *cache, Word_t section, unsigned datafile_
2479
size_t current_extent_index_id;
2480
Pvoid_t *PValue = JudyLIns(&JudyL_extents_pos, xio->pos, PJE0);
2481
if(!PValue || PValue == PJERR)
2474
- fatal("Corrupted JudyL extents pos");
2482
+ fatal("CACHE: JudyLIns(JudyL_extents_pos, %" PRIu64 ") failed, JudyL_extents_pos = %p, result = %p",
2483
+ xio->pos, JudyL_extents_pos, PValue);
2484
2485
struct jv2_extents_info *ei;
2486
if(!*PValue) {
@@ -2495,7 +2504,8 @@ void pgc_open_cache_to_journal_v2(PGC *cache, Word_t section, unsigned datafile_
2504
2505
PValue = JudyLIns(&JudyL_metrics, page->metric_id, PJE0);
2506
if(!PValue || PValue == PJERR)
2498
- fatal("Corrupted JudyL metrics");
2507
+ fatal("CACHE: JudyLIns(JudyL_metrics, 0x%lx) failed, JudyL_metrics = %p, result = %p",
2508
+ (long unsigned)page->metric_id, JudyL_metrics, PValue);
2509
2510
struct jv2_metrics_info *mi;
2511
if(!*PValue) {
@@ -2521,7 +2531,8 @@ void pgc_open_cache_to_journal_v2(PGC *cache, Word_t section, unsigned datafile_
2531
2532
PValue = JudyLIns(&mi->JudyL_pages_by_start_time, page->start_time_s, PJE0);
2533
if(!PValue || PValue == PJERR)
2524
- fatal("Corrupted JudyL metric pages");
2534
+ fatal("CACHE: JudyLIns(JudyL_pages_by_start_time, %ld) failed, JudyL_pages_by_start_time = %p, result = %p",
2535
+ (long)page->start_time_s, mi->JudyL_pages_by_start_time, PValue);
2536
2537
if(!*PValue) {
2538
struct jv2_page_info *pi = aral_mallocz(ar_pi); // callocz(1, (sizeof(struct jv2_page_info)));
src/libnetdata/threads/threads.c
+37
-15
@@ -9,6 +9,12 @@
9
10
typedef void (*nd_thread_canceller)(void *data);
11
12
+typedef enum __attribute__((packed)) {
13
+ ND_THREAD_LIST_NONE = 0,
14
+ ND_THREAD_LIST_RUNNING,
15
+ ND_THREAD_LIST_EXITED,
16
+} ND_THREAD_LIST;
17
+
18
struct nd_thread {
19
void *arg;
20
pid_t tid;
@@ -36,6 +42,7 @@ struct nd_thread {
42
void *data;
43
} canceller;
44
45
+ ND_THREAD_LIST list;
46
struct nd_thread *prev, *next;
47
};
48
@@ -262,8 +269,10 @@ static void nd_thread_join_exited_detached_threads(void) {
269
while (nti && nd_thread_status_check(nti, NETDATA_THREAD_OPTION_JOINABLE) == 0)
270
nti = nti->next;
271
265
- if(nti)
272
+ if(nti) {
273
DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(threads_globals.exited.list, nti, prev, next);
274
+ nti->list = ND_THREAD_LIST_NONE;
275
+ }
276
277
spinlock_unlock(&threads_globals.exited.spinlock);
278
@@ -276,8 +285,7 @@ static void nd_thread_join_exited_detached_threads(void) {
285
}
286
}
287
279
-static void nd_thread_exit(void *pptr) {
280
- ND_THREAD *nti = CLEANUP_FUNCTION_GET_PTR(pptr);
288
+static void nd_thread_exit(ND_THREAD *nti) {
289
290
if(nti != _nd_thread_info || !nti || !_nd_thread_info) {
291
nd_log(NDLS_DAEMON, NDLP_ERR,
@@ -328,12 +336,16 @@ static void nd_thread_exit(void *pptr) {
336
nd_thread_status_set(nti, NETDATA_THREAD_STATUS_FINISHED);
337
338
spinlock_lock(&threads_globals.running.spinlock);
331
- DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(threads_globals.running.list, nti, prev, next);
339
+ if(nti->list == ND_THREAD_LIST_RUNNING) {
340
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(threads_globals.running.list, nti, prev, next);
341
+ nti->list = ND_THREAD_LIST_NONE;
342
+ }
343
spinlock_unlock(&threads_globals.running.spinlock);
344
345
if (nd_thread_status_check(nti, NETDATA_THREAD_OPTION_JOINABLE) != NETDATA_THREAD_OPTION_JOINABLE) {
346
spinlock_lock(&threads_globals.exited.spinlock);
347
DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(threads_globals.exited.list, nti, prev, next);
348
+ nti->list = ND_THREAD_LIST_EXITED;
349
spinlock_unlock(&threads_globals.exited.spinlock);
350
}
351
}
@@ -354,13 +366,17 @@ static void *nd_thread_starting_point(void *ptr) {
366
if(pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL) != 0)
367
nd_log(NDLS_DAEMON, NDLP_WARNING, "cannot set pthread cancel state to ENABLE.");
368
357
- CLEANUP_FUNCTION_REGISTER(nd_thread_exit) cleanup_ptr = nti;
358
-
369
signals_block_all_except_deadly();
370
371
+ spinlock_lock(&threads_globals.running.spinlock);
372
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(threads_globals.running.list, nti, prev, next);
373
+ nti->list = ND_THREAD_LIST_RUNNING;
374
+ spinlock_unlock(&threads_globals.running.spinlock);
375
+
376
// run the thread code
377
nti->ret = nti->start_routine(nti->arg);
378
379
+ nd_thread_exit(nti);
380
return nti;
381
}
382
@@ -382,19 +398,12 @@ ND_THREAD *nd_thread_create(const char *tag, NETDATA_THREAD_OPTIONS options, voi
398
nti->options = options & NETDATA_THREAD_OPTIONS_ALL;
399
strncpyz(nti->tag, tag, ND_THREAD_TAG_MAX);
400
385
- spinlock_lock(&threads_globals.running.spinlock);
386
- DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(threads_globals.running.list, nti, prev, next);
387
- spinlock_unlock(&threads_globals.running.spinlock);
388
-
401
int ret = pthread_create(&nti->thread, &threads_globals.attr, nd_thread_starting_point, nti);
402
if(ret != 0) {
403
nd_log(NDLS_DAEMON, NDLP_ERR,
404
"failed to create new thread for %s. pthread_create() failed with code %d",
405
tag, ret);
406
395
- spinlock_lock(&threads_globals.running.spinlock);
396
- DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(threads_globals.running.list, nti, prev, next);
397
- spinlock_unlock(&threads_globals.running.spinlock);
407
freez(nti);
408
return NULL;
409
}
@@ -440,16 +449,29 @@ int nd_thread_join(ND_THREAD *nti) {
449
450
int ret = pthread_join(nti->thread, NULL);
451
if(ret != 0) {
452
+ // we can't join the thread
453
+
454
nd_log(NDLS_DAEMON, NDLP_WARNING,
455
"cannot join thread. pthread_join() failed with code %d. (tag=%s)",
456
ret, nti->tag);
457
}
458
else {
459
+ // we successfully joined the thread
460
+
461
nd_thread_status_set(nti, NETDATA_THREAD_STATUS_JOINED);
462
463
+ spinlock_lock(&threads_globals.running.spinlock);
464
+ if(nti->list == ND_THREAD_LIST_RUNNING) {
465
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(threads_globals.running.list, nti, prev, next);
466
+ nti->list = ND_THREAD_LIST_NONE;
467
+ }
468
+ spinlock_unlock(&threads_globals.running.spinlock);
469
+
470
spinlock_lock(&threads_globals.exited.spinlock);
451
- if(nti->prev)
452
- DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(threads_globals.exited.list, nti, prev, next);
471
+ if(nti->list == ND_THREAD_LIST_EXITED) {
472
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(threads_globals.exited.list, nti, prev, next);
473
+ nti->list = ND_THREAD_LIST_NONE;
474
+ }
475
spinlock_unlock(&threads_globals.exited.spinlock);
476
477
freez(nti);