@cryptotaxi247 / netdata-1 / commits / 3fc34a5e3

Fix 1.37 crashes (#14081)

* Wait for pending read to complete before destroying the page * fix page alignment crash * Compare copy of descriptor * prevent workers crashes by disabling cancellability on critical areas and separate sqlite3 statistics to its own worker job * do not update sqlite3 stats when they are slow * do not query sqlite3 statistics when they are slow * flipped condition * sqlite3 proper timeout calculation Co-authored-by: Costa Tsaousis <costa@netdata.cloud>

Stelios Fragkakis committed Dec 2, 2022 at 16:12 UTC 3fc34a5e32fc16c7c832a0caefddf913c4e56eac
9 files changed +246 -103
daemon/global_statistics.c
+165 -79
@@ -4,17 +4,18 @@
4
5 #define GLOBAL_STATS_RESET_WEB_USEC_MAX 0x01
6
7 -#define WORKER_JOB_GLOBAL 0
8 -#define WORKER_JOB_REGISTRY 1
9 -#define WORKER_JOB_DBENGINE 2
10 -#define WORKER_JOB_HEARTBEAT 3
11 -#define WORKER_JOB_STRINGS 4
12 -#define WORKER_JOB_DICTIONARIES 5
13 -#define WORKER_JOB_MALLOC_TRACE 6
14 -#define WORKER_JOB_WORKERS 7
15 -
16 -#if WORKER_UTILIZATION_MAX_JOB_TYPES < 8
17 -#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 8
7 +#define WORKER_JOB_GLOBAL 0
8 +#define WORKER_JOB_REGISTRY 1
9 +#define WORKER_JOB_WORKERS 2
10 +#define WORKER_JOB_DBENGINE 3
11 +#define WORKER_JOB_HEARTBEAT 4
12 +#define WORKER_JOB_STRINGS 5
13 +#define WORKER_JOB_DICTIONARIES 6
14 +#define WORKER_JOB_MALLOC_TRACE 7
15 +#define WORKER_JOB_SQLITE3 8
16 +
17 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 9
18 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 9
19 #endif
20
21 bool global_statistics_enabled = true;
@@ -60,21 +61,6 @@ static struct global_statistics {
61
62 uint64_t db_points_stored_per_tier[RRD_STORAGE_TIERS];
63
63 - uint64_t sqlite3_queries_made;
64 - uint64_t sqlite3_queries_ok;
65 - uint64_t sqlite3_queries_failed;
66 - uint64_t sqlite3_queries_failed_busy;
67 - uint64_t sqlite3_queries_failed_locked;
68 - uint64_t sqlite3_rows;
69 - uint64_t sqlite3_metadata_cache_hit;
70 - uint64_t sqlite3_context_cache_hit;
71 - uint64_t sqlite3_metadata_cache_miss;
72 - uint64_t sqlite3_context_cache_miss;
73 - uint64_t sqlite3_metadata_cache_spill;
74 - uint64_t sqlite3_context_cache_spill;
75 - uint64_t sqlite3_metadata_cache_write;
76 - uint64_t sqlite3_context_cache_write;
77 -
64 } global_statistics = {
65 .connected_clients = 0,
66 .web_requests = 0,
@@ -112,27 +98,6 @@ void global_statistics_backfill_query_completed(size_t points_read) {
98 __atomic_fetch_add(&global_statistics.backfill_db_points_read, points_read, __ATOMIC_RELAXED);
99 }
100
115 -void global_statistics_sqlite3_query_completed(bool success, bool busy, bool locked) {
116 - __atomic_fetch_add(&global_statistics.sqlite3_queries_made, 1, __ATOMIC_RELAXED);
117 -
118 - if(success) {
119 - __atomic_fetch_add(&global_statistics.sqlite3_queries_ok, 1, __ATOMIC_RELAXED);
120 - }
121 - else {
122 - __atomic_fetch_add(&global_statistics.sqlite3_queries_failed, 1, __ATOMIC_RELAXED);
123 -
124 - if(busy)
125 - __atomic_fetch_add(&global_statistics.sqlite3_queries_failed_busy, 1, __ATOMIC_RELAXED);
126 -
127 - if(locked)
128 - __atomic_fetch_add(&global_statistics.sqlite3_queries_failed_locked, 1, __ATOMIC_RELAXED);
129 - }
130 -}
131 -
132 -void global_statistics_sqlite3_row_completed(void) {
133 - __atomic_fetch_add(&global_statistics.sqlite3_rows, 1, __ATOMIC_RELAXED);
134 -}
135 -
101 void global_statistics_rrdr_query_completed(size_t queries, uint64_t db_points_read, uint64_t result_points_generated, QUERY_SOURCE query_source) {
102 switch(query_source) {
103 case QUERY_SOURCE_API_DATA:
@@ -241,25 +206,6 @@ static inline void global_statistics_copy(struct global_statistics *gs, uint8_t
206 uint64_t n = 0;
207 __atomic_compare_exchange(&global_statistics.web_usec_max, (uint64_t *) &gs->web_usec_max, &n, 1, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
208 }
244 -
245 - gs->sqlite3_queries_made = __atomic_load_n(&global_statistics.sqlite3_queries_made, __ATOMIC_RELAXED);
246 - gs->sqlite3_queries_ok = __atomic_load_n(&global_statistics.sqlite3_queries_ok, __ATOMIC_RELAXED);
247 - gs->sqlite3_queries_failed = __atomic_load_n(&global_statistics.sqlite3_queries_failed, __ATOMIC_RELAXED);
248 - gs->sqlite3_queries_failed_busy = __atomic_load_n(&global_statistics.sqlite3_queries_failed_busy, __ATOMIC_RELAXED);
249 - gs->sqlite3_queries_failed_locked = __atomic_load_n(&global_statistics.sqlite3_queries_failed_locked, __ATOMIC_RELAXED);
250 - gs->sqlite3_rows = __atomic_load_n(&global_statistics.sqlite3_rows, __ATOMIC_RELAXED);
251 -
252 - gs->sqlite3_metadata_cache_hit = (uint64_t) sql_metadata_cache_stats(SQLITE_DBSTATUS_CACHE_HIT);
253 - gs->sqlite3_context_cache_hit = (uint64_t) sql_context_cache_stats(SQLITE_DBSTATUS_CACHE_HIT);
254 -
255 - gs->sqlite3_metadata_cache_miss = (uint64_t) sql_metadata_cache_stats(SQLITE_DBSTATUS_CACHE_MISS);
256 - gs->sqlite3_context_cache_miss = (uint64_t) sql_context_cache_stats(SQLITE_DBSTATUS_CACHE_MISS);
257 -
258 - gs->sqlite3_metadata_cache_spill = (uint64_t) sql_metadata_cache_stats(SQLITE_DBSTATUS_CACHE_SPILL);
259 - gs->sqlite3_context_cache_spill = (uint64_t) sql_context_cache_stats(SQLITE_DBSTATUS_CACHE_SPILL);
260 -
261 - gs->sqlite3_metadata_cache_write = (uint64_t) sql_metadata_cache_stats(SQLITE_DBSTATUS_CACHE_WRITE);
262 - gs->sqlite3_context_cache_write = (uint64_t) sql_context_cache_stats(SQLITE_DBSTATUS_CACHE_WRITE);
209 }
210
211 static void global_statistics_charts(void) {
@@ -707,8 +653,129 @@ static void global_statistics_charts(void) {
653
654 rrdset_done(st_points_stored);
655 }
656 +}
657
711 - // ----------------------------------------------------------------
658 +// ----------------------------------------------------------------------------
659 +// sqlite3 statistics
660 +
661 +struct sqlite3_statistics {
662 + uint64_t sqlite3_queries_made;
663 + uint64_t sqlite3_queries_ok;
664 + uint64_t sqlite3_queries_failed;
665 + uint64_t sqlite3_queries_failed_busy;
666 + uint64_t sqlite3_queries_failed_locked;
667 + uint64_t sqlite3_rows;
668 + uint64_t sqlite3_metadata_cache_hit;
669 + uint64_t sqlite3_context_cache_hit;
670 + uint64_t sqlite3_metadata_cache_miss;
671 + uint64_t sqlite3_context_cache_miss;
672 + uint64_t sqlite3_metadata_cache_spill;
673 + uint64_t sqlite3_context_cache_spill;
674 + uint64_t sqlite3_metadata_cache_write;
675 + uint64_t sqlite3_context_cache_write;
676 +
677 +} sqlite3_statistics = { };
678 +
679 +void global_statistics_sqlite3_query_completed(bool success, bool busy, bool locked) {
680 + __atomic_fetch_add(&sqlite3_statistics.sqlite3_queries_made, 1, __ATOMIC_RELAXED);
681 +
682 + if(success) {
683 + __atomic_fetch_add(&sqlite3_statistics.sqlite3_queries_ok, 1, __ATOMIC_RELAXED);
684 + }
685 + else {
686 + __atomic_fetch_add(&sqlite3_statistics.sqlite3_queries_failed, 1, __ATOMIC_RELAXED);
687 +
688 + if(busy)
689 + __atomic_fetch_add(&sqlite3_statistics.sqlite3_queries_failed_busy, 1, __ATOMIC_RELAXED);
690 +
691 + if(locked)
692 + __atomic_fetch_add(&sqlite3_statistics.sqlite3_queries_failed_locked, 1, __ATOMIC_RELAXED);
693 + }
694 +}
695 +
696 +void global_statistics_sqlite3_row_completed(void) {
697 + __atomic_fetch_add(&sqlite3_statistics.sqlite3_rows, 1, __ATOMIC_RELAXED);
698 +}
699 +
700 +static inline void sqlite3_statistics_copy(struct sqlite3_statistics *gs) {
701 + static usec_t last_run = 0;
702 +
703 + gs->sqlite3_queries_made = __atomic_load_n(&sqlite3_statistics.sqlite3_queries_made, __ATOMIC_RELAXED);
704 + gs->sqlite3_queries_ok = __atomic_load_n(&sqlite3_statistics.sqlite3_queries_ok, __ATOMIC_RELAXED);
705 + gs->sqlite3_queries_failed = __atomic_load_n(&sqlite3_statistics.sqlite3_queries_failed, __ATOMIC_RELAXED);
706 + gs->sqlite3_queries_failed_busy = __atomic_load_n(&sqlite3_statistics.sqlite3_queries_failed_busy, __ATOMIC_RELAXED);
707 + gs->sqlite3_queries_failed_locked = __atomic_load_n(&sqlite3_statistics.sqlite3_queries_failed_locked, __ATOMIC_RELAXED);
708 + gs->sqlite3_rows = __atomic_load_n(&sqlite3_statistics.sqlite3_rows, __ATOMIC_RELAXED);
709 +
710 + usec_t timeout = default_rrd_update_every * USEC_PER_SEC + default_rrd_update_every * USEC_PER_SEC / 3;
711 + usec_t now = now_monotonic_usec();
712 + if(!last_run)
713 + last_run = now;
714 + usec_t delta = now - last_run;
715 + bool query_sqlite3 = delta < timeout;
716 +
717 + if(query_sqlite3 && now_monotonic_usec() - last_run < timeout)
718 + gs->sqlite3_metadata_cache_hit = (uint64_t) sql_metadata_cache_stats(SQLITE_DBSTATUS_CACHE_HIT);
719 + else {
720 + gs->sqlite3_metadata_cache_hit = UINT64_MAX;
721 + query_sqlite3 = false;
722 + }
723 +
724 + if(query_sqlite3 && now_monotonic_usec() - last_run < timeout)
725 + gs->sqlite3_context_cache_hit = (uint64_t) sql_context_cache_stats(SQLITE_DBSTATUS_CACHE_HIT);
726 + else {
727 + gs->sqlite3_context_cache_hit = UINT64_MAX;
728 + query_sqlite3 = false;
729 + }
730 +
731 + if(query_sqlite3 && now_monotonic_usec() - last_run < timeout)
732 + gs->sqlite3_metadata_cache_miss = (uint64_t) sql_metadata_cache_stats(SQLITE_DBSTATUS_CACHE_MISS);
733 + else {
734 + gs->sqlite3_metadata_cache_miss = UINT64_MAX;
735 + query_sqlite3 = false;
736 + }
737 +
738 + if(query_sqlite3 && now_monotonic_usec() - last_run < timeout)
739 + gs->sqlite3_context_cache_miss = (uint64_t) sql_context_cache_stats(SQLITE_DBSTATUS_CACHE_MISS);
740 + else {
741 + gs->sqlite3_context_cache_miss = UINT64_MAX;
742 + query_sqlite3 = false;
743 + }
744 +
745 + if(query_sqlite3 && now_monotonic_usec() - last_run < timeout)
746 + gs->sqlite3_metadata_cache_spill = (uint64_t) sql_metadata_cache_stats(SQLITE_DBSTATUS_CACHE_SPILL);
747 + else {
748 + gs->sqlite3_metadata_cache_spill = UINT64_MAX;
749 + query_sqlite3 = false;
750 + }
751 +
752 + if(query_sqlite3 && now_monotonic_usec() - last_run < timeout)
753 + gs->sqlite3_context_cache_spill = (uint64_t) sql_context_cache_stats(SQLITE_DBSTATUS_CACHE_SPILL);
754 + else {
755 + gs->sqlite3_context_cache_spill = UINT64_MAX;
756 + query_sqlite3 = false;
757 + }
758 +
759 + if(query_sqlite3 && now_monotonic_usec() - last_run < timeout)
760 + gs->sqlite3_metadata_cache_write = (uint64_t) sql_metadata_cache_stats(SQLITE_DBSTATUS_CACHE_WRITE);
761 + else {
762 + gs->sqlite3_metadata_cache_write = UINT64_MAX;
763 + query_sqlite3 = false;
764 + }
765 +
766 + if(query_sqlite3 && now_monotonic_usec() - last_run < timeout)
767 + gs->sqlite3_context_cache_write = (uint64_t) sql_context_cache_stats(SQLITE_DBSTATUS_CACHE_WRITE);
768 + else {
769 + gs->sqlite3_context_cache_write = UINT64_MAX;
770 + query_sqlite3 = false;
771 + }
772 +
773 + last_run = now_monotonic_usec();
774 +}
775 +
776 +static void sqlite3_statistics_charts(void) {
777 + struct sqlite3_statistics gs;
778 + sqlite3_statistics_copy(&gs);
779
780 if(gs.sqlite3_queries_made) {
781 static RRDSET *st_sqlite3_queries = NULL;
@@ -833,10 +900,17 @@ static void global_statistics_charts(void) {
900 rd_cache_write = rrddim_add(st_sqlite3_cache, "cache_write", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
901 }
902
836 - rrddim_set_by_pointer(st_sqlite3_cache, rd_cache_hit, (collected_number)gs.sqlite3_metadata_cache_hit);
837 - rrddim_set_by_pointer(st_sqlite3_cache, rd_cache_miss, (collected_number)gs.sqlite3_metadata_cache_miss);
838 - rrddim_set_by_pointer(st_sqlite3_cache, rd_cache_spill, (collected_number)gs.sqlite3_metadata_cache_spill);
839 - rrddim_set_by_pointer(st_sqlite3_cache, rd_cache_write, (collected_number)gs.sqlite3_metadata_cache_write);
903 + if(gs.sqlite3_metadata_cache_hit != UINT64_MAX)
904 + rrddim_set_by_pointer(st_sqlite3_cache, rd_cache_hit, (collected_number)gs.sqlite3_metadata_cache_hit);
905 +
906 + if(gs.sqlite3_metadata_cache_miss != UINT64_MAX)
907 + rrddim_set_by_pointer(st_sqlite3_cache, rd_cache_miss, (collected_number)gs.sqlite3_metadata_cache_miss);
908 +
909 + if(gs.sqlite3_metadata_cache_spill != UINT64_MAX)
910 + rrddim_set_by_pointer(st_sqlite3_cache, rd_cache_spill, (collected_number)gs.sqlite3_metadata_cache_spill);
911 +
912 + if(gs.sqlite3_metadata_cache_write != UINT64_MAX)
913 + rrddim_set_by_pointer(st_sqlite3_cache, rd_cache_write, (collected_number)gs.sqlite3_metadata_cache_write);
914
915 rrdset_done(st_sqlite3_cache);
916 }
@@ -870,10 +944,17 @@ static void global_statistics_charts(void) {
944 rd_cache_write = rrddim_add(st_sqlite3_cache, "cache_write", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
945 }
946
873 - rrddim_set_by_pointer(st_sqlite3_cache, rd_cache_hit, (collected_number)gs.sqlite3_context_cache_hit);
874 - rrddim_set_by_pointer(st_sqlite3_cache, rd_cache_miss, (collected_number)gs.sqlite3_context_cache_miss);
875 - rrddim_set_by_pointer(st_sqlite3_cache, rd_cache_spill, (collected_number)gs.sqlite3_context_cache_spill);
876 - rrddim_set_by_pointer(st_sqlite3_cache, rd_cache_write, (collected_number)gs.sqlite3_context_cache_write);
947 + if(gs.sqlite3_context_cache_hit != UINT64_MAX)
948 + rrddim_set_by_pointer(st_sqlite3_cache, rd_cache_hit, (collected_number)gs.sqlite3_context_cache_hit);
949 +
950 + if(gs.sqlite3_context_cache_miss != UINT64_MAX)
951 + rrddim_set_by_pointer(st_sqlite3_cache, rd_cache_miss, (collected_number)gs.sqlite3_context_cache_miss);
952 +
953 + if(gs.sqlite3_context_cache_spill != UINT64_MAX)
954 + rrddim_set_by_pointer(st_sqlite3_cache, rd_cache_spill, (collected_number)gs.sqlite3_context_cache_spill);
955 +
956 + if(gs.sqlite3_context_cache_write != UINT64_MAX)
957 + rrddim_set_by_pointer(st_sqlite3_cache, rd_cache_write, (collected_number)gs.sqlite3_context_cache_write);
958
959 rrdset_done(st_sqlite3_cache);
960 }
@@ -2490,8 +2571,6 @@ static int read_thread_cpu_time_from_proc_stat(pid_t pid __maybe_unused, kernel_
2571 static Pvoid_t workers_by_pid_JudyL_array = NULL;
2572
2573 static void workers_threads_cleanup(struct worker_utilization *wu) {
2493 - netdata_thread_disable_cancelability();
2494 -
2574 struct worker_thread *t = wu->threads;
2575 while(t) {
2576 struct worker_thread *next = t->next;
@@ -2503,8 +2582,6 @@ static void workers_threads_cleanup(struct worker_utilization *wu) {
2582 }
2583 t = next;
2584 }
2506 -
2507 - netdata_thread_enable_cancelability();
2585 }
2586
2587 static struct worker_thread *worker_thread_find(struct worker_utilization *wu __maybe_unused, pid_t pid) {
@@ -2639,13 +2716,18 @@ static void worker_utilization_charts(void) {
2716
2717 for(int i = 0; all_workers_utilization[i].name ;i++) {
2718 workers_utilization_reset_statistics(&all_workers_utilization[i]);
2719 +
2720 + netdata_thread_disable_cancelability();
2721 workers_foreach(all_workers_utilization[i].name, worker_utilization_charts_callback, &all_workers_utilization[i]);
2722 + netdata_thread_enable_cancelability();
2723
2724 // skip the first iteration, so that we don't accumulate startup utilization to our charts
2725 if(likely(iterations > 1))
2726 workers_utilization_update_chart(&all_workers_utilization[i]);
2727
2728 + netdata_thread_disable_cancelability();
2729 workers_threads_cleanup(&all_workers_utilization[i]);
2730 + netdata_thread_enable_cancelability();
2731 }
2732
2733 workers_total_cpu_utilization_chart();
@@ -2692,6 +2774,7 @@ static void global_statistics_register_workers(void) {
2774 worker_register_job_name(WORKER_JOB_DICTIONARIES, "dictionaries");
2775 worker_register_job_name(WORKER_JOB_MALLOC_TRACE, "malloc_trace");
2776 worker_register_job_name(WORKER_JOB_WORKERS, "workers");
2777 + worker_register_job_name(WORKER_JOB_SQLITE3, "sqlite3");
2778 }
2779
2780 static void global_statistics_cleanup(void *ptr)
@@ -2734,6 +2817,9 @@ void *global_statistics_main(void *ptr)
2817 worker_is_busy(WORKER_JOB_GLOBAL);
2818 global_statistics_charts();
2819
2820 + worker_is_busy(WORKER_JOB_SQLITE3);
2821 + sqlite3_statistics_charts();
2822 +
2823 worker_is_busy(WORKER_JOB_REGISTRY);
2824 registry_statistics();
2825
daemon/service.c
+3 -3
@@ -46,19 +46,19 @@ static void svc_rrddim_obsolete_to_archive(RRDDIM *rd) {
46
47 /* only a collector can mark a chart as obsolete, so we must remove the reference */
48
49 - size_t tiers_available = 0, tiers_said_yes = 0;
49 + size_t tiers_available = 0, tiers_said_no_retention = 0;
50 for(size_t tier = 0; tier < storage_tiers ;tier++) {
51 if(rd->tiers[tier]) {
52 tiers_available++;
53
54 if(rd->tiers[tier]->collect_ops->finalize(rd->tiers[tier]->db_collection_handle))
55 - tiers_said_yes++;
55 + tiers_said_no_retention++;
56
57 rd->tiers[tier]->db_collection_handle = NULL;
58 }
59 }
60
61 - if (tiers_available == tiers_said_yes && tiers_said_yes) {
61 + if (tiers_available == tiers_said_no_retention && tiers_said_no_retention) {
62 /* This metric has no data and no references */
63 metaqueue_delete_dimension_uuid(&rd->metric_uuid);
64 }
database/engine/pagecache.c
+8
@@ -524,6 +524,14 @@ uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_d
524 }
525 rrdeng_page_descr_mutex_unlock(ctx, descr);
526
527 + while (unlikely(pg_cache_descr->flags & RRD_PAGE_READ_PENDING)) {
528 + error_limit_static_global_var(erl, 1, 0);
529 + error_limit(&erl, "%s: Found page with READ PENDING, waiting for read to complete", __func__);
530 + if (unlikely(debug_flags & D_RRDENGINE))
531 + print_page_cache_descr(descr, "", true);
532 + pg_cache_wait_event_unsafe(descr);
533 + }
534 +
535 if (pg_cache_descr->flags & RRD_PAGE_POPULATED) {
536 /* only after locking can it be safely deleted from LRU */
537 pg_cache_replaceQ_delete(ctx, descr);
database/engine/rrdengine.c
+29 -7
@@ -272,9 +272,19 @@ static void fill_page_with_nulls(void *page, uint32_t page_length, uint8_t type)
272 }
273 }
274
275 +struct rrdeng_page_descr *get_descriptor(struct pg_cache_page_index *page_index, time_t start_time_s)
276 +{
277 + uv_rwlock_rdlock(&page_index->lock);
278 + Pvoid_t *PValue = JudyLGet(page_index->JudyL_array, start_time_s, PJE0);
279 + struct rrdeng_page_descr *descr = unlikely(NULL == PValue) ? NULL : *PValue;
280 + uv_rwlock_rdunlock(&page_index->lock);
281 + return descr;
282 +};
283 +
284 static void do_extent_processing (struct rrdengine_worker_config *wc, struct extent_io_descriptor *xt_io_descr, bool read_failed)
285 {
286 struct rrdengine_instance *ctx = wc->ctx;
287 + struct page_cache *pg_cache = &ctx->pg_cache;
288 struct rrdeng_page_descr *descr;
289 struct page_cache_descr *pg_cache_descr;
290 int ret;
@@ -365,19 +375,30 @@ after_crc_check:
375 }
376 }
377
378 + uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
379 + Pvoid_t *PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, xt_io_descr->descr_array[0]->id, sizeof(uuid_t));
380 + struct pg_cache_page_index *page_index = likely( NULL != PValue) ? *PValue : NULL;
381 + uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
382 +
383 +
384 for (i = 0, page_offset = 0; i < count; page_offset += header->descr[i++].page_length) {
385 uint8_t is_prefetched_page;
386 descr = NULL;
387 for (j = 0 ; j < xt_io_descr->descr_count; ++j) {
372 - struct rrdeng_page_descr *descrj;
388 + struct rrdeng_page_descr descrj;
389
374 - descrj = xt_io_descr->descr_array[j];
390 + descrj = xt_io_descr->descr_read_array[j];
391 /* care, we don't hold the descriptor mutex */
376 - if (!uuid_compare(*(uuid_t *) header->descr[i].uuid, *descrj->id) &&
377 - header->descr[i].page_length == descrj->page_length &&
378 - header->descr[i].start_time_ut == descrj->start_time_ut &&
379 - header->descr[i].end_time_ut == descrj->end_time_ut) {
380 - descr = descrj;
392 + if (!uuid_compare(*(uuid_t *) header->descr[i].uuid, *descrj.id) &&
393 + header->descr[i].page_length == descrj.page_length &&
394 + header->descr[i].start_time_ut == descrj.start_time_ut &&
395 + header->descr[i].end_time_ut == descrj.end_time_ut) {
396 + //descr = descrj;
397 + descr = get_descriptor(page_index, (time_t) (descrj.start_time_ut / USEC_PER_SEC));
398 + if (unlikely(!descr)) {
399 + error_limit_static_thread_var(erl, 1, 0);
400 + error_limit(&erl, "%s: Required descriptor is not in the page index anymore", __FUNCTION__);
401 + }
402 break;
403 }
404 }
@@ -506,6 +527,7 @@ static void do_read_extent(struct rrdengine_worker_config* wc,
527 pg_cache_descr->flags |= RRD_PAGE_READ_PENDING;
528 rrdeng_page_descr_mutex_unlock(ctx, descr[i]);
529 xt_io_descr->descr_array[i] = descr[i];
530 + xt_io_descr->descr_read_array[i] = *(descr[i]);
531 }
532 xt_io_descr->descr_count = count;
533 xt_io_descr->file = datafile->file;
database/engine/rrdengine.h
+1
@@ -117,6 +117,7 @@ struct extent_io_descriptor {
117 unsigned descr_count;
118 int release_descr;
119 struct rrdeng_page_descr *descr_array[MAX_PAGES_PER_EXTENT];
120 + struct rrdeng_page_descr descr_read_array[MAX_PAGES_PER_EXTENT];
121 Word_t descr_commit_idx_array[MAX_PAGES_PER_EXTENT];
122 struct extent_io_descriptor *next; /* multiple requests to be served by the same cached extent */
123 };
database/engine/rrdengineapi.c
+33 -11
@@ -39,20 +39,39 @@ uint8_t rrdeng_drop_metrics_under_page_cache_pressure = 1;
39 // ----------------------------------------------------------------------------
40 // metrics groups
41
42 +static inline void rrdeng_page_alignment_acquire(struct pg_alignment *pa) {
43 + if(unlikely(!pa)) return;
44 + __atomic_add_fetch(&pa->refcount, 1, __ATOMIC_SEQ_CST);
45 +}
46 +
47 +static inline bool rrdeng_page_alignment_release(struct pg_alignment *pa) {
48 + if(unlikely(!pa)) return true;
49 +
50 + if(__atomic_sub_fetch(&pa->refcount, 1, __ATOMIC_SEQ_CST) == 0) {
51 + freez(pa);
52 + return true;
53 + }
54 +
55 + return false;
56 +}
57 +
58 +// charts call this
59 STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *db_instance __maybe_unused, uuid_t *uuid __maybe_unused) {
43 - return callocz(1, sizeof(struct pg_alignment));
60 + struct pg_alignment *pa = callocz(1, sizeof(struct pg_alignment));
61 + rrdeng_page_alignment_acquire(pa);
62 + return (STORAGE_METRICS_GROUP *)pa;
63 }
64
65 +// charts call this
66 void rrdeng_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg) {
47 - if(!smg) return;
67 + if(unlikely(!smg)) return;
68
69 struct rrdengine_instance *ctx = (struct rrdengine_instance *)db_instance;
70 struct pg_alignment *pa = (struct pg_alignment *)smg;
71 struct page_cache *pg_cache = &ctx->pg_cache;
72
73 uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
54 - if(pa->refcount == 0)
55 - freez(pa);
74 + rrdeng_page_alignment_release(pa);
75 uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
76 }
77
@@ -134,12 +153,13 @@ STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *db_instance, uuid_t *
153 __atomic_add_fetch(&page_index->refcount, 1, __ATOMIC_SEQ_CST);
154
155 if(pa) {
137 - if(page_index->alignment && page_index->alignment != pa && page_index->writers > 0)
138 - fatal("DBENGINE: page_index has a different alignment (page_index refcount is %u, writers is %u).",
139 - page_index->refcount, page_index->writers);
156 + if(page_index->alignment != pa) {
157 + if(!rrdeng_page_alignment_release(page_index->alignment)) // NULL is ok
158 + error("DBENGINE: metric switched alignment, but the previous is still used.");
159
141 - page_index->alignment = pa;
142 - __atomic_add_fetch(&pa->refcount, 1, __ATOMIC_SEQ_CST);
160 + rrdeng_page_alignment_acquire(pa);
161 + page_index->alignment = pa;
162 + }
163 }
164 }
165
@@ -162,8 +182,7 @@ STORAGE_METRIC_HANDLE *rrdeng_metric_create(STORAGE_INSTANCE *db_instance, uuid_
182 pg_cache->metrics_index.last_page_index = page_index;
183 page_index->alignment = pa;
184 page_index->refcount = 1;
165 - if(pa)
166 - pa->refcount++;
185 + rrdeng_page_alignment_acquire(pa);
186 uv_rwlock_wrunlock(&pg_cache->metrics_index.lock);
187
188 return (STORAGE_METRIC_HANDLE *)page_index;
@@ -532,6 +551,9 @@ int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle) {
551 uv_rwlock_wrlock(&page_index->lock);
552 if (!--page_index->writers && !page_index->page_count) {
553 can_delete_metric = 1;
554 +
555 + rrdeng_page_alignment_release(page_index->alignment);
556 + page_index->alignment = NULL;
557 }
558 uv_rwlock_wrunlock(&page_index->lock);
559 freez(handle);
database/rrddim.c
+3 -3
@@ -195,19 +195,19 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
195
196 debug(D_RRD_CALLS, "rrddim_free() %s.%s", rrdset_name(st), rrddim_name(rd));
197
198 - size_t tiers_available = 0, tiers_said_yes = 0;
198 + size_t tiers_available = 0, tiers_said_no_retention = 0;
199 for(size_t tier = 0; tier < storage_tiers ;tier++) {
200 if(rd->tiers[tier] && rd->tiers[tier]->db_collection_handle) {
201 tiers_available++;
202
203 if(rd->tiers[tier]->collect_ops->finalize(rd->tiers[tier]->db_collection_handle))
204 - tiers_said_yes++;
204 + tiers_said_no_retention++;
205
206 rd->tiers[tier]->db_collection_handle = NULL;
207 }
208 }
209
210 - if (tiers_available == tiers_said_yes && tiers_said_yes && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
210 + if (tiers_available == tiers_said_no_retention && tiers_said_no_retention && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
211 /* This metric has no data and no references */
212 metaqueue_delete_dimension_uuid(&rd->metric_uuid);
213 }
database/sqlite/sqlite_context.c
+2
@@ -449,7 +449,9 @@ skip_delete:
449 int sql_context_cache_stats(int op)
450 {
451 int count, dummy;
452 + netdata_thread_disable_cancelability();
453 sqlite3_db_status(db_context_meta, op, &count, &dummy, 0);
454 + netdata_thread_enable_cancelability();
455 return count;
456 }
457
database/sqlite/sqlite_functions.c
+2
@@ -1263,7 +1263,9 @@ int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_b
1263 int sql_metadata_cache_stats(int op)
1264 {
1265 int count, dummy;
1266 + netdata_thread_disable_cancelability();
1267 sqlite3_db_status(db_meta, op, &count, &dummy, 0);
1268 + netdata_thread_enable_cancelability();
1269 return count;
1270 }
1271