Add spinlock to protect metric release (#16989)
* Add spinlock to protect metric release Cleanup service thread * Adjust formatting of the shutdown message During shutdown service thread should exit faster
Stelios Fragkakis committed
Feb 12, 2024 at 16:58 UTC
140d437446d783f14972ccd19f27a99587ad74ad
4 files changed
+19
-23
src/daemon/main.c
+1
-1
@@ -303,7 +303,7 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {
303
do { \
304
usec_t now_ut = now_monotonic_usec(); \
305
if(prev_msg) \
306
- netdata_log_info("NETDATA SHUTDOWN: in %7llu ms, %s%s - next: %s", (now_ut - last_ut) / USEC_PER_MS, (timeout)?"(TIMEOUT) ":"", prev_msg, msg); \
306
+ netdata_log_info("NETDATA SHUTDOWN: in %llu ms, %s%s - next: %s", (now_ut - last_ut) / USEC_PER_MS, (timeout)?"(TIMEOUT) ":"", prev_msg, msg); \
307
else \
308
netdata_log_info("NETDATA SHUTDOWN: next: %s", msg); \
309
last_ut = now_ut; \
src/daemon/service.c
+8
-15
@@ -36,20 +36,7 @@ static void svc_rrddim_obsolete_to_archive(RRDDIM *rd) {
36
37
if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
38
/* only a collector can mark a chart as obsolete, so we must remove the reference */
39
-
40
- size_t tiers_available = 0, tiers_said_no_retention = 0;
41
- for(size_t tier = 0; tier < storage_tiers ;tier++) {
42
- if(rd->tiers[tier].sch) {
43
- tiers_available++;
44
-
45
- if(storage_engine_store_finalize(rd->tiers[tier].sch))
46
- tiers_said_no_retention++;
47
-
48
- rd->tiers[tier].sch = NULL;
49
- }
50
- }
51
-
52
- if (tiers_available == tiers_said_no_retention && tiers_said_no_retention) {
39
+ if (!rrddim_finalize_collection_and_check_retention(rd)) {
40
/* This metric has no data and no references */
41
metaqueue_delete_dimension_uuid(&rd->metric_uuid);
42
}
@@ -204,6 +191,10 @@ static void svc_rrd_cleanup_obsolete_charts_from_all_hosts() {
191
192
RRDHOST *host;
193
rrdhost_foreach_read(host) {
194
+
195
+ if (!service_running(SERVICE_MAINTENANCE))
196
+ break;
197
+
198
if(rrdhost_receiver_replicating_charts(host) || rrdhost_sender_replicating_charts(host))
199
continue;
200
@@ -321,7 +312,9 @@ void *service_main(void *ptr)
312
real_step = USEC_PER_SEC;
313
314
svc_rrd_cleanup_obsolete_charts_from_all_hosts();
324
- svc_rrdhost_cleanup_orphan_hosts(localhost);
315
+
316
+ if (service_running(SERVICE_MAINTENANCE))
317
+ svc_rrdhost_cleanup_orphan_hosts(localhost);
318
}
319
320
netdata_thread_cleanup_pop(1);
src/database/rrd.h
+1
@@ -260,6 +260,7 @@ typedef struct storage_collect_handle {
260
struct rrddim_tier {
261
STORAGE_POINT virtual_point;
262
STORAGE_ENGINE_BACKEND seb;
263
+ SPINLOCK spinlock;
264
uint32_t tier_grouping;
265
time_t next_point_end_time_s;
266
STORAGE_METRIC_HANDLE *smh; // the metric handle inside the database
src/database/rrddim.c
+9
-7
@@ -95,6 +95,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
95
rd->tiers[tier].seb = eng->seb;
96
rd->tiers[tier].tier_grouping = host->db[tier].tier_grouping;
97
rd->tiers[tier].smh = eng->api.metric_get_or_create(rd, host->db[tier].si);
98
+ rd->tiers[tier].spinlock.locked = false;
99
storage_point_unset(rd->tiers[tier].virtual_point);
100
initialized++;
101
@@ -169,15 +170,16 @@ bool rrddim_finalize_collection_and_check_retention(RRDDIM *rd) {
170
size_t tiers_available = 0, tiers_said_no_retention = 0;
171
172
for(size_t tier = 0; tier < storage_tiers ;tier++) {
172
- if(!rd->tiers[tier].sch)
173
- continue;
173
+ spinlock_lock(&rd->tiers[tier].spinlock);
174
+ if(rd->tiers[tier].sch) {
175
+ tiers_available++;
176
175
- tiers_available++;
177
+ if (storage_engine_store_finalize(rd->tiers[tier].sch))
178
+ tiers_said_no_retention++;
179
177
- if(storage_engine_store_finalize(rd->tiers[tier].sch))
178
- tiers_said_no_retention++;
179
-
180
- rd->tiers[tier].sch = NULL;
180
+ rd->tiers[tier].sch = NULL;
181
+ }
182
+ spinlock_unlock(&rd->tiers[tier].spinlock);
183
}
184
185
// return true if the dimension has retention in the db