fix rrdcontexts left in the post-processing queue from the garbage collector (#13645)
* fix rrdcontexts left in the post-processing queue from the garbage collector * set the queuing flags atomically, using the dictionary callbacks
Costa Tsaousis committed
Sep 7, 2022 at 23:02 UTC
642e00348d5be1af256aef4ce0ff922ecf1d82e4
2 files changed
+67
-49
database/rrdcontext.c
+61
-48
@@ -106,9 +106,7 @@ typedef enum {
106
#define RRD_FLAGS_PREVENTING_DELETIONS ( \
107
RRD_FLAG_QUEUED_FOR_HUB \
108
|RRD_FLAG_COLLECTED \
109
- \
110
- /* RRD_FLAG_QUEUED_FOR_POST_PROCESSING */ \
111
- /* should not be here or nothing will be deleted */ \
109
+ |RRD_FLAG_QUEUED_FOR_POST_PROCESSING \
110
)
111
112
// get all the flags of an object
@@ -325,18 +323,10 @@ static inline void rrdmetric_release(RRDMETRIC_ACQUIRED *rma) {
323
// ----------------------------------------------------------------------------
324
// helper one-liners for RRDINSTANCE
325
328
-static inline RRDINSTANCE_ACQUIRED *rrdinstance_dup(RRDINSTANCE_ACQUIRED *ria) {
329
- return (RRDINSTANCE_ACQUIRED *)dictionary_acquired_item_dup((DICTIONARY_ITEM *)ria);
330
-}
331
-
326
static inline RRDINSTANCE *rrdinstance_acquired_value(RRDINSTANCE_ACQUIRED *ria) {
327
return dictionary_acquired_item_value((DICTIONARY_ITEM *)ria);
328
}
329
336
-static inline const char *rrdinstance_acquired_name(RRDINSTANCE_ACQUIRED *ria) {
337
- return dictionary_acquired_item_name((DICTIONARY_ITEM *)ria);
338
-}
339
-
330
static inline void rrdinstance_release(RRDINSTANCE_ACQUIRED *ria) {
331
RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
332
dictionary_acquired_item_release(ri->rc->rrdinstances, (DICTIONARY_ITEM *)ria);
@@ -345,22 +335,10 @@ static inline void rrdinstance_release(RRDINSTANCE_ACQUIRED *ria) {
335
// ----------------------------------------------------------------------------
336
// helper one-liners for RRDCONTEXT
337
348
-static inline RRDCONTEXT_ACQUIRED *rrdcontext_dup(RRDCONTEXT_ACQUIRED *rca) {
349
- return (RRDCONTEXT_ACQUIRED *)dictionary_acquired_item_dup((DICTIONARY_ITEM *)rca);
350
-}
351
-
352
-static inline const char *rrdcontext_acquired_name(RRDCONTEXT_ACQUIRED *rca) {
353
- return dictionary_acquired_item_name((DICTIONARY_ITEM *)rca);
354
-}
355
-
338
static inline RRDCONTEXT *rrdcontext_acquired_value(RRDCONTEXT_ACQUIRED *rca) {
339
return dictionary_acquired_item_value((DICTIONARY_ITEM *)rca);
340
}
341
360
-static inline RRDCONTEXT_ACQUIRED *rrdcontext_acquire(RRDHOST *host, const char *name) {
361
- return (RRDCONTEXT_ACQUIRED *)dictionary_get_and_acquire_item((DICTIONARY *)host->rrdctx, name);
362
-}
363
-
342
static inline void rrdcontext_release(RRDCONTEXT_ACQUIRED *rca) {
343
RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
344
dictionary_acquired_item_release((DICTIONARY *)rc->rrdhost->rrdctx, (DICTIONARY_ITEM *)rca);
@@ -374,7 +352,6 @@ static uint64_t rrdcontext_version_hash_with_callback(RRDHOST *host, void (*call
352
353
static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker_jobs);
354
static void rrdcontext_garbage_collect_for_all_hosts(void);
377
-void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc);
355
356
#define rrdcontext_lock(rc) netdata_mutex_lock(&((rc)->mutex))
357
#define rrdcontext_unlock(rc) netdata_mutex_unlock(&((rc)->mutex))
@@ -386,6 +363,9 @@ static uint64_t rrdcontext_get_next_version(RRDCONTEXT *rc);
363
static bool check_if_cloud_version_changed_unsafe(RRDCONTEXT *rc, bool sending __maybe_unused);
364
static void rrdcontext_message_send_unsafe(RRDCONTEXT *rc, bool snapshot __maybe_unused, void *bundle __maybe_unused);
365
366
+static void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc);
367
+
368
+static void rrdcontext_dequeue_from_post_processing(RRDCONTEXT *rc);
369
static void rrdcontext_queue_for_post_processing(RRDCONTEXT *rc, const char *function, RRD_FLAGS flags);
370
static void rrdcontext_post_process_updates(RRDCONTEXT *rc, bool force, RRD_FLAGS reason, bool worker_jobs);
371
@@ -1279,6 +1259,37 @@ static void rrdcontext_trigger_updates(RRDCONTEXT *rc, const char *function) {
1259
rrdcontext_queue_for_post_processing(rc, function, rc->flags);
1260
}
1261
1262
+static void rrdcontext_hub_queue_insert_callback(const char *name __maybe_unused, void *context, void *data __maybe_unused) {
1263
+ RRDCONTEXT *rc = context;
1264
+ rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_HUB);
1265
+ rc->queue.queued_ut = now_realtime_usec();
1266
+ rc->queue.queued_flags = rrd_flags_get(rc);
1267
+}
1268
+
1269
+static void rrdcontext_hub_queue_delete_callback(const char *name __maybe_unused, void *context, void *data __maybe_unused) {
1270
+ RRDCONTEXT *rc = context;
1271
+ rrd_flag_clear(rc, RRD_FLAG_QUEUED_FOR_HUB);
1272
+}
1273
+
1274
+static void rrdcontext_hub_queue_conflict_callback(const char *name __maybe_unused, void *context, void *new_context __maybe_unused, void *data __maybe_unused) {
1275
+ // context and new_context are the same
1276
+ // we just need to update the timings
1277
+ RRDCONTEXT *rc = context;
1278
+ rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_HUB);
1279
+ rc->queue.queued_ut = now_realtime_usec();
1280
+ rc->queue.queued_flags |= rrd_flags_get(rc);
1281
+}
1282
+
1283
+static void rrdcontext_post_processing_queue_insert_callback(const char *name __maybe_unused, void *context, void *data __maybe_unused) {
1284
+ RRDCONTEXT *rc = context;
1285
+ rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_POST_PROCESSING);
1286
+}
1287
+
1288
+static void rrdcontext_post_processing_queue_delete_callback(const char *name __maybe_unused, void *context, void *data __maybe_unused) {
1289
+ RRDCONTEXT *rc = context;
1290
+ rrd_flag_clear(rc, RRD_FLAG_QUEUED_FOR_POST_PROCESSING);
1291
+}
1292
+
1293
void rrdhost_create_rrdcontexts(RRDHOST *host) {
1294
if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
1295
return;
@@ -1292,8 +1303,20 @@ void rrdhost_create_rrdcontexts(RRDHOST *host) {
1303
dictionary_register_conflict_callback((DICTIONARY *)host->rrdctx, rrdcontext_conflict_callback, (void *)host);
1304
dictionary_register_react_callback((DICTIONARY *)host->rrdctx, rrdcontext_react_callback, (void *)host);
1305
1295
- host->rrdctx_hub_queue = (RRDCONTEXTS *)dictionary_create(DICTIONARY_FLAG_DONT_OVERWRITE_VALUE | DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE);
1296
- host->rrdctx_post_processing_queue = (RRDCONTEXTS *)dictionary_create(DICTIONARY_FLAG_DONT_OVERWRITE_VALUE | DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE);
1306
+ host->rrdctx_hub_queue = (RRDCONTEXTS *)dictionary_create(
1307
+ DICTIONARY_FLAG_DONT_OVERWRITE_VALUE
1308
+ |DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE);
1309
+
1310
+ dictionary_register_insert_callback((DICTIONARY *)host->rrdctx_hub_queue, rrdcontext_hub_queue_insert_callback, NULL);
1311
+ dictionary_register_delete_callback((DICTIONARY *)host->rrdctx_hub_queue, rrdcontext_hub_queue_delete_callback, NULL);
1312
+ dictionary_register_conflict_callback((DICTIONARY *)host->rrdctx_hub_queue, rrdcontext_hub_queue_conflict_callback, NULL);
1313
+
1314
+ host->rrdctx_post_processing_queue = (RRDCONTEXTS *)dictionary_create(
1315
+ DICTIONARY_FLAG_DONT_OVERWRITE_VALUE
1316
+ |DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE);
1317
+
1318
+ dictionary_register_insert_callback((DICTIONARY *)host->rrdctx_hub_queue, rrdcontext_post_processing_queue_insert_callback, NULL);
1319
+ dictionary_register_delete_callback((DICTIONARY *)host->rrdctx_hub_queue, rrdcontext_post_processing_queue_delete_callback, NULL);
1320
}
1321
1322
void rrdhost_destroy_rrdcontexts(RRDHOST *host) {
@@ -1309,7 +1332,6 @@ void rrdhost_destroy_rrdcontexts(RRDHOST *host) {
1332
RRDCONTEXT *rc;
1333
dfe_start_write(old, rc) {
1334
dictionary_del_having_write_lock(old, string2str(rc->id));
1312
- rrdset_flag_clear(rc, RRD_FLAG_QUEUED_FOR_HUB);
1335
}
1336
dfe_done(rc);
1337
dictionary_destroy(old);
@@ -1322,7 +1344,6 @@ void rrdhost_destroy_rrdcontexts(RRDHOST *host) {
1344
RRDCONTEXT *rc;
1345
dfe_start_write(old, rc) {
1346
dictionary_del_having_write_lock(old, string2str(rc->id));
1325
- rrdset_flag_clear(rc, RRD_FLAG_QUEUED_FOR_POST_PROCESSING);
1347
}
1348
dfe_done(rc);
1349
dictionary_destroy(old);
@@ -2381,6 +2402,7 @@ static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker_jo
2402
2403
if(unlikely(rrdcontext_should_be_deleted(rc))) {
2404
if(worker_jobs) worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
2405
+ rrdcontext_dequeue_from_post_processing(rc);
2406
rrdcontext_delete_from_sql_unsafe(rc);
2407
2408
if(dictionary_del((DICTIONARY *)host->rrdctx, string2str(rc->id)) != 0)
@@ -2669,17 +2691,8 @@ static void rrdcontext_post_process_updates(RRDCONTEXT *rc, bool force, RRD_FLAG
2691
if(unlikely(rrd_flag_is_updated(rc) && rc->rrdhost->rrdctx_hub_queue)) {
2692
if(check_if_cloud_version_changed_unsafe(rc, false)) {
2693
rc->version = rrdcontext_get_next_version(rc);
2672
-
2673
- if(rrd_flag_check(rc, RRD_FLAG_QUEUED_FOR_HUB)) {
2674
- rc->queue.queued_ut = now_realtime_usec();
2675
- rc->queue.queued_flags |= rrd_flags_get(rc);
2676
- }
2677
- else {
2678
- rc->queue.queued_ut = now_realtime_usec();
2679
- rc->queue.queued_flags = rrd_flags_get(rc);
2680
- rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_HUB);
2681
- dictionary_set((DICTIONARY *)rc->rrdhost->rrdctx_hub_queue, string2str(rc->id), rc, sizeof(*rc));
2682
- }
2694
+ dictionary_set((DICTIONARY *)rc->rrdhost->rrdctx_hub_queue,
2695
+ string2str(rc->id), rc, sizeof(*rc));
2696
}
2697
}
2698
@@ -2691,8 +2704,10 @@ static void rrdcontext_queue_for_post_processing(RRDCONTEXT *rc, const char *fun
2704
if(unlikely(!rc->rrdhost->rrdctx_post_processing_queue)) return;
2705
2706
if(!rrd_flag_check(rc, RRD_FLAG_QUEUED_FOR_POST_PROCESSING)) {
2694
- rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_POST_PROCESSING);
2695
- dictionary_set((DICTIONARY *)rc->rrdhost->rrdctx_post_processing_queue, string2str(rc->id), rc, sizeof(*rc));
2707
+ dictionary_set((DICTIONARY *)rc->rrdhost->rrdctx_post_processing_queue,
2708
+ string2str(rc->id),
2709
+ rc,
2710
+ sizeof(*rc));
2711
2712
#ifdef NETDATA_INTERNAL_CHECKS
2713
{
@@ -2711,14 +2726,11 @@ static void rrdcontext_queue_for_post_processing(RRDCONTEXT *rc, const char *fun
2726
buffer_free(wb_flags);
2727
}
2728
#endif
2714
-
2729
}
2730
}
2731
2732
static void rrdcontext_dequeue_from_post_processing(RRDCONTEXT *rc) {
2733
if(unlikely(!rc->rrdhost->rrdctx_post_processing_queue)) return;
2720
-
2721
- rrd_flag_clear(rc, RRD_FLAG_QUEUED_FOR_POST_PROCESSING);
2734
dictionary_del((DICTIONARY *)rc->rrdhost->rrdctx_post_processing_queue, string2str(rc->id));
2735
}
2736
@@ -2884,6 +2896,10 @@ static inline usec_t rrdcontext_calculate_queued_dispatch_time_ut(RRDCONTEXT *rc
2896
return dispatch_ut;
2897
}
2898
2899
+static void rrdcontext_dequeue_from_hub_queue(RRDCONTEXT *rc) {
2900
+ dictionary_del((DICTIONARY *)rc->rrdhost->rrdctx_hub_queue, string2str(rc->id));
2901
+}
2902
+
2903
static void rrdcontext_dispatch_queued_contexts_to_hub(RRDHOST *host, usec_t now_ut) {
2904
2905
// check if we have received a streaming command for this host
@@ -2938,12 +2954,9 @@ static void rrdcontext_dispatch_queued_contexts_to_hub(RRDHOST *host, usec_t now
2954
else
2955
rc->version = rc->hub.version;
2956
2941
- // remove the queued flag, so that it can be queued again
2942
- rrd_flag_clear(rc, RRD_FLAG_QUEUED_FOR_HUB);
2943
-
2957
// remove it from the queue
2958
worker_is_busy(WORKER_JOB_DEQUEUE);
2946
- dictionary_del((DICTIONARY *)host->rrdctx_hub_queue, string2str(rc->id));
2959
+ rrdcontext_dequeue_from_hub_queue(rc);
2960
2961
if(unlikely(rrdcontext_should_be_deleted(rc))) {
2962
// this is a deleted context - delete it forever...
libnetdata/dictionary/dictionary.c
+6
-1
@@ -914,6 +914,10 @@ static NAME_VALUE *dictionary_set_name_value_unsafe(DICTIONARY *dict, const char
914
// so, either we will return the old one
915
// or overwrite the value, depending on dictionary flags
916
917
+ // We should not compare the values here!
918
+ // even if they are the same, we have to do the whole job
919
+ // so that the callbacks will be called.
920
+
921
nv = *pnv;
922
923
if(!(dict->flags & DICTIONARY_FLAG_DONT_OVERWRITE_VALUE)) {
@@ -927,7 +931,8 @@ static NAME_VALUE *dictionary_set_name_value_unsafe(DICTIONARY *dict, const char
931
}
932
933
else {
930
- // make sure this flag is not set
934
+ // we did really nothing!
935
+ // make sure this flag is not set.
936
nv->flags &= ~NAME_VALUE_FLAG_NEW_OR_UPDATED;
937
}
938
}