@cryptotaxi247 / netdata-1 / commits / 58c79fd32

Faster rrdcontext (#13629)

* moved rrdcontexts processing to worker thread * added loggings * check for aclk deeper in the code * removed unessesary logs * code re-organization; cleanup; more comments; better error handling; rrdcontext locks optimization; more clarity * updated 2 comments * make instances walkthrough reentrant; move context lock to the place is really needed * created macro for reentrant dictionary walkthrough * incremental updates on instances and metrics * renamed family of rrdcontext workers * prevent crash in case RRDINSTANCE or RRDMETRIC is freed during shutdown * prevent crash during rrddim save, on out of memory fatal() * always post-process contexts * added tracing for tracking the caller that trigger updates * more details on tracing info * fix for charts that are collected without metrics

Costa Tsaousis committed Sep 6, 2022 at 19:02 UTC 58c79fd329df7d2187e4aee56fb4a58a9c02c3ae
8 files changed +1096 -985
daemon/global_statistics.c
+1 -55
@@ -39,10 +39,6 @@ static struct global_statistics {
39 volatile uint64_t sqlite3_queries_failed_locked;
40 volatile uint64_t sqlite3_rows;
41
42 - volatile uint64_t rrdcontext_metric_triggers;
43 - volatile uint64_t rrdcontext_instance_triggers;
44 - volatile uint64_t rrdcontext_context_triggers;
45 -
42 } global_statistics = {
43 .connected_clients = 0,
44 .web_requests = 0,
@@ -85,18 +81,6 @@ void rrdr_query_completed(uint64_t db_points_read, uint64_t result_points_genera
81 __atomic_fetch_add(&global_statistics.rrdr_result_points_generated, result_points_generated, __ATOMIC_RELAXED);
82 }
83
88 -void rrdcontext_triggered_update_on_rrdmetric() {
89 - __atomic_fetch_add(&global_statistics.rrdcontext_metric_triggers, 1, __ATOMIC_RELAXED);
90 -}
91 -
92 -void rrdcontext_triggered_update_on_rrdinstance() {
93 - __atomic_fetch_add(&global_statistics.rrdcontext_instance_triggers, 1, __ATOMIC_RELAXED);
94 -}
95 -
96 -void rrdcontext_triggered_update_on_rrdcontext() {
97 - __atomic_fetch_add(&global_statistics.rrdcontext_context_triggers, 1, __ATOMIC_RELAXED);
98 -}
99 -
84 void finished_web_request_statistics(uint64_t dt,
85 uint64_t bytes_received,
86 uint64_t bytes_sent,
@@ -150,10 +134,6 @@ static inline void global_statistics_copy(struct global_statistics *gs, uint8_t
134 gs->sqlite3_queries_failed_busy = __atomic_load_n(&global_statistics.sqlite3_queries_failed_busy, __ATOMIC_RELAXED);
135 gs->sqlite3_queries_failed_locked = __atomic_load_n(&global_statistics.sqlite3_queries_failed_locked, __ATOMIC_RELAXED);
136 gs->sqlite3_rows = __atomic_load_n(&global_statistics.sqlite3_rows, __ATOMIC_RELAXED);
153 -
154 - gs->rrdcontext_metric_triggers = __atomic_load_n(&global_statistics.rrdcontext_metric_triggers, __ATOMIC_RELAXED);
155 - gs->rrdcontext_instance_triggers = __atomic_load_n(&global_statistics.rrdcontext_instance_triggers, __ATOMIC_RELAXED);
156 - gs->rrdcontext_context_triggers = __atomic_load_n(&global_statistics.rrdcontext_context_triggers, __ATOMIC_RELAXED);
137 }
138
139 static void global_statistics_charts(void) {
@@ -602,40 +582,6 @@ static void global_statistics_charts(void) {
582 }
583
584 // ----------------------------------------------------------------
605 -
606 - if(gs.rrdcontext_context_triggers || gs.rrdcontext_instance_triggers || gs.rrdcontext_metric_triggers) {
607 - static RRDSET *st_rrdcontext_triggers = NULL;
608 - static RRDDIM *rd_context = NULL, *rd_instance = NULL, *rd_metric = NULL;
609 -
610 - if (unlikely(!st_rrdcontext_triggers)) {
611 - st_rrdcontext_triggers = rrdset_create_localhost(
612 - "netdata"
613 - , "rrdcontext_triggers"
614 - , NULL
615 - , "rrdcontext"
616 - , NULL
617 - , "Netdata RRD context triggers"
618 - , "triggers/s"
619 - , "netdata"
620 - , "stats"
621 - , 131200
622 - , localhost->rrd_update_every
623 - , RRDSET_TYPE_LINE
624 - );
625 -
626 - rd_metric = rrddim_add(st_rrdcontext_triggers, "metric", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
627 - rd_instance = rrddim_add(st_rrdcontext_triggers, "instance", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
628 - rd_context = rrddim_add(st_rrdcontext_triggers, "context", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
629 - }
630 - else
631 - rrdset_next(st_rrdcontext_triggers);
632 -
633 - rrddim_set_by_pointer(st_rrdcontext_triggers, rd_metric, (collected_number)gs.rrdcontext_metric_triggers);
634 - rrddim_set_by_pointer(st_rrdcontext_triggers, rd_instance, (collected_number)gs.rrdcontext_instance_triggers);
635 - rrddim_set_by_pointer(st_rrdcontext_triggers, rd_context, (collected_number)gs.rrdcontext_context_triggers);
636 -
637 - rrdset_done(st_rrdcontext_triggers);
638 - }
585 }
586
587 static void dbengine_statistics_charts(void) {
@@ -1289,7 +1235,7 @@ static struct worker_utilization all_workers_utilization[] = {
1235 { .name = "TC", .family = "workers plugin tc", .priority = 1000000 },
1236 { .name = "TIMEX", .family = "workers plugin timex", .priority = 1000000 },
1237 { .name = "IDLEJITTER", .family = "workers plugin idlejitter", .priority = 1000000 },
1292 - { .name = "RRDCONTEXT", .family = "workers aclk contexts", .priority = 1000000 },
1238 + { .name = "RRDCONTEXT", .family = "workers contexts", .priority = 1000000 },
1239
1240 // has to be terminated with a NULL
1241 { .name = NULL, .family = NULL }
daemon/global_statistics.h
-4
@@ -12,10 +12,6 @@ extern void rrdr_query_completed(uint64_t db_points_read, uint64_t result_points
12 extern void sqlite3_query_completed(bool success, bool busy, bool locked);
13 extern void sqlite3_row_completed(void);
14
15 -extern void rrdcontext_triggered_update_on_rrdmetric(void);
16 -extern void rrdcontext_triggered_update_on_rrdinstance(void);
17 -extern void rrdcontext_triggered_update_on_rrdcontext(void);
18 -
15 extern void finished_web_request_statistics(uint64_t dt,
16 uint64_t bytes_received,
17 uint64_t bytes_sent,
database/rrd.h
+2 -1
@@ -916,7 +916,8 @@ struct rrdhost {
916
917 STORAGE_INSTANCE *storage_instance[RRD_STORAGE_TIERS]; // the database instances of the storage tiers
918
919 - RRDCONTEXTS *rrdctx_queue;
919 + RRDCONTEXTS *rrdctx_hub_queue;
920 + RRDCONTEXTS *rrdctx_post_processing_queue;
921 RRDCONTEXTS *rrdctx;
922
923 uuid_t host_uuid; // Global GUID for this host
database/rrdcontext.c
+1077 -919
@@ -10,10 +10,23 @@ int rrdcontext_enabled = CONFIG_BOOLEAN_YES;
10
11 #define MESSAGES_PER_BUNDLE_TO_SEND_TO_HUB_PER_HOST 5000
12 #define FULL_RETENTION_SCAN_DELAY_AFTER_DB_ROTATION_SECS 120
13 -#define RRDCONTEXT_WORKER_THREAD_HEARTBEAT_SECS 1
13 +#define RRDCONTEXT_WORKER_THREAD_HEARTBEAT_USEC (1000 * USEC_PER_MS)
14 #define RRDCONTEXT_MINIMUM_ALLOWED_PRIORITY 10
15
16 -// #define LOG_TRANSITIONS 1
16 +#define WORKER_JOB_HOSTS 1
17 +#define WORKER_JOB_CHECK 2
18 +#define WORKER_JOB_SEND 3
19 +#define WORKER_JOB_DEQUEUE 4
20 +#define WORKER_JOB_RETENTION 5
21 +#define WORKER_JOB_QUEUED 6
22 +#define WORKER_JOB_CLEANUP 7
23 +#define WORKER_JOB_CLEANUP_DELETE 8
24 +#define WORKER_JOB_PP_METRIC 9 // post-processing metrics
25 +#define WORKER_JOB_PP_INSTANCE 10 // post-processing instances
26 +#define WORKER_JOB_PP_CONTEXT 11 // post-processing contexts
27 +#define WORKER_JOB_HUB_QUEUE_SIZE 12
28 +#define WORKER_JOB_PP_QUEUE_SIZE 13
29 +
30
31 typedef enum {
32 RRD_FLAG_NONE = 0,
@@ -23,10 +36,11 @@ typedef enum {
36 RRD_FLAG_ARCHIVED = (1 << 3), // this object is not currently being collected
37 RRD_FLAG_OWN_LABELS = (1 << 4), // this instance has its own labels - not linked to an RRDSET
38 RRD_FLAG_LIVE_RETENTION = (1 << 5), // we have got live retention from the database
26 - RRD_FLAG_QUEUED = (1 << 6), // this context is currently queued to be dispatched to hub
27 - RRD_FLAG_DONT_PROCESS = (1 << 7), // don't process updates for this object
39 + RRD_FLAG_QUEUED_FOR_HUB = (1 << 6), // this context is currently queued to be dispatched to hub
40 + RRD_FLAG_QUEUED_FOR_POST_PROCESSING = (1 << 7), // this context is currently queued to be post-processed
41 RRD_FLAG_HIDDEN = (1 << 8), // don't expose this to the hub or the API
42
43 + RRD_FLAG_UPDATE_REASON_TRIGGERED = (1 << 9), // the update was triggered by the child object
44 RRD_FLAG_UPDATE_REASON_LOAD_SQL = (1 << 10), // this object has just been loaded from SQL
45 RRD_FLAG_UPDATE_REASON_NEW_OBJECT = (1 << 11), // this object has just been created
46 RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT = (1 << 12), // we received an update on this object
@@ -54,7 +68,8 @@ typedef enum {
68 } RRD_FLAGS;
69
70 #define RRD_FLAG_ALL_UPDATE_REASONS ( \
57 - RRD_FLAG_UPDATE_REASON_LOAD_SQL \
71 + RRD_FLAG_UPDATE_REASON_TRIGGERED \
72 + |RRD_FLAG_UPDATE_REASON_LOAD_SQL \
73 |RRD_FLAG_UPDATE_REASON_NEW_OBJECT \
74 |RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT \
75 |RRD_FLAG_UPDATE_REASON_CHANGED_LINKING \
@@ -79,51 +94,110 @@ typedef enum {
94
95 #define RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS ( \
96 RRD_FLAG_ARCHIVED \
82 - |RRD_FLAG_DONT_PROCESS \
97 |RRD_FLAG_HIDDEN \
98 |RRD_FLAG_ALL_UPDATE_REASONS \
99 )
100
101 +#define RRD_FLAGS_REQUIRED_FOR_DELETIONS ( \
102 + RRD_FLAG_DELETED \
103 + |RRD_FLAG_LIVE_RETENTION \
104 +)
105 +
106 #define RRD_FLAGS_PREVENTING_DELETIONS ( \
88 - RRD_FLAG_QUEUED \
107 + RRD_FLAG_QUEUED_FOR_HUB \
108 |RRD_FLAG_COLLECTED \
109 + \
110 + /* RRD_FLAG_QUEUED_FOR_POST_PROCESSING */ \
111 + /* should not be here or nothing will be deleted */ \
112 )
113
92 -#define rrd_flag_set_updated(obj, reason) (obj)->flags |= (RRD_FLAG_UPDATED | (reason))
93 -#define rrd_flag_unset_updated(obj) (obj)->flags &= ~(RRD_FLAG_UPDATED | RRD_FLAG_ALL_UPDATE_REASONS)
94 -
95 -#define rrd_flag_set_collected(obj) do { \
96 - if(likely( !((obj)->flags & RRD_FLAG_COLLECTED))) \
97 - (obj)->flags |= (RRD_FLAG_COLLECTED | RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED | RRD_FLAG_UPDATED); \
98 - if(likely( ((obj)->flags & (RRD_FLAG_ARCHIVED | RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED)))) \
99 - (obj)->flags &= ~(RRD_FLAG_ARCHIVED | RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED); \
100 - if(unlikely(((obj)->flags & (RRD_FLAG_DELETED | RRD_FLAG_UPDATE_REASON_ZERO_RETENTION)))) \
101 - (obj)->flags &= ~(RRD_FLAG_DELETED | RRD_FLAG_UPDATE_REASON_ZERO_RETENTION); \
102 - if(unlikely(((obj)->flags & RRD_FLAG_DONT_PROCESS))) \
103 - (obj)->flags &= ~RRD_FLAG_DONT_PROCESS; \
104 -} while(0)
105 -
106 -#define rrd_flag_set_archived(obj) do { \
107 - if(likely( !((obj)->flags & RRD_FLAG_ARCHIVED))) \
108 - (obj)->flags |= (RRD_FLAG_ARCHIVED | RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED | RRD_FLAG_UPDATED); \
109 - if(likely( ((obj)->flags & (RRD_FLAG_COLLECTED | RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED)))) \
110 - (obj)->flags &= ~(RRD_FLAG_COLLECTED | RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED); \
111 - if(unlikely(((obj)->flags & (RRD_FLAG_DELETED | RRD_FLAG_UPDATE_REASON_ZERO_RETENTION)))) \
112 - (obj)->flags &= ~(RRD_FLAG_DELETED | RRD_FLAG_UPDATE_REASON_ZERO_RETENTION); \
113 -} while(0)
114 -
115 -#define rrd_flag_set_deleted(obj, reason) do { \
116 - if(likely( !((obj)->flags & RRD_FLAG_DELETED))) \
117 - (obj)->flags |= (RRD_FLAG_DELETED | RRD_FLAG_UPDATE_REASON_ZERO_RETENTION | RRD_FLAG_UPDATED | (reason)); \
118 - if(unlikely(((obj)->flags & RRD_FLAG_ARCHIVED))) \
119 - (obj)->flags &= ~RRD_FLAG_ARCHIVED; \
120 - if(likely( ((obj)->flags & RRD_FLAG_COLLECTED))) \
121 - (obj)->flags &= ~RRD_FLAG_COLLECTED; \
122 -} while(0)
123 -
124 -
125 -#define rrd_flag_is_collected(obj) ((obj)->flags & RRD_FLAG_COLLECTED)
126 -#define rrd_flag_is_archived(obj) ((obj)->flags & RRD_FLAG_ARCHIVED)
114 +// get all the flags of an object
115 +#define rrd_flags_get(obj) __atomic_load_n(&((obj)->flags), __ATOMIC_SEQ_CST)
116 +
117 +// check if ANY of the given flags (bits) is set
118 +#define rrd_flag_check(obj, flag) (rrd_flags_get(obj) & (flag))
119 +
120 +// check if ALL of the given flags (bits) are set
121 +#define rrd_flag_check_all(obj, flag) (rrd_flag_check(obj, flag) == (flag))
122 +
123 +// set one or more flags (bits)
124 +#define rrd_flag_set(obj, flag) __atomic_or_fetch(&((obj)->flags), flag, __ATOMIC_SEQ_CST)
125 +
126 +// clear one or more flags (bits)
127 +#define rrd_flag_clear(obj, flag) __atomic_and_fetch(&((obj)->flags), ~(flag), __ATOMIC_SEQ_CST)
128 +
129 +// replace the flags of an object, with the supplied ones
130 +#define rrd_flags_replace(obj, all_flags) __atomic_store_n(&((obj)->flags), all_flags, __ATOMIC_SEQ_CST)
131 +
132 +static inline void
133 +rrd_flag_add_remove_atomic(RRD_FLAGS *flags, RRD_FLAGS check, RRD_FLAGS conditionally_add, RRD_FLAGS always_remove) {
134 + RRD_FLAGS expected, desired;
135 + do {
136 + expected = *flags;
137 +
138 + desired = expected;
139 + desired &= ~(always_remove);
140 +
141 + if(!(expected & check))
142 + desired |= (check | conditionally_add);
143 +
144 + } while(!__atomic_compare_exchange_n(flags, &expected, desired, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST));
145 +}
146 +
147 +#define rrd_flag_set_collected(obj) \
148 + rrd_flag_add_remove_atomic(&((obj)->flags) \
149 + /* check this flag */ \
150 + , RRD_FLAG_COLLECTED \
151 + \
152 + /* add these flags together with the above, if the above is not already set */ \
153 + , RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED | RRD_FLAG_UPDATED \
154 + \
155 + /* always remove these flags */ \
156 + , RRD_FLAG_ARCHIVED \
157 + | RRD_FLAG_DELETED \
158 + | RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED \
159 + | RRD_FLAG_UPDATE_REASON_ZERO_RETENTION \
160 + )
161 +
162 +#define rrd_flag_set_archived(obj) \
163 + rrd_flag_add_remove_atomic(&((obj)->flags) \
164 + /* check this flag */ \
165 + , RRD_FLAG_ARCHIVED \
166 + \
167 + /* add these flags together with the above, if the above is not already set */ \
168 + , RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED | RRD_FLAG_UPDATED \
169 + \
170 + /* always remove these flags */ \
171 + , RRD_FLAG_COLLECTED \
172 + | RRD_FLAG_DELETED \
173 + | RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED \
174 + | RRD_FLAG_UPDATE_REASON_ZERO_RETENTION \
175 + )
176 +
177 +#define rrd_flag_set_deleted(obj, reason) \
178 + rrd_flag_add_remove_atomic(&((obj)->flags) \
179 + /* check this flag */ \
180 + , RRD_FLAG_DELETED \
181 + \
182 + /* add these flags together with the above, if the above is not already set */ \
183 + , RRD_FLAG_UPDATE_REASON_ZERO_RETENTION | RRD_FLAG_UPDATED | (reason) \
184 + \
185 + /* always remove these flags */ \
186 + , RRD_FLAG_ARCHIVED \
187 + | RRD_FLAG_DELETED \
188 + )
189 +
190 +#define rrd_flag_is_collected(obj) rrd_flag_check(obj, RRD_FLAG_COLLECTED)
191 +#define rrd_flag_is_archived(obj) rrd_flag_check(obj, RRD_FLAG_ARCHIVED)
192 +#define rrd_flag_is_deleted(obj) rrd_flag_check(obj, RRD_FLAG_DELETED)
193 +#define rrd_flag_is_updated(obj) rrd_flag_check(obj, RRD_FLAG_UPDATED)
194 +
195 +// mark an object as updated, providing reasons (additional bits)
196 +#define rrd_flag_set_updated(obj, reason) rrd_flag_set(obj, RRD_FLAG_UPDATED | (reason))
197 +
198 +// clear an object as being updated, clearing also all the reasons
199 +#define rrd_flag_unset_updated(obj) rrd_flag_clear(obj, RRD_FLAG_UPDATED | RRD_FLAG_ALL_UPDATE_REASONS)
200 +
201
202 static struct rrdcontext_reason {
203 RRD_FLAGS flag;
@@ -131,6 +205,7 @@ static struct rrdcontext_reason {
205 usec_t delay_ut;
206 } rrdcontext_reasons[] = {
207 // context related
208 + { RRD_FLAG_UPDATE_REASON_TRIGGERED, "triggered transition", 60 * USEC_PER_SEC },
209 { RRD_FLAG_UPDATE_REASON_NEW_OBJECT, "object created", 60 * USEC_PER_SEC },
210 { RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT, "object updated", 60 * USEC_PER_SEC },
211 { RRD_FLAG_UPDATE_REASON_LOAD_SQL, "loaded from sql", 60 * USEC_PER_SEC },
@@ -173,8 +248,6 @@ typedef struct rrdmetric {
248 RRD_FLAGS flags;
249
250 struct rrdinstance *ri;
176 -
177 - usec_t created_ut; // the time this object was created
251 } RRDMETRIC;
252
253 typedef struct rrdinstance {
@@ -199,6 +272,12 @@ typedef struct rrdinstance {
272
273 struct rrdcontext *rc;
274 DICTIONARY *rrdmetrics;
275 +
276 + struct {
277 + uint32_t collected_metrics; // a temporary variable to detect BEGIN/END without SET
278 + // don't use it for other purposes
279 + // it goes up and then resets to zero, on every iteration
280 + } internal;
281 } RRDINSTANCE;
282
283 typedef struct rrdcontext {
@@ -287,31 +366,38 @@ static inline void rrdcontext_release(RRDCONTEXT_ACQUIRED *rca) {
366 dictionary_acquired_item_release((DICTIONARY *)rc->rrdhost->rrdctx, (DICTIONARY_ITEM *)rca);
367 }
368
290 -static void rrdcontext_recalculate_context_retention(RRDCONTEXT *rc, RRD_FLAGS reason, int job_id);
291 -static void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reason, int job_id);
369 +static void rrdcontext_recalculate_context_retention(RRDCONTEXT *rc, RRD_FLAGS reason, bool worker_jobs);
370 +static void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reason, bool worker_jobs);
371
372 #define rrdcontext_version_hash(host) rrdcontext_version_hash_with_callback(host, NULL, false, NULL)
373 static uint64_t rrdcontext_version_hash_with_callback(RRDHOST *host, void (*callback)(RRDCONTEXT *, bool, void *), bool snapshot, void *bundle);
374
296 -static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker);
297 -static void rrdcontext_garbage_collect(void);
375 +static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker_jobs);
376 +static void rrdcontext_garbage_collect_for_all_hosts(void);
377 void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc);
378
379 #define rrdcontext_lock(rc) netdata_mutex_lock(&((rc)->mutex))
380 #define rrdcontext_unlock(rc) netdata_mutex_unlock(&((rc)->mutex))
381
382 // ----------------------------------------------------------------------------
304 -// Updates triggers
383 +// Forward definitions
384 +
385 +static uint64_t rrdcontext_get_next_version(RRDCONTEXT *rc);
386 +static bool check_if_cloud_version_changed_unsafe(RRDCONTEXT *rc, bool sending __maybe_unused);
387 +static void rrdcontext_message_send_unsafe(RRDCONTEXT *rc, bool snapshot __maybe_unused, void *bundle __maybe_unused);
388
306 -static void rrdmetric_trigger_updates(RRDMETRIC *rm, bool force, bool escalate, const char *function);
307 -static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escalate, const char *function);
308 -static void rrdcontext_trigger_updates(RRDCONTEXT *rc, bool force, const char *function);
389 +static void rrdcontext_queue_for_post_processing(RRDCONTEXT *rc, const char *function, RRD_FLAGS flags);
390 +static void rrdcontext_post_process_updates(RRDCONTEXT *rc, bool force, RRD_FLAGS reason, bool worker_jobs);
391 +
392 +static void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function);
393 +static void rrdinstance_trigger_updates(RRDINSTANCE *ri, const char *function);
394 +static void rrdcontext_trigger_updates(RRDCONTEXT *rc, const char *function);
395
396 // ----------------------------------------------------------------------------
397 // visualizing flags
398
399 static void rrd_flags_to_buffer(RRD_FLAGS flags, BUFFER *wb) {
314 - if(flags & RRD_FLAG_QUEUED)
400 + if(flags & RRD_FLAG_QUEUED_FOR_HUB)
401 buffer_strcat(wb, "QUEUED ");
402
403 if(flags & RRD_FLAG_DELETED)
@@ -332,11 +418,11 @@ static void rrd_flags_to_buffer(RRD_FLAGS flags, BUFFER *wb) {
418 if(flags & RRD_FLAG_LIVE_RETENTION)
419 buffer_strcat(wb, "LIVE_RETENTION ");
420
335 - if(flags & RRD_FLAG_DONT_PROCESS)
336 - buffer_strcat(wb, "DONT_PROCESS ");
337 -
421 if(flags & RRD_FLAG_HIDDEN)
422 buffer_strcat(wb, "HIDDEN ");
423 +
424 + if(flags & RRD_FLAG_QUEUED_FOR_POST_PROCESSING)
425 + buffer_strcat(wb, "PENDING_UPDATES ");
426 }
427
428 static void rrd_reasons_to_buffer(RRD_FLAGS flags, BUFFER *wb) {
@@ -350,55 +436,11 @@ static void rrd_reasons_to_buffer(RRD_FLAGS flags, BUFFER *wb) {
436 }
437 }
438
353 -// ----------------------------------------------------------------------------
354 -// logging of all data collected
355 -
356 -#ifdef LOG_TRANSITIONS
357 -static void log_transition(RRDMETRIC *rm, RRDINSTANCE *ri, RRDCONTEXT *rc, const char *function) {
358 - BUFFER *wb = buffer_create(1000);
359 - const char *triggered_on = "triggered on ";
360 -
361 - buffer_sprintf(wb, "RRD TRANSITION: %s() ", function);
362 -
363 - if(rm) {
364 - buffer_sprintf(wb, "%smetric '%s' of ", triggered_on, string2str(rm->id));
365 - triggered_on = "";
366 - }
367 -
368 - if(ri) {
369 - buffer_sprintf(wb, "%sinstance '%s' of ", triggered_on, string2str(ri->id));
370 - triggered_on = "";
371 - }
372 -
373 - buffer_sprintf(wb, "%scontext '%s' ", triggered_on, string2str(rc->id));
374 -
375 - RRD_FLAGS flags = rc->flags;
376 - const char *we_are = "context";
377 - if(ri) {
378 - flags = ri->flags;
379 - we_are = "instance";
380 - }
381 - if(rm) {
382 - flags = rm->flags;
383 - we_are = "metric";
384 - }
385 -
386 - buffer_sprintf(wb, "%s flags: ", we_are);
387 - rrd_flags_to_buffer(flags, wb);
388 -
389 - buffer_strcat(wb, ", having reasons: ");
390 - rrd_reasons_to_buffer(flags, wb);
391 -
392 - internal_error(true, "%s", buffer_tostring(wb));
393 - buffer_free(wb);
394 -}
395 -#else
396 -#define log_transition(rm, ri, rc, function) debug_dummy()
397 -#endif
398 -
439 // ----------------------------------------------------------------------------
440 // RRDMETRIC
441
442 +// free the contents of RRDMETRIC.
443 +// RRDMETRIC itself is managed by DICTIONARY - no need to free it here.
444 static void rrdmetric_free(RRDMETRIC *rm) {
445 string_freez(rm->id);
446 string_freez(rm->name);
@@ -408,60 +450,8 @@ static void rrdmetric_free(RRDMETRIC *rm) {
450 rm->ri = NULL;
451 }
452
411 -static void rrdmetric_update_retention(RRDMETRIC *rm) {
412 - time_t min_first_time_t = LONG_MAX, max_last_time_t = 0;
413 -
414 - if(rm->rrddim) {
415 - min_first_time_t = rrddim_first_entry_t(rm->rrddim);
416 - max_last_time_t = rrddim_last_entry_t(rm->rrddim);
417 - }
418 -#ifdef ENABLE_DBENGINE
419 - else {
420 - RRDHOST *rrdhost = rm->ri->rc->rrdhost;
421 - for (int tier = 0; tier < storage_tiers; tier++) {
422 - if(!rrdhost->storage_instance[tier]) continue;
423 -
424 - time_t first_time_t, last_time_t;
425 - if (rrdeng_metric_retention_by_uuid(rrdhost->storage_instance[tier], &rm->uuid, &first_time_t, &last_time_t) == 0) {
426 - if (first_time_t < min_first_time_t)
427 - min_first_time_t = first_time_t;
428 -
429 - if (last_time_t > max_last_time_t)
430 - max_last_time_t = last_time_t;
431 - }
432 - }
433 - }
434 -#endif
435 -
436 - if(min_first_time_t == LONG_MAX)
437 - min_first_time_t = 0;
438 -
439 - if(min_first_time_t > max_last_time_t) {
440 - internal_error(true, "RRDMETRIC: retention of '%s' is flipped", string2str(rm->id));
441 - time_t tmp = min_first_time_t;
442 - min_first_time_t = max_last_time_t;
443 - max_last_time_t = tmp;
444 - }
445 -
446 - // check if retention changed
447 -
448 - if (min_first_time_t != rm->first_time_t) {
449 - rm->first_time_t = min_first_time_t;
450 - rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
451 - }
452 -
453 - if (max_last_time_t != rm->last_time_t) {
454 - rm->last_time_t = max_last_time_t;
455 - rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
456 - }
457 -
458 - if(unlikely(!rm->first_time_t && !rm->last_time_t))
459 - rrd_flag_set_deleted(rm, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
460 -
461 - rm->flags |= RRD_FLAG_LIVE_RETENTION;
462 -}
463 -
453 // called when this rrdmetric is inserted to the rrdmetrics dictionary of a rrdinstance
454 +// the constructor of the rrdmetric object
455 static void rrdmetric_insert_callback(const char *id __maybe_unused, void *value, void *data) {
456 RRDMETRIC *rm = value;
457
@@ -469,15 +459,14 @@ static void rrdmetric_insert_callback(const char *id __maybe_unused, void *value
459 rm->ri = data;
460
461 // remove flags that we need to figure out at runtime
472 - rm->flags = rm->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS;
473 -
474 - rm->created_ut = now_realtime_usec();
462 + rm->flags = rm->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS; // no need for atomics
463
464 // signal the react callback to do the job
465 rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_NEW_OBJECT);
466 }
467
468 // called when this rrdmetric is deleted from the rrdmetrics dictionary of a rrdinstance
469 +// the destructor of the rrdmetric object
470 static void rrdmetric_delete_callback(const char *id __maybe_unused, void *value, void *data __maybe_unused) {
471 RRDMETRIC *rm = value;
472
@@ -488,6 +477,7 @@ static void rrdmetric_delete_callback(const char *id __maybe_unused, void *value
477 }
478
479 // called when the same rrdmetric is inserted again to the rrdmetrics dictionary of a rrdinstance
480 +// while this is called, the dictionary is write locked, but there may be other users of the object
481 static void rrdmetric_conflict_callback(const char *id __maybe_unused, void *oldv, void *newv, void *data __maybe_unused) {
482 RRDMETRIC *rm = oldv;
483 RRDMETRIC *rm_new = newv;
@@ -537,26 +527,27 @@ static void rrdmetric_conflict_callback(const char *id __maybe_unused, void *old
527 rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
528 }
529
540 - rm->flags |= (rm_new->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS);
530 + rrd_flag_set(rm, rm_new->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS); // no needs for atomics on rm_new
531
532 if(rrd_flag_is_collected(rm) && rrd_flag_is_archived(rm))
533 rrd_flag_set_collected(rm);
534
545 - if(rm->flags & RRD_FLAG_UPDATED)
546 - rm->flags |= RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT;
535 + if(rrd_flag_check(rm, RRD_FLAG_UPDATED))
536 + rrd_flag_set(rm, RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT);
537
538 rrdmetric_free(rm_new);
539
540 // the react callback will continue from here
541 }
542
543 +// this is called after the insert or the conflict callbacks,
544 +// but the dictionary is now unlocked
545 static void rrdmetric_react_callback(const char *id __maybe_unused, void *value, void *data __maybe_unused) {
546 RRDMETRIC *rm = value;
555 -
556 - rrdmetric_trigger_updates(rm, false, true, __FUNCTION__);
547 + rrdmetric_trigger_updates(rm, __FUNCTION__ );
548 }
549
559 -static void rrdmetrics_create(RRDINSTANCE *ri) {
550 +static void rrdmetrics_create_in_rrdinstance(RRDINSTANCE *ri) {
551 if(unlikely(!ri)) return;
552 if(likely(ri->rrdmetrics)) return;
553
@@ -567,51 +558,26 @@ static void rrdmetrics_create(RRDINSTANCE *ri) {
558 dictionary_register_react_callback(ri->rrdmetrics, rrdmetric_react_callback, (void *)ri);
559 }
560
570 -static void rrdmetrics_destroy(RRDINSTANCE *ri) {
561 +static void rrdmetrics_destroy_from_rrdinstance(RRDINSTANCE *ri) {
562 if(unlikely(!ri || !ri->rrdmetrics)) return;
563 dictionary_destroy(ri->rrdmetrics);
564 ri->rrdmetrics = NULL;
565 }
566
576 -static inline bool rrdmetric_should_be_deleted(RRDMETRIC *rm) {
577 - if(likely(!(rm->flags & RRD_FLAG_DELETED)))
578 - return false;
579 -
580 - if(likely(!(rm->flags & RRD_FLAG_LIVE_RETENTION)))
581 - return false;
582 -
583 - if(unlikely(rm->flags & RRD_FLAGS_PREVENTING_DELETIONS))
584 - return false;
585 -
586 - if(likely(rm->rrddim))
587 - return false;
588 -
589 - //if((now_realtime_usec() - rm->created_ut) < 600 * USEC_PER_SEC)
590 - // return false;
591 -
592 - rrdmetric_update_retention(rm);
593 - if(rm->first_time_t || rm->last_time_t)
594 - return false;
595 -
596 - return true;
597 -}
598 -
599 -static void rrdmetric_trigger_updates(RRDMETRIC *rm, bool force, bool escalate, const char *function __maybe_unused) {
600 - if(likely(!force && !(rm->flags & RRD_FLAG_UPDATED))) return;
601 -
602 - // logs and statistics
603 - log_transition(rm, rm->ri, rm->ri->rc, function);
604 - rrdcontext_triggered_update_on_rrdmetric();
605 -
606 - if(unlikely(rrd_flag_is_collected(rm)) && (!rm->rrddim || rm->flags & RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD))
567 +// trigger post-processing of the rrdmetric, escalating changes to the rrdinstance it belongs
568 +static void rrdmetric_trigger_updates(RRDMETRIC *rm, const char *function) {
569 + if(unlikely(rrd_flag_is_collected(rm)) && (!rm->rrddim || rrd_flag_check(rm, RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD)))
570 rrd_flag_set_archived(rm);
571
609 - rrdmetric_update_retention(rm);
610 -
611 - if(unlikely(escalate && rm->flags & RRD_FLAG_UPDATED))
612 - rrdinstance_trigger_updates(rm->ri, true, true, __FUNCTION__);
572 + if(rrd_flag_is_updated(rm) || !rrd_flag_check(rm, RRD_FLAG_LIVE_RETENTION)) {
573 + rrd_flag_set_updated(rm->ri, RRD_FLAG_UPDATE_REASON_TRIGGERED);
574 + rrdcontext_queue_for_post_processing(rm->ri->rc, function, rm->flags);
575 + }
576 }
577
578 +// ----------------------------------------------------------------------------
579 +// RRDMETRIC HOOKS ON RRDDIM
580 +
581 static inline void rrdmetric_from_rrddim(RRDDIM *rd) {
582 if(unlikely(!rd->rrdset))
583 fatal("RRDMETRIC: rrddim '%s' does not have a rrdset.", rrddim_id(rd));
@@ -627,7 +593,7 @@ static inline void rrdmetric_from_rrddim(RRDDIM *rd) {
593 RRDMETRIC trm = {
594 .id = string_dup(rd->id),
595 .name = string_dup(rd->name),
630 - .flags = RRD_FLAG_NONE,
596 + .flags = RRD_FLAG_NONE, // no need for atomics
597 .rrddim = rd,
598 };
599 uuid_copy(trm.uuid, rd->metric_uuid);
@@ -648,6 +614,10 @@ static inline RRDMETRIC *rrddim_get_rrdmetric_with_trace(RRDDIM *rd, const char
614 }
615
616 RRDMETRIC *rm = rrdmetric_acquired_value(rd->rrdmetric);
617 + if(unlikely(!rm)) {
618 + error("RRDMETRIC: RRDDIM '%s' lost the link to its RRDMETRIC at %s()", rrddim_id(rd), function);
619 + return NULL;
620 + }
621
622 if(unlikely(rm->rrddim != rd))
623 fatal("RRDMETRIC: '%s' is not linked to RRDDIM '%s' at %s()", string2str(rm->id), rrddim_id(rd), function);
@@ -663,7 +633,7 @@ static inline void rrdmetric_rrddim_is_freed(RRDDIM *rd) {
633 rrd_flag_set_archived(rm);
634
635 rm->rrddim = NULL;
666 - rrdmetric_trigger_updates(rm, false, true, __FUNCTION__);
636 + rrdmetric_trigger_updates(rm, __FUNCTION__ );
637 rrdmetric_release(rd->rrdmetric);
638 rd->rrdmetric = NULL;
639 }
@@ -677,7 +647,7 @@ static inline void rrdmetric_updated_rrddim_flags(RRDDIM *rd) {
647 rrd_flag_set_archived(rm);
648 }
649
680 - rrdmetric_trigger_updates(rm, false, true, __FUNCTION__);
650 + rrdmetric_trigger_updates(rm, __FUNCTION__ );
651 }
652
653 static inline void rrdmetric_collected_rrddim(RRDDIM *rd) {
@@ -687,7 +657,10 @@ static inline void rrdmetric_collected_rrddim(RRDDIM *rd) {
657 if(unlikely(!rrd_flag_is_collected(rm)))
658 rrd_flag_set_collected(rm);
659
690 - rrdmetric_trigger_updates(rm, false, true, __FUNCTION__);
660 + // we use this variable to detect BEGIN/END without SET
661 + rm->ri->internal.collected_metrics++;
662 +
663 + rrdmetric_trigger_updates(rm, __FUNCTION__ );
664 }
665
666 // ----------------------------------------------------------------------------
@@ -695,10 +668,10 @@ static inline void rrdmetric_collected_rrddim(RRDDIM *rd) {
668
669 static void rrdinstance_free(RRDINSTANCE *ri) {
670
698 - if(ri->flags & RRD_FLAG_OWN_LABELS)
671 + if(rrd_flag_check(ri, RRD_FLAG_OWN_LABELS))
672 dictionary_destroy(ri->rrdlabels);
673
701 - rrdmetrics_destroy(ri);
674 + rrdmetrics_destroy_from_rrdinstance(ri);
675 string_freez(ri->id);
676 string_freez(ri->name);
677 string_freez(ri->title);
@@ -727,33 +700,32 @@ static void rrdinstance_insert_callback(const char *id __maybe_unused, void *val
700 // link it to its parent
701 ri->rc = data;
702
730 - ri->flags = ri->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS;
703 + ri->flags = ri->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS; // no need for atomics
704
705 if(!ri->name)
706 ri->name = string_dup(ri->id);
707
708 if(ri->rrdset && ri->rrdset->state) {
709 ri->rrdlabels = ri->rrdset->state->chart_labels;
737 - if(ri->flags & RRD_FLAG_OWN_LABELS)
738 - ri->flags &= ~RRD_FLAG_OWN_LABELS;
710 + ri->flags &= ~RRD_FLAG_OWN_LABELS; // no need of atomics at the constructor
711 }
712 else {
713 ri->rrdlabels = rrdlabels_create();
742 - ri->flags |= RRD_FLAG_OWN_LABELS;
714 + ri->flags |= RRD_FLAG_OWN_LABELS; // no need of atomics at the constructor
715 }
716
717 if(ri->rrdset) {
718 if(unlikely((rrdset_flag_check(ri->rrdset, RRDSET_FLAG_HIDDEN)) || (ri->rrdset->state && ri->rrdset->state->is_ar_chart)))
747 - ri->flags |= RRD_FLAG_HIDDEN;
719 + ri->flags |= RRD_FLAG_HIDDEN; // no need of atomics at the constructor
720 else
749 - ri->flags &= ~RRD_FLAG_HIDDEN;
721 + ri->flags &= ~RRD_FLAG_HIDDEN; // no need of atomics at the constructor
722 }
723
724 // we need this when loading from SQL
725 if(unlikely(ri->id == ml_anomaly_rates_id))
754 - ri->flags |= RRD_FLAG_HIDDEN;
726 + ri->flags |= RRD_FLAG_HIDDEN; // no need of atomics at the constructor
727
756 - rrdmetrics_create(ri);
728 + rrdmetrics_create_in_rrdinstance(ri);
729
730 // signal the react callback to do the job
731 rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_NEW_OBJECT);
@@ -840,32 +812,32 @@ static void rrdinstance_conflict_callback(const char *id __maybe_unused, void *o
812 if(ri->rrdset != ri_new->rrdset) {
813 ri->rrdset = ri_new->rrdset;
814
843 - if(ri->rrdset && (ri->flags & RRD_FLAG_OWN_LABELS)) {
815 + if(ri->rrdset && rrd_flag_check(ri, RRD_FLAG_OWN_LABELS)) {
816 DICTIONARY *old = ri->rrdlabels;
817 ri->rrdlabels = ri->rrdset->state->chart_labels;
846 - ri->flags &= ~RRD_FLAG_OWN_LABELS;
818 + rrd_flag_clear(ri, RRD_FLAG_OWN_LABELS);
819 rrdlabels_destroy(old);
820 }
849 - else if(!ri->rrdset && !(ri->flags & RRD_FLAG_OWN_LABELS)) {
821 + else if(!ri->rrdset && !rrd_flag_check(ri, RRD_FLAG_OWN_LABELS)) {
822 ri->rrdlabels = rrdlabels_create();
851 - ri->flags |= RRD_FLAG_OWN_LABELS;
823 + rrd_flag_set(ri, RRD_FLAG_OWN_LABELS);
824 }
825 }
826
827 if(ri->rrdset) {
828 if(unlikely((rrdset_flag_check(ri->rrdset, RRDSET_FLAG_HIDDEN)) || (ri->rrdset->state && ri->rrdset->state->is_ar_chart)))
857 - ri->flags |= RRD_FLAG_HIDDEN;
829 + rrd_flag_set(ri, RRD_FLAG_HIDDEN);
830 else
859 - ri->flags &= ~RRD_FLAG_HIDDEN;
831 + rrd_flag_clear(ri, RRD_FLAG_HIDDEN);
832 }
833
862 - ri->flags |= (ri_new->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS);
834 + rrd_flag_set(ri, ri_new->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS); // no need for atomics on ri_new
835
836 if(rrd_flag_is_collected(ri) && rrd_flag_is_archived(ri))
837 rrd_flag_set_collected(ri);
838
867 - if(ri->flags & RRD_FLAG_UPDATED)
868 - ri->flags |= RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT;
839 + if(rrd_flag_is_updated(ri))
840 + rrd_flag_set(ri, RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT);
841
842 // free the new one
843 rrdinstance_free(ri_new);
@@ -876,10 +848,10 @@ static void rrdinstance_conflict_callback(const char *id __maybe_unused, void *o
848 static void rrdinstance_react_callback(const char *id __maybe_unused, void *value, void *data __maybe_unused) {
849 RRDINSTANCE *ri = value;
850
879 - rrdinstance_trigger_updates(ri, false, true, __FUNCTION__);
851 + rrdinstance_trigger_updates(ri, __FUNCTION__ );
852 }
853
882 -void rrdinstances_create(RRDCONTEXT *rc) {
854 +void rrdinstances_create_in_rrdcontext(RRDCONTEXT *rc) {
855 if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
856 return;
857
@@ -892,156 +864,42 @@ void rrdinstances_create(RRDCONTEXT *rc) {
864 dictionary_register_react_callback(rc->rrdinstances, rrdinstance_react_callback, (void *)rc);
865 }
866
895 -void rrdinstances_destroy(RRDCONTEXT *rc) {
867 +void rrdinstances_destroy_from_rrdcontext(RRDCONTEXT *rc) {
868 if(unlikely(!rc || !rc->rrdinstances)) return;
869
870 dictionary_destroy(rc->rrdinstances);
871 rc->rrdinstances = NULL;
872 }
873
902 -static inline bool rrdinstance_should_be_deleted(RRDINSTANCE *ri) {
903 - if(likely(!(ri->flags & RRD_FLAG_DELETED)))
904 - return false;
905 -
906 - if(likely(!(ri->flags & RRD_FLAG_LIVE_RETENTION)))
907 - return false;
908 -
909 - if(unlikely(ri->flags & RRD_FLAGS_PREVENTING_DELETIONS))
910 - return false;
911 -
912 - if(likely(ri->rrdset))
913 - return false;
914 -
915 - if(unlikely(dictionary_stats_referenced_items(ri->rrdmetrics) != 0))
916 - return false;
917 -
918 - if(unlikely(dictionary_stats_entries(ri->rrdmetrics) != 0))
919 - return false;
920 -
921 - if(ri->first_time_t || ri->last_time_t)
922 - return false;
923 -
924 - return true;
925 -}
926 -
927 -static void rrdinstance_trigger_updates(RRDINSTANCE *ri, bool force, bool escalate, const char *function __maybe_unused) {
928 - if(unlikely(ri->flags & RRD_FLAG_DONT_PROCESS)) return;
929 - if(unlikely(!force && !(ri->flags & RRD_FLAG_UPDATED))) return;
930 -
931 - // logs and stats
932 - log_transition(NULL, ri, ri->rc, function);
933 - rrdcontext_triggered_update_on_rrdinstance();
874 +static void rrdinstance_trigger_updates(RRDINSTANCE *ri, const char *function) {
875 + RRDSET *st = ri->rrdset;
876
935 - if(likely(ri->rrdset)) {
936 - if(unlikely(ri->rrdset->priority != ri->priority)) {
937 - ri->priority = ri->rrdset->priority;
877 + if(likely(st)) {
878 + if(unlikely(st->priority != ri->priority)) {
879 + ri->priority = st->priority;
880 rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_PRIORITY);
881 }
940 - if(unlikely(ri->rrdset->update_every != ri->update_every)) {
941 - ri->update_every = ri->rrdset->update_every;
882 + if(unlikely(st->update_every != ri->update_every)) {
883 + ri->update_every = st->update_every;
884 rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_UPDATE_EVERY);
885 }
886 }
887 else if(unlikely(rrd_flag_is_collected(ri))) {
888 + // there is no rrdset, but we have it as collected!
889 +
890 rrd_flag_set_archived(ri);
891 rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LINKING);
892 }
893
950 - time_t min_first_time_t = LONG_MAX, max_last_time_t = 0;
951 - size_t metrics_active = 0, metrics_deleted = 0;
952 - bool live_retention = true, currently_collected = false;
953 - if(dictionary_stats_entries(ri->rrdmetrics) > 0) {
954 - RRDMETRIC *rm;
955 - dfe_start_read((DICTIONARY *)ri->rrdmetrics, rm) {
956 - if(unlikely(!(rm->flags & RRD_FLAG_LIVE_RETENTION)))
957 - live_retention = false;
958 -
959 - if (unlikely((rrdmetric_should_be_deleted(rm)))) {
960 - metrics_deleted++;
961 - rrd_flag_unset_updated(rm);
962 - continue;
963 - }
964 -
965 - if(rm->flags & RRD_FLAG_COLLECTED && rm->first_time_t)
966 - currently_collected = true;
967 -
968 - metrics_active++;
969 -
970 - if (rm->first_time_t && rm->first_time_t < min_first_time_t)
971 - min_first_time_t = rm->first_time_t;
972 -
973 - if (rm->last_time_t && rm->last_time_t > max_last_time_t)
974 - max_last_time_t = rm->last_time_t;
975 -
976 - rrd_flag_unset_updated(rm);
977 - }
978 - dfe_done(rm);
979 - }
980 -
981 - if(unlikely(live_retention && !(ri->flags & RRD_FLAG_LIVE_RETENTION)))
982 - ri->flags |= RRD_FLAG_LIVE_RETENTION;
983 - else if(unlikely(!live_retention && (ri->flags & RRD_FLAG_LIVE_RETENTION)))
984 - ri->flags &= ~RRD_FLAG_LIVE_RETENTION;
985 -
986 - if(unlikely(!metrics_active)) {
987 - // no metrics available
988 -
989 - if(ri->first_time_t) {
990 - ri->first_time_t = 0;
991 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
992 - }
993 -
994 - if(ri->last_time_t) {
995 - ri->last_time_t = 0;
996 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
997 - }
998 -
999 - rrd_flag_set_deleted(ri, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
1000 - }
1001 - else {
1002 - // we have active metrics...
1003 -
1004 - if (unlikely(min_first_time_t == LONG_MAX))
1005 - min_first_time_t = 0;
1006 -
1007 - if (unlikely(min_first_time_t == 0 || max_last_time_t == 0)) {
1008 - if(ri->first_time_t) {
1009 - ri->first_time_t = 0;
1010 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
1011 - }
1012 -
1013 - if(ri->last_time_t) {
1014 - ri->last_time_t = 0;
1015 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
1016 - }
1017 -
1018 - if(likely(live_retention))
1019 - rrd_flag_set_deleted(ri, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
1020 - }
1021 - else {
1022 - ri->flags &= ~RRD_FLAG_UPDATE_REASON_ZERO_RETENTION;
1023 -
1024 - if (unlikely(ri->first_time_t != min_first_time_t)) {
1025 - ri->first_time_t = min_first_time_t;
1026 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
1027 - }
1028 -
1029 - if (unlikely(ri->last_time_t != max_last_time_t)) {
1030 - ri->last_time_t = max_last_time_t;
1031 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
1032 - }
1033 -
1034 - if(likely(currently_collected))
1035 - rrd_flag_set_collected(ri);
1036 - else
1037 - rrd_flag_set_archived(ri);
1038 - }
894 + if(rrd_flag_is_updated(ri) || !rrd_flag_check(ri, RRD_FLAG_LIVE_RETENTION)) {
895 + rrd_flag_set_updated(ri->rc, RRD_FLAG_UPDATE_REASON_TRIGGERED);
896 + rrdcontext_queue_for_post_processing(ri->rc, function, ri->flags);
897 }
1040 -
1041 - if(unlikely(escalate && ri->flags & RRD_FLAG_UPDATED))
1042 - rrdcontext_trigger_updates(ri->rc, true, __FUNCTION__);
898 }
899
900 +// ----------------------------------------------------------------------------
901 +// RRDINSTANCE HOOKS ON RRDSET
902 +
903 static inline void rrdinstance_from_rrdset(RRDSET *st) {
904 RRDCONTEXT trc = {
905 .id = string_dup(st->context),
@@ -1050,7 +908,7 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) {
908 .family = string_dup(st->family),
909 .priority = st->priority,
910 .chart_type = st->chart_type,
1053 - .flags = RRD_FLAG_NONE,
911 + .flags = RRD_FLAG_NONE, // no need for atomics
912 .rrdhost = st->rrdhost,
913 };
914
@@ -1066,7 +924,7 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) {
924 .chart_type = st->chart_type,
925 .priority = st->priority,
926 .update_every = st->update_every,
1069 - .flags = RRD_FLAG_DONT_PROCESS,
927 + .flags = RRD_FLAG_NONE, // no need for atomics
928 .rrdset = st,
929 };
930 uuid_copy(tri.uuid, *st->chart_uuid);
@@ -1090,8 +948,9 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) {
948 }
949
950 if(rca_old && ria_old) {
1093 - // the chart changed context
1094 - RRDCONTEXT *rc_old = rrdcontext_acquired_value(rca_old);
951 + // Ooops! The chart changed context!
952 +
953 + // RRDCONTEXT *rc_old = rrdcontext_acquired_value(rca_old);
954 RRDINSTANCE *ri_old = rrdinstance_acquired_value(ria_old);
955
956 // migrate all dimensions to the new metrics
@@ -1101,7 +960,7 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) {
960 if (!rd->rrdmetric) continue;
961
962 RRDMETRIC *rm_old = rrdmetric_acquired_value(rd->rrdmetric);
1104 - rm_old->flags = RRD_FLAG_DELETED|RRD_FLAG_UPDATED|RRD_FLAG_LIVE_RETENTION|RRD_FLAG_UPDATE_REASON_UNUSED|RRD_FLAG_UPDATE_REASON_ZERO_RETENTION;
963 + rrd_flags_replace(rm_old, RRD_FLAG_DELETED|RRD_FLAG_UPDATED|RRD_FLAG_LIVE_RETENTION|RRD_FLAG_UPDATE_REASON_UNUSED|RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
964 rm_old->rrddim = NULL;
965 rm_old->first_time_t = 0;
966 rm_old->last_time_t = 0;
@@ -1114,20 +973,15 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) {
973 rrdset_unlock(st);
974
975 // mark the old instance, ready to be deleted
1117 - if(!(ri_old->flags & RRD_FLAG_OWN_LABELS))
976 + if(!rrd_flag_check(ri_old, RRD_FLAG_OWN_LABELS))
977 ri_old->rrdlabels = rrdlabels_create();
978
1120 - ri_old->flags = RRD_FLAG_OWN_LABELS|RRD_FLAG_DELETED|RRD_FLAG_UPDATED|RRD_FLAG_LIVE_RETENTION|RRD_FLAG_UPDATE_REASON_UNUSED|RRD_FLAG_UPDATE_REASON_ZERO_RETENTION;
979 + rrd_flags_replace(ri_old, RRD_FLAG_OWN_LABELS|RRD_FLAG_DELETED|RRD_FLAG_UPDATED|RRD_FLAG_LIVE_RETENTION|RRD_FLAG_UPDATE_REASON_UNUSED|RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
980 ri_old->rrdset = NULL;
981 ri_old->first_time_t = 0;
982 ri_old->last_time_t = 0;
983
1125 - ri_old->flags &= ~RRD_FLAG_DONT_PROCESS;
1126 - rc_old->flags &= ~RRD_FLAG_DONT_PROCESS;
1127 -
1128 - rrdinstance_trigger_updates(ri_old, true, true, __FUNCTION__);
1129 -
1130 - ri_old->flags |= RRD_FLAG_DONT_PROCESS;
984 + rrdinstance_trigger_updates(ri_old, __FUNCTION__ );
985 rrdinstance_release(ria_old);
986
987 /*
@@ -1138,10 +992,10 @@ static inline void rrdinstance_from_rrdset(RRDSET *st) {
992 rc_old->first_time_t = 0;
993 rc_old->last_time_t = 0;
994 rrdcontext_unlock(rc_old);
1141 - rrdcontext_trigger_updates(rc_old, true, __FUNCTION__);
995 + rrdcontext_trigger_updates(rc_old, __FUNCTION__ );
996 }
997 else
1144 - rrdcontext_trigger_updates(rc_old, true, __FUNCTION__);
998 + rrdcontext_trigger_updates(rc_old, __FUNCTION__ );
999 */
1000
1001 rrdcontext_release(rca_old);
@@ -1161,6 +1015,10 @@ static inline RRDINSTANCE *rrdset_get_rrdinstance_with_trace(RRDSET *st, const c
1015 }
1016
1017 RRDINSTANCE *ri = rrdinstance_acquired_value(st->rrdinstance);
1018 + if(unlikely(!ri)) {
1019 + error("RRDINSTANCE: RRDSET '%s' lost its link to an RRDINSTANCE at %s()", rrdset_id(st), function);
1020 + return NULL;
1021 + }
1022
1023 if(unlikely(ri->rrdset != st))
1024 fatal("RRDINSTANCE: '%s' is not linked to RRDSET '%s' at %s()", string2str(ri->id), rrdset_id(st), function);
@@ -1174,17 +1032,15 @@ static inline void rrdinstance_rrdset_is_freed(RRDSET *st) {
1032
1033 rrd_flag_set_archived(ri);
1034
1177 - if(!(ri->flags & RRD_FLAG_OWN_LABELS)) {
1178 - ri->flags |= RRD_FLAG_OWN_LABELS;
1035 + if(!rrd_flag_check(ri, RRD_FLAG_OWN_LABELS)) {
1036 ri->rrdlabels = rrdlabels_create();
1037 rrdlabels_copy(ri->rrdlabels, st->state->chart_labels);
1038 + rrd_flag_set(ri, RRD_FLAG_OWN_LABELS);
1039 }
1040
1041 ri->rrdset = NULL;
1042
1185 - ri->flags &= ~RRD_FLAG_DONT_PROCESS;
1186 - rrdinstance_trigger_updates(ri, false, true, __FUNCTION__);
1187 - ri->flags |= RRD_FLAG_DONT_PROCESS;
1043 + rrdinstance_trigger_updates(ri, __FUNCTION__ );
1044
1045 rrdinstance_release(st->rrdinstance);
1046 st->rrdinstance = NULL;
@@ -1206,7 +1062,7 @@ static inline void rrdinstance_updated_rrdset_name(RRDSET *st) {
1062 string_freez(old);
1063
1064 rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_NAME);
1209 - rrdinstance_trigger_updates(ri, false, true, __FUNCTION__);
1065 + rrdinstance_trigger_updates(ri, __FUNCTION__ );
1066 }
1067 }
1068
@@ -1215,13 +1071,17 @@ static inline void rrdinstance_updated_rrdset_flags_no_action(RRDINSTANCE *ri, R
1071 fatal("RRDCONTEXT: instance '%s' is not linked to chart '%s' on host '%s'",
1072 string2str(ri->id), rrdset_id(st), rrdhost_hostname(st->rrdhost));
1073
1218 - if(unlikely((rrdset_flag_check(st, RRDSET_FLAG_HIDDEN)) && !(ri->flags & RRD_FLAG_HIDDEN))) {
1219 - ri->flags |= RRD_FLAG_HIDDEN;
1220 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS);
1221 - }
1222 - else if(unlikely(!(rrdset_flag_check(st, RRDSET_FLAG_HIDDEN)) && (ri->flags & RRD_FLAG_HIDDEN))) {
1223 - ri->flags &= ~RRD_FLAG_HIDDEN;
1224 - rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS);
1074 + bool st_is_hidden = rrdset_flag_check(st, RRDSET_FLAG_HIDDEN);
1075 + bool ri_is_hidden = rrd_flag_check(ri, RRD_FLAG_HIDDEN);
1076 +
1077 + if(unlikely(st_is_hidden != ri_is_hidden)) {
1078 + if (unlikely(st_is_hidden && !ri_is_hidden))
1079 + rrd_flag_set_updated(ri, RRD_FLAG_HIDDEN | RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS);
1080 +
1081 + else if (unlikely(!st_is_hidden && ri_is_hidden)) {
1082 + rrd_flag_clear(ri, RRD_FLAG_HIDDEN);
1083 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS);
1084 + }
1085 }
1086 }
1087
@@ -1234,9 +1094,7 @@ static inline void rrdinstance_updated_rrdset_flags(RRDSET *st) {
1094
1095 rrdinstance_updated_rrdset_flags_no_action(ri, st);
1096
1237 - ri->flags &= ~RRD_FLAG_DONT_PROCESS;
1238 - rrdinstance_trigger_updates(ri, false, true, __FUNCTION__);
1239 - ri->flags |= RRD_FLAG_DONT_PROCESS;
1097 + rrdinstance_trigger_updates(ri, __FUNCTION__ );
1098 }
1099
1100 static inline void rrdinstance_collected_rrdset(RRDSET *st) {
@@ -1245,16 +1103,13 @@ static inline void rrdinstance_collected_rrdset(RRDSET *st) {
1103
1104 rrdinstance_updated_rrdset_flags_no_action(ri, st);
1105
1248 - if(dictionary_stats_entries(ri->rrdmetrics) > 0) {
1249 -
1250 - if(unlikely(!rrd_flag_is_collected(ri)))
1251 - rrd_flag_set_collected(ri);
1106 + if(unlikely(ri->internal.collected_metrics && !rrd_flag_is_collected(ri)))
1107 + rrd_flag_set_collected(ri);
1108
1253 - if(unlikely(ri->flags & RRD_FLAG_DONT_PROCESS))
1254 - ri->flags &= ~RRD_FLAG_DONT_PROCESS;
1109 + // we use this variable to detect BEGIN/END without SET
1110 + ri->internal.collected_metrics = 0;
1111
1256 - rrdinstance_trigger_updates(ri, false, true, __FUNCTION__);
1257 - }
1112 + rrdinstance_trigger_updates(ri, __FUNCTION__ );
1113 }
1114
1115 // ----------------------------------------------------------------------------
@@ -1267,131 +1122,13 @@ static void rrdcontext_freez(RRDCONTEXT *rc) {
1122 string_freez(rc->family);
1123 }
1124
1270 -static uint64_t rrdcontext_get_next_version(RRDCONTEXT *rc) {
1271 - time_t now = now_realtime_sec();
1272 - uint64_t version = MAX(rc->version, rc->hub.version);
1273 - version = MAX((uint64_t)now, version);
1274 - version++;
1275 - return version;
1276 -}
1125 +static void rrdcontext_insert_callback(const char *id, void *value, void *data) {
1126 + (void)id;
1127 + RRDHOST *host = (RRDHOST *)data;
1128 + RRDCONTEXT *rc = (RRDCONTEXT *)value;
1129
1278 -static void rrdcontext_message_send_unsafe(RRDCONTEXT *rc, bool snapshot __maybe_unused, void *bundle __maybe_unused) {
1279 -
1280 - // save it, so that we know the last version we sent to hub
1281 - rc->version = rc->hub.version = rrdcontext_get_next_version(rc);
1282 - rc->hub.id = string2str(rc->id);
1283 - rc->hub.title = string2str(rc->title);
1284 - rc->hub.units = string2str(rc->units);
1285 - rc->hub.family = string2str(rc->family);
1286 - rc->hub.chart_type = rrdset_type_name(rc->chart_type);
1287 - rc->hub.priority = rc->priority;
1288 - rc->hub.first_time_t = rc->first_time_t;
1289 - rc->hub.last_time_t = rrd_flag_is_collected(rc) ? 0 : rc->last_time_t;
1290 - rc->hub.deleted = (rc->flags & RRD_FLAG_DELETED) ? true : false;
1291 -
1292 -#ifdef ENABLE_ACLK
1293 - struct context_updated message = {
1294 - .id = rc->hub.id,
1295 - .version = rc->hub.version,
1296 - .title = rc->hub.title,
1297 - .units = rc->hub.units,
1298 - .family = rc->hub.family,
1299 - .chart_type = rc->hub.chart_type,
1300 - .priority = rc->hub.priority,
1301 - .first_entry = rc->hub.first_time_t,
1302 - .last_entry = rc->hub.last_time_t,
1303 - .deleted = rc->hub.deleted,
1304 - };
1305 -
1306 - if(likely(!(rc->flags & RRD_FLAG_HIDDEN))) {
1307 - if (snapshot) {
1308 - if (!rc->hub.deleted)
1309 - contexts_snapshot_add_ctx_update(bundle, &message);
1310 - }
1311 - else
1312 - contexts_updated_add_ctx_update(bundle, &message);
1313 - }
1314 -#endif
1315 -
1316 - // store it to SQL
1317 -
1318 - if(rc->flags & RRD_FLAG_DELETED) {
1319 - rrdcontext_delete_from_sql_unsafe(rc);
1320 - }
1321 - else {
1322 - if (ctx_store_context(&rc->rrdhost->host_uuid, &rc->hub) != 0)
1323 - error("RRDCONTEXT: failed to save context '%s' version %"PRIu64" to SQL.", rc->hub.id, rc->hub.version);
1324 - }
1325 -}
1326 -
1327 -static bool check_if_cloud_version_changed_unsafe(RRDCONTEXT *rc, bool sending __maybe_unused) {
1328 - bool id_changed = false,
1329 - title_changed = false,
1330 - units_changed = false,
1331 - family_changed = false,
1332 - chart_type_changed = false,
1333 - priority_changed = false,
1334 - first_time_changed = false,
1335 - last_time_changed = false,
1336 - deleted_changed = false;
1337 -
1338 - if(unlikely(string2str(rc->id) != rc->hub.id))
1339 - id_changed = true;
1340 -
1341 - if(unlikely(string2str(rc->title) != rc->hub.title))
1342 - title_changed = true;
1343 -
1344 - if(unlikely(string2str(rc->units) != rc->hub.units))
1345 - units_changed = true;
1346 -
1347 - if(unlikely(string2str(rc->family) != rc->hub.family))
1348 - family_changed = true;
1349 -
1350 - if(unlikely(rrdset_type_name(rc->chart_type) != rc->hub.chart_type))
1351 - chart_type_changed = true;
1352 -
1353 - if(unlikely(rc->priority != rc->hub.priority))
1354 - priority_changed = true;
1355 -
1356 - if(unlikely((uint64_t)rc->first_time_t != rc->hub.first_time_t))
1357 - first_time_changed = true;
1358 -
1359 - if(unlikely((uint64_t)(rrd_flag_is_collected(rc) ? 0 : rc->last_time_t) != rc->hub.last_time_t))
1360 - last_time_changed = true;
1361 -
1362 - if(unlikely(((rc->flags & RRD_FLAG_DELETED) ? true : false) != rc->hub.deleted))
1363 - deleted_changed = true;
1364 -
1365 - if(unlikely(id_changed || title_changed || units_changed || family_changed || chart_type_changed || priority_changed || first_time_changed || last_time_changed || deleted_changed)) {
1366 -
1367 - internal_error(true, "RRDCONTEXT: %s NEW VERSION '%s'%s, version %"PRIu64", title '%s'%s, units '%s'%s, family '%s'%s, chart type '%s'%s, priority %u%s, first_time_t %ld%s, last_time_t %ld%s, deleted '%s'%s, (queued for %llu ms, expected %llu ms)",
1368 - sending?"SENDING":"QUEUE",
1369 - string2str(rc->id), id_changed ? " (CHANGED)" : "",
1370 - rc->version,
1371 - string2str(rc->title), title_changed ? " (CHANGED)" : "",
1372 - string2str(rc->units), units_changed ? " (CHANGED)" : "",
1373 - string2str(rc->family), family_changed ? " (CHANGED)" : "",
1374 - rrdset_type_name(rc->chart_type), chart_type_changed ? " (CHANGED)" : "",
1375 - rc->priority, priority_changed ? " (CHANGED)" : "",
1376 - rc->first_time_t, first_time_changed ? " (CHANGED)" : "",
1377 - rrd_flag_is_collected(rc) ? 0 : rc->last_time_t, last_time_changed ? " (CHANGED)" : "",
1378 - (rc->flags & RRD_FLAG_DELETED) ? "true" : "false", deleted_changed ? " (CHANGED)" : "",
1379 - sending ? (now_realtime_usec() - rc->queue.queued_ut) / USEC_PER_MS : 0,
1380 - sending ? (rc->queue.scheduled_dispatch_ut - rc->queue.queued_ut) / USEC_PER_SEC : 0
1381 - );
1382 - return true;
1383 - }
1384 -
1385 - return false;
1386 -}
1387 -
1388 -static void rrdcontext_insert_callback(const char *id, void *value, void *data) {
1389 - (void)id;
1390 - RRDHOST *host = (RRDHOST *)data;
1391 - RRDCONTEXT *rc = (RRDCONTEXT *)value;
1392 -
1393 - rc->rrdhost = host;
1394 - rc->flags = rc->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS;
1130 + rc->rrdhost = host;
1131 + rc->flags = rc->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS; // no need for atomics at constructor
1132
1133 if(rc->hub.version) {
1134 // we are loading data from the SQL database
@@ -1428,7 +1165,7 @@ static void rrdcontext_insert_callback(const char *id, void *value, void *data)
1165 rc->last_time_t = rc->hub.last_time_t;
1166
1167 if(rc->hub.deleted || !rc->hub.first_time_t)
1431 - rrd_flag_set_deleted(rc, 0);
1168 + rrd_flag_set_deleted(rc, RRD_FLAG_NONE);
1169 else {
1170 if (rc->last_time_t == 0)
1171 rrd_flag_set_collected(rc);
@@ -1436,14 +1173,14 @@ static void rrdcontext_insert_callback(const char *id, void *value, void *data)
1173 rrd_flag_set_archived(rc);
1174 }
1175
1439 - rc->flags |= RRD_FLAG_UPDATE_REASON_LOAD_SQL;
1176 + rc->flags |= RRD_FLAG_UPDATE_REASON_LOAD_SQL; // no need for atomics at constructor
1177 }
1178 else {
1179 // we are adding this context now for the first time
1180 rc->version = now_realtime_sec();
1181 }
1182
1446 - rrdinstances_create(rc);
1183 + rrdinstances_create_in_rrdcontext(rc);
1184 netdata_mutex_init(&rc->mutex);
1185
1186 // signal the react callback to do the job
@@ -1457,7 +1194,7 @@ static void rrdcontext_delete_callback(const char *id, void *value, void *data)
1194
1195 RRDCONTEXT *rc = (RRDCONTEXT *)value;
1196
1460 - rrdinstances_destroy(rc);
1197 + rrdinstances_destroy_from_rrdcontext(rc);
1198 netdata_mutex_destroy(&rc->mutex);
1199 rrdcontext_freez(rc);
1200 }
@@ -1515,13 +1252,13 @@ static void rrdcontext_conflict_callback(const char *id, void *oldv, void *newv,
1252 rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_PRIORITY);
1253 }
1254
1518 - rc->flags |= (rc_new->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS);
1255 + rrd_flag_set(rc, rc_new->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS); // no need for atomics on rc_new
1256
1257 if(rrd_flag_is_collected(rc) && rrd_flag_is_archived(rc))
1258 rrd_flag_set_collected(rc);
1259
1523 - if(rc->flags & RRD_FLAG_UPDATED)
1524 - rc->flags |= RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT;
1260 + if(rrd_flag_is_updated(rc))
1261 + rrd_flag_set(rc, RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT);
1262
1263 rrdcontext_unlock(rc);
1264
@@ -1534,7 +1271,12 @@ static void rrdcontext_conflict_callback(const char *id, void *oldv, void *newv,
1271 static void rrdcontext_react_callback(const char *id __maybe_unused, void *value, void *data __maybe_unused) {
1272 RRDCONTEXT *rc = (RRDCONTEXT *)value;
1273
1537 - rrdcontext_trigger_updates(rc, false, __FUNCTION__);
1274 + rrdcontext_trigger_updates(rc, __FUNCTION__ );
1275 +}
1276 +
1277 +static void rrdcontext_trigger_updates(RRDCONTEXT *rc, const char *function) {
1278 + if(rrd_flag_is_updated(rc) || !rrd_flag_check(rc, RRD_FLAG_LIVE_RETENTION))
1279 + rrdcontext_queue_for_post_processing(rc, function, rc->flags);
1280 }
1281
1282 void rrdhost_create_rrdcontexts(RRDHOST *host) {
@@ -1550,187 +1292,45 @@ void rrdhost_create_rrdcontexts(RRDHOST *host) {
1292 dictionary_register_conflict_callback((DICTIONARY *)host->rrdctx, rrdcontext_conflict_callback, (void *)host);
1293 dictionary_register_react_callback((DICTIONARY *)host->rrdctx, rrdcontext_react_callback, (void *)host);
1294
1553 - host->rrdctx_queue = (RRDCONTEXTS *)dictionary_create(DICTIONARY_FLAG_DONT_OVERWRITE_VALUE | DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE);
1295 + host->rrdctx_hub_queue = (RRDCONTEXTS *)dictionary_create(DICTIONARY_FLAG_DONT_OVERWRITE_VALUE | DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE);
1296 + host->rrdctx_post_processing_queue = (RRDCONTEXTS *)dictionary_create(DICTIONARY_FLAG_DONT_OVERWRITE_VALUE | DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE);
1297 }
1298
1299 void rrdhost_destroy_rrdcontexts(RRDHOST *host) {
1300 if(unlikely(!host)) return;
1301 if(unlikely(!host->rrdctx)) return;
1302
1560 - if(host->rrdctx_queue) {
1561 - dictionary_destroy((DICTIONARY *)host->rrdctx_queue);
1562 - host->rrdctx_queue = NULL;
1563 - }
1564 -
1565 - dictionary_destroy((DICTIONARY *)host->rrdctx);
1566 - host->rrdctx = NULL;
1567 -}
1568 -
1569 -static inline bool rrdcontext_should_be_deleted(RRDCONTEXT *rc) {
1570 - if(likely(!(rc->flags & RRD_FLAG_DELETED)))
1571 - return false;
1572 -
1573 - if(likely(!(rc->flags & RRD_FLAG_LIVE_RETENTION)))
1574 - return false;
1575 -
1576 - if(unlikely(rc->flags & RRD_FLAGS_PREVENTING_DELETIONS))
1577 - return false;
1578 -
1579 - if(unlikely(dictionary_stats_referenced_items(rc->rrdinstances) != 0))
1580 - return false;
1581 -
1582 - if(unlikely(dictionary_stats_entries(rc->rrdinstances) != 0))
1583 - return false;
1584 -
1585 - if(unlikely(rc->first_time_t || rc->last_time_t))
1586 - return false;
1587 -
1588 - return true;
1589 -}
1590 -
1591 -static void rrdcontext_trigger_updates(RRDCONTEXT *rc, bool force, const char *function __maybe_unused) {
1592 - if(unlikely(rc->flags & RRD_FLAG_DONT_PROCESS)) return;
1593 - if(unlikely(!force && !(rc->flags & RRD_FLAG_UPDATED))) return;
1594 -
1595 - // logs and stats
1596 - log_transition(NULL, NULL, rc, function);
1597 - rrdcontext_triggered_update_on_rrdcontext();
1598 -
1599 - rrdcontext_lock(rc);
1600 -
1601 - size_t min_priority = LONG_MAX;
1602 - time_t min_first_time_t = LONG_MAX, max_last_time_t = 0;
1603 - size_t instances_active = 0, instances_deleted = 0;
1604 - bool live_retention = true, currently_collected = false, hidden = true;
1605 - if(dictionary_stats_entries(rc->rrdinstances) > 0) {
1606 - RRDINSTANCE *ri;
1607 - dfe_start_read(rc->rrdinstances, ri) {
1608 - if(likely(!(ri->flags & RRD_FLAG_HIDDEN)))
1609 - hidden = false;
1610 -
1611 - if(!(ri->flags & RRD_FLAG_LIVE_RETENTION))
1612 - live_retention = false;
1613 -
1614 - if (unlikely(rrdinstance_should_be_deleted(ri))) {
1615 - instances_deleted++;
1616 - rrd_flag_unset_updated(ri);
1617 - continue;
1618 - }
1619 -
1620 - if(ri->flags & RRD_FLAG_COLLECTED && ri->first_time_t)
1621 - currently_collected = true;
1622 -
1623 - internal_error(rc->units != ri->units,
1624 - "RRDCONTEXT: '%s' rrdinstance '%s' has different units, context '%s', instance '%s'",
1625 - string2str(rc->id), string2str(ri->id),
1626 - string2str(rc->units), string2str(ri->units));
1627 -
1628 - instances_active++;
1629 -
1630 - if (ri->priority >= RRDCONTEXT_MINIMUM_ALLOWED_PRIORITY && ri->priority < min_priority)
1631 - min_priority = ri->priority;
1632 -
1633 - if (ri->first_time_t && ri->first_time_t < min_first_time_t)
1634 - min_first_time_t = ri->first_time_t;
1635 -
1636 - if (ri->last_time_t && ri->last_time_t > max_last_time_t)
1637 - max_last_time_t = ri->last_time_t;
1638 -
1639 - rrd_flag_unset_updated(ri);
1640 - }
1641 - dfe_done(ri);
1642 - }
1643 -
1644 - if(hidden && !(rc->flags & RRD_FLAG_HIDDEN))
1645 - rc->flags |= RRD_FLAG_HIDDEN;
1646 - else if(!hidden && (rc->flags & RRD_FLAG_HIDDEN))
1647 - rc->flags &= ~RRD_FLAG_HIDDEN;
1648 -
1649 - if(live_retention && !(rc->flags & RRD_FLAG_LIVE_RETENTION))
1650 - rc->flags |= RRD_FLAG_LIVE_RETENTION;
1651 - else if(!live_retention && (rc->flags & RRD_FLAG_LIVE_RETENTION))
1652 - rc->flags &= ~RRD_FLAG_LIVE_RETENTION;
1653 -
1654 - if(unlikely(!instances_active)) {
1655 - // we had some instances, but they are gone now...
1656 -
1657 - if(rc->first_time_t) {
1658 - rc->first_time_t = 0;
1659 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
1660 - }
1661 -
1662 - if(rc->last_time_t) {
1663 - rc->last_time_t = 0;
1664 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
1665 - }
1666 -
1667 - rrd_flag_set_deleted(rc, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
1668 - }
1669 - else {
1670 - // we have some active instances...
1671 -
1672 - if (unlikely(min_first_time_t == LONG_MAX))
1673 - min_first_time_t = 0;
1674 -
1675 - if (unlikely(min_first_time_t == 0 && max_last_time_t == 0)) {
1676 - if(rc->first_time_t) {
1677 - rc->first_time_t = 0;
1678 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
1679 - }
1680 -
1681 - if(rc->last_time_t) {
1682 - rc->last_time_t = 0;
1683 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
1684 - }
1685 -
1686 - rrd_flag_set_deleted(rc, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
1687 - }
1688 - else {
1689 - rc->flags &= ~RRD_FLAG_UPDATE_REASON_ZERO_RETENTION;
1690 -
1691 - if (unlikely(rc->first_time_t != min_first_time_t)) {
1692 - rc->first_time_t = min_first_time_t;
1693 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
1694 - }
1695 -
1696 - if (rc->last_time_t != max_last_time_t) {
1697 - rc->last_time_t = max_last_time_t;
1698 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
1699 - }
1303 + DICTIONARY *old;
1304
1701 - if(likely(currently_collected))
1702 - rrd_flag_set_collected(rc);
1703 - else
1704 - rrd_flag_set_archived(rc);
1705 - }
1305 + if(host->rrdctx_hub_queue) {
1306 + old = (DICTIONARY *)host->rrdctx_hub_queue;
1307 + host->rrdctx_hub_queue = NULL;
1308
1707 - if (min_priority != LONG_MAX && rc->priority != min_priority) {
1708 - rc->priority = min_priority;
1709 - rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_PRIORITY);
1309 + RRDCONTEXT *rc;
1310 + dfe_start_write(old, rc) {
1311 + dictionary_del_having_write_lock(old, string2str(rc->id));
1312 + rrdset_flag_clear(rc, RRD_FLAG_QUEUED_FOR_HUB);
1313 }
1314 + dfe_done(rc);
1315 + dictionary_destroy(old);
1316 }
1317
1713 - if(unlikely(rc->flags & RRD_FLAG_UPDATED)) {
1714 - if(check_if_cloud_version_changed_unsafe(rc, false)) {
1715 - rc->version = rrdcontext_get_next_version(rc);
1716 -
1717 - if(rc->flags & RRD_FLAG_QUEUED) {
1718 - rc->queue.queued_ut = now_realtime_usec();
1719 - rc->queue.queued_flags |= rc->flags;
1720 - }
1721 - else {
1722 - rc->queue.queued_ut = now_realtime_usec();
1723 - rc->queue.queued_flags = rc->flags;
1318 + if(host->rrdctx_post_processing_queue) {
1319 + old = (DICTIONARY *)host->rrdctx_post_processing_queue;
1320 + host->rrdctx_post_processing_queue = NULL;
1321
1725 - rc->flags |= RRD_FLAG_QUEUED;
1726 - dictionary_set((DICTIONARY *)rc->rrdhost->rrdctx_queue, string2str(rc->id), rc, sizeof(*rc));
1727 - }
1322 + RRDCONTEXT *rc;
1323 + dfe_start_write(old, rc) {
1324 + dictionary_del_having_write_lock(old, string2str(rc->id));
1325 + rrdset_flag_clear(rc, RRD_FLAG_QUEUED_FOR_POST_PROCESSING);
1326 }
1729 -
1730 - rrd_flag_unset_updated(rc);
1327 + dfe_done(rc);
1328 + dictionary_destroy(old);
1329 }
1330
1733 - rrdcontext_unlock(rc);
1331 + old = (DICTIONARY *)host->rrdctx;
1332 + host->rrdctx = NULL;
1333 + dictionary_destroy(old);
1334 }
1335
1336 // ----------------------------------------------------------------------------
@@ -1834,7 +1434,13 @@ void rrdcontext_host_child_disconnected(RRDHOST *host) {
1434 if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
1435 return;
1436
1837 - rrdcontext_recalculate_host_retention(host, RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD, -1);
1437 + rrdcontext_recalculate_host_retention(host, RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD, false);
1438 +}
1439 +
1440 +static usec_t rrdcontext_next_db_rotation_ut = 0;
1441 +void rrdcontext_db_rotation(void) {
1442 + // called when the db rotates its database
1443 + rrdcontext_next_db_rotation_ut = now_realtime_usec() + FULL_RETENTION_SCAN_DELAY_AFTER_DB_ROTATION_SECS * USEC_PER_SEC;
1444 }
1445
1446 // ----------------------------------------------------------------------------
@@ -1905,7 +1511,7 @@ void rrdcontext_hub_checkpoint_command(void *ptr) {
1511 contexts_snapshot_t bundle = contexts_snapshot_new(cmd->claim_id, uuid, our_version_hash);
1512
1513 // do a deep scan on every metric of the host to make sure all our data are updated
1908 - rrdcontext_recalculate_host_retention(host, RRD_FLAG_NONE, -1);
1514 + rrdcontext_recalculate_host_retention(host, RRD_FLAG_NONE, false);
1515
1516 // calculate version hash and pack all the messages together in one go
1517 our_version_hash = rrdcontext_version_hash_with_callback(host, rrdcontext_message_send_unsafe, true, bundle);
@@ -1982,7 +1588,7 @@ static inline int rrdmetric_to_json_callback(const char *id, void *value, void *
1588 time_t after = t->after;
1589 time_t before = t->before;
1590
1985 - if((rm->flags & RRD_FLAG_DELETED) && !(options & RRDCONTEXT_OPTION_SHOW_DELETED))
1591 + if(unlikely(rrd_flag_is_deleted(rm) && !(options & RRDCONTEXT_OPTION_SHOW_DELETED)))
1592 return 0;
1593
1594 if(after && (!rm->last_time_t || after > rm->last_time_t))
@@ -2000,13 +1606,13 @@ static inline int rrdmetric_to_json_callback(const char *id, void *value, void *
1606 buffer_strcat(wb, ",\n");
1607 t->combined_first_time_t = MIN(t->combined_first_time_t, rm->first_time_t);
1608 t->combined_last_time_t = MAX(t->combined_last_time_t, rm->last_time_t);
2003 - t->combined_flags |= rm->flags;
1609 + t->combined_flags |= rrd_flags_get(rm);
1610 }
1611 else {
1612 buffer_strcat(wb, "\n");
1613 t->combined_first_time_t = rm->first_time_t;
1614 t->combined_last_time_t = rm->last_time_t;
2009 - t->combined_flags = rm->flags;
1615 + t->combined_flags = rrd_flags_get(rm);
1616 }
1617
1618 buffer_sprintf(wb, "\t\t\t\t\t\t\"%s\": {", id);
@@ -2025,19 +1631,19 @@ static inline int rrdmetric_to_json_callback(const char *id, void *value, void *
1631 , string2str(rm->name)
1632 , rm->first_time_t
1633 , rrd_flag_is_collected(rm) ? t->now : rm->last_time_t
2028 - , rm->flags & RRD_FLAG_COLLECTED ? "true" : "false"
1634 + , rrd_flag_is_collected(rm) ? "true" : "false"
1635 );
1636
1637 if(options & RRDCONTEXT_OPTION_SHOW_DELETED) {
1638 buffer_sprintf(wb,
1639 ",\n\t\t\t\t\t\t\t\"deleted\":%s"
2034 - , rm->flags & RRD_FLAG_DELETED ? "true" : "false"
1640 + , rrd_flag_is_deleted(rm) ? "true" : "false"
1641 );
1642 }
1643
1644 if(options & RRDCONTEXT_OPTION_SHOW_FLAGS) {
1645 buffer_strcat(wb, ",\n\t\t\t\t\t\t\t\"flags\":\"");
2040 - rrd_flags_to_buffer(rm->flags, wb);
1646 + rrd_flags_to_buffer(rrd_flags_get(rm), wb);
1647 buffer_strcat(wb, "\"");
1648 }
1649
@@ -2055,7 +1661,7 @@ static inline int rrdinstance_to_json_callback(const char *id, void *value, void
1661 time_t before = t_parent->before;
1662 bool has_filter = t_parent->chart_label_key || t_parent->chart_labels_filter || t_parent->chart_dimensions;
1663
2058 - if((ri->flags & RRD_FLAG_DELETED) && !(options & RRDCONTEXT_OPTION_SHOW_DELETED))
1664 + if(unlikely(rrd_flag_is_deleted(ri) && !(options & RRDCONTEXT_OPTION_SHOW_DELETED)))
1665 return 0;
1666
1667 if(after && (!ri->last_time_t || after > ri->last_time_t))
@@ -2072,7 +1678,7 @@ static inline int rrdinstance_to_json_callback(const char *id, void *value, void
1678
1679 time_t first_time_t = ri->first_time_t;
1680 time_t last_time_t = ri->last_time_t;
2075 - RRD_FLAGS flags = ri->flags;
1681 + RRD_FLAGS flags = rrd_flags_get(ri);
1682
1683 BUFFER *wb_metrics = NULL;
1684 if(options & RRDCONTEXT_OPTION_SHOW_METRICS || t_parent->chart_dimensions) {
@@ -2151,13 +1757,13 @@ static inline int rrdinstance_to_json_callback(const char *id, void *value, void
1757 if(options & RRDCONTEXT_OPTION_SHOW_DELETED) {
1758 buffer_sprintf(wb,
1759 ",\n\t\t\t\t\t\"deleted\":%s"
2154 - , (ri->flags & RRD_FLAG_DELETED) ? "true" : "false"
1760 + , rrd_flag_is_deleted(ri) ? "true" : "false"
1761 );
1762 }
1763
1764 if(options & RRDCONTEXT_OPTION_SHOW_FLAGS) {
1765 buffer_strcat(wb, ",\n\t\t\t\t\t\"flags\":\"");
2160 - rrd_flags_to_buffer(ri->flags, wb);
1766 + rrd_flags_to_buffer(rrd_flags_get(ri), wb);
1767 buffer_strcat(wb, "\"");
1768 }
1769
@@ -2189,14 +1795,14 @@ static inline int rrdcontext_to_json_callback(const char *id, void *value, void
1795 time_t before = t_parent->before;
1796 bool has_filter = t_parent->chart_label_key || t_parent->chart_labels_filter || t_parent->chart_dimensions;
1797
2192 - if(unlikely((rc->flags & RRD_FLAG_HIDDEN) && !(options & RRDCONTEXT_OPTION_SHOW_HIDDEN)))
1798 + if(unlikely(rrd_flag_check(rc, RRD_FLAG_HIDDEN) && !(options & RRDCONTEXT_OPTION_SHOW_HIDDEN)))
1799 return 0;
1800
2195 - if((rc->flags & RRD_FLAG_DELETED) && !(options & RRDCONTEXT_OPTION_SHOW_DELETED))
1801 + if(unlikely(rrd_flag_is_deleted(rc) && !(options & RRDCONTEXT_OPTION_SHOW_DELETED)))
1802 return 0;
1803
1804 if(options & RRDCONTEXT_OPTION_DEEPSCAN)
2199 - rrdcontext_recalculate_context_retention(rc, RRD_FLAG_NONE, -1);
1805 + rrdcontext_recalculate_context_retention(rc, RRD_FLAG_NONE, false);
1806
1807 if(after && (!rc->last_time_t || after > rc->last_time_t))
1808 return 0;
@@ -2206,7 +1812,7 @@ static inline int rrdcontext_to_json_callback(const char *id, void *value, void
1812
1813 time_t first_time_t = rc->first_time_t;
1814 time_t last_time_t = rc->last_time_t;
2209 - RRD_FLAGS flags = rc->flags;
1815 + RRD_FLAGS flags = rrd_flags_get(rc);
1816
1817 BUFFER *wb_instances = NULL;
1818 if((options & (RRDCONTEXT_OPTION_SHOW_LABELS|RRDCONTEXT_OPTION_SHOW_INSTANCES|RRDCONTEXT_OPTION_SHOW_METRICS))
@@ -2273,13 +1879,13 @@ static inline int rrdcontext_to_json_callback(const char *id, void *value, void
1879 if(options & RRDCONTEXT_OPTION_SHOW_DELETED) {
1880 buffer_sprintf(wb,
1881 ",\n\t\t\t\"deleted\":%s"
2276 - , (rc->flags & RRD_FLAG_DELETED) ? "true" : "false"
1882 + , rrd_flag_is_deleted(rc) ? "true" : "false"
1883 );
1884 }
1885
1886 if(options & RRDCONTEXT_OPTION_SHOW_FLAGS) {
1887 buffer_strcat(wb, ",\n\t\t\t\"flags\":\"");
2282 - rrd_flags_to_buffer(rc->flags, wb);
1888 + rrd_flags_to_buffer(rrd_flags_get(rc), wb);
1889 buffer_strcat(wb, "\"");
1890 }
1891
@@ -2427,7 +2033,7 @@ static void rrdinstance_load_dimension(SQL_DIMENSION_DATA *sd, void *data) {
2033 RRDMETRIC trm = {
2034 .id = string_strdupz(sd->id),
2035 .name = string_strdupz(sd->name),
2430 - .flags = RRD_FLAG_ARCHIVED | RRD_FLAG_UPDATE_REASON_LOAD_SQL,
2036 + .flags = RRD_FLAG_ARCHIVED | RRD_FLAG_UPDATE_REASON_LOAD_SQL, // no need for atomic
2037 };
2038 uuid_copy(trm.uuid, sd->dim_id);
2039
@@ -2444,7 +2050,7 @@ static void rrdinstance_load_chart_callback(SQL_CHART_DATA *sc, void *data) {
2050 .family = string_strdupz(sc->family),
2051 .priority = sc->priority,
2052 .chart_type = sc->chart_type,
2447 - .flags = RRD_FLAG_ARCHIVED | RRD_FLAG_DONT_PROCESS | RRD_FLAG_UPDATE_REASON_LOAD_SQL,
2053 + .flags = RRD_FLAG_ARCHIVED | RRD_FLAG_UPDATE_REASON_LOAD_SQL, // no need for atomics
2054 .rrdhost = host,
2055 };
2056
@@ -2460,7 +2066,7 @@ static void rrdinstance_load_chart_callback(SQL_CHART_DATA *sc, void *data) {
2066 .chart_type = sc->chart_type,
2067 .priority = sc->priority,
2068 .update_every = sc->update_every,
2463 - .flags = RRD_FLAG_ARCHIVED | RRD_FLAG_DONT_PROCESS | RRD_FLAG_UPDATE_REASON_LOAD_SQL,
2069 + .flags = RRD_FLAG_ARCHIVED | RRD_FLAG_UPDATE_REASON_LOAD_SQL, // no need for atomics
2070 };
2071 uuid_copy(tri.uuid, sc->chart_id);
2072
@@ -2469,13 +2075,7 @@ static void rrdinstance_load_chart_callback(SQL_CHART_DATA *sc, void *data) {
2075
2076 ctx_get_dimension_list(&ri->uuid, rrdinstance_load_dimension, ri);
2077 ctx_get_label_list(&ri->uuid, rrdinstance_load_clabel, ri);
2472 - ri->flags &= ~RRD_FLAG_DONT_PROCESS;
2473 - rrdinstance_trigger_updates(ri, true, true, __FUNCTION__);
2474 -
2475 - // let the instance be in "don't process" mode
2476 - // so that we process it once, when it is collected
2477 - ri->flags |= RRD_FLAG_DONT_PROCESS;
2478 -
2078 + rrdinstance_trigger_updates(ri, __FUNCTION__ );
2079 rrdinstance_release(ria);
2080 rrdcontext_release(rca);
2081 }
@@ -2486,7 +2086,7 @@ static void rrdcontext_load_context_callback(VERSIONED_CONTEXT_DATA *ctx_data, v
2086
2087 RRDCONTEXT trc = {
2088 .id = string_strdupz(ctx_data->id),
2489 - .flags = RRD_FLAG_ARCHIVED | RRD_FLAG_DONT_PROCESS | RRD_FLAG_UPDATE_REASON_LOAD_SQL,
2089 + .flags = RRD_FLAG_ARCHIVED | RRD_FLAG_UPDATE_REASON_LOAD_SQL, // no need for atomics
2090
2091 // no need to set more data here
2092 // we only need the hub data
@@ -2508,8 +2108,7 @@ void rrdhost_load_rrdcontext_data(RRDHOST *host) {
2108
2109 RRDCONTEXT *rc;
2110 dfe_start_read((DICTIONARY *)host->rrdctx, rc) {
2511 - rc->flags &= ~RRD_FLAG_DONT_PROCESS;
2512 - rrdcontext_trigger_updates(rc, true, __FUNCTION__);
2111 + rrdcontext_trigger_updates(rc, __FUNCTION__ );
2112 }
2113 dfe_done(rc);
2114
@@ -2517,67 +2116,25 @@ void rrdhost_load_rrdcontext_data(RRDHOST *host) {
2116 }
2117
2118 // ----------------------------------------------------------------------------
2520 -// the worker thread
2119 +// version hash calculation
2120
2522 -static inline usec_t rrdcontext_calculate_queued_dispatch_time_ut(RRDCONTEXT *rc, usec_t now_ut) {
2121 +static uint64_t rrdcontext_version_hash_with_callback(
2122 + RRDHOST *host,
2123 + void (*callback)(RRDCONTEXT *, bool, void *),
2124 + bool snapshot,
2125 + void *bundle) {
2126
2524 - if(likely(rc->queue.delay_calc_ut >= rc->queue.queued_ut))
2525 - return rc->queue.scheduled_dispatch_ut;
2127 + if(unlikely(!host || !host->rrdctx)) return 0;
2128
2527 - RRD_FLAGS flags = rc->queue.queued_flags;
2528 -
2529 - usec_t delay = LONG_MAX;
2530 - int i;
2531 - struct rrdcontext_reason *reason;
2532 - for(i = 0, reason = &rrdcontext_reasons[i]; reason->name ; reason = &rrdcontext_reasons[++i]) {
2533 - if(unlikely(flags & reason->flag)) {
2534 - if(reason->delay_ut < delay)
2535 - delay = reason->delay_ut;
2536 - }
2537 - }
2538 -
2539 - if(unlikely(delay == LONG_MAX)) {
2540 - internal_error(true, "RRDCONTEXT: '%s', cannot find minimum delay of flags %x", string2str(rc->id), (unsigned int)flags);
2541 - delay = 60 * USEC_PER_SEC;
2542 - }
2543 -
2544 - rc->queue.delay_calc_ut = now_ut;
2545 - usec_t dispatch_ut = rc->queue.scheduled_dispatch_ut = rc->queue.queued_ut + delay;
2546 - return dispatch_ut;
2547 -}
2548 -
2549 -#define WORKER_JOB_HOSTS 1
2550 -#define WORKER_JOB_CHECK 2
2551 -#define WORKER_JOB_SEND 3
2552 -#define WORKER_JOB_DEQUEUE 4
2553 -#define WORKER_JOB_RETENTION 5
2554 -#define WORKER_JOB_QUEUED 6
2555 -#define WORKER_JOB_CLEANUP 7
2556 -#define WORKER_JOB_CLEANUP_DELETE 8
2557 -
2558 -static usec_t rrdcontext_next_db_rotation_ut = 0;
2559 -void rrdcontext_db_rotation(void) {
2560 - // called when the db rotates its database
2561 - rrdcontext_next_db_rotation_ut = now_realtime_usec() + FULL_RETENTION_SCAN_DELAY_AFTER_DB_ROTATION_SECS * USEC_PER_SEC;
2562 -}
2563 -
2564 -static uint64_t rrdcontext_version_hash_with_callback(
2565 - RRDHOST *host,
2566 - void (*callback)(RRDCONTEXT *, bool, void *),
2567 - bool snapshot,
2568 - void *bundle) {
2569 -
2570 - if(unlikely(!host || !host->rrdctx)) return 0;
2571 -
2572 - RRDCONTEXT *rc;
2573 - uint64_t hash = 0;
2129 + RRDCONTEXT *rc;
2130 + uint64_t hash = 0;
2131
2132 // loop through all contexts of the host
2133 dfe_start_read((DICTIONARY *)host->rrdctx, rc) {
2134
2135 rrdcontext_lock(rc);
2136
2580 - if(unlikely(rc->flags & RRD_FLAG_HIDDEN)) {
2137 + if(unlikely(rrd_flag_check(rc, RRD_FLAG_HIDDEN))) {
2138 rrdcontext_unlock(rc);
2139 continue;
2140 }
@@ -2586,7 +2143,7 @@ static uint64_t rrdcontext_version_hash_with_callback(
2143 callback(rc, snapshot, bundle);
2144
2145 // skip any deleted contexts
2589 - if(unlikely(rc->flags & RRD_FLAG_DELETED)) {
2146 + if(unlikely(rrd_flag_is_deleted(rc))) {
2147 rrdcontext_unlock(rc);
2148 continue;
2149 }
@@ -2611,51 +2168,148 @@ static uint64_t rrdcontext_version_hash_with_callback(
2168 return hash;
2169 }
2170
2614 -static void rrdcontext_recalculate_context_retention(RRDCONTEXT *rc, RRD_FLAGS reason, int job_id) {
2615 - RRDINSTANCE *ri;
2616 - dfe_start_read(rc->rrdinstances, ri) {
2617 - RRDMETRIC *rm;
2618 - dfe_start_read(ri->rrdmetrics, rm) {
2619 -
2620 - if(job_id >= 0)
2621 - worker_is_busy(job_id);
2622 -
2623 - rrd_flag_set_updated(rm, reason);
2624 -
2625 - rrdmetric_trigger_updates(rm, true, false, __FUNCTION__);
2626 - }
2627 - dfe_done(rm);
2628 -
2629 - ri->flags &= ~RRD_FLAG_DONT_PROCESS;
2630 - rrdinstance_trigger_updates(ri, true, false, __FUNCTION__);
2631 - ri->flags |= RRD_FLAG_DONT_PROCESS;
2632 - }
2633 - dfe_done(ri);
2171 +// ----------------------------------------------------------------------------
2172 +// retention recalculation
2173
2635 - rc->flags &= ~RRD_FLAG_DONT_PROCESS;
2636 - rrdcontext_trigger_updates(rc, true, __FUNCTION__);
2174 +static void rrdcontext_recalculate_context_retention(RRDCONTEXT *rc, RRD_FLAGS reason, bool worker_jobs) {
2175 + rrdcontext_post_process_updates(rc, true, reason, worker_jobs);
2176 }
2177
2639 -static void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reason, int job_id) {
2178 +static void rrdcontext_recalculate_host_retention(RRDHOST *host, RRD_FLAGS reason, bool worker_jobs) {
2179 if(unlikely(!host || !host->rrdctx)) return;
2180
2181 RRDCONTEXT *rc;
2182 dfe_start_read((DICTIONARY *)host->rrdctx, rc) {
2644 - rrdcontext_recalculate_context_retention(rc, reason, job_id);
2183 + rrdcontext_recalculate_context_retention(rc, reason, worker_jobs);
2184 }
2185 dfe_done(rc);
2186 }
2187
2649 -static void rrdcontext_recalculate_retention(int job_id) {
2188 +static void rrdcontext_recalculate_retention_all_hosts(void) {
2189 rrdcontext_next_db_rotation_ut = 0;
2190 rrd_rdlock();
2191 RRDHOST *host;
2192 rrdhost_foreach_read(host) {
2654 - rrdcontext_recalculate_host_retention(host, RRD_FLAG_UPDATE_REASON_DB_ROTATION, job_id);
2193 + worker_is_busy(WORKER_JOB_RETENTION);
2194 + rrdcontext_recalculate_host_retention(host, RRD_FLAG_UPDATE_REASON_DB_ROTATION, true);
2195 }
2196 rrd_unlock();
2197 }
2198
2199 +// ----------------------------------------------------------------------------
2200 +// garbage collector
2201 +
2202 +static void rrdmetric_update_retention(RRDMETRIC *rm) {
2203 + time_t min_first_time_t = LONG_MAX, max_last_time_t = 0;
2204 +
2205 + if(rm->rrddim) {
2206 + min_first_time_t = rrddim_first_entry_t(rm->rrddim);
2207 + max_last_time_t = rrddim_last_entry_t(rm->rrddim);
2208 + }
2209 +#ifdef ENABLE_DBENGINE
2210 + else {
2211 + RRDHOST *rrdhost = rm->ri->rc->rrdhost;
2212 + for (int tier = 0; tier < storage_tiers; tier++) {
2213 + if(!rrdhost->storage_instance[tier]) continue;
2214 +
2215 + time_t first_time_t, last_time_t;
2216 + if (rrdeng_metric_retention_by_uuid(rrdhost->storage_instance[tier], &rm->uuid, &first_time_t, &last_time_t) == 0) {
2217 + if (first_time_t < min_first_time_t)
2218 + min_first_time_t = first_time_t;
2219 +
2220 + if (last_time_t > max_last_time_t)
2221 + max_last_time_t = last_time_t;
2222 + }
2223 + }
2224 + }
2225 +#endif
2226 +
2227 + if(min_first_time_t == LONG_MAX)
2228 + min_first_time_t = 0;
2229 +
2230 + if(min_first_time_t > max_last_time_t) {
2231 + internal_error(true, "RRDMETRIC: retention of '%s' is flipped", string2str(rm->id));
2232 + time_t tmp = min_first_time_t;
2233 + min_first_time_t = max_last_time_t;
2234 + max_last_time_t = tmp;
2235 + }
2236 +
2237 + // check if retention changed
2238 +
2239 + if (min_first_time_t != rm->first_time_t) {
2240 + rm->first_time_t = min_first_time_t;
2241 + rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
2242 + }
2243 +
2244 + if (max_last_time_t != rm->last_time_t) {
2245 + rm->last_time_t = max_last_time_t;
2246 + rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
2247 + }
2248 +
2249 + if(unlikely(!rm->first_time_t && !rm->last_time_t))
2250 + rrd_flag_set_deleted(rm, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
2251 +
2252 + rrd_flag_set(rm, RRD_FLAG_LIVE_RETENTION);
2253 +}
2254 +
2255 +static inline bool rrdmetric_should_be_deleted(RRDMETRIC *rm) {
2256 + if(likely(!rrd_flag_check(rm, RRD_FLAGS_REQUIRED_FOR_DELETIONS)))
2257 + return false;
2258 +
2259 + if(likely(rrd_flag_check(rm, RRD_FLAGS_PREVENTING_DELETIONS)))
2260 + return false;
2261 +
2262 + if(likely(rm->rrddim))
2263 + return false;
2264 +
2265 + rrdmetric_update_retention(rm);
2266 + if(rm->first_time_t || rm->last_time_t)
2267 + return false;
2268 +
2269 + return true;
2270 +}
2271 +
2272 +static inline bool rrdinstance_should_be_deleted(RRDINSTANCE *ri) {
2273 + if(likely(!rrd_flag_check(ri, RRD_FLAGS_REQUIRED_FOR_DELETIONS)))
2274 + return false;
2275 +
2276 + if(likely(rrd_flag_check(ri, RRD_FLAGS_PREVENTING_DELETIONS)))
2277 + return false;
2278 +
2279 + if(likely(ri->rrdset))
2280 + return false;
2281 +
2282 + if(unlikely(dictionary_stats_referenced_items(ri->rrdmetrics) != 0))
2283 + return false;
2284 +
2285 + if(unlikely(dictionary_stats_entries(ri->rrdmetrics) != 0))
2286 + return false;
2287 +
2288 + if(ri->first_time_t || ri->last_time_t)
2289 + return false;
2290 +
2291 + return true;
2292 +}
2293 +
2294 +static inline bool rrdcontext_should_be_deleted(RRDCONTEXT *rc) {
2295 + if(likely(!rrd_flag_check(rc, RRD_FLAGS_REQUIRED_FOR_DELETIONS)))
2296 + return false;
2297 +
2298 + if(likely(rrd_flag_check(rc, RRD_FLAGS_PREVENTING_DELETIONS)))
2299 + return false;
2300 +
2301 + if(unlikely(dictionary_stats_referenced_items(rc->rrdinstances) != 0))
2302 + return false;
2303 +
2304 + if(unlikely(dictionary_stats_entries(rc->rrdinstances) != 0))
2305 + return false;
2306 +
2307 + if(unlikely(rc->first_time_t || rc->last_time_t))
2308 + return false;
2309 +
2310 + return true;
2311 +}
2312 +
2313 void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc) {
2314 // we need to refresh the string pointers in rc->hub
2315 // in case the context changed values
@@ -2669,22 +2323,26 @@ void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc) {
2323 error("RRDCONTEXT: failed to delete context '%s' version %"PRIu64" from SQL.", rc->hub.id, rc->hub.version);
2324 }
2325
2672 -static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker) {
2326 +static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker_jobs) {
2327
2328 internal_error(true, "RRDCONTEXT: garbage collecting context structures of host '%s'", rrdhost_hostname(host));
2329
2330 RRDCONTEXT *rc;
2677 - dfe_start_write((DICTIONARY *)host->rrdctx, rc) {
2678 - if(worker) worker_is_busy(WORKER_JOB_CLEANUP);
2331 + dfe_start_reentrant((DICTIONARY *)host->rrdctx, rc) {
2332 + if(unlikely(netdata_exit)) break;
2333 +
2334 + if(worker_jobs) worker_is_busy(WORKER_JOB_CLEANUP);
2335
2336 rrdcontext_lock(rc);
2337
2338 RRDINSTANCE *ri;
2683 - dfe_start_write(rc->rrdinstances, ri) {
2339 + dfe_start_reentrant(rc->rrdinstances, ri) {
2340 + if(unlikely(netdata_exit)) break;
2341 +
2342 RRDMETRIC *rm;
2343 dfe_start_write(ri->rrdmetrics, rm) {
2344 if(rrdmetric_should_be_deleted(rm)) {
2687 - if(worker) worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
2345 + if(worker_jobs) worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
2346 if(dictionary_del_having_write_lock(ri->rrdmetrics, string2str(rm->id)) != 0)
2347 error("RRDCONTEXT: metric '%s' of instance '%s' of context '%s' of host '%s', failed to be deleted from rrdmetrics dictionary.",
2348 string2str(rm->id),
@@ -2704,8 +2362,8 @@ static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker) {
2362 dfe_done(rm);
2363
2364 if(rrdinstance_should_be_deleted(ri)) {
2707 - if(worker) worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
2708 - if(dictionary_del_having_write_lock(rc->rrdinstances, string2str(ri->id)) != 0)
2365 + if(worker_jobs) worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
2366 + if(dictionary_del(rc->rrdinstances, string2str(ri->id)) != 0)
2367 error("RRDCONTEXT: instance '%s' of context '%s' of host '%s', failed to be deleted from rrdmetrics dictionary.",
2368 string2str(ri->id),
2369 string2str(rc->id),
@@ -2722,10 +2380,10 @@ static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker) {
2380 dfe_done(ri);
2381
2382 if(unlikely(rrdcontext_should_be_deleted(rc))) {
2725 - if(worker) worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
2383 + if(worker_jobs) worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
2384 rrdcontext_delete_from_sql_unsafe(rc);
2385
2728 - if(dictionary_del_having_write_lock((DICTIONARY *)host->rrdctx, string2str(rc->id)) != 0)
2386 + if(dictionary_del((DICTIONARY *)host->rrdctx, string2str(rc->id)) != 0)
2387 error("RRDCONTEXT: context '%s' of host '%s', failed to be deleted from rrdmetrics dictionary.",
2388 string2str(rc->id),
2389 rrdhost_hostname(host));
@@ -2746,7 +2404,7 @@ static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker) {
2404 dfe_done(rc);
2405 }
2406
2749 -static void rrdcontext_garbage_collect(void) {
2407 +static void rrdcontext_garbage_collect_for_all_hosts(void) {
2408 rrd_rdlock();
2409 RRDHOST *host;
2410 rrdhost_foreach_read(host) {
@@ -2755,153 +2413,653 @@ static void rrdcontext_garbage_collect(void) {
2413 rrd_unlock();
2414 }
2415
2758 -static void rrdcontext_main_cleanup(void *ptr) {
2759 - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
2760 - static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
2761 - // custom code
2762 - static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
2763 -}
2764 -
2765 -void *rrdcontext_main(void *ptr) {
2766 - netdata_thread_cleanup_push(rrdcontext_main_cleanup, ptr);
2416 +// ----------------------------------------------------------------------------
2417 +// post processing
2418
2768 - if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
2769 - goto exit;
2419 +static void rrdmetric_process_updates(RRDMETRIC *rm, bool force, RRD_FLAGS reason, bool worker_jobs) {
2420 + if(reason != RRD_FLAG_NONE)
2421 + rrd_flag_set_updated(rm, reason);
2422
2771 - worker_register("RRDCONTEXT");
2772 - worker_register_job_name(WORKER_JOB_HOSTS, "hosts");
2773 - worker_register_job_name(WORKER_JOB_CHECK, "dedup checks");
2774 - worker_register_job_name(WORKER_JOB_SEND, "sent contexts");
2775 - worker_register_job_name(WORKER_JOB_DEQUEUE, "deduped contexts");
2776 - worker_register_job_name(WORKER_JOB_RETENTION, "metrics retention");
2777 - worker_register_job_name(WORKER_JOB_QUEUED, "queued contexts");
2778 - worker_register_job_name(WORKER_JOB_CLEANUP, "cleanups");
2779 - worker_register_job_name(WORKER_JOB_CLEANUP_DELETE, "deletes");
2423 + if(!force && !rrd_flag_is_updated(rm) && rrd_flag_check(rm, RRD_FLAG_LIVE_RETENTION))
2424 + return;
2425
2781 - heartbeat_t hb;
2782 - heartbeat_init(&hb);
2783 - usec_t step = USEC_PER_SEC * RRDCONTEXT_WORKER_THREAD_HEARTBEAT_SECS;
2426 + if(worker_jobs)
2427 + worker_is_busy(WORKER_JOB_PP_METRIC);
2428
2785 - while (!netdata_exit) {
2786 - worker_is_idle();
2787 - heartbeat_next(&hb, step);
2429 + rrdmetric_update_retention(rm);
2430
2789 - if(unlikely(netdata_exit)) break;
2431 + rrd_flag_unset_updated(rm);
2432 +}
2433
2791 - if(!aclk_connected) continue;
2434 +static void rrdinstance_post_process_updates(RRDINSTANCE *ri, bool force, RRD_FLAGS reason, bool worker_jobs) {
2435 + if(reason != RRD_FLAG_NONE)
2436 + rrd_flag_set_updated(ri, reason);
2437
2793 - usec_t now_ut = now_realtime_usec();
2438 + if(!force && !rrd_flag_is_updated(ri) && rrd_flag_check(ri, RRD_FLAG_LIVE_RETENTION))
2439 + return;
2440
2795 - if(rrdcontext_next_db_rotation_ut && now_ut > rrdcontext_next_db_rotation_ut) {
2796 - rrdcontext_recalculate_retention(WORKER_JOB_RETENTION);
2797 - rrdcontext_garbage_collect();
2798 - rrdcontext_next_db_rotation_ut = 0;
2799 - }
2441 + if(worker_jobs)
2442 + worker_is_busy(WORKER_JOB_PP_INSTANCE);
2443
2801 - rrd_rdlock();
2802 - RRDHOST *host;
2803 - rrdhost_foreach_read(host) {
2444 + time_t min_first_time_t = LONG_MAX, max_last_time_t = 0;
2445 + size_t metrics_active = 0, metrics_deleted = 0;
2446 + bool live_retention = true, currently_collected = false;
2447 + if(dictionary_stats_entries(ri->rrdmetrics) > 0) {
2448 + RRDMETRIC *rm;
2449 + dfe_start_read((DICTIONARY *)ri->rrdmetrics, rm) {
2450 if(unlikely(netdata_exit)) break;
2451
2806 - worker_is_busy(WORKER_JOB_HOSTS);
2452 + rrdmetric_process_updates(rm, force, reason, worker_jobs);
2453
2808 - // check if we have received a streaming command for this host
2809 - if(!rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS))
2810 - continue;
2454 + if(unlikely(!rrd_flag_check(rm, RRD_FLAG_LIVE_RETENTION)))
2455 + live_retention = false;
2456
2812 - // check if there are queued items to send
2813 - if(!dictionary_stats_entries((DICTIONARY *)host->rrdctx_queue))
2457 + if (unlikely((rrdmetric_should_be_deleted(rm)))) {
2458 + metrics_deleted++;
2459 continue;
2460 + }
2461
2816 - if(!host->node_id)
2817 - continue;
2462 + if(!currently_collected && rrd_flag_check(rm, RRD_FLAG_COLLECTED) && rm->first_time_t)
2463 + currently_collected = true;
2464
2819 - size_t messages_added = 0;
2820 - contexts_updated_t bundle = NULL;
2465 + metrics_active++;
2466
2822 - RRDCONTEXT *rc;
2823 - dfe_start_write((DICTIONARY *)host->rrdctx_queue, rc) {
2824 - if(unlikely(netdata_exit)) break;
2467 + if (rm->first_time_t && rm->first_time_t < min_first_time_t)
2468 + min_first_time_t = rm->first_time_t;
2469
2826 - if(unlikely(messages_added >= MESSAGES_PER_BUNDLE_TO_SEND_TO_HUB_PER_HOST))
2827 - break;
2470 + if (rm->last_time_t && rm->last_time_t > max_last_time_t)
2471 + max_last_time_t = rm->last_time_t;
2472 + }
2473 + dfe_done(rm);
2474 + }
2475
2829 - worker_is_busy(WORKER_JOB_QUEUED);
2830 - usec_t dispatch_ut = rrdcontext_calculate_queued_dispatch_time_ut(rc, now_ut);
2831 - char *claim_id = get_agent_claimid();
2832 - if(unlikely(now_ut >= dispatch_ut) && claim_id) {
2833 - worker_is_busy(WORKER_JOB_CHECK);
2476 + if(unlikely(live_retention && !rrd_flag_check(ri, RRD_FLAG_LIVE_RETENTION)))
2477 + rrd_flag_set(ri, RRD_FLAG_LIVE_RETENTION);
2478 + else if(unlikely(!live_retention && rrd_flag_check(ri, RRD_FLAG_LIVE_RETENTION)))
2479 + rrd_flag_clear(ri, RRD_FLAG_LIVE_RETENTION);
2480
2835 - rrdcontext_lock(rc);
2481 + if(unlikely(!metrics_active)) {
2482 + // no metrics available
2483
2837 - if(check_if_cloud_version_changed_unsafe(rc, true)) {
2838 - worker_is_busy(WORKER_JOB_SEND);
2484 + if(ri->first_time_t) {
2485 + ri->first_time_t = 0;
2486 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
2487 + }
2488
2840 -#ifdef ENABLE_ACLK
2841 - if(!bundle) {
2842 - // prepare the bundle to send the messages
2843 - char uuid[UUID_STR_LEN];
2844 - uuid_unparse_lower(*host->node_id, uuid);
2489 + if(ri->last_time_t) {
2490 + ri->last_time_t = 0;
2491 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
2492 + }
2493
2846 - bundle = contexts_updated_new(claim_id, uuid, 0, now_ut);
2847 - }
2848 -#endif
2849 - // update the hub data of the context, give a new version, pack the message
2850 - // and save an update to SQL
2851 - rrdcontext_message_send_unsafe(rc, false, bundle);
2852 - messages_added++;
2494 + rrd_flag_set_deleted(ri, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
2495 + }
2496 + else {
2497 + // we have active metrics...
2498
2854 - rc->queue.dequeued_ut = now_ut;
2855 - }
2856 - else
2857 - rc->version = rc->hub.version;
2858 -
2859 - // remove the queued flag, so that it can be queued again
2860 - rc->flags &= ~RRD_FLAG_QUEUED;
2499 + if (unlikely(min_first_time_t == LONG_MAX))
2500 + min_first_time_t = 0;
2501
2862 - // remove it from the queue
2863 - worker_is_busy(WORKER_JOB_DEQUEUE);
2864 - dictionary_del_having_write_lock((DICTIONARY *)host->rrdctx_queue, string2str(rc->id));
2502 + if (unlikely(min_first_time_t == 0 || max_last_time_t == 0)) {
2503 + if(ri->first_time_t) {
2504 + ri->first_time_t = 0;
2505 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
2506 + }
2507
2866 - if(unlikely(rrdcontext_should_be_deleted(rc))) {
2867 - // this is a deleted context - delete it forever...
2508 + if(ri->last_time_t) {
2509 + ri->last_time_t = 0;
2510 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
2511 + }
2512
2869 - worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
2870 - rrdcontext_delete_from_sql_unsafe(rc);
2513 + if(likely(live_retention))
2514 + rrd_flag_set_deleted(ri, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
2515 + }
2516 + else {
2517 + rrd_flag_clear(ri, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
2518
2872 - STRING *id = string_dup(rc->id);
2873 - rrdcontext_unlock(rc);
2519 + if (unlikely(ri->first_time_t != min_first_time_t)) {
2520 + ri->first_time_t = min_first_time_t;
2521 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
2522 + }
2523
2875 - // delete it from the master dictionary
2876 - if(dictionary_del((DICTIONARY *)host->rrdctx, string2str(rc->id)) != 0)
2877 - error("RRDCONTEXT: '%s' of host '%s' failed to be deleted from rrdcontext dictionary.",
2878 - string2str(id), rrdhost_hostname(host));
2524 + if (unlikely(ri->last_time_t != max_last_time_t)) {
2525 + ri->last_time_t = max_last_time_t;
2526 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
2527 + }
2528
2880 - string_freez(id);
2881 - }
2882 - else
2883 - rrdcontext_unlock(rc);
2884 - }
2885 - freez(claim_id);
2529 + if(likely(currently_collected))
2530 + rrd_flag_set_collected(ri);
2531 + else
2532 + rrd_flag_set_archived(ri);
2533 + }
2534 + }
2535 +
2536 + rrd_flag_unset_updated(ri);
2537 +}
2538 +
2539 +static void rrdcontext_post_process_updates(RRDCONTEXT *rc, bool force, RRD_FLAGS reason, bool worker_jobs) {
2540 + if(reason != RRD_FLAG_NONE)
2541 + rrd_flag_set_updated(rc, reason);
2542 +
2543 + if(worker_jobs)
2544 + worker_is_busy(WORKER_JOB_PP_CONTEXT);
2545 +
2546 + size_t min_priority = LONG_MAX;
2547 + time_t min_first_time_t = LONG_MAX, max_last_time_t = 0;
2548 + size_t instances_active = 0, instances_deleted = 0;
2549 + bool live_retention = true, currently_collected = false, hidden = true;
2550 + if(dictionary_stats_entries(rc->rrdinstances) > 0) {
2551 + RRDINSTANCE *ri;
2552 + dfe_start_reentrant(rc->rrdinstances, ri) {
2553 + if(unlikely(netdata_exit)) break;
2554 +
2555 + rrdinstance_post_process_updates(ri, force, reason, worker_jobs);
2556 +
2557 + if(unlikely(hidden && !rrd_flag_check(ri, RRD_FLAG_HIDDEN)))
2558 + hidden = false;
2559 +
2560 + if(unlikely(live_retention && !rrd_flag_check(ri, RRD_FLAG_LIVE_RETENTION)))
2561 + live_retention = false;
2562 +
2563 + if (unlikely(rrdinstance_should_be_deleted(ri))) {
2564 + instances_deleted++;
2565 + continue;
2566 }
2887 - dfe_done(rc);
2567 +
2568 + if(unlikely(!currently_collected && rrd_flag_is_collected(ri) && ri->first_time_t))
2569 + currently_collected = true;
2570 +
2571 + internal_error(rc->units != ri->units,
2572 + "RRDCONTEXT: '%s' rrdinstance '%s' has different units, context '%s', instance '%s'",
2573 + string2str(rc->id), string2str(ri->id),
2574 + string2str(rc->units), string2str(ri->units));
2575 +
2576 + instances_active++;
2577 +
2578 + if (ri->priority >= RRDCONTEXT_MINIMUM_ALLOWED_PRIORITY && ri->priority < min_priority)
2579 + min_priority = ri->priority;
2580 +
2581 + if (ri->first_time_t && ri->first_time_t < min_first_time_t)
2582 + min_first_time_t = ri->first_time_t;
2583 +
2584 + if (ri->last_time_t && ri->last_time_t > max_last_time_t)
2585 + max_last_time_t = ri->last_time_t;
2586 + }
2587 + dfe_done(ri);
2588 + }
2589 +
2590 + {
2591 + bool previous_hidden = rrd_flag_check(rc, RRD_FLAG_HIDDEN);
2592 + if (hidden != previous_hidden) {
2593 + if (hidden && !rrd_flag_check(rc, RRD_FLAG_HIDDEN))
2594 + rrd_flag_set(rc, RRD_FLAG_HIDDEN);
2595 + else if (!hidden && rrd_flag_check(rc, RRD_FLAG_HIDDEN))
2596 + rrd_flag_clear(rc, RRD_FLAG_HIDDEN);
2597 + }
2598 +
2599 + bool previous_live_retention = rrd_flag_check(rc, RRD_FLAG_LIVE_RETENTION);
2600 + if (live_retention != previous_live_retention) {
2601 + if (live_retention && !rrd_flag_check(rc, RRD_FLAG_LIVE_RETENTION))
2602 + rrd_flag_set(rc, RRD_FLAG_LIVE_RETENTION);
2603 + else if (!live_retention && rrd_flag_check(rc, RRD_FLAG_LIVE_RETENTION))
2604 + rrd_flag_clear(rc, RRD_FLAG_LIVE_RETENTION);
2605 + }
2606 + }
2607 +
2608 + rrdcontext_lock(rc);
2609 +
2610 + if(unlikely(!instances_active)) {
2611 + // we had some instances, but they are gone now...
2612 +
2613 + if(rc->first_time_t) {
2614 + rc->first_time_t = 0;
2615 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
2616 + }
2617 +
2618 + if(rc->last_time_t) {
2619 + rc->last_time_t = 0;
2620 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
2621 + }
2622 +
2623 + rrd_flag_set_deleted(rc, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
2624 + }
2625 + else {
2626 + // we have some active instances...
2627 +
2628 + if (unlikely(min_first_time_t == LONG_MAX))
2629 + min_first_time_t = 0;
2630 +
2631 + if (unlikely(min_first_time_t == 0 && max_last_time_t == 0)) {
2632 + if(rc->first_time_t) {
2633 + rc->first_time_t = 0;
2634 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
2635 + }
2636 +
2637 + if(rc->last_time_t) {
2638 + rc->last_time_t = 0;
2639 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
2640 + }
2641 +
2642 + rrd_flag_set_deleted(rc, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
2643 + }
2644 + else {
2645 + rrd_flag_clear(rc, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
2646 +
2647 + if (unlikely(rc->first_time_t != min_first_time_t)) {
2648 + rc->first_time_t = min_first_time_t;
2649 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T);
2650 + }
2651 +
2652 + if (rc->last_time_t != max_last_time_t) {
2653 + rc->last_time_t = max_last_time_t;
2654 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T);
2655 + }
2656 +
2657 + if(likely(currently_collected))
2658 + rrd_flag_set_collected(rc);
2659 + else
2660 + rrd_flag_set_archived(rc);
2661 + }
2662 +
2663 + if (min_priority != LONG_MAX && rc->priority != min_priority) {
2664 + rc->priority = min_priority;
2665 + rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_PRIORITY);
2666 + }
2667 + }
2668 +
2669 + if(unlikely(rrd_flag_is_updated(rc) && rc->rrdhost->rrdctx_hub_queue)) {
2670 + if(check_if_cloud_version_changed_unsafe(rc, false)) {
2671 + rc->version = rrdcontext_get_next_version(rc);
2672 +
2673 + if(rrd_flag_check(rc, RRD_FLAG_QUEUED_FOR_HUB)) {
2674 + rc->queue.queued_ut = now_realtime_usec();
2675 + rc->queue.queued_flags |= rrd_flags_get(rc);
2676 + }
2677 + else {
2678 + rc->queue.queued_ut = now_realtime_usec();
2679 + rc->queue.queued_flags = rrd_flags_get(rc);
2680 + rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_HUB);
2681 + dictionary_set((DICTIONARY *)rc->rrdhost->rrdctx_hub_queue, string2str(rc->id), rc, sizeof(*rc));
2682 + }
2683 + }
2684 + }
2685 +
2686 + rrd_flag_unset_updated(rc);
2687 + rrdcontext_unlock(rc);
2688 +}
2689 +
2690 +static void rrdcontext_queue_for_post_processing(RRDCONTEXT *rc, const char *function __maybe_unused, RRD_FLAGS flags __maybe_unused) {
2691 + if(unlikely(!rc->rrdhost->rrdctx_post_processing_queue)) return;
2692 +
2693 + if(!rrd_flag_check(rc, RRD_FLAG_QUEUED_FOR_POST_PROCESSING)) {
2694 + rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_POST_PROCESSING);
2695 + dictionary_set((DICTIONARY *)rc->rrdhost->rrdctx_post_processing_queue, string2str(rc->id), rc, sizeof(*rc));
2696 +
2697 +#ifdef NETDATA_INTERNAL_CHECKS
2698 + {
2699 + BUFFER *wb_flags = buffer_create(1000);
2700 + rrd_flags_to_buffer(flags, wb_flags);
2701 +
2702 + BUFFER *wb_reasons = buffer_create(1000);
2703 + rrd_reasons_to_buffer(flags, wb_reasons);
2704 +
2705 + internal_error(true, "RRDCONTEXT: '%s' update triggered by function %s(), due to flags: %s, reasons: %s",
2706 + string2str(rc->id), function,
2707 + buffer_tostring(wb_flags),
2708 + buffer_tostring(wb_reasons));
2709 +
2710 + buffer_free(wb_reasons);
2711 + buffer_free(wb_flags);
2712 + }
2713 +#endif
2714 +
2715 + }
2716 +}
2717 +
2718 +static void rrdcontext_dequeue_from_post_processing(RRDCONTEXT *rc) {
2719 + if(unlikely(!rc->rrdhost->rrdctx_post_processing_queue)) return;
2720 +
2721 + rrd_flag_clear(rc, RRD_FLAG_QUEUED_FOR_POST_PROCESSING);
2722 + dictionary_del((DICTIONARY *)rc->rrdhost->rrdctx_post_processing_queue, string2str(rc->id));
2723 +}
2724 +
2725 +static void rrdcontext_post_process_queued_contexts(RRDHOST *host) {
2726 + if(unlikely(!host->rrdctx_post_processing_queue)) return;
2727 +
2728 + RRDCONTEXT *rc;
2729 + dfe_start_reentrant((DICTIONARY *)host->rrdctx_post_processing_queue, rc) {
2730 + if(unlikely(netdata_exit)) break;
2731 +
2732 + rrdcontext_dequeue_from_post_processing(rc);
2733 + rrdcontext_post_process_updates(rc, false, RRD_FLAG_NONE, true);
2734 + }
2735 + dfe_done(rc);
2736 +}
2737 +
2738 +// ----------------------------------------------------------------------------
2739 +// dispatching contexts to cloud
2740 +
2741 +static uint64_t rrdcontext_get_next_version(RRDCONTEXT *rc) {
2742 + time_t now = now_realtime_sec();
2743 + uint64_t version = MAX(rc->version, rc->hub.version);
2744 + version = MAX((uint64_t)now, version);
2745 + version++;
2746 + return version;
2747 +}
2748 +
2749 +static void rrdcontext_message_send_unsafe(RRDCONTEXT *rc, bool snapshot __maybe_unused, void *bundle __maybe_unused) {
2750 +
2751 + // save it, so that we know the last version we sent to hub
2752 + rc->version = rc->hub.version = rrdcontext_get_next_version(rc);
2753 + rc->hub.id = string2str(rc->id);
2754 + rc->hub.title = string2str(rc->title);
2755 + rc->hub.units = string2str(rc->units);
2756 + rc->hub.family = string2str(rc->family);
2757 + rc->hub.chart_type = rrdset_type_name(rc->chart_type);
2758 + rc->hub.priority = rc->priority;
2759 + rc->hub.first_time_t = rc->first_time_t;
2760 + rc->hub.last_time_t = rrd_flag_is_collected(rc) ? 0 : rc->last_time_t;
2761 + rc->hub.deleted = rrd_flag_is_deleted(rc) ? true : false;
2762 +
2763 +#ifdef ENABLE_ACLK
2764 + struct context_updated message = {
2765 + .id = rc->hub.id,
2766 + .version = rc->hub.version,
2767 + .title = rc->hub.title,
2768 + .units = rc->hub.units,
2769 + .family = rc->hub.family,
2770 + .chart_type = rc->hub.chart_type,
2771 + .priority = rc->hub.priority,
2772 + .first_entry = rc->hub.first_time_t,
2773 + .last_entry = rc->hub.last_time_t,
2774 + .deleted = rc->hub.deleted,
2775 + };
2776 +
2777 + if(likely(!rrd_flag_check(rc, RRD_FLAG_HIDDEN))) {
2778 + if (snapshot) {
2779 + if (!rc->hub.deleted)
2780 + contexts_snapshot_add_ctx_update(bundle, &message);
2781 + }
2782 + else
2783 + contexts_updated_add_ctx_update(bundle, &message);
2784 + }
2785 +#endif
2786 +
2787 + // store it to SQL
2788 +
2789 + if(rrd_flag_is_deleted(rc))
2790 + rrdcontext_delete_from_sql_unsafe(rc);
2791 +
2792 + else if (ctx_store_context(&rc->rrdhost->host_uuid, &rc->hub) != 0)
2793 + error("RRDCONTEXT: failed to save context '%s' version %"PRIu64" to SQL.", rc->hub.id, rc->hub.version);
2794 +}
2795 +
2796 +static bool check_if_cloud_version_changed_unsafe(RRDCONTEXT *rc, bool sending __maybe_unused) {
2797 + bool id_changed = false,
2798 + title_changed = false,
2799 + units_changed = false,
2800 + family_changed = false,
2801 + chart_type_changed = false,
2802 + priority_changed = false,
2803 + first_time_changed = false,
2804 + last_time_changed = false,
2805 + deleted_changed = false;
2806 +
2807 + RRD_FLAGS flags = rrd_flags_get(rc);
2808 +
2809 + if(unlikely(string2str(rc->id) != rc->hub.id))
2810 + id_changed = true;
2811 +
2812 + if(unlikely(string2str(rc->title) != rc->hub.title))
2813 + title_changed = true;
2814 +
2815 + if(unlikely(string2str(rc->units) != rc->hub.units))
2816 + units_changed = true;
2817 +
2818 + if(unlikely(string2str(rc->family) != rc->hub.family))
2819 + family_changed = true;
2820 +
2821 + if(unlikely(rrdset_type_name(rc->chart_type) != rc->hub.chart_type))
2822 + chart_type_changed = true;
2823 +
2824 + if(unlikely(rc->priority != rc->hub.priority))
2825 + priority_changed = true;
2826 +
2827 + if(unlikely((uint64_t)rc->first_time_t != rc->hub.first_time_t))
2828 + first_time_changed = true;
2829 +
2830 + if(unlikely((uint64_t)((flags & RRD_FLAG_COLLECTED) ? 0 : rc->last_time_t) != rc->hub.last_time_t))
2831 + last_time_changed = true;
2832 +
2833 + if(unlikely(((flags & RRD_FLAG_DELETED) ? true : false) != rc->hub.deleted))
2834 + deleted_changed = true;
2835 +
2836 + if(unlikely(id_changed || title_changed || units_changed || family_changed || chart_type_changed || priority_changed || first_time_changed || last_time_changed || deleted_changed)) {
2837 +
2838 + internal_error(true, "RRDCONTEXT: %s NEW VERSION '%s'%s, version %"PRIu64", title '%s'%s, units '%s'%s, family '%s'%s, chart type '%s'%s, priority %u%s, first_time_t %ld%s, last_time_t %ld%s, deleted '%s'%s, (queued for %llu ms, expected %llu ms)",
2839 + sending?"SENDING":"QUEUE",
2840 + string2str(rc->id), id_changed ? " (CHANGED)" : "",
2841 + rc->version,
2842 + string2str(rc->title), title_changed ? " (CHANGED)" : "",
2843 + string2str(rc->units), units_changed ? " (CHANGED)" : "",
2844 + string2str(rc->family), family_changed ? " (CHANGED)" : "",
2845 + rrdset_type_name(rc->chart_type), chart_type_changed ? " (CHANGED)" : "",
2846 + rc->priority, priority_changed ? " (CHANGED)" : "",
2847 + rc->first_time_t, first_time_changed ? " (CHANGED)" : "",
2848 + (flags & RRD_FLAG_COLLECTED) ? 0 : rc->last_time_t, last_time_changed ? " (CHANGED)" : "",
2849 + (flags & RRD_FLAG_DELETED) ? "true" : "false", deleted_changed ? " (CHANGED)" : "",
2850 + sending ? (now_realtime_usec() - rc->queue.queued_ut) / USEC_PER_MS : 0,
2851 + sending ? (rc->queue.scheduled_dispatch_ut - rc->queue.queued_ut) / USEC_PER_MS : 0
2852 + );
2853 +
2854 + return true;
2855 + }
2856 +
2857 + return false;
2858 +}
2859 +
2860 +static inline usec_t rrdcontext_calculate_queued_dispatch_time_ut(RRDCONTEXT *rc, usec_t now_ut) {
2861 +
2862 + if(likely(rc->queue.delay_calc_ut >= rc->queue.queued_ut))
2863 + return rc->queue.scheduled_dispatch_ut;
2864 +
2865 + RRD_FLAGS flags = rc->queue.queued_flags;
2866 +
2867 + usec_t delay = LONG_MAX;
2868 + int i;
2869 + struct rrdcontext_reason *reason;
2870 + for(i = 0, reason = &rrdcontext_reasons[i]; reason->name ; reason = &rrdcontext_reasons[++i]) {
2871 + if(unlikely(flags & reason->flag)) {
2872 + if(reason->delay_ut < delay)
2873 + delay = reason->delay_ut;
2874 + }
2875 + }
2876 +
2877 + if(unlikely(delay == LONG_MAX)) {
2878 + internal_error(true, "RRDCONTEXT: '%s', cannot find minimum delay of flags %x", string2str(rc->id), (unsigned int)flags);
2879 + delay = 60 * USEC_PER_SEC;
2880 + }
2881 +
2882 + rc->queue.delay_calc_ut = now_ut;
2883 + usec_t dispatch_ut = rc->queue.scheduled_dispatch_ut = rc->queue.queued_ut + delay;
2884 + return dispatch_ut;
2885 +}
2886 +
2887 +static void rrdcontext_dispatch_queued_contexts_to_hub(RRDHOST *host, usec_t now_ut) {
2888 +
2889 + // check if we have received a streaming command for this host
2890 + if(!rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS) || !aclk_connected || !host->rrdctx_hub_queue)
2891 + return;
2892 +
2893 + // check if there are queued items to send
2894 + if(!dictionary_stats_entries((DICTIONARY *)host->rrdctx_hub_queue))
2895 + return;
2896 +
2897 + if(!host->node_id)
2898 + return;
2899 +
2900 + size_t messages_added = 0;
2901 + contexts_updated_t bundle = NULL;
2902 +
2903 + RRDCONTEXT *rc;
2904 + dfe_start_reentrant((DICTIONARY *)host->rrdctx_hub_queue, rc) {
2905 + if(unlikely(netdata_exit)) break;
2906 +
2907 + if(unlikely(messages_added >= MESSAGES_PER_BUNDLE_TO_SEND_TO_HUB_PER_HOST))
2908 + break;
2909 +
2910 + worker_is_busy(WORKER_JOB_QUEUED);
2911 + usec_t dispatch_ut = rrdcontext_calculate_queued_dispatch_time_ut(rc, now_ut);
2912 + char *claim_id = get_agent_claimid();
2913 +
2914 + if(unlikely(now_ut >= dispatch_ut) && claim_id) {
2915 + worker_is_busy(WORKER_JOB_CHECK);
2916 +
2917 + rrdcontext_lock(rc);
2918 +
2919 + if(check_if_cloud_version_changed_unsafe(rc, true)) {
2920 + worker_is_busy(WORKER_JOB_SEND);
2921
2922 #ifdef ENABLE_ACLK
2890 - if(!netdata_exit && bundle) {
2891 - // we have a bundle to send messages
2923 + if(!bundle) {
2924 + // prepare the bundle to send the messages
2925 + char uuid[UUID_STR_LEN];
2926 + uuid_unparse_lower(*host->node_id, uuid);
2927 +
2928 + bundle = contexts_updated_new(claim_id, uuid, 0, now_ut);
2929 + }
2930 +#endif
2931 + // update the hub data of the context, give a new version, pack the message
2932 + // and save an update to SQL
2933 + rrdcontext_message_send_unsafe(rc, false, bundle);
2934 + messages_added++;
2935 +
2936 + rc->queue.dequeued_ut = now_ut;
2937 + }
2938 + else
2939 + rc->version = rc->hub.version;
2940 +
2941 + // remove the queued flag, so that it can be queued again
2942 + rrd_flag_clear(rc, RRD_FLAG_QUEUED_FOR_HUB);
2943 +
2944 + // remove it from the queue
2945 + worker_is_busy(WORKER_JOB_DEQUEUE);
2946 + dictionary_del((DICTIONARY *)host->rrdctx_hub_queue, string2str(rc->id));
2947 +
2948 + if(unlikely(rrdcontext_should_be_deleted(rc))) {
2949 + // this is a deleted context - delete it forever...
2950 +
2951 + worker_is_busy(WORKER_JOB_CLEANUP_DELETE);
2952 +
2953 + rrdcontext_dequeue_from_post_processing(rc);
2954 + rrdcontext_delete_from_sql_unsafe(rc);
2955
2893 - // update the version hash
2894 - contexts_updated_update_version_hash(bundle, rrdcontext_version_hash(host));
2956 + STRING *id = string_dup(rc->id);
2957 + rrdcontext_unlock(rc);
2958
2896 - // send it
2897 - aclk_send_contexts_updated(bundle);
2959 + // delete it from the master dictionary
2960 + if(dictionary_del((DICTIONARY *)host->rrdctx, string2str(rc->id)) != 0)
2961 + error("RRDCONTEXT: '%s' of host '%s' failed to be deleted from rrdcontext dictionary.",
2962 + string2str(id), rrdhost_hostname(host));
2963 +
2964 + string_freez(id);
2965 }
2899 - else if(bundle)
2900 - contexts_updated_delete(bundle);
2966 + else
2967 + rrdcontext_unlock(rc);
2968 + }
2969 + freez(claim_id);
2970 + }
2971 + dfe_done(rc);
2972 +
2973 +#ifdef ENABLE_ACLK
2974 + if(!netdata_exit && bundle) {
2975 + // we have a bundle to send messages
2976 +
2977 + // update the version hash
2978 + contexts_updated_update_version_hash(bundle, rrdcontext_version_hash(host));
2979 +
2980 + // send it
2981 + aclk_send_contexts_updated(bundle);
2982 + }
2983 + else if(bundle)
2984 + contexts_updated_delete(bundle);
2985 #endif
2986 +
2987 +}
2988 +
2989 +// ----------------------------------------------------------------------------
2990 +// worker thread
2991 +
2992 +static void rrdcontext_main_cleanup(void *ptr) {
2993 + struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
2994 + static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
2995 + // custom code
2996 + static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
2997 +}
2998 +
2999 +void *rrdcontext_main(void *ptr) {
3000 + netdata_thread_cleanup_push(rrdcontext_main_cleanup, ptr);
3001 +
3002 + if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
3003 + goto exit;
3004 +
3005 + worker_register("RRDCONTEXT");
3006 + worker_register_job_name(WORKER_JOB_HOSTS, "hosts");
3007 + worker_register_job_name(WORKER_JOB_CHECK, "dedup checks");
3008 + worker_register_job_name(WORKER_JOB_SEND, "sent contexts");
3009 + worker_register_job_name(WORKER_JOB_DEQUEUE, "deduped contexts");
3010 + worker_register_job_name(WORKER_JOB_RETENTION, "metrics retention");
3011 + worker_register_job_name(WORKER_JOB_QUEUED, "queued contexts");
3012 + worker_register_job_name(WORKER_JOB_CLEANUP, "cleanups");
3013 + worker_register_job_name(WORKER_JOB_CLEANUP_DELETE, "deletes");
3014 + worker_register_job_name(WORKER_JOB_PP_METRIC, "check metrics");
3015 + worker_register_job_name(WORKER_JOB_PP_INSTANCE, "check instances");
3016 + worker_register_job_name(WORKER_JOB_PP_CONTEXT, "check contexts");
3017 +
3018 + worker_register_job_custom_metric(WORKER_JOB_HUB_QUEUE_SIZE, "hub queue size", "contexts", WORKER_METRIC_ABSOLUTE);
3019 + worker_register_job_custom_metric(WORKER_JOB_PP_QUEUE_SIZE, "post processing queue size", "contexts", WORKER_METRIC_ABSOLUTE);
3020 +
3021 + heartbeat_t hb;
3022 + heartbeat_init(&hb);
3023 + usec_t step = RRDCONTEXT_WORKER_THREAD_HEARTBEAT_USEC;
3024 +
3025 + while (!netdata_exit) {
3026 + worker_is_idle();
3027 + heartbeat_next(&hb, step);
3028 +
3029 + if(unlikely(netdata_exit)) break;
3030 +
3031 + usec_t now_ut = now_realtime_usec();
3032 +
3033 + if(rrdcontext_next_db_rotation_ut && now_ut > rrdcontext_next_db_rotation_ut) {
3034 + rrdcontext_recalculate_retention_all_hosts();
3035 + rrdcontext_garbage_collect_for_all_hosts();
3036 + rrdcontext_next_db_rotation_ut = 0;
3037 + }
3038 +
3039 + size_t hub_queued_contexts_for_all_hosts = 0;
3040 + size_t pp_queued_contexts_for_all_hosts = 0;
3041 +
3042 + rrd_rdlock();
3043 + RRDHOST *host;
3044 + rrdhost_foreach_read(host) {
3045 + if(unlikely(netdata_exit)) break;
3046 +
3047 + worker_is_busy(WORKER_JOB_HOSTS);
3048 +
3049 + if(host->rrdctx_post_processing_queue) {
3050 + pp_queued_contexts_for_all_hosts += dictionary_stats_entries((DICTIONARY *)host->rrdctx_post_processing_queue);
3051 + rrdcontext_post_process_queued_contexts(host);
3052 + }
3053 +
3054 + if(host->rrdctx_hub_queue) {
3055 + hub_queued_contexts_for_all_hosts += dictionary_stats_entries((DICTIONARY *)host->rrdctx_hub_queue);
3056 + rrdcontext_dispatch_queued_contexts_to_hub(host, now_ut);
3057 + }
3058 }
3059 rrd_unlock();
3060
3061 + worker_set_metric(WORKER_JOB_HUB_QUEUE_SIZE, (NETDATA_DOUBLE)hub_queued_contexts_for_all_hosts);
3062 + worker_set_metric(WORKER_JOB_PP_QUEUE_SIZE, (NETDATA_DOUBLE)pp_queued_contexts_for_all_hosts);
3063 }
3064
3065 exit:
database/rrddim.c
+4 -4
@@ -571,7 +571,7 @@ size_t rrddim_memory_file_header_size(void) {
571 }
572
573 void rrddim_memory_file_update(RRDDIM *rd) {
574 - if(!rd->rd_on_file) return;
574 + if(!rd || !rd->rd_on_file) return;
575 struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;
576
577 rd_on_file->last_collected_time.tv_sec = rd->last_collected_time.tv_sec;
@@ -579,7 +579,7 @@ void rrddim_memory_file_update(RRDDIM *rd) {
579 }
580
581 void rrddim_memory_file_free(RRDDIM *rd) {
582 - if(!rd->rd_on_file) return;
582 + if(!rd || !rd->rd_on_file) return;
583
584 // needed for memory mode map, to save the latest state
585 rrddim_memory_file_update(rd);
@@ -594,13 +594,13 @@ void rrddim_memory_file_free(RRDDIM *rd) {
594 }
595
596 const char *rrddim_cache_filename(RRDDIM *rd) {
597 - if(!rd->rd_on_file) return NULL;
597 + if(!rd || !rd->rd_on_file) return NULL;
598 struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;
599 return rd_on_file->cache_filename;
600 }
601
602 void rrddim_memory_file_save(RRDDIM *rd) {
603 - if(!rd->rd_on_file) return;
603 + if(!rd || !rd->rd_on_file) return;
604
605 rrddim_memory_file_update(rd);
606
database/rrdvar.c
+1 -1
@@ -99,7 +99,7 @@ inline RRDVAR *rrdvar_create_and_index(const char *scope __maybe_unused, DICTION
99
100 void rrdvar_free_remaining_variables(RRDHOST *host, DICTIONARY *dict) {
101 RRDVAR *rv;
102 - dfe_start_rw(dict, rv, DICTIONARY_LOCK_REENTRANT) {
102 + dfe_start_reentrant(dict, rv) {
103 rrdvar_free(host, dict, rv);
104 }
105 dfe_done(rv);
libnetdata/dictionary/dictionary.c
+9
@@ -121,30 +121,39 @@ void dictionary_register_react_callback(DICTIONARY *dict, void (*react_callback)
121 // dictionary statistics maintenance
122
123 long int dictionary_stats_allocated_memory(DICTIONARY *dict) {
124 + if(unlikely(!dict)) return 0;
125 return dict->memory;
126 }
127 long int dictionary_stats_entries(DICTIONARY *dict) {
128 + if(unlikely(!dict)) return 0;
129 return dict->entries;
130 }
131 size_t dictionary_stats_version(DICTIONARY *dict) {
132 + if(unlikely(!dict)) return 0;
133 return dict->version;
134 }
135 size_t dictionary_stats_searches(DICTIONARY *dict) {
136 + if(unlikely(!dict)) return 0;
137 return dict->searches;
138 }
139 size_t dictionary_stats_inserts(DICTIONARY *dict) {
140 + if(unlikely(!dict)) return 0;
141 return dict->inserts;
142 }
143 size_t dictionary_stats_deletes(DICTIONARY *dict) {
144 + if(unlikely(!dict)) return 0;
145 return dict->deletes;
146 }
147 size_t dictionary_stats_resets(DICTIONARY *dict) {
148 + if(unlikely(!dict)) return 0;
149 return dict->resets;
150 }
151 size_t dictionary_stats_walkthroughs(DICTIONARY *dict) {
152 + if(unlikely(!dict)) return 0;
153 return dict->walkthroughs;
154 }
155 size_t dictionary_stats_referenced_items(DICTIONARY *dict) {
156 + if(unlikely(!dict)) return 0;
157 return __atomic_load_n(&dict->referenced_items, __ATOMIC_SEQ_CST);
158 }
159
libnetdata/dictionary/dictionary.h
+2 -1
@@ -198,10 +198,11 @@ typedef DICTFE_CONST struct dictionary_foreach {
198
199 #define dfe_start_read(dict, value) dfe_start_rw(dict, value, DICTIONARY_LOCK_READ)
200 #define dfe_start_write(dict, value) dfe_start_rw(dict, value, DICTIONARY_LOCK_WRITE)
201 +#define dfe_start_reentrant(dict, value) dfe_start_rw(dict, value, DICTIONARY_LOCK_REENTRANT)
202 #define dfe_start_rw(dict, value, mode) \
203 do { \
204 DICTFE value ## _dfe = {}; \
204 - const char *value ## _name; (void)(value ## _name); (void)value; \
205 + const char *value ## _name; (void)(value ## _name); (void)(value); \
206 for((value) = dictionary_foreach_start_rw(&value ## _dfe, (dict), (mode)), ( value ## _name ) = value ## _dfe.name; \
207 (value ## _dfe.name) ;\
208 (value) = dictionary_foreach_next(&value ## _dfe), ( value ## _name ) = value ## _dfe.name) \