replication improvements (#13989)
contexts are not set to collected until replication finishes for them; sender thread disables cancelability while replication queries are executed;
Costa Tsaousis committed
Nov 14, 2022 at 01:22 UTC
cbebc18ca3e52ec5ab41272ef85a5fd45d8d3b10
6 files changed
+43
-11
collectors/plugins.d/pluginsd_parser.c
+2
@@ -1144,6 +1144,8 @@ PARSER_RC pluginsd_replay_end(char **words, size_t num_words, void *user)
1144
return PARSER_RC_OK;
1145
}
1146
1147
+ rrdcontext_updated_retention_rrdset(st);
1148
+
1149
bool ok = replicate_chart_request(send_to_plugin, user_object->parser, host, st, first_entry_child, last_entry_child,
1150
first_entry_requested, last_entry_requested);
1151
return ok ? PARSER_RC_OK : PARSER_RC_ERROR;
database/rrdcontext.c
+27
-7
@@ -63,9 +63,7 @@ typedef enum {
63
RRD_FLAG_UPDATE_REASON_DB_ROTATION = (1 << 28), // this context changed because of a db rotation
64
RRD_FLAG_UPDATE_REASON_UNUSED = (1 << 29), // this context is not used anymore
65
RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS = (1 << 30), // this context is not used anymore
66
-
67
- // DO NOT ADD (1 << 31) or bigger!
68
- // runtime error: left shift of 1 by 31 places cannot be represented in type 'int'
66
+ RRD_FLAG_UPDATE_REASON_UPDATED_RETENTION = (1 << 31), // this object has updated retention
67
} RRD_FLAGS;
68
69
#define RRD_FLAG_ALL_UPDATE_REASONS ( \
@@ -229,6 +227,7 @@ static struct rrdcontext_reason {
227
{ RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD, "child disconnected", 65 * USEC_PER_SEC },
228
{ RRD_FLAG_UPDATE_REASON_DB_ROTATION, "db rotation", 65 * USEC_PER_SEC },
229
{ RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS, "changed flags", 65 * USEC_PER_SEC },
230
+ { RRD_FLAG_UPDATE_REASON_UPDATED_RETENTION, "updated retention", 65 * USEC_PER_SEC },
231
232
// terminator
233
{ 0, NULL, 0 },
@@ -1097,6 +1096,14 @@ static inline void rrdinstance_rrdset_is_freed(RRDSET *st) {
1096
st->rrdcontext = NULL;
1097
}
1098
1099
+static inline void rrdinstance_rrdset_has_updated_retention(RRDSET *st) {
1100
+ RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
1101
+ if(unlikely(!ri)) return;
1102
+
1103
+ rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_UPDATED_RETENTION);
1104
+ rrdinstance_trigger_updates(ri, __FUNCTION__ );
1105
+}
1106
+
1107
static inline void rrdinstance_updated_rrdset_name(RRDSET *st) {
1108
// the chart may not be initialized when this is called
1109
if(unlikely(!st->rrdinstance)) return;
@@ -1467,6 +1474,10 @@ void rrdcontext_removed_rrdset(RRDSET *st) {
1474
rrdinstance_rrdset_is_freed(st);
1475
}
1476
1477
+void rrdcontext_updated_retention_rrdset(RRDSET *st) {
1478
+ rrdinstance_rrdset_has_updated_retention(st);
1479
+}
1480
+
1481
void rrdcontext_updated_rrdset_name(RRDSET *st) {
1482
rrdinstance_updated_rrdset_name(st);
1483
}
@@ -2743,9 +2754,10 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {
2754
QUERY_TARGET *qt = &thread_query_target;
2755
2756
if(qt->used)
2746
- fatal("QUERY TARGET: this query target is already used.");
2757
+ fatal("QUERY TARGET: this query target is already used (%zu queries made with this QUERY_TARGET so far).", qt->queries);
2758
2759
qt->used = true;
2760
+ qt->queries++;
2761
2762
// copy the request into query_thread_target
2763
qt->request = *qtr;
@@ -3248,7 +3260,7 @@ static void rrdmetric_process_updates(RRDMETRIC *rm, bool force, RRD_FLAGS reaso
3260
if(reason != RRD_FLAG_NONE)
3261
rrd_flag_set_updated(rm, reason);
3262
3251
- if(!force && !rrd_flag_is_updated(rm) && rrd_flag_check(rm, RRD_FLAG_LIVE_RETENTION))
3263
+ if(!force && !rrd_flag_is_updated(rm) && rrd_flag_check(rm, RRD_FLAG_LIVE_RETENTION) && !rrd_flag_check(rm, RRD_FLAG_UPDATE_REASON_UPDATED_RETENTION))
3264
return;
3265
3266
if(worker_jobs)
@@ -3282,7 +3294,11 @@ static void rrdinstance_post_process_updates(RRDINSTANCE *ri, bool force, RRD_FL
3294
dfe_start_read((DICTIONARY *)ri->rrdmetrics, rm) {
3295
if(unlikely(netdata_exit)) break;
3296
3285
- rrdmetric_process_updates(rm, force, reason, worker_jobs);
3297
+ RRD_FLAGS reason_to_pass = reason;
3298
+ if(rrd_flag_check(ri, RRD_FLAG_UPDATE_REASON_UPDATED_RETENTION))
3299
+ reason_to_pass |= RRD_FLAG_UPDATE_REASON_UPDATED_RETENTION;
3300
+
3301
+ rrdmetric_process_updates(rm, force, reason_to_pass, worker_jobs);
3302
3303
if(unlikely(!rrd_flag_check(rm, RRD_FLAG_LIVE_RETENTION)))
3304
live_retention = false;
@@ -3385,7 +3401,11 @@ static void rrdcontext_post_process_updates(RRDCONTEXT *rc, bool force, RRD_FLAG
3401
dfe_start_reentrant(rc->rrdinstances, ri) {
3402
if(unlikely(netdata_exit)) break;
3403
3388
- rrdinstance_post_process_updates(ri, force, reason, worker_jobs);
3404
+ RRD_FLAGS reason_to_pass = reason;
3405
+ if(rrd_flag_check(rc, RRD_FLAG_UPDATE_REASON_UPDATED_RETENTION))
3406
+ reason_to_pass |= RRD_FLAG_UPDATE_REASON_UPDATED_RETENTION;
3407
+
3408
+ rrdinstance_post_process_updates(ri, force, reason_to_pass, worker_jobs);
3409
3410
if(unlikely(hidden && !rrd_flag_check(ri, RRD_FLAG_HIDDEN)))
3411
hidden = false;
database/rrdcontext.h
+2
@@ -87,6 +87,7 @@ void rrdcontext_updated_rrdset(RRDSET *st);
87
void rrdcontext_removed_rrdset(RRDSET *st);
88
void rrdcontext_updated_rrdset_name(RRDSET *st);
89
void rrdcontext_updated_rrdset_flags(RRDSET *st);
90
+void rrdcontext_updated_retention_rrdset(RRDSET *st);
91
void rrdcontext_collected_rrdset(RRDSET *st);
92
int rrdcontext_find_chart_uuid(RRDSET *st, uuid_t *store_uuid);
93
@@ -177,6 +178,7 @@ typedef struct query_target {
178
QUERY_TARGET_REQUEST request;
179
180
bool used; // when true, this query is currently being used
181
+ size_t queries; // how many query we have done so far
182
183
struct {
184
bool relative; // true when the request made with relative timestamps, true if it was absolute
database/rrdset.c
+3
-2
@@ -1097,8 +1097,6 @@ void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n,
1097
1098
store_metric_at_tier(rd, t, sp, point_end_time_ut);
1099
}
1100
-
1101
- rrdcontext_collected_rrddim(rd);
1100
}
1101
1102
// caching of dimensions rrdset_done() and rrdset_done_interpolate() loop through
@@ -1257,6 +1255,7 @@ static inline size_t rrdset_done_interpolate(
1255
if(unlikely(!store_this_entry)) {
1256
(void) ml_is_anomalous(rd, 0, false);
1257
rrddim_store_metric(rd, next_store_ut, NAN, SN_FLAG_NONE);
1258
+ rrdcontext_collected_rrddim(rd);
1259
continue;
1260
}
1261
@@ -1269,6 +1268,7 @@ static inline size_t rrdset_done_interpolate(
1268
}
1269
1270
rrddim_store_metric(rd, next_store_ut, new_value, dim_storage_flags);
1271
+ rrdcontext_collected_rrddim(rd);
1272
rd->last_stored_value = new_value;
1273
}
1274
else {
@@ -1277,6 +1277,7 @@ static inline size_t rrdset_done_interpolate(
1277
rrdset_debug(st, "%s: STORE[%ld] = NON EXISTING ", rrddim_name(rd), current_entry);
1278
1279
rrddim_store_metric(rd, next_store_ut, NAN, SN_FLAG_NONE);
1280
+ rrdcontext_collected_rrddim(rd);
1281
rd->last_stored_value = NAN;
1282
}
1283
streaming/replication.c
+5
-2
@@ -153,10 +153,13 @@ bool replicate_chart_response(RRDHOST *host, RRDSET *st, bool start_streaming, t
153
time_t query_after = after;
154
time_t query_before = before;
155
time_t now = now_realtime_sec();
156
+ time_t tolerance = 2; // sometimes from the time we get this value, to the time we check,
157
+ // a data collection has been made
158
+ // so, we give this tolerance to detect invalid timestamps
159
160
// find the first entry we have
161
time_t first_entry_local = rrdset_first_entry_t(st);
159
- if(first_entry_local > now) {
162
+ if(first_entry_local > now + tolerance) {
163
internal_error(true,
164
"RRDSET: '%s' first time %llu is in the future (now is %llu)",
165
rrdset_id(st), (unsigned long long)first_entry_local, (unsigned long long)now);
@@ -175,7 +178,7 @@ bool replicate_chart_response(RRDHOST *host, RRDSET *st, bool start_streaming, t
178
last_entry_local = rrdset_last_entry_t(st);
179
}
180
178
- if(last_entry_local > now) {
181
+ if(last_entry_local > now + tolerance) {
182
internal_error(true,
183
"RRDSET: '%s' last updated time %llu is in the future (now is %llu)",
184
rrdset_id(st), (unsigned long long)last_entry_local, (unsigned long long)now);
streaming/sender.c
+4
@@ -1132,10 +1132,14 @@ static void process_replication_requests(struct sender_state *s) {
1132
continue;
1133
}
1134
1135
+ netdata_thread_disable_cancelability();
1136
+
1137
// send the replication data
1138
bool start_streaming = replicate_chart_response(st->rrdhost, st,
1139
rr->start_streaming, rr->after, rr->before);
1140
1141
+ netdata_thread_enable_cancelability();
1142
+
1143
// enable normal streaming if we have to
1144
if (start_streaming) {
1145
debug(D_REPLICATION, "Enabling metric streaming for chart %s.%s",