streaming compression, query planner and replication fixes (#14023)
* streaming compression, query planner and replication fixes * remove journal v2 stats from global statistics * disable sql for checking past sql UUIDs * single threaded replication * final replication thread using dictionaries and JudyL for sorting the pending requests * do not timeout the sending socket when there are pending replication requests * streaming receiver using read() instead of fread() * remove FILE * from streaming - now using posix read() and write() * increase timeouts to 10 minutes * apply sender timeout only when there are metrics that are supposed to be streamed * error handling in replication * remove retries on socket read timeout; better error messages * take into account inbound traffic too to detect that a connection is stale * remove race conditions from replication thread * make sure deleted entries are marked as executed, so that even if deletion fails, they will not be executed * 2 minutes timeout to retry streaming to a parent that already has this node * remove unecessary condition check * fix compilation warnings * include judy in replication * wrappers to handle retries for SSL_read and SSL_write * compressed bytes read monitoring * recursive locks on replication to make it faster during flush or cleanup * replication completion chart at the receiver side * simplified recursive mutex * simplified recursive mutex again
Costa Tsaousis committed
Nov 20, 2022 at 23:47 UTC
284f6f3aa4f36cefad2601c490510621496c2b53
36 files changed
+1672
-871
collectors/plugins.d/pluginsd_parser.c
+143
-46
@@ -13,20 +13,44 @@ static int send_to_plugin(const char *txt, void *data) {
13
#ifdef ENABLE_HTTPS
14
struct netdata_ssl *ssl = parser->ssl_output;
15
if(ssl) {
16
- if(ssl->conn && ssl->flags == NETDATA_SSL_HANDSHAKE_COMPLETE) {
17
- size_t size = strlen(txt);
18
- return SSL_write(ssl->conn, txt, (int)size);
19
- }
16
+ if(ssl->conn && ssl->flags == NETDATA_SSL_HANDSHAKE_COMPLETE)
17
+ return (int)netdata_ssl_write(ssl->conn, (void *)txt, strlen(txt));
18
21
- error("cannot write to SSL connection - connection is not ready.");
19
+ error("PLUGINSD: cannot send command (SSL)");
20
return -1;
21
}
22
#endif
23
26
- FILE *fp = parser->output;
27
- int ret = fprintf(fp, "%s", txt);
28
- fflush(fp);
29
- return ret;
24
+ if(parser->fp_output) {
25
+ int bytes = fprintf(parser->fp_output, "%s", txt);
26
+ if(bytes <= 0) {
27
+ error("PLUGINSD: cannot send command (FILE)");
28
+ return -2;
29
+ }
30
+ fflush(parser->fp_output);
31
+ return bytes;
32
+ }
33
+
34
+ if(parser->fd != -1) {
35
+ size_t bytes = 0;
36
+ size_t total = strlen(txt);
37
+ ssize_t sent;
38
+
39
+ do {
40
+ sent = write(parser->fd, &txt[bytes], total - bytes);
41
+ if(sent <= 0) {
42
+ error("PLUGINSD: cannot send command (fd)");
43
+ return -3;
44
+ }
45
+ bytes += sent;
46
+ }
47
+ while(bytes < total);
48
+
49
+ return (int)bytes;
50
+ }
51
+
52
+ error("PLUGINSD: cannot send command (no output socket/pipe/file given to plugins.d parser)");
53
+ return -4;
54
}
55
56
PARSER_RC pluginsd_set(char **words, size_t num_words, void *user)
@@ -293,9 +317,25 @@ PARSER_RC pluginsd_chart_definition_end(char **words, size_t num_words, void *us
317
// rrdhost_hostname(host), rrdset_id(st),
318
// (unsigned long long)first_entry_child, (unsigned long long)last_entry_child);
319
296
- rrdset_flag_clear(st, RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED);
320
+ bool ok = true;
321
+ if(!rrdset_flag_check(st, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS)) {
322
+
323
+#ifdef NETDATA_INTERNAL_CHECKS
324
+ st->replay.start_streaming = false;
325
+ st->replay.after = 0;
326
+ st->replay.before = 0;
327
+#endif
328
+
329
+ rrdset_flag_clear(st, RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED);
330
+ rrdset_flag_set(st, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS);
331
+
332
+ ok = replicate_chart_request(send_to_plugin, user_object->parser, host, st, first_entry_child,
333
+ last_entry_child, 0, 0);
334
+ }
335
+ else {
336
+ internal_error(true, "RRDSET: not sending duplicate replication request for chart '%s'", rrdset_id(st));
337
+ }
338
298
- bool ok = replicate_chart_request(send_to_plugin, user_object->parser, host, st, first_entry_child, last_entry_child, 0, 0);
339
return ok ? PARSER_RC_OK : PARSER_RC_ERROR;
340
}
341
@@ -425,7 +465,7 @@ static void inflight_functions_insert_callback(const DICTIONARY_ITEM *item, void
465
pf->sent_ut = now_realtime_usec();
466
467
if(ret < 0) {
428
- error("FUNCTION: failed to send function to plugin, fprintf() returned error %d", ret);
468
+ error("FUNCTION: failed to send function to plugin, error %d", ret);
469
rrd_call_function_error(pf->destination_wb, "Failed to communicate with collector", HTTP_RESP_BACKEND_FETCH_FAILED);
470
}
471
else {
@@ -847,42 +887,54 @@ PARSER_RC pluginsd_replay_rrdset_begin(char **words, size_t num_words, void *use
887
char *id = get_word(words, num_words, 1);
888
char *start_time_str = get_word(words, num_words, 2);
889
char *end_time_str = get_word(words, num_words, 3);
890
+ char *child_now_str = get_word(words, num_words, 4);
891
892
RRDSET *st = ((PARSER_USER_OBJECT *) user)->st;
893
RRDHOST *host = ((PARSER_USER_OBJECT *)user)->host;
894
895
if (unlikely(!id || (!st && !*id))) {
855
- error("REPLAY: requested a " PLUGINSD_KEYWORD_REPLAY_BEGIN " without a chart id for host '%s'. Disabling it.", rrdhost_hostname(host));
896
+ error("REPLAY: got a " PLUGINSD_KEYWORD_REPLAY_BEGIN " without a chart id for host '%s'. Disabling it.", rrdhost_hostname(host));
897
goto disable;
898
}
899
900
if(*id) {
901
st = rrdset_find(host, id);
902
if (unlikely(!st)) {
862
- error("requested a " PLUGINSD_KEYWORD_REPLAY_BEGIN " on chart '%s', which does not exist on host '%s'. Disabling it.",
903
+ error("REPLAY: got a " PLUGINSD_KEYWORD_REPLAY_BEGIN " on chart '%s', which does not exist on host '%s'. Disabling it.",
904
id, rrdhost_hostname(host));
905
goto disable;
906
}
907
908
((PARSER_USER_OBJECT *) user)->st = st;
868
- ((PARSER_USER_OBJECT *) user)->replay.start_time = 0;
869
- ((PARSER_USER_OBJECT *) user)->replay.end_time = 0;
870
- ((PARSER_USER_OBJECT *) user)->replay.start_time_ut = 0;
871
- ((PARSER_USER_OBJECT *) user)->replay.end_time_ut = 0;
909
+ }
910
+
911
+ if(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) {
912
+ error("REPLAY: chart '%s' on host '%s' has the OBSOLETE flag set, but it is collected.", rrdset_id(st), rrdhost_hostname(host));
913
+ rrdset_isnot_obsolete(st);
914
}
915
916
if(start_time_str && end_time_str) {
917
time_t start_time = strtol(start_time_str, NULL, 0);
918
time_t end_time = strtol(end_time_str, NULL, 0);
919
878
- if(start_time && end_time) {
879
- if (start_time > end_time) {
880
- error("REPLAY: requested a " PLUGINSD_KEYWORD_REPLAY_BEGIN " on chart '%s' ('%s') on host '%s', but timings are invalid (%ld to %ld). Disabling it.",
881
- rrdset_name(st), rrdset_id(st), rrdhost_hostname(st->rrdhost), start_time, end_time);
882
- goto disable;
883
- }
920
+ time_t wall_clock_time = 0, tolerance;
921
+ if(child_now_str) {
922
+ wall_clock_time = strtol(child_now_str, NULL, 0);
923
+ tolerance = 1;
924
+ }
925
+
926
+ if(wall_clock_time <= 0) {
927
+ wall_clock_time = now_realtime_sec();
928
+ tolerance = st->update_every + 60;
929
+ }
930
+
931
+ internal_error(
932
+ (!st->replay.start_streaming && (end_time < st->replay.after || start_time > st->replay.before)),
933
+ "REPLAY: got a " PLUGINSD_KEYWORD_REPLAY_BEGIN " on chart '%s' ('%s') on host '%s', from %ld to %ld, which does not match our request (%ld to %ld).",
934
+ rrdset_name(st), rrdset_id(st), rrdhost_hostname(st->rrdhost), start_time, end_time, st->replay.after, st->replay.before);
935
885
- if (end_time - start_time != st->update_every)
936
+ if(start_time && end_time && start_time < wall_clock_time + tolerance && end_time < wall_clock_time + tolerance && start_time < end_time) {
937
+ if (unlikely(end_time - start_time != st->update_every))
938
rrdset_set_update_every(st, end_time - start_time);
939
940
st->last_collected_time.tv_sec = end_time;
@@ -891,11 +943,6 @@ PARSER_RC pluginsd_replay_rrdset_begin(char **words, size_t num_words, void *use
943
st->last_updated.tv_sec = end_time;
944
st->last_updated.tv_usec = 0;
945
894
- ((PARSER_USER_OBJECT *) user)->replay.start_time = start_time;
895
- ((PARSER_USER_OBJECT *) user)->replay.end_time = end_time;
896
- ((PARSER_USER_OBJECT *) user)->replay.start_time_ut = (usec_t) start_time * USEC_PER_SEC;
897
- ((PARSER_USER_OBJECT *) user)->replay.end_time_ut = (usec_t) end_time * USEC_PER_SEC;
898
-
946
st->counter++;
947
st->counter_done++;
948
@@ -903,9 +950,31 @@ PARSER_RC pluginsd_replay_rrdset_begin(char **words, size_t num_words, void *use
950
st->current_entry++;
951
if(st->current_entry >= st->entries)
952
st->current_entry -= st->entries;
953
+
954
+ ((PARSER_USER_OBJECT *) user)->replay.start_time = start_time;
955
+ ((PARSER_USER_OBJECT *) user)->replay.end_time = end_time;
956
+ ((PARSER_USER_OBJECT *) user)->replay.start_time_ut = (usec_t) start_time * USEC_PER_SEC;
957
+ ((PARSER_USER_OBJECT *) user)->replay.end_time_ut = (usec_t) end_time * USEC_PER_SEC;
958
+ ((PARSER_USER_OBJECT *) user)->replay.wall_clock_time = wall_clock_time;
959
+ ((PARSER_USER_OBJECT *) user)->replay.rset_enabled = true;
960
+
961
+ return PARSER_RC_OK;
962
}
963
+
964
+ internal_error(true,
965
+ "REPLAY: got a " PLUGINSD_KEYWORD_REPLAY_BEGIN " on chart '%s' ('%s') on host '%s', from %ld to %ld, but timestamps are invalid (now is %ld).",
966
+ rrdset_name(st), rrdset_id(st), rrdhost_hostname(st->rrdhost), start_time, end_time, wall_clock_time);
967
}
968
969
+ // the child sends an RBEGIN without any parameters initially
970
+ // setting rset_enabled to false, means the RSET should not store any metrics
971
+ // to store metrics, the RBEGIN needs to have timestamps
972
+ ((PARSER_USER_OBJECT *) user)->replay.start_time = 0;
973
+ ((PARSER_USER_OBJECT *) user)->replay.end_time = 0;
974
+ ((PARSER_USER_OBJECT *) user)->replay.start_time_ut = 0;
975
+ ((PARSER_USER_OBJECT *) user)->replay.end_time_ut = 0;
976
+ ((PARSER_USER_OBJECT *) user)->replay.wall_clock_time = 0;
977
+ ((PARSER_USER_OBJECT *) user)->replay.rset_enabled = false;
978
return PARSER_RC_OK;
979
980
disable:
@@ -915,6 +984,9 @@ disable:
984
985
PARSER_RC pluginsd_replay_set(char **words, size_t num_words, void *user)
986
{
987
+ if(!((PARSER_USER_OBJECT *) user)->replay.rset_enabled)
988
+ return PARSER_RC_OK;
989
+
990
char *dimension = get_word(words, num_words, 1);
991
char *value_str = get_word(words, num_words, 2);
992
char *flags_str = get_word(words, num_words, 3);
@@ -923,20 +995,22 @@ PARSER_RC pluginsd_replay_set(char **words, size_t num_words, void *user)
995
RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
996
997
if (unlikely(!st)) {
926
- error("REPLAY: requested a " PLUGINSD_KEYWORD_REPLAY_SET " on dimension '%s' on host '%s', without a " PLUGINSD_KEYWORD_REPLAY_BEGIN ". Disabling it.",
998
+ error("REPLAY: got a " PLUGINSD_KEYWORD_REPLAY_SET " on dimension '%s' on host '%s', without a " PLUGINSD_KEYWORD_REPLAY_BEGIN ". Disabling it.",
999
dimension, rrdhost_hostname(host));
1000
goto disable;
1001
}
1002
1003
if (unlikely(!dimension || !*dimension)) {
932
- error("REPLAY: requested a " PLUGINSD_KEYWORD_REPLAY_SET " on chart '%s' of host '%s', without a dimension. Disabling it.",
1004
+ error("REPLAY: got a " PLUGINSD_KEYWORD_REPLAY_SET " on chart '%s' of host '%s', without a dimension. Disabling it.",
1005
rrdset_id(st), rrdhost_hostname(host));
1006
goto disable;
1007
}
1008
1009
if (unlikely(!((PARSER_USER_OBJECT *) user)->replay.start_time || !((PARSER_USER_OBJECT *) user)->replay.end_time)) {
938
- error("REPLAY: requested a " PLUGINSD_KEYWORD_REPLAY_SET " on dimension '%s' on host '%s', without timings from a " PLUGINSD_KEYWORD_REPLAY_BEGIN ". Disabling it.",
939
- dimension, rrdhost_hostname(host));
1010
+ error("REPLAY: got a " PLUGINSD_KEYWORD_REPLAY_SET " on dimension '%s' on host '%s', with invalid timestamps %ld to %ld from a " PLUGINSD_KEYWORD_REPLAY_BEGIN ". Disabling it.",
1011
+ dimension, rrdhost_hostname(host),
1012
+ ((PARSER_USER_OBJECT *) user)->replay.start_time,
1013
+ ((PARSER_USER_OBJECT *) user)->replay.end_time);
1014
goto disable;
1015
}
1016
@@ -946,14 +1020,11 @@ PARSER_RC pluginsd_replay_set(char **words, size_t num_words, void *user)
1020
if(unlikely(!flags_str))
1021
flags_str = "";
1022
949
- if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
950
- debug(D_PLUGINSD, "REPLAY: is replaying dimension '%s'/'%s' to '%s'", rrdset_id(st), dimension, value_str);
951
-
1023
if (likely(value_str)) {
1024
RRDDIM_ACQUIRED *rda = rrddim_find_and_acquire(st, dimension);
1025
RRDDIM *rd = rrddim_acquired_to_rrddim(rda);
1026
if(unlikely(!rd)) {
956
- error("REPLAY: requested a " PLUGINSD_KEYWORD_REPLAY_SET " to dimension with id '%s' on chart '%s' ('%s') on host '%s', which does not exist. Disabling it.",
1027
+ error("REPLAY: got a " PLUGINSD_KEYWORD_REPLAY_SET " to dimension '%s' on chart '%s' ('%s') on host '%s', which does not exist. Disabling it.",
1028
dimension, rrdset_name(st), rrdset_id(st), rrdhost_hostname(st->rrdhost));
1029
goto disable;
1030
}
@@ -961,7 +1032,7 @@ PARSER_RC pluginsd_replay_set(char **words, size_t num_words, void *user)
1032
RRDDIM_FLAGS rd_flags = rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE | RRDDIM_FLAG_ARCHIVED);
1033
1034
if(unlikely(rd_flags & RRDDIM_FLAG_OBSOLETE)) {
964
- error("Dimension %s in chart '%s' has the OBSOLETE flag set, but it is collected.", rrddim_name(rd), rrdset_id(st));
1035
+ error("REPLAY: dimension '%s' in chart '%s' has the OBSOLETE flag set, but it is collected.", rrddim_name(rd), rrdset_id(st));
1036
rrddim_isnot_obsolete(st, rd);
1037
}
1038
@@ -998,7 +1069,7 @@ PARSER_RC pluginsd_replay_set(char **words, size_t num_words, void *user)
1069
rd->collections_counter++;
1070
}
1071
else
1001
- error("Dimension %s in chart '%s' has the ARCHIVED flag set, but it is collected. Ignoring data.", rrddim_name(rd), rrdset_id(st));
1072
+ error("REPLAY: dimension '%s' in chart '%s' has the ARCHIVED flag set, but it is collected. Ignoring data.", rrddim_name(rd), rrdset_id(st));
1073
1074
rrddim_acquired_release(rda);
1075
}
@@ -1021,13 +1092,13 @@ PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, size_t num_words
1092
RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
1093
1094
if (unlikely(!st)) {
1024
- error("REPLAY: requested a " PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE " on dimension '%s' on host '%s', without a " PLUGINSD_KEYWORD_REPLAY_BEGIN ". Disabling it.",
1095
+ error("REPLAY: got a " PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE " on dimension '%s' on host '%s', without a " PLUGINSD_KEYWORD_REPLAY_BEGIN ". Disabling it.",
1096
dimension, rrdhost_hostname(host));
1097
goto disable;
1098
}
1099
1100
if (unlikely(!dimension || !*dimension)) {
1030
- error("REPLAY: requested a " PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE " on chart '%s' of host '%s', without a dimension. Disabling it.",
1101
+ error("REPLAY: got a " PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE " on chart '%s' of host '%s', without a dimension. Disabling it.",
1102
rrdset_id(st), rrdhost_hostname(host));
1103
goto disable;
1104
}
@@ -1035,7 +1106,7 @@ PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, size_t num_words
1106
RRDDIM_ACQUIRED *rda = rrddim_find_and_acquire(st, dimension);
1107
RRDDIM *rd = rrddim_acquired_to_rrddim(rda);
1108
if(unlikely(!rd)) {
1038
- error("REPLAY: requested a " PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE " to dimension with id '%s' on chart '%s' ('%s') on host '%s', which does not exist. Disabling it.",
1109
+ error("REPLAY: got a " PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE " to dimension with id '%s' on chart '%s' ('%s') on host '%s', which does not exist. Disabling it.",
1110
dimension, rrdset_name(st), rrdset_id(st), rrdhost_hostname(st->rrdhost));
1111
goto disable;
1112
}
@@ -1067,7 +1138,7 @@ PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words
1138
RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
1139
1140
if (unlikely(!st)) {
1070
- error("REPLAY: requested a " PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE " on host '%s', without a " PLUGINSD_KEYWORD_REPLAY_BEGIN ". Disabling it.",
1141
+ error("REPLAY: got a " PLUGINSD_KEYWORD_REPLAY_RRDSET_STATE " on host '%s', without a " PLUGINSD_KEYWORD_REPLAY_BEGIN ". Disabling it.",
1142
rrdhost_hostname(host));
1143
goto disable;
1144
}
@@ -1117,7 +1188,7 @@ PARSER_RC pluginsd_replay_end(char **words, size_t num_words, void *user)
1188
RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
1189
1190
if (unlikely(!st)) {
1120
- error("REPLAY: requested a " PLUGINSD_KEYWORD_REPLAY_END " on host '%s', without a " PLUGINSD_KEYWORD_REPLAY_BEGIN ". Disabling it.",
1191
+ error("REPLAY: got a " PLUGINSD_KEYWORD_REPLAY_END " on host '%s', without a " PLUGINSD_KEYWORD_REPLAY_BEGIN ". Disabling it.",
1192
rrdhost_hostname(host));
1193
return PARSER_RC_ERROR;
1194
}
@@ -1132,15 +1203,41 @@ PARSER_RC pluginsd_replay_end(char **words, size_t num_words, void *user)
1203
((PARSER_USER_OBJECT *) user)->st = NULL;
1204
((PARSER_USER_OBJECT *) user)->count++;
1205
1206
+ if(((PARSER_USER_OBJECT *) user)->replay.rset_enabled && st->rrdhost->receiver) {
1207
+ time_t now = now_realtime_sec();
1208
+ time_t started = st->rrdhost->receiver->replication_first_time_t;
1209
+ time_t current = ((PARSER_USER_OBJECT *) user)->replay.end_time;
1210
+
1211
+ worker_set_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION,
1212
+ (NETDATA_DOUBLE)(current - started) * 100.0 / (NETDATA_DOUBLE)(now - started));
1213
+ }
1214
+
1215
+ ((PARSER_USER_OBJECT *) user)->replay.start_time = 0;
1216
+ ((PARSER_USER_OBJECT *) user)->replay.end_time = 0;
1217
+ ((PARSER_USER_OBJECT *) user)->replay.start_time_ut = 0;
1218
+ ((PARSER_USER_OBJECT *) user)->replay.end_time_ut = 0;
1219
+ ((PARSER_USER_OBJECT *) user)->replay.wall_clock_time = 0;
1220
+ ((PARSER_USER_OBJECT *) user)->replay.rset_enabled = false;
1221
+
1222
st->counter++;
1223
st->counter_done++;
1224
1225
+#ifdef NETDATA_INTERNAL_CHECKS
1226
+ st->replay.start_streaming = false;
1227
+ st->replay.after = 0;
1228
+ st->replay.before = 0;
1229
+#endif
1230
+
1231
if (start_streaming) {
1232
if (st->update_every != update_every_child)
1233
rrdset_set_update_every(st, update_every_child);
1234
1235
rrdset_flag_set(st, RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED);
1236
+ rrdset_flag_clear(st, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS);
1237
rrdset_flag_clear(st, RRDSET_FLAG_SYNC_CLOCK);
1238
+
1239
+ worker_set_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION, 100.0);
1240
+
1241
return PARSER_RC_OK;
1242
}
1243
@@ -1191,7 +1288,7 @@ inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp_plugi
1288
};
1289
1290
// fp_plugin_output = our input; fp_plugin_input = our output
1194
- PARSER *parser = parser_init(host, &user, fp_plugin_output, fp_plugin_input, PARSER_INPUT_SPLIT, NULL);
1291
+ PARSER *parser = parser_init(host, &user, fp_plugin_output, fp_plugin_input, -1, PARSER_INPUT_SPLIT, NULL);
1292
1293
rrd_collector_started();
1294
collectors/plugins.d/pluginsd_parser.h
+4
@@ -26,6 +26,10 @@ typedef struct parser_user_object {
26
27
usec_t start_time_ut;
28
usec_t end_time_ut;
29
+
30
+ time_t wall_clock_time;
31
+
32
+ bool rset_enabled;
33
} replay;
34
} PARSER_USER_OBJECT;
35
daemon/global_statistics.c
+2
-1
@@ -1848,6 +1848,7 @@ static struct worker_utilization all_workers_utilization[] = {
1848
{ .name = "TIMEX", .family = "workers plugin timex", .priority = 1000000 },
1849
{ .name = "IDLEJITTER", .family = "workers plugin idlejitter", .priority = 1000000 },
1850
{ .name = "RRDCONTEXT", .family = "workers contexts", .priority = 1000000 },
1851
+ { .name = "REPLICATION", .family = "workers replication sender", .priority = 1000000 },
1852
{ .name = "SERVICE", .family = "workers service", .priority = 1000000 },
1853
1854
// has to be terminated with a NULL
@@ -2203,7 +2204,7 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
2204
{
2205
size_t i;
2206
for (i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES; i++) {
2206
- if(wu->per_job_type[i].type != WORKER_METRIC_INCREMENTAL)
2207
+ if(wu->per_job_type[i].type != WORKER_METRIC_INCREMENT && wu->per_job_type[i].type != WORKER_METRIC_INCREMENTAL_TOTAL)
2208
continue;
2209
2210
if(!wu->per_job_type[i].count_value)
daemon/main.c
+11
-1
@@ -4,6 +4,7 @@
4
#include "buildinfo.h"
5
#include "static_threads.h"
6
7
+bool unittest_running = false;
8
int netdata_zero_metrics_enabled;
9
int netdata_anonymous_statistics_enabled;
10
@@ -678,7 +679,7 @@ static void get_netdata_configured_variables() {
679
// ------------------------------------------------------------------------
680
// get default Database Engine page cache size in MiB
681
681
- db_engine_use_malloc = config_get_boolean(CONFIG_SECTION_DB, "dbengine page cache with malloc", CONFIG_BOOLEAN_NO);
682
+ db_engine_use_malloc = config_get_boolean(CONFIG_SECTION_DB, "dbengine page cache with malloc", CONFIG_BOOLEAN_YES);
683
default_rrdeng_page_cache_mb = (int) config_get_number(CONFIG_SECTION_DB, "dbengine page cache size MB", default_rrdeng_page_cache_mb);
684
if(default_rrdeng_page_cache_mb < RRDENG_MIN_PAGE_CACHE_SIZE_MB) {
685
error("Invalid page cache size %d given. Defaulting to %d.", default_rrdeng_page_cache_mb, RRDENG_MIN_PAGE_CACHE_SIZE_MB);
@@ -982,6 +983,8 @@ int main(int argc, char **argv) {
983
}
984
985
if(strcmp(optarg, "unittest") == 0) {
986
+ unittest_running = true;
987
+
988
if (unit_test_static_threads())
989
return 1;
990
if (unit_test_buffer())
@@ -1028,24 +1031,31 @@ int main(int argc, char **argv) {
1031
#endif
1032
#ifdef ENABLE_DBENGINE
1033
else if(strcmp(optarg, "mctest") == 0) {
1034
+ unittest_running = true;
1035
return mc_unittest();
1036
}
1037
else if(strcmp(optarg, "ctxtest") == 0) {
1038
+ unittest_running = true;
1039
return ctx_unittest();
1040
}
1041
else if(strcmp(optarg, "dicttest") == 0) {
1042
+ unittest_running = true;
1043
return dictionary_unittest(10000);
1044
}
1045
else if(strcmp(optarg, "araltest") == 0) {
1046
+ unittest_running = true;
1047
return aral_unittest(10000);
1048
}
1049
else if(strcmp(optarg, "stringtest") == 0) {
1050
+ unittest_running = true;
1051
return string_unittest(10000);
1052
}
1053
else if(strcmp(optarg, "rrdlabelstest") == 0) {
1054
+ unittest_running = true;
1055
return rrdlabels_unittest();
1056
}
1057
else if(strcmp(optarg, "metatest") == 0) {
1058
+ unittest_running = true;
1059
return metadata_unittest();
1060
}
1061
else if(strncmp(optarg, createdataset_string, strlen(createdataset_string)) == 0) {
daemon/service.c
+7
-4
@@ -156,17 +156,20 @@ static void svc_rrdhost_cleanup_obsolete_charts(RRDHOST *host) {
156
static void svc_rrdset_check_obsoletion(RRDHOST *host) {
157
worker_is_busy(WORKER_JOB_CHILD_CHART_OBSOLETION_CHECK);
158
159
+ time_t now = now_realtime_sec();
160
time_t last_entry_t;
161
RRDSET *st;
162
rrdset_foreach_read(st, host) {
163
+ if(!rrdset_flag_check(st, RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED))
164
+ continue;
165
+
166
last_entry_t = rrdset_last_entry_t(st);
167
164
- if(last_entry_t && last_entry_t < host->senders_connect_time && host->senders_connect_time
165
- + TIME_TO_RUN_OBSOLETIONS_ON_CHILD_CONNECT + ITERATIONS_TO_RUN_OBSOLETIONS_ON_CHILD_CONNECT * st->update_every
166
- < now_realtime_sec())
168
+ if(last_entry_t && last_entry_t < host->senders_connect_time &&
169
+ host->senders_connect_time + TIME_TO_RUN_OBSOLETIONS_ON_CHILD_CONNECT + ITERATIONS_TO_RUN_OBSOLETIONS_ON_CHILD_CONNECT * st->update_every
170
+ < now)
171
172
rrdset_is_obsolete(st);
169
-
173
}
174
rrdset_foreach_done(st);
175
}
daemon/static_threads.c
+21
-10
@@ -2,16 +2,17 @@
2
3
#include "common.h"
4
5
-extern void *aclk_main(void *ptr);
6
-extern void *analytics_main(void *ptr);
7
-extern void *checks_main(void *ptr);
8
-extern void *cpuidlejitter_main(void *ptr);
9
-extern void *global_statistics_main(void *ptr);
10
-extern void *health_main(void *ptr);
11
-extern void *pluginsd_main(void *ptr);
12
-extern void *service_main(void *ptr);
13
-extern void *statsd_main(void *ptr);
14
-extern void *timex_main(void *ptr);
5
+void *aclk_main(void *ptr);
6
+void *analytics_main(void *ptr);
7
+void *checks_main(void *ptr);
8
+void *cpuidlejitter_main(void *ptr);
9
+void *global_statistics_main(void *ptr);
10
+void *health_main(void *ptr);
11
+void *pluginsd_main(void *ptr);
12
+void *service_main(void *ptr);
13
+void *statsd_main(void *ptr);
14
+void *timex_main(void *ptr);
15
+void *replication_thread_main(void *ptr __maybe_unused);
16
17
extern bool global_statistics_enabled;
18
@@ -140,6 +141,16 @@ const struct netdata_static_thread static_threads_common[] = {
141
.start_routine = rrdcontext_main
142
},
143
144
+ {
145
+ .name = "replication",
146
+ .config_section = NULL,
147
+ .config_name = NULL,
148
+ .enabled = 1,
149
+ .thread = NULL,
150
+ .init_routine = NULL,
151
+ .start_routine = replication_thread_main
152
+ },
153
+
154
// terminator
155
{
156
.name = NULL,
database/rrd.h
+13
-2
@@ -55,6 +55,7 @@ struct pg_cache_page_index;
55
#include "sqlite/sqlite_health.h"
56
#include "rrdcontext.h"
57
58
+extern bool unittest_running;
59
extern bool dbengine_enabled;
60
extern size_t storage_tiers;
61
extern size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS];
@@ -533,8 +534,9 @@ typedef enum rrdset_flags {
534
535
RRDSET_FLAG_SENDER_REPLICATION_FINISHED = (1 << 22), // the sending side has completed replication
536
RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED = (1 << 23), // the receiving side has completed replication
537
+ RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS = (1 << 24), // the receiving side has replication in progress
538
537
- RRDSET_FLAG_UPSTREAM_SEND_VARIABLES = (1 << 24), // a custom variable has been updated and needs to be exposed to parent
539
+ RRDSET_FLAG_UPSTREAM_SEND_VARIABLES = (1 << 25), // a custom variable has been updated and needs to be exposed to parent
540
} RRDSET_FLAGS;
541
542
#define rrdset_flag_check(st, flag) (__atomic_load_n(&((st)->flags), __ATOMIC_SEQ_CST) & (flag))
@@ -658,6 +660,14 @@ struct rrdset {
660
netdata_rwlock_t rwlock; // protection for RRDCALC *base
661
RRDCALC *base; // double linked list of RRDCALC related to this RRDSET
662
} alerts;
663
+
664
+#ifdef NETDATA_INTERNAL_CHECKS
665
+ struct {
666
+ bool start_streaming;
667
+ time_t after;
668
+ time_t before;
669
+ } replay;
670
+#endif
671
};
672
673
#define rrdset_plugin_name(st) string2str((st)->plugin_name)
@@ -757,6 +767,8 @@ typedef enum {
767
// Configuration options
768
RRDHOST_OPTION_DELETE_OBSOLETE_CHARTS = (1 << 3), // delete files of obsolete charts
769
RRDHOST_OPTION_DELETE_ORPHAN_HOST = (1 << 4), // delete the entire host when orphan
770
+
771
+ RRDHOST_OPTION_REPLICATION = (1 << 5), // when set, we support replication for this host
772
} RRDHOST_OPTIONS;
773
774
#define rrdhost_option_check(host, flag) ((host)->options & (flag))
@@ -937,7 +949,6 @@ struct rrdhost {
949
struct rrdpush_destinations *destination; // the current destination from the above list
950
SIMPLE_PATTERN *rrdpush_send_charts_matching; // pattern to match the charts to be sent
951
940
- bool rrdpush_enable_replication; // enable replication
952
time_t rrdpush_seconds_to_replicate; // max time we want to replicate from the child
953
time_t rrdpush_replication_step; // seconds per replication step
954
database/rrdcalc.c
+20
-2
@@ -408,6 +408,8 @@ struct rrdcalc_constructor {
408
RRDCALC_REACT_NONE,
409
RRDCALC_REACT_NEW,
410
} react_action;
411
+
412
+ bool existing_from_template;
413
};
414
415
static void rrdcalc_rrdhost_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdcalc, void *constructor_data) {
@@ -543,6 +545,20 @@ static void rrdcalc_rrdhost_insert_callback(const DICTIONARY_ITEM *item __maybe_
545
ctr->react_action = RRDCALC_REACT_NEW;
546
}
547
548
+static bool rrdcalc_rrdhost_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdcalc, void *rrdcalc_new __maybe_unused, void *constructor_data ) {
549
+ RRDCALC *rc = rrdcalc;
550
+ struct rrdcalc_constructor *ctr = constructor_data;
551
+
552
+ if(rc->run_flags & RRDCALC_FLAG_FROM_TEMPLATE)
553
+ ctr->existing_from_template = true;
554
+ else
555
+ ctr->existing_from_template = false;
556
+
557
+ ctr->react_action = RRDCALC_REACT_NONE;
558
+
559
+ return false;
560
+}
561
+
562
static void rrdcalc_rrdhost_react_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdcalc, void *constructor_data) {
563
RRDCALC *rc = rrdcalc;
564
struct rrdcalc_constructor *ctr = constructor_data;
@@ -612,6 +628,7 @@ void rrdcalc_rrdhost_index_init(RRDHOST *host) {
628
host->rrdcalc_root_index = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);
629
630
dictionary_register_insert_callback(host->rrdcalc_root_index, rrdcalc_rrdhost_insert_callback, NULL);
631
+ dictionary_register_conflict_callback(host->rrdcalc_root_index, rrdcalc_rrdhost_conflict_callback, NULL);
632
dictionary_register_react_callback(host->rrdcalc_root_index, rrdcalc_rrdhost_react_callback, NULL);
633
dictionary_register_delete_callback(host->rrdcalc_root_index, rrdcalc_rrdhost_delete_callback, host);
634
}
@@ -635,11 +652,12 @@ void rrdcalc_add_from_rrdcalctemplate(RRDHOST *host, RRDCALCTEMPLATE *rt, RRDSET
652
.overwrite_alert_name = overwrite_alert_name,
653
.overwrite_dimensions = overwrite_dimensions,
654
.react_action = RRDCALC_REACT_NONE,
655
+ .existing_from_template = false,
656
};
657
658
dictionary_set_advanced(host->rrdcalc_root_index, key, (ssize_t)(key_len + 1), NULL, sizeof(RRDCALC), &tmp);
641
- if(tmp.react_action != RRDCALC_REACT_NEW)
642
- error("RRDCALC: from template '%s' on chart '%s' with key '%s', failed to be added to host '%s'. It already exists.",
659
+ if(tmp.react_action != RRDCALC_REACT_NEW && tmp.existing_from_template == false)
660
+ error("RRDCALC: from template '%s' on chart '%s' with key '%s', failed to be added to host '%s'. It is manually configured.",
661
string2str(rt->name), rrdset_id(st), key, rrdhost_hostname(host));
662
}
663
database/rrdcontext.c
+112
-91
@@ -38,7 +38,7 @@ typedef enum {
38
RRD_FLAG_OWN_LABELS = (1 << 4), // this instance has its own labels - not linked to an RRDSET
39
RRD_FLAG_LIVE_RETENTION = (1 << 5), // we have got live retention from the database
40
RRD_FLAG_QUEUED_FOR_HUB = (1 << 6), // this context is currently queued to be dispatched to hub
41
- RRD_FLAG_QUEUED_FOR_POST_PROCESSING = (1 << 7), // this context is currently queued to be post-processed
41
+ RRD_FLAG_QUEUED_FOR_PP = (1 << 7), // this context is currently queued to be post-processed
42
RRD_FLAG_HIDDEN = (1 << 8), // don't expose this to the hub or the API
43
44
RRD_FLAG_UPDATE_REASON_TRIGGERED = (1 << 9), // the update was triggered by the child object
@@ -46,24 +46,18 @@ typedef enum {
46
RRD_FLAG_UPDATE_REASON_NEW_OBJECT = (1 << 11), // this object has just been created
47
RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT = (1 << 12), // we received an update on this object
48
RRD_FLAG_UPDATE_REASON_CHANGED_LINKING = (1 << 13), // an instance or a metric switched RRDSET or RRDDIM
49
- RRD_FLAG_UPDATE_REASON_CHANGED_UUID = (1 << 14), // an instance or a metric changed UUID
50
- RRD_FLAG_UPDATE_REASON_CHANGED_NAME = (1 << 15), // an instance or a metric changed name
51
- RRD_FLAG_UPDATE_REASON_CHANGED_UNITS = (1 << 16), // this context or instance changed units
52
- RRD_FLAG_UPDATE_REASON_CHANGED_TITLE = (1 << 17), // this context or instance changed title
53
- RRD_FLAG_UPDATE_REASON_CHANGED_FAMILY = (1 << 18), // the context or the instance changed family
54
- RRD_FLAG_UPDATE_REASON_CHANGED_CHART_TYPE = (1 << 19), // this context or instance changed chart type
55
- RRD_FLAG_UPDATE_REASON_CHANGED_PRIORITY = (1 << 20), // this context or instance changed its priority
56
- RRD_FLAG_UPDATE_REASON_CHANGED_UPDATE_EVERY = (1 << 21), // the instance or the metric changed update frequency
57
- RRD_FLAG_UPDATE_REASON_ZERO_RETENTION = (1 << 22), // this object has not retention
58
- RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T = (1 << 23), // this object changed its oldest time in the db
59
- RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T = (1 << 24), // this object change its latest time in the db
60
- RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED = (1 << 25), // this object has stopped being collected
61
- RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED = (1 << 26), // this object has started being collected
62
- RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD = (1 << 27), // this context belongs to a host that just disconnected
63
- RRD_FLAG_UPDATE_REASON_DB_ROTATION = (1 << 28), // this context changed because of a db rotation
64
- RRD_FLAG_UPDATE_REASON_UNUSED = (1 << 29), // this context is not used anymore
65
- RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS = (1 << 30), // this context is not used anymore
66
- RRD_FLAG_UPDATE_REASON_UPDATED_RETENTION = (1 << 31), // this object has updated retention
49
+ RRD_FLAG_UPDATE_REASON_CHANGED_METADATA = (1 << 14), // this context or instance changed uuid, name, units, title, family, chart type, priority, update every, rrd changed flags
50
+ RRD_FLAG_UPDATE_REASON_ZERO_RETENTION = (1 << 15), // this object has no retention
51
+ RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T = (1 << 16), // this object changed its oldest time in the db
52
+ RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T = (1 << 17), // this object change its latest time in the db
53
+ RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED = (1 << 18), // this object has stopped being collected
54
+ RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED = (1 << 19), // this object has started being collected
55
+ RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD = (1 << 20), // this context belongs to a host that just disconnected
56
+ RRD_FLAG_UPDATE_REASON_UNUSED = (1 << 21), // this context is not used anymore
57
+ RRD_FLAG_UPDATE_REASON_DB_ROTATION = (1 << 22), // this context changed because of a db rotation
58
+
59
+ // action to perform on an object
60
+ RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION = (1 << 30), // this object has to update its retention from the db
61
} RRD_FLAGS;
62
63
#define RRD_FLAG_ALL_UPDATE_REASONS ( \
@@ -72,14 +66,7 @@ typedef enum {
66
|RRD_FLAG_UPDATE_REASON_NEW_OBJECT \
67
|RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT \
68
|RRD_FLAG_UPDATE_REASON_CHANGED_LINKING \
75
- |RRD_FLAG_UPDATE_REASON_CHANGED_UUID \
76
- |RRD_FLAG_UPDATE_REASON_CHANGED_NAME \
77
- |RRD_FLAG_UPDATE_REASON_CHANGED_UNITS \
78
- |RRD_FLAG_UPDATE_REASON_CHANGED_TITLE \
79
- |RRD_FLAG_UPDATE_REASON_CHANGED_FAMILY \
80
- |RRD_FLAG_UPDATE_REASON_CHANGED_CHART_TYPE \
81
- |RRD_FLAG_UPDATE_REASON_CHANGED_PRIORITY \
82
- |RRD_FLAG_UPDATE_REASON_CHANGED_UPDATE_EVERY \
69
+ |RRD_FLAG_UPDATE_REASON_CHANGED_METADATA \
70
|RRD_FLAG_UPDATE_REASON_ZERO_RETENTION \
71
|RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T \
72
|RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T \
@@ -88,7 +75,6 @@ typedef enum {
75
|RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD \
76
|RRD_FLAG_UPDATE_REASON_DB_ROTATION \
77
|RRD_FLAG_UPDATE_REASON_UNUSED \
91
- |RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS \
78
)
79
80
#define RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS ( \
@@ -105,7 +91,7 @@ typedef enum {
91
#define RRD_FLAGS_PREVENTING_DELETIONS ( \
92
RRD_FLAG_QUEUED_FOR_HUB \
93
|RRD_FLAG_COLLECTED \
108
- |RRD_FLAG_QUEUED_FOR_POST_PROCESSING \
94
+ |RRD_FLAG_QUEUED_FOR_PP \
95
)
96
97
// get all the flags of an object
@@ -203,34 +189,26 @@ static struct rrdcontext_reason {
189
usec_t delay_ut;
190
} rrdcontext_reasons[] = {
191
// context related
206
- { RRD_FLAG_UPDATE_REASON_TRIGGERED, "triggered transition", 65 * USEC_PER_SEC },
207
- { RRD_FLAG_UPDATE_REASON_NEW_OBJECT, "object created", 65 * USEC_PER_SEC },
208
- { RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT, "object updated", 65 * USEC_PER_SEC },
209
- { RRD_FLAG_UPDATE_REASON_LOAD_SQL, "loaded from sql", 65 * USEC_PER_SEC },
210
- { RRD_FLAG_UPDATE_REASON_CHANGED_TITLE, "changed title", 65 * USEC_PER_SEC },
211
- { RRD_FLAG_UPDATE_REASON_CHANGED_UNITS, "changed units", 65 * USEC_PER_SEC },
212
- { RRD_FLAG_UPDATE_REASON_CHANGED_FAMILY, "changed family", 65 * USEC_PER_SEC },
213
- { RRD_FLAG_UPDATE_REASON_CHANGED_PRIORITY, "changed priority", 65 * USEC_PER_SEC },
214
- { RRD_FLAG_UPDATE_REASON_ZERO_RETENTION, "has no retention", 65 * USEC_PER_SEC },
215
- { RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T, "updated first_time_t", 65 * USEC_PER_SEC },
216
- { RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T, "updated last_time_t", 65 * USEC_PER_SEC },
217
- { RRD_FLAG_UPDATE_REASON_CHANGED_CHART_TYPE, "changed chart type", 65 * USEC_PER_SEC },
218
- { RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED, "stopped collected", 65 * USEC_PER_SEC },
219
- { RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED, "started collected", 5 * USEC_PER_SEC },
220
- { RRD_FLAG_UPDATE_REASON_UNUSED, "unused", 5 * USEC_PER_SEC },
192
+ {RRD_FLAG_UPDATE_REASON_TRIGGERED, "triggered transition", 65 * USEC_PER_SEC },
193
+ {RRD_FLAG_UPDATE_REASON_NEW_OBJECT, "object created", 65 * USEC_PER_SEC },
194
+ {RRD_FLAG_UPDATE_REASON_UPDATED_OBJECT, "object updated", 65 * USEC_PER_SEC },
195
+ {RRD_FLAG_UPDATE_REASON_LOAD_SQL, "loaded from sql", 65 * USEC_PER_SEC },
196
+ {RRD_FLAG_UPDATE_REASON_CHANGED_METADATA, "changed metadata", 65 * USEC_PER_SEC },
197
+ {RRD_FLAG_UPDATE_REASON_ZERO_RETENTION, "has no retention", 65 * USEC_PER_SEC },
198
+ {RRD_FLAG_UPDATE_REASON_CHANGED_FIRST_TIME_T, "updated first_time_t", 65 * USEC_PER_SEC },
199
+ {RRD_FLAG_UPDATE_REASON_CHANGED_LAST_TIME_T, "updated last_time_t", 65 * USEC_PER_SEC },
200
+ {RRD_FLAG_UPDATE_REASON_STOPPED_BEING_COLLECTED, "stopped collected", 65 * USEC_PER_SEC },
201
+ {RRD_FLAG_UPDATE_REASON_STARTED_BEING_COLLECTED, "started collected", 5 * USEC_PER_SEC },
202
+ {RRD_FLAG_UPDATE_REASON_UNUSED, "unused", 5 * USEC_PER_SEC },
203
204
// not context related
223
- { RRD_FLAG_UPDATE_REASON_CHANGED_UUID, "changed uuid", 65 * USEC_PER_SEC },
224
- { RRD_FLAG_UPDATE_REASON_CHANGED_UPDATE_EVERY, "changed updated every",65 * USEC_PER_SEC },
225
- { RRD_FLAG_UPDATE_REASON_CHANGED_LINKING, "changed rrd link", 65 * USEC_PER_SEC },
226
- { RRD_FLAG_UPDATE_REASON_CHANGED_NAME, "changed name", 65 * USEC_PER_SEC },
227
- { RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD, "child disconnected", 65 * USEC_PER_SEC },
228
- { RRD_FLAG_UPDATE_REASON_DB_ROTATION, "db rotation", 65 * USEC_PER_SEC },
229
- { RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS, "changed flags", 65 * USEC_PER_SEC },
230
- { RRD_FLAG_UPDATE_REASON_UPDATED_RETENTION, "updated retention", 65 * USEC_PER_SEC },
205
+ {RRD_FLAG_UPDATE_REASON_CHANGED_LINKING, "changed rrd link", 65 * USEC_PER_SEC },
206
+ {RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD, "child disconnected", 65 * USEC_PER_SEC },
207
+ {RRD_FLAG_UPDATE_REASON_DB_ROTATION, "db rotation", 65 * USEC_PER_SEC },
208
+ {RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION, "updated retention", 65 * USEC_PER_SEC },
209
210
// terminator
233
- { 0, NULL, 0 },
211
+ {0, NULL, 0 },
212
};
213
214
@@ -320,7 +298,7 @@ typedef struct rrdcontext {
298
// ----------------------------------------------------------------------------
299
// helper one-liners for RRDMETRIC
300
323
-static void rrdmetric_update_retention(RRDMETRIC *rm);
301
+static bool rrdmetric_update_retention(RRDMETRIC *rm);
302
303
static inline RRDMETRIC *rrdmetric_acquired_value(RRDMETRIC_ACQUIRED *rma) {
304
return dictionary_acquired_item_value((DICTIONARY_ITEM *)rma);
@@ -472,7 +450,7 @@ static void rrd_flags_to_buffer(RRD_FLAGS flags, BUFFER *wb) {
450
if(flags & RRD_FLAG_HIDDEN)
451
buffer_strcat(wb, "HIDDEN ");
452
475
- if(flags & RRD_FLAG_QUEUED_FOR_POST_PROCESSING)
453
+ if(flags & RRD_FLAG_QUEUED_FOR_PP)
454
buffer_strcat(wb, "PENDING_UPDATES ");
455
}
456
@@ -538,12 +516,39 @@ static bool rrdmetric_conflict_callback(const DICTIONARY_ITEM *item __maybe_unus
516
string2str(rm->id), string2str(rm_new->id));
517
518
if(uuid_compare(rm->uuid, rm_new->uuid) != 0) {
519
+#ifdef NETDATA_INTERNAL_CHECKS
520
char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN];
521
uuid_unparse(rm->uuid, uuid1);
522
uuid_unparse(rm_new->uuid, uuid2);
544
- internal_error(true, "RRDMETRIC: '%s' of instance '%s' changed uuid from '%s' to '%s'", string2str(rm->id), string2str(rm->ri->id), uuid1, uuid2);
523
+
524
+ time_t old_first_time_t = 0;
525
+ time_t old_last_time_t = 0;
526
+ if(rrdmetric_update_retention(rm)) {
527
+ old_first_time_t = rm->first_time_t;
528
+ old_last_time_t = rm->last_time_t;
529
+ }
530
+
531
+ uuid_copy(rm->uuid, rm_new->uuid);
532
+
533
+ time_t new_first_time_t = 0;
534
+ time_t new_last_time_t = 0;
535
+ if(rrdmetric_update_retention(rm)) {
536
+ new_first_time_t = rm->first_time_t;
537
+ new_last_time_t = rm->last_time_t;
538
+ }
539
+
540
+ internal_error(true,
541
+ "RRDMETRIC: '%s' of instance '%s' of host '%s' changed UUID from '%s' (retention %ld to %ld, %ld secs) to '%s' (retention %ld to %ld, %ld secs)"
542
+ , string2str(rm->id)
543
+ , string2str(rm->ri->id)
544
+ , rrdhost_hostname(rm->ri->rc->rrdhost)
545
+ , uuid1, old_first_time_t, old_last_time_t, old_last_time_t - old_first_time_t
546
+ , uuid2, new_first_time_t, new_last_time_t, new_last_time_t - new_first_time_t
547
+ );
548
+#else
549
uuid_copy(rm->uuid, rm_new->uuid);
546
- rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_UUID);
550
+#endif
551
+ rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
552
}
553
554
if(rm->rrddim && rm_new->rrddim && rm->rrddim != rm_new->rrddim) {
@@ -551,12 +556,14 @@ static bool rrdmetric_conflict_callback(const DICTIONARY_ITEM *item __maybe_unus
556
rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_LINKING);
557
}
558
559
+#ifdef NETDATA_INTERNAL_CHECKS
560
if(rm->rrddim && uuid_compare(rm->uuid, rm->rrddim->metric_uuid) != 0) {
561
char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN];
562
uuid_unparse(rm->uuid, uuid1);
563
uuid_unparse(rm_new->uuid, uuid2);
564
internal_error(true, "RRDMETRIC: '%s' is linked to RRDDIM '%s' but they have different UUIDs. RRDMETRIC has '%s', RRDDIM has '%s'", string2str(rm->id), rrddim_id(rm->rrddim), uuid1, uuid2);
565
}
566
+#endif
567
568
if(rm->rrddim != rm_new->rrddim)
569
rm->rrddim = rm_new->rrddim;
@@ -565,7 +572,7 @@ static bool rrdmetric_conflict_callback(const DICTIONARY_ITEM *item __maybe_unus
572
STRING *old = rm->name;
573
rm->name = string_dup(rm_new->name);
574
string_freez(old);
568
- rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_NAME);
575
+ rrd_flag_set_updated(rm, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
576
}
577
578
if(!rm->first_time_t || (rm_new->first_time_t && rm_new->first_time_t < rm->first_time_t)) {
@@ -800,8 +807,16 @@ static bool rrdinstance_conflict_callback(const DICTIONARY_ITEM *item __maybe_un
807
string2str(ri->id), string2str(ri_new->id));
808
809
if(uuid_compare(ri->uuid, ri_new->uuid) != 0) {
810
+#ifdef NETDATA_INTERNAL_CHECKS
811
+ char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN];
812
+ uuid_unparse(ri->uuid, uuid1);
813
+ uuid_unparse(ri_new->uuid, uuid2);
814
+ internal_error(true, "RRDINSTANCE: '%s' of host '%s' changed UUID from '%s' to '%s'",
815
+ string2str(ri->id), rrdhost_hostname(ri->rc->rrdhost), uuid1, uuid2);
816
+#endif
817
+
818
uuid_copy(ri->uuid, ri_new->uuid);
804
- rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_UUID);
819
+ rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
820
}
821
822
if(ri->rrdset && ri_new->rrdset && ri->rrdset != ri_new->rrdset) {
@@ -809,54 +824,56 @@ static bool rrdinstance_conflict_callback(const DICTIONARY_ITEM *item __maybe_un
824
rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_LINKING);
825
}
826
827
+#ifdef NETDATA_INTERNAL_CHECKS
828
if(ri->rrdset && uuid_compare(ri->uuid, ri->rrdset->chart_uuid) != 0) {
829
char uuid1[UUID_STR_LEN], uuid2[UUID_STR_LEN];
830
uuid_unparse(ri->uuid, uuid1);
831
uuid_unparse(ri->rrdset->chart_uuid, uuid2);
832
internal_error(true, "RRDINSTANCE: '%s' is linked to RRDSET '%s' but they have different UUIDs. RRDINSTANCE has '%s', RRDSET has '%s'", string2str(ri->id), rrdset_id(ri->rrdset), uuid1, uuid2);
833
}
834
+#endif
835
836
if(ri->name != ri_new->name) {
837
STRING *old = ri->name;
838
ri->name = string_dup(ri_new->name);
839
string_freez(old);
823
- rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_NAME);
840
+ rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
841
}
842
843
if(ri->title != ri_new->title) {
844
STRING *old = ri->title;
845
ri->title = string_dup(ri_new->title);
846
string_freez(old);
830
- rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_TITLE);
847
+ rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
848
}
849
850
if(ri->units != ri_new->units) {
851
STRING *old = ri->units;
852
ri->units = string_dup(ri_new->units);
853
string_freez(old);
837
- rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_UNITS);
854
+ rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
855
}
856
857
if(ri->family != ri_new->family) {
858
STRING *old = ri->family;
859
ri->family = string_dup(ri_new->family);
860
string_freez(old);
844
- rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FAMILY);
861
+ rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
862
}
863
864
if(ri->chart_type != ri_new->chart_type) {
865
ri->chart_type = ri_new->chart_type;
849
- rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_CHART_TYPE);
866
+ rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
867
}
868
869
if(ri->priority != ri_new->priority) {
870
ri->priority = ri_new->priority;
854
- rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_PRIORITY);
871
+ rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
872
}
873
874
if(ri->update_every != ri_new->update_every) {
875
ri->update_every = ri_new->update_every;
859
- rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_UPDATE_EVERY);
876
+ rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
877
}
878
879
if(ri->rrdset != ri_new->rrdset) {
@@ -925,11 +942,11 @@ static void rrdinstance_trigger_updates(RRDINSTANCE *ri, const char *function) {
942
if(likely(st)) {
943
if(unlikely((unsigned int) st->priority != ri->priority)) {
944
ri->priority = st->priority;
928
- rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_PRIORITY);
945
+ rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
946
}
947
if(unlikely(st->update_every != ri->update_every)) {
948
ri->update_every = st->update_every;
932
- rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_UPDATE_EVERY);
949
+ rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
950
}
951
}
952
else if(unlikely(rrd_flag_is_collected(ri))) {
@@ -1100,7 +1117,7 @@ static inline void rrdinstance_rrdset_has_updated_retention(RRDSET *st) {
1117
RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
1118
if(unlikely(!ri)) return;
1119
1103
- rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_UPDATED_RETENTION);
1120
+ rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION);
1121
rrdinstance_trigger_updates(ri, __FUNCTION__ );
1122
}
1123
@@ -1116,7 +1133,7 @@ static inline void rrdinstance_updated_rrdset_name(RRDSET *st) {
1133
ri->name = string_dup(st->name);
1134
string_freez(old);
1135
1119
- rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_NAME);
1136
+ rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
1137
rrdinstance_trigger_updates(ri, __FUNCTION__ );
1138
}
1139
}
@@ -1131,11 +1148,11 @@ static inline void rrdinstance_updated_rrdset_flags_no_action(RRDINSTANCE *ri, R
1148
1149
if(unlikely(st_is_hidden != ri_is_hidden)) {
1150
if (unlikely(st_is_hidden && !ri_is_hidden))
1134
- rrd_flag_set_updated(ri, RRD_FLAG_HIDDEN | RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS);
1151
+ rrd_flag_set_updated(ri, RRD_FLAG_HIDDEN | RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
1152
1153
else if (unlikely(!st_is_hidden && ri_is_hidden)) {
1154
rrd_flag_clear(ri, RRD_FLAG_HIDDEN);
1138
- rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS);
1155
+ rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
1156
}
1157
}
1158
}
@@ -1269,14 +1286,14 @@ static bool rrdcontext_conflict_callback(const DICTIONARY_ITEM *item __maybe_unu
1286
else
1287
rc->title = string_2way_merge(rc->title, rc_new->title);
1288
string_freez(old_title);
1272
- rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_TITLE);
1289
+ rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
1290
}
1291
1292
if(rc->units != rc_new->units) {
1293
STRING *old_units = rc->units;
1294
rc->units = string_dup(rc_new->units);
1295
string_freez(old_units);
1279
- rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_UNITS);
1296
+ rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
1297
}
1298
1299
if(rc->family != rc_new->family) {
@@ -1286,17 +1303,17 @@ static bool rrdcontext_conflict_callback(const DICTIONARY_ITEM *item __maybe_unu
1303
else
1304
rc->family = string_2way_merge(rc->family, rc_new->family);
1305
string_freez(old_family);
1289
- rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_FAMILY);
1306
+ rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
1307
}
1308
1309
if(rc->chart_type != rc_new->chart_type) {
1310
rc->chart_type = rc_new->chart_type;
1294
- rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_CHART_TYPE);
1311
+ rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
1312
}
1313
1314
if(rc->priority != rc_new->priority) {
1315
rc->priority = rc_new->priority;
1299
- rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_PRIORITY);
1316
+ rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
1317
}
1318
1319
rrd_flag_set(rc, rc_new->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS); // no need for atomics on rc_new
@@ -1351,14 +1368,14 @@ static bool rrdcontext_hub_queue_conflict_callback(const DICTIONARY_ITEM *item _
1368
1369
static void rrdcontext_post_processing_queue_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *nothing __maybe_unused) {
1370
RRDCONTEXT *rc = context;
1354
- rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_POST_PROCESSING);
1371
+ rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_PP);
1372
rc->pp.queued_flags = rc->flags;
1373
rc->pp.queued_ut = now_realtime_usec();
1374
}
1375
1376
static void rrdcontext_post_processing_queue_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *context, void *nothing __maybe_unused) {
1377
RRDCONTEXT *rc = context;
1361
- rrd_flag_clear(rc, RRD_FLAG_QUEUED_FOR_POST_PROCESSING);
1378
+ rrd_flag_clear(rc, RRD_FLAG_QUEUED_FOR_PP);
1379
rc->pp.dequeued_ut = now_realtime_usec();
1380
}
1381
@@ -1366,8 +1383,8 @@ static bool rrdcontext_post_processing_queue_conflict_callback(const DICTIONARY_
1383
RRDCONTEXT *rc = context;
1384
bool changed = false;
1385
1369
- if(!(rc->flags & RRD_FLAG_QUEUED_FOR_POST_PROCESSING)) {
1370
- rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_POST_PROCESSING);
1386
+ if(!(rc->flags & RRD_FLAG_QUEUED_FOR_PP)) {
1387
+ rrd_flag_set(rc, RRD_FLAG_QUEUED_FOR_PP);
1388
changed = true;
1389
}
1390
@@ -3035,7 +3052,7 @@ static void rrdcontext_recalculate_retention_all_hosts(void) {
3052
// ----------------------------------------------------------------------------
3053
// garbage collector
3054
3038
-static void rrdmetric_update_retention(RRDMETRIC *rm) {
3055
+static bool rrdmetric_update_retention(RRDMETRIC *rm) {
3056
time_t min_first_time_t = LONG_MAX, max_last_time_t = 0;
3057
3058
if(rm->rrddim) {
@@ -3060,7 +3077,7 @@ static void rrdmetric_update_retention(RRDMETRIC *rm) {
3077
}
3078
else {
3079
// cannot get retention
3063
- return;
3080
+ return false;
3081
}
3082
#endif
3083
@@ -3090,6 +3107,8 @@ static void rrdmetric_update_retention(RRDMETRIC *rm) {
3107
rrd_flag_set_deleted(rm, RRD_FLAG_UPDATE_REASON_ZERO_RETENTION);
3108
3109
rrd_flag_set(rm, RRD_FLAG_LIVE_RETENTION);
3110
+
3111
+ return true;
3112
}
3113
3114
static inline bool rrdmetric_should_be_deleted(RRDMETRIC *rm) {
@@ -3261,16 +3280,18 @@ static void rrdmetric_process_updates(RRDMETRIC *rm, bool force, RRD_FLAGS reaso
3280
if(reason != RRD_FLAG_NONE)
3281
rrd_flag_set_updated(rm, reason);
3282
3264
- if(!force && !rrd_flag_is_updated(rm) && rrd_flag_check(rm, RRD_FLAG_LIVE_RETENTION) && !rrd_flag_check(rm, RRD_FLAG_UPDATE_REASON_UPDATED_RETENTION))
3283
+ if(!force && !rrd_flag_is_updated(rm) && rrd_flag_check(rm, RRD_FLAG_LIVE_RETENTION) && !rrd_flag_check(rm, RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION))
3284
return;
3285
3286
if(worker_jobs)
3287
worker_is_busy(WORKER_JOB_PP_METRIC);
3288
3270
- if(reason == RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD) {
3289
+ if(reason & RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD) {
3290
rrd_flag_set_archived(rm);
3291
rrd_flag_set(rm, RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD);
3292
}
3293
+ if(rrd_flag_is_deleted(rm) && (reason & RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION))
3294
+ rrd_flag_set_archived(rm);
3295
3296
rrdmetric_update_retention(rm);
3297
@@ -3296,8 +3317,8 @@ static void rrdinstance_post_process_updates(RRDINSTANCE *ri, bool force, RRD_FL
3317
if(unlikely(netdata_exit)) break;
3318
3319
RRD_FLAGS reason_to_pass = reason;
3299
- if(rrd_flag_check(ri, RRD_FLAG_UPDATE_REASON_UPDATED_RETENTION))
3300
- reason_to_pass |= RRD_FLAG_UPDATE_REASON_UPDATED_RETENTION;
3320
+ if(rrd_flag_check(ri, RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION))
3321
+ reason_to_pass |= RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION;
3322
3323
rrdmetric_process_updates(rm, force, reason_to_pass, worker_jobs);
3324
@@ -3403,8 +3424,8 @@ static void rrdcontext_post_process_updates(RRDCONTEXT *rc, bool force, RRD_FLAG
3424
if(unlikely(netdata_exit)) break;
3425
3426
RRD_FLAGS reason_to_pass = reason;
3406
- if(rrd_flag_check(rc, RRD_FLAG_UPDATE_REASON_UPDATED_RETENTION))
3407
- reason_to_pass |= RRD_FLAG_UPDATE_REASON_UPDATED_RETENTION;
3427
+ if(rrd_flag_check(rc, RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION))
3428
+ reason_to_pass |= RRD_FLAG_UPDATE_REASON_UPDATE_RETENTION;
3429
3430
rrdinstance_post_process_updates(ri, force, reason_to_pass, worker_jobs);
3431
@@ -3517,7 +3538,7 @@ static void rrdcontext_post_process_updates(RRDCONTEXT *rc, bool force, RRD_FLAG
3538
3539
if (min_priority != LONG_MAX && rc->priority != min_priority) {
3540
rc->priority = min_priority;
3520
- rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_PRIORITY);
3541
+ rrd_flag_set_updated(rc, RRD_FLAG_UPDATE_REASON_CHANGED_METADATA);
3542
}
3543
}
3544
@@ -3536,7 +3557,7 @@ static void rrdcontext_post_process_updates(RRDCONTEXT *rc, bool force, RRD_FLAG
3557
static void rrdcontext_queue_for_post_processing(RRDCONTEXT *rc, const char *function __maybe_unused, RRD_FLAGS flags __maybe_unused) {
3558
if(unlikely(!rc->rrdhost->rrdctx_post_processing_queue)) return;
3559
3539
- if(!rrd_flag_check(rc, RRD_FLAG_QUEUED_FOR_POST_PROCESSING)) {
3560
+ if(!rrd_flag_check(rc, RRD_FLAG_QUEUED_FOR_PP)) {
3561
dictionary_set((DICTIONARY *)rc->rrdhost->rrdctx_post_processing_queue,
3562
string2str(rc->id),
3563
rc,
database/rrddim.c
+17
@@ -84,6 +84,23 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
84
85
if (unlikely(rrdcontext_find_dimension_uuid(st, rrddim_id(rd), &(rd->metric_uuid)))) {
86
uuid_generate(rd->metric_uuid);
87
+ bool found_in_sql = false; (void)found_in_sql;
88
+
89
+// bool found_in_sql = true;
90
+// if(unlikely(sql_find_dimension_uuid(st, rd, &rd->metric_uuid))) {
91
+// found_in_sql = false;
92
+// uuid_generate(rd->metric_uuid);
93
+// }
94
+
95
+#ifdef NETDATA_INTERNAL_CHECKS
96
+ char uuid_str[UUID_STR_LEN];
97
+ uuid_unparse_lower(rd->metric_uuid, uuid_str);
98
+ error_report("Dimension UUID for host %s chart [%s] dimension [%s] not found in context. It is now set to %s (%s)",
99
+ string2str(host->hostname),
100
+ string2str(st->name),
101
+ string2str(rd->name),
102
+ uuid_str, found_in_sql ? "found in sqlite" : "newly generated");
103
+#endif
104
}
105
106
// initialize the db tiers
database/rrdhost.c
+39
-25
@@ -4,7 +4,7 @@
4
#include "rrd.h"
5
6
bool dbengine_enabled = false; // will become true if and when dbengine is initialized
7
-size_t storage_tiers = 1;
7
+size_t storage_tiers = 3;
8
size_t storage_tiers_grouping_iterations[RRD_STORAGE_TIERS] = { 1, 60, 60, 60, 60 };
9
RRD_BACKFILL storage_tiers_backfill[RRD_STORAGE_TIERS] = { RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW, RRD_BACKFILL_NEW };
10
@@ -290,7 +290,11 @@ int is_legacy = 1;
290
host, rrdpush_enabled, rrdpush_destination, rrdpush_api_key, rrdpush_send_charts_matching);
291
}
292
293
- host->rrdpush_enable_replication = rrdpush_enable_replication;
293
+ if(rrdpush_enable_replication)
294
+ rrdhost_option_set(host, RRDHOST_OPTION_REPLICATION);
295
+ else
296
+ rrdhost_option_clear(host, RRDHOST_OPTION_REPLICATION);
297
+
298
host->rrdpush_seconds_to_replicate = rrdpush_seconds_to_replicate;
299
host->rrdpush_replication_step = rrdpush_replication_step;
300
@@ -616,11 +620,14 @@ void rrdhost_update(RRDHOST *host
620
rrdcalctemplate_index_init(host);
621
rrdcalc_rrdhost_index_init(host);
622
619
- host->rrdpush_enable_replication = rrdpush_enable_replication;
623
+ if(rrdpush_enable_replication)
624
+ rrdhost_option_set(host, RRDHOST_OPTION_REPLICATION);
625
+ else
626
+ rrdhost_option_clear(host, RRDHOST_OPTION_REPLICATION);
627
+
628
host->rrdpush_seconds_to_replicate = rrdpush_seconds_to_replicate;
629
host->rrdpush_replication_step = rrdpush_replication_step;
630
623
-
631
rrd_hosts_available++;
632
ml_new_host(host);
633
rrdhost_load_rrdcontext_data(host);
@@ -783,6 +790,7 @@ void dbengine_init(char *hostname) {
790
size_t created_tiers = 0;
791
char dbenginepath[FILENAME_MAX + 1];
792
char dbengineconfig[200 + 1];
793
+ int divisor = 1;
794
for(size_t tier = 0; tier < storage_tiers ;tier++) {
795
if(tier == 0)
796
snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", netdata_configured_cache_dir);
@@ -795,8 +803,11 @@ void dbengine_init(char *hostname) {
803
break;
804
}
805
798
- int page_cache_mb = default_rrdeng_page_cache_mb;
799
- int disk_space_mb = default_multidb_disk_quota_mb;
806
+ if(tier > 0)
807
+ divisor *= 2;
808
+
809
+ int page_cache_mb = default_rrdeng_page_cache_mb / divisor;
810
+ int disk_space_mb = default_multidb_disk_quota_mb / divisor;
811
size_t grouping_iterations = storage_tiers_grouping_iterations[tier];
812
RRD_BACKFILL backfill = storage_tiers_backfill[tier];
813
@@ -863,6 +874,7 @@ void dbengine_init(char *hostname) {
874
storage_tiers = 1;
875
config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
876
}
877
+ dbengine_enabled = false;
878
#endif
879
}
880
@@ -881,32 +893,34 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
893
894
if (unlikely(strcmp(hostname, "unittest") == 0)) {
895
dbengine_enabled = true;
884
- goto unittest;
885
- }
886
-
887
- health_init();
888
- rrdpush_init();
889
-
890
- if(default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE || storage_tiers > 1 || rrdpush_receiver_needs_dbengine()) {
891
- info("Initializing dbengine...");
892
- dbengine_init(hostname);
896
}
894
- else
895
- info("Not initializing dbengine...");
897
+ else {
898
+ health_init();
899
+ rrdpush_init();
900
897
- if(!dbengine_enabled) {
898
- if (storage_tiers > 1) {
899
- error("dbengine is not enabled, but %zu tiers have been requested. Resetting tiers to 1", storage_tiers);
901
+ if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE || rrdpush_receiver_needs_dbengine()) {
902
+ info("Initializing dbengine...");
903
+ dbengine_init(hostname);
904
+ }
905
+ else {
906
+ info("Not initializing dbengine...");
907
storage_tiers = 1;
908
}
909
903
- if(default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
904
- error("dbengine is not enabled, but it has been given as the default db mode. Resetting db mode to alloc");
905
- default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC;
910
+ if (!dbengine_enabled) {
911
+ if (storage_tiers > 1) {
912
+ error("dbengine is not enabled, but %zu tiers have been requested. Resetting tiers to 1",
913
+ storage_tiers);
914
+ storage_tiers = 1;
915
+ }
916
+
917
+ if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
918
+ error("dbengine is not enabled, but it has been given as the default db mode. Resetting db mode to alloc");
919
+ default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC;
920
+ }
921
}
922
}
923
909
-unittest:
924
metadata_sync_init();
925
debug(D_RRDHOST, "Initializing localhost with hostname '%s'", hostname);
926
rrd_wrlock();
@@ -1041,7 +1055,7 @@ void stop_streaming_sender(RRDHOST *host)
1055
if (host->sender->compressor)
1056
host->sender->compressor->destroy(&host->sender->compressor);
1057
#endif
1044
- dictionary_destroy(host->sender->replication_requests);
1058
+ replication_cleanup_sender(host->sender);
1059
freez(host->sender);
1060
host->sender = NULL;
1061
rrdhost_flag_clear(host, RRDHOST_FLAG_RRDPUSH_SENDER_INITIALIZED);
database/rrdset.c
+15
@@ -366,6 +366,21 @@ static void rrdset_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
366
if (ctr->react_action & RRDSET_REACT_NEW) {
367
if(unlikely(rrdcontext_find_chart_uuid(st, &st->chart_uuid))) {
368
uuid_generate(st->chart_uuid);
369
+ bool found_in_sql = false; (void)found_in_sql;
370
+
371
+// bool found_in_sql = true;
372
+// if(unlikely(sql_find_chart_uuid(host, st, &st->chart_uuid))) {
373
+// uuid_generate(st->chart_uuid);
374
+// found_in_sql = false;
375
+// }
376
+
377
+#ifdef NETDATA_INTERNAL_CHECKS
378
+ char uuid_str[UUID_STR_LEN];
379
+ uuid_unparse_lower(st->chart_uuid, uuid_str);
380
+ error_report("Chart UUID for host %s chart [%s] not found in context. It is now set to %s (%s)",
381
+ string2str(host->hostname),
382
+ string2str(st->name), uuid_str, found_in_sql ? "found in sqlite" : "newly generated");
383
+#endif
384
}
385
}
386
rrdset_flag_set(st, RRDSET_FLAG_METADATA_UPDATE);
database/sqlite/sqlite_functions.c
+109
@@ -1266,3 +1266,112 @@ int sql_metadata_cache_stats(int op)
1266
sqlite3_db_status(db_meta, op, &count, &dummy, 0);
1267
return count;
1268
}
1269
+
1270
+#define SQL_FIND_CHART_UUID \
1271
+ "SELECT chart_id FROM chart WHERE host_id = @host AND type=@type AND id=@id AND (name IS NULL OR name=@name) AND chart_id IS NOT NULL;"
1272
+
1273
+#define SQL_FIND_DIMENSION_UUID \
1274
+ "SELECT dim_id FROM dimension WHERE chart_id=@chart AND id=@id AND name=@name AND LENGTH(dim_id)=16;"
1275
+
1276
+
1277
+//Do a database lookup to find the UUID of a chart
1278
+//If found store it in store_uuid and return 0
1279
+int sql_find_chart_uuid(RRDHOST *host, RRDSET *st, uuid_t *store_uuid)
1280
+{
1281
+ static __thread sqlite3_stmt *res = NULL;
1282
+ int rc;
1283
+
1284
+ const char *name = string2str(st->parts.name);
1285
+
1286
+ if (unlikely(!db_meta) && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
1287
+ return 1;
1288
+
1289
+ if (unlikely(!res)) {
1290
+ rc = prepare_statement(db_meta, SQL_FIND_CHART_UUID, &res);
1291
+ if (rc != SQLITE_OK) {
1292
+ error_report("Failed to prepare statement to lookup chart UUID in the database");
1293
+ return 1;
1294
+ }
1295
+ }
1296
+
1297
+ rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
1298
+ if (unlikely(rc != SQLITE_OK))
1299
+ goto bind_fail;
1300
+
1301
+ rc = sqlite3_bind_text(res, 2, string2str(st->parts.type), -1, SQLITE_STATIC);
1302
+ if (unlikely(rc != SQLITE_OK))
1303
+ goto bind_fail;
1304
+
1305
+ rc = sqlite3_bind_text(res, 3, string2str(st->parts.id), -1, SQLITE_STATIC);
1306
+ if (unlikely(rc != SQLITE_OK))
1307
+ goto bind_fail;
1308
+
1309
+ rc = sqlite3_bind_text(res, 4, name && *name ? name : string2str(st->parts.id), -1, SQLITE_STATIC);
1310
+ if (unlikely(rc != SQLITE_OK))
1311
+ goto bind_fail;
1312
+
1313
+ int status = 1;
1314
+ rc = sqlite3_step_monitored(res);
1315
+ if (likely(rc == SQLITE_ROW)) {
1316
+ uuid_copy(*store_uuid, sqlite3_column_blob(res, 0));
1317
+ status = 0;
1318
+ }
1319
+
1320
+ rc = sqlite3_reset(res);
1321
+ if (unlikely(rc != SQLITE_OK))
1322
+ error_report("Failed to reset statement when searching for a chart UUID, rc = %d", rc);
1323
+
1324
+ return status;
1325
+
1326
+bind_fail:
1327
+ error_report("Failed to bind input parameter to perform chart UUID database lookup, rc = %d", rc);
1328
+ rc = sqlite3_reset(res);
1329
+ if (unlikely(rc != SQLITE_OK))
1330
+ error_report("Failed to reset statement when searching for a chart UUID, rc = %d", rc);
1331
+ return 1;
1332
+}
1333
+
1334
+int sql_find_dimension_uuid(RRDSET *st, RRDDIM *rd, uuid_t *store_uuid)
1335
+{
1336
+ static __thread sqlite3_stmt *res = NULL;
1337
+ int rc;
1338
+ int status = 1;
1339
+
1340
+ if (unlikely(!db_meta) && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
1341
+ return 1;
1342
+
1343
+ if (unlikely(!res)) {
1344
+ rc = prepare_statement(db_meta, SQL_FIND_DIMENSION_UUID, &res);
1345
+ if (rc != SQLITE_OK) {
1346
+ error_report("Failed to bind prepare statement to lookup dimension UUID in the database");
1347
+ return 1;
1348
+ }
1349
+ }
1350
+
1351
+ rc = sqlite3_bind_blob(res, 1, st->chart_uuid, sizeof(*st->chart_uuid), SQLITE_STATIC);
1352
+ if (unlikely(rc != SQLITE_OK))
1353
+ goto bind_fail;
1354
+
1355
+ rc = sqlite3_bind_text(res, 2, rrddim_id(rd), -1, SQLITE_STATIC);
1356
+ if (unlikely(rc != SQLITE_OK))
1357
+ goto bind_fail;
1358
+
1359
+ rc = sqlite3_bind_text(res, 3, rrddim_name(rd), -1, SQLITE_STATIC);
1360
+ if (unlikely(rc != SQLITE_OK))
1361
+ goto bind_fail;
1362
+
1363
+ rc = sqlite3_step_monitored(res);
1364
+ if (likely(rc == SQLITE_ROW)) {
1365
+ uuid_copy(*store_uuid, *((uuid_t *) sqlite3_column_blob(res, 0)));
1366
+ status = 0;
1367
+ }
1368
+
1369
+ rc = sqlite3_reset(res);
1370
+ if (unlikely(rc != SQLITE_OK))
1371
+ error_report("Failed to reset statement find dimension uuid, rc = %d", rc);
1372
+ return status;
1373
+
1374
+bind_fail:
1375
+ error_report("Failed to bind input parameter to perform dimension UUID database lookup, rc = %d", rc);
1376
+ return 1;
1377
+}
database/sqlite/sqlite_functions.h
+2
-3
@@ -65,16 +65,15 @@ int get_host_id(uuid_t *node_id, uuid_t *host_id);
65
struct node_instance_list *get_node_list(void);
66
void sql_load_node_id(RRDHOST *host);
67
char *get_hostname_by_node_id(char *node_id);
68
+int sql_find_chart_uuid(RRDHOST *host, RRDSET *st, uuid_t *store_uuid);
69
+int sql_find_dimension_uuid(RRDSET *st, RRDDIM *rd, uuid_t *store_uuid);
70
71
// Help build archived hosts in memory when agent starts
72
void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info);
73
DICTIONARY *sql_load_host_labels(uuid_t *host_id);
74
75
// For queries: To be removed when context queries are implemented
74
-RRDHOST *sql_create_host_by_uuid(char *guid);
76
void sql_rrdset2json(RRDHOST *host, BUFFER *wb);
76
-void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDHOST *host, char *context, char *chart);
77
-void free_temporary_host(RRDHOST *host);
77
78
// TODO: move to metadata
79
int update_node_id(uuid_t *host_id, uuid_t *node_id);
health/health.c
+1
-1
@@ -692,7 +692,7 @@ static void initialize_health(RRDHOST *host, int is_localhost) {
692
health_silencers_init();
693
}
694
695
-static void health_sleep(time_t next_run, unsigned int loop, RRDHOST *host) {
695
+static void health_sleep(time_t next_run, unsigned int loop __maybe_unused, RRDHOST *host) {
696
time_t now = now_realtime_sec();
697
if(now < next_run) {
698
worker_is_idle();
libnetdata/arrayalloc/arrayalloc.c
+1
-1
@@ -344,7 +344,7 @@ void arrayalloc_freez(ARAL *ar, void *ptr) {
344
#endif
345
}
346
347
-#ifdef NETDATA_INTERNAL_CHECKS
347
+#ifdef NETDATA_ARRAYALLOC_INTERNAL_CHECKS
348
{
349
// find the page ptr belongs
350
ARAL_PAGE *page2 = find_page_with_allocation_internal_check(ar, ptr);
libnetdata/dictionary/dictionary.c
+7
-2
@@ -2171,11 +2171,13 @@ int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const D
2171
// ----------------------------------------------------------------------------
2172
// sorted walkthrough
2173
2174
+typedef int (*qsort_compar)(const void *item1, const void *item2);
2175
+
2176
static int dictionary_sort_compar(const void *item1, const void *item2) {
2177
return strcmp(item_get_name((*(DICTIONARY_ITEM **)item1)), item_get_name((*(DICTIONARY_ITEM **)item2)));
2178
}
2179
2178
-int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const DICTIONARY_ITEM *item, void *entry, void *data), void *data) {
2180
+int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const DICTIONARY_ITEM *item, void *entry, void *data), void *data, dictionary_sorted_compar compar) {
2181
if(unlikely(!dict || !callback)) return 0;
2182
2183
if(unlikely(is_dictionary_destroyed(dict))) {
@@ -2200,7 +2202,10 @@ int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(
2202
if(unlikely(i != entries))
2203
entries = i;
2204
2203
- qsort(array, entries, sizeof(DICTIONARY_ITEM *), dictionary_sort_compar);
2205
+ if(compar)
2206
+ qsort(array, entries, sizeof(DICTIONARY_ITEM *), (qsort_compar)compar);
2207
+ else
2208
+ qsort(array, entries, sizeof(DICTIONARY_ITEM *), dictionary_sort_compar);
2209
2210
bool callit = true;
2211
int ret = 0, r;
libnetdata/dictionary/dictionary.h
+5
-3
@@ -230,9 +230,11 @@ size_t dictionary_acquired_item_references(DICT_ITEM_CONST DICTIONARY_ITEM *item
230
#define dictionary_walkthrough_write(dict, callback, data) dictionary_walkthrough_rw(dict, 'w', callback, data)
231
int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const DICTIONARY_ITEM *item, void *value, void *data), void *data);
232
233
-#define dictionary_sorted_walkthrough_read(dict, callback, data) dictionary_sorted_walkthrough_rw(dict, 'r', callback, data)
234
-#define dictionary_sorted_walkthrough_write(dict, callback, data) dictionary_sorted_walkthrough_rw(dict, 'w', callback, data)
235
-int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const DICTIONARY_ITEM *item, void *entry, void *data), void *data);
233
+typedef int (*dictionary_sorted_compar)(const DICTIONARY_ITEM **item1, const DICTIONARY_ITEM **item2);
234
+
235
+#define dictionary_sorted_walkthrough_read(dict, callback, data) dictionary_sorted_walkthrough_rw(dict, 'r', callback, data, NULL)
236
+#define dictionary_sorted_walkthrough_write(dict, callback, data) dictionary_sorted_walkthrough_rw(dict, 'w', callback, data, NULL)
237
+int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const DICTIONARY_ITEM *item, void *entry, void *data), void *data, dictionary_sorted_compar compar);
238
239
// ----------------------------------------------------------------------------
240
// Traverse with foreach
libnetdata/ebpf/ebpf.c
+1
-1
@@ -471,7 +471,7 @@ void ebpf_update_pid_table(ebpf_local_maps_t *pid, ebpf_module_t *em)
471
* @param em the structure with information about how the module/thread is working.
472
* @param map_name the name of the file used to log.
473
*/
474
-void ebpf_update_map_size(struct bpf_map *map, ebpf_local_maps_t *lmap, ebpf_module_t *em, const char *map_name)
474
+void ebpf_update_map_size(struct bpf_map *map, ebpf_local_maps_t *lmap, ebpf_module_t *em, const char *map_name __maybe_unused)
475
{
476
uint32_t define_size = 0;
477
uint32_t apps_type = NETDATA_EBPF_MAP_PID | NETDATA_EBPF_MAP_RESIZABLE;
libnetdata/inlined.h
+1
-1
@@ -45,7 +45,7 @@ static inline uint32_t simple_uhash(const char *name) {
45
static inline int str2i(const char *s) {
46
int n = 0;
47
char c, negative = (char)(*s == '-');
48
- const char *e = &s[30]; // max number of character to iterate
48
+ const char *e = s + 30; // max number of character to iterate
49
50
for(c = (char)((negative)?*(++s):*s); c >= '0' && c <= '9' && s < e ; c = *(++s)) {
51
n *= 10;
libnetdata/socket/socket.c
+51
-6
@@ -919,6 +919,53 @@ int connect_to_one_of_urls(const char *destination, int default_port, struct tim
919
}
920
921
922
+#ifdef ENABLE_HTTPS
923
+ssize_t netdata_ssl_read(SSL *ssl, void *buf, size_t num) {
924
+ error_limit_static_thread_var(erl, 1, 0);
925
+
926
+ int bytes, err, retries = 0;
927
+
928
+ do {
929
+ bytes = SSL_read(ssl, buf, (int)num);
930
+ err = SSL_get_error(ssl, bytes);
931
+ retries++;
932
+ } while (bytes <= 0 && (err == SSL_ERROR_WANT_READ));
933
+
934
+ if(unlikely(bytes <= 0))
935
+ error("SSL_read() returned %d bytes, SSL error %d", bytes, err);
936
+
937
+ if(retries > 1)
938
+ error_limit(&erl, "SSL_read() retried %d times", retries);
939
+
940
+ return bytes;
941
+}
942
+
943
+ssize_t netdata_ssl_write(SSL *ssl, const void *buf, size_t num) {
944
+ error_limit_static_thread_var(erl, 1, 0);
945
+
946
+ int bytes, err, retries = 0;
947
+ size_t total = 0;
948
+
949
+ do {
950
+ bytes = SSL_write(ssl, (uint8_t *)buf + total, (int)(num - total));
951
+ err = SSL_get_error(ssl, bytes);
952
+ retries++;
953
+
954
+ if(bytes > 0)
955
+ total += bytes;
956
+
957
+ } while ((bytes <= 0 && (err == SSL_ERROR_WANT_WRITE)) || (bytes > 0 && total < num));
958
+
959
+ if(unlikely(bytes <= 0))
960
+ error("SSL_read() returned %d bytes, SSL error %d", bytes, err);
961
+
962
+ if(retries > 1)
963
+ error_limit(&erl, "SSL_read() retried %d times", retries);
964
+
965
+ return bytes;
966
+}
967
+#endif
968
+
969
// --------------------------------------------------------------------------------------------------------------------
970
// helpers to send/receive data in one call, in blocking mode, with a timeout
971
@@ -956,12 +1003,10 @@ ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout)
1003
}
1004
1005
#ifdef ENABLE_HTTPS
959
- if (ssl->conn) {
960
- if (!ssl->flags) {
961
- return SSL_read(ssl->conn,buf,len);
962
- }
963
- }
1006
+ if (ssl->conn && ssl->flags == NETDATA_SSL_HANDSHAKE_COMPLETE)
1007
+ return netdata_ssl_read(ssl->conn, buf, len);
1008
#endif
1009
+
1010
return recv(sockfd, buf, len, flags);
1011
}
1012
@@ -1001,7 +1046,7 @@ ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout)
1046
#ifdef ENABLE_HTTPS
1047
if(ssl->conn) {
1048
if (ssl->flags == NETDATA_SSL_HANDSHAKE_COMPLETE) {
1004
- return SSL_write(ssl->conn, buf, len);
1049
+ return netdata_ssl_write(ssl->conn, buf, len);
1050
}
1051
else {
1052
error("cannot write to SSL connection - connection is not ready.");
libnetdata/socket/socket.h
+2
@@ -67,6 +67,8 @@ int connect_to_one_of_urls(const char *destination, int default_port, struct tim
67
#ifdef ENABLE_HTTPS
68
ssize_t recv_timeout(struct netdata_ssl *ssl,int sockfd, void *buf, size_t len, int flags, int timeout);
69
ssize_t send_timeout(struct netdata_ssl *ssl,int sockfd, void *buf, size_t len, int flags, int timeout);
70
+ssize_t netdata_ssl_read(SSL *ssl, void *buf, size_t num);
71
+ssize_t netdata_ssl_write(SSL *ssl, const void *buf, size_t num);
72
#else
73
ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout);
74
ssize_t send_timeout(int sockfd, void *buf, size_t len, int flags, int timeout);
libnetdata/worker_utilization/worker_utilization.c
+17
-7
@@ -151,10 +151,17 @@ void worker_set_metric(size_t job_id, NETDATA_DOUBLE value) {
151
if(unlikely(job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES))
152
return;
153
154
- if(worker->per_job_type[job_id].type == WORKER_METRIC_INCREMENTAL)
155
- worker->per_job_type[job_id].custom_value += value;
156
- else
157
- worker->per_job_type[job_id].custom_value = value;
154
+ switch(worker->per_job_type[job_id].type) {
155
+ case WORKER_METRIC_INCREMENT:
156
+ worker->per_job_type[job_id].custom_value += value;
157
+ break;
158
+
159
+ case WORKER_METRIC_INCREMENTAL_TOTAL:
160
+ case WORKER_METRIC_ABSOLUTE:
161
+ default:
162
+ worker->per_job_type[job_id].custom_value = value;
163
+ break;
164
+ }
165
}
166
167
// statistics interface
@@ -200,11 +207,12 @@ void workers_foreach(const char *workname, void (*callback)(
207
208
switch(p->per_job_type[i].type) {
209
default:
203
- case WORKER_METRIC_EMPTY:
210
+ case WORKER_METRIC_EMPTY: {
211
per_job_type_jobs_started[i] = 0;
212
per_job_type_busy_time[i] = 0;
213
per_job_custom_values[i] = NAN;
214
break;
215
+ }
216
217
case WORKER_METRIC_IDLE_BUSY: {
218
size_t tmp_jobs_started = p->per_job_type[i].worker_jobs_started;
@@ -219,14 +227,16 @@ void workers_foreach(const char *workname, void (*callback)(
227
break;
228
}
229
222
- case WORKER_METRIC_ABSOLUTE:
230
+ case WORKER_METRIC_ABSOLUTE: {
231
per_job_type_jobs_started[i] = 0;
232
per_job_type_busy_time[i] = 0;
233
234
per_job_custom_values[i] = p->per_job_type[i].custom_value;
235
break;
236
+ }
237
229
- case WORKER_METRIC_INCREMENTAL: {
238
+ case WORKER_METRIC_INCREMENTAL_TOTAL:
239
+ case WORKER_METRIC_INCREMENT: {
240
per_job_type_jobs_started[i] = 0;
241
per_job_type_busy_time[i] = 0;
242
libnetdata/worker_utilization/worker_utilization.h
+2
-1
@@ -11,7 +11,8 @@ typedef enum {
11
WORKER_METRIC_EMPTY = 0,
12
WORKER_METRIC_IDLE_BUSY = 1,
13
WORKER_METRIC_ABSOLUTE = 2,
14
- WORKER_METRIC_INCREMENTAL = 3,
14
+ WORKER_METRIC_INCREMENT = 3,
15
+ WORKER_METRIC_INCREMENTAL_TOTAL = 4,
16
} WORKER_METRIC_TYPE;
17
18
void worker_register(const char *workname);
parser/parser.c
+39
-20
@@ -29,14 +29,15 @@ inline int find_first_keyword(const char *str, char *keyword, int max_size, int
29
*
30
*/
31
32
-PARSER *parser_init(RRDHOST *host, void *user, void *input, void *output, PARSER_INPUT_TYPE flags, void *ssl __maybe_unused)
32
+PARSER *parser_init(RRDHOST *host, void *user, FILE *fp_input, FILE *fp_output, int fd, PARSER_INPUT_TYPE flags, void *ssl __maybe_unused)
33
{
34
PARSER *parser;
35
36
parser = callocz(1, sizeof(*parser));
37
parser->user = user;
38
- parser->input = input;
39
- parser->output = output;
38
+ parser->fd = fd;
39
+ parser->fp_input = fp_input;
40
+ parser->fp_output = fp_output;
41
#ifdef ENABLE_HTTPS
42
parser->ssl_output = ssl;
43
#endif
@@ -222,19 +223,21 @@ int parser_next(PARSER *parser)
223
}
224
225
if (unlikely(parser->read_function))
225
- tmp = parser->read_function(parser->buffer, PLUGINSD_LINE_MAX, parser->input);
226
+ tmp = parser->read_function(parser->buffer, PLUGINSD_LINE_MAX, parser->fp_input);
227
+ else if(likely(parser->fp_input))
228
+ tmp = fgets(parser->buffer, PLUGINSD_LINE_MAX, (FILE *)parser->fp_input);
229
else
227
- tmp = fgets(parser->buffer, PLUGINSD_LINE_MAX, (FILE *)parser->input);
230
+ tmp = NULL;
231
232
if (unlikely(!tmp)) {
233
if (unlikely(parser->eof_function)) {
231
- int rc = parser->eof_function(parser->input);
234
+ int rc = parser->eof_function(parser->fp_input);
235
error("read failed: user defined function returned %d", rc);
236
}
237
else {
235
- if (feof((FILE *)parser->input))
238
+ if (feof((FILE *)parser->fp_input))
239
error("read failed: end of file");
237
- else if (ferror((FILE *)parser->input))
240
+ else if (ferror((FILE *)parser->fp_input))
241
error("read failed: input error");
242
else
243
error("read failed: unknown error");
@@ -253,6 +256,8 @@ int parser_next(PARSER *parser)
256
257
inline int parser_action(PARSER *parser, char *input)
258
{
259
+ parser->line++;
260
+
261
PARSER_RC rc = PARSER_RC_OK;
262
char *words[PLUGINSD_MAX_WORDS];
263
char command[PLUGINSD_LINE_MAX + 1];
@@ -288,7 +293,7 @@ inline int parser_action(PARSER *parser, char *input)
293
if(buffer_strlen(parser->defer.response) > 10 * 1024 * 1024) {
294
// more than 10MB of data
295
// a bad plugin that did not send the end_keyword
291
- internal_error(true, "Deferred response is too big (%zu bytes). Stopping this plugin.", buffer_strlen(parser->defer.response));
296
+ internal_error(true, "PLUGINSD: deferred response is too big (%zu bytes). Stopping this plugin.", buffer_strlen(parser->defer.response));
297
return 1;
298
}
299
}
@@ -321,11 +326,10 @@ inline int parser_action(PARSER *parser, char *input)
326
327
size_t worker_job_id = WORKER_UTILIZATION_MAX_JOB_TYPES + 1; // set an invalid value by default
328
while(tmp_keyword) {
324
- if (command_hash == tmp_keyword->keyword_hash &&
325
- (!strcmp(command, tmp_keyword->keyword))) {
326
- action_function_list = &tmp_keyword->func[0];
327
- worker_job_id = tmp_keyword->worker_job_id;
328
- break;
329
+ if (command_hash == tmp_keyword->keyword_hash && (!strcmp(command, tmp_keyword->keyword))) {
330
+ action_function_list = &tmp_keyword->func[0];
331
+ worker_job_id = tmp_keyword->worker_job_id;
332
+ break;
333
}
334
tmp_keyword = tmp_keyword->next;
335
}
@@ -335,17 +339,14 @@ inline int parser_action(PARSER *parser, char *input)
339
rc = parser->unknown_function(words, num_words, parser->user);
340
else
341
rc = PARSER_RC_ERROR;
338
-
339
- internal_error(rc != PARSER_RC_OK, "Unknown keyword [%s]", input);
342
}
343
else {
344
worker_is_busy(worker_job_id);
345
while ((action_function = *action_function_list) != NULL) {
346
rc = action_function(words, num_words, parser->user);
345
- if (unlikely(rc == PARSER_RC_ERROR || rc == PARSER_RC_STOP)) {
346
- internal_error(true, "action_function() failed with rc = %u", rc);
347
+ if (unlikely(rc == PARSER_RC_ERROR || rc == PARSER_RC_STOP))
348
break;
348
- }
349
+
350
action_function_list++;
351
}
352
worker_is_idle();
@@ -354,7 +355,25 @@ inline int parser_action(PARSER *parser, char *input)
355
if (likely(input == parser->buffer))
356
parser->flags |= PARSER_INPUT_PROCESSED;
357
357
- internal_error(rc == PARSER_RC_ERROR, "parser_action() failed.");
358
+#ifdef NETDATA_INTERNAL_CHECKS
359
+ if(rc == PARSER_RC_ERROR) {
360
+ BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX);
361
+ for(size_t i = 0; i < num_words ;i++) {
362
+ if(i) buffer_fast_strcat(wb, " ", 1);
363
+
364
+ buffer_fast_strcat(wb, "\"", 1);
365
+ const char *s = get_word(words, num_words, i);
366
+ buffer_strcat(wb, s?s:"");
367
+ buffer_fast_strcat(wb, "\"", 1);
368
+ }
369
+
370
+ internal_error(true, "PLUGINSD: parser_action('%s') failed on line %zu: { %s } (quotes added to show parsing)",
371
+ command, parser->line, buffer_tostring(wb));
372
+
373
+ buffer_free(wb);
374
+ }
375
+#endif
376
+
377
return (rc == PARSER_RC_ERROR);
378
}
379
parser/parser.h
+9
-4
@@ -7,7 +7,10 @@
7
8
#define PARSER_MAX_CALLBACKS 20
9
#define PARSER_MAX_RECOVER_KEYWORDS 128
10
-#define WORKER_PARSER_FIRST_JOB 1
10
+#define WORKER_PARSER_FIRST_JOB 3
11
+
12
+// this has to be in-sync with the same at receiver.c
13
+#define WORKER_RECEIVER_JOB_REPLICATION_COMPLETION (WORKER_PARSER_FIRST_JOB - 3)
14
15
// PARSER return codes
16
typedef enum parser_rc {
@@ -47,8 +50,9 @@ typedef struct parser {
50
size_t worker_job_next_id;
51
uint8_t version; // Parser version
52
RRDHOST *host;
50
- void *input; // Input source e.g. stream
51
- void *output; // Stream to send commands to plugin
53
+ int fd; // Socket
54
+ FILE *fp_input; // Input source e.g. stream
55
+ FILE *fp_output; // Stream to send commands to plugin
56
#ifdef ENABLE_HTTPS
57
struct netdata_ssl *ssl_output;
58
#endif
@@ -56,6 +60,7 @@ typedef struct parser {
60
PARSER_KEYWORD *keyword; // List of parse keywords and functions
61
void *user; // User defined structure to hold extra state between calls
62
uint32_t flags;
63
+ size_t line;
64
65
char *(*read_function)(char *buffer, long unsigned int, void *input);
66
int (*eof_function)(void *input);
@@ -85,7 +90,7 @@ typedef struct parser {
90
91
int find_first_keyword(const char *str, char *keyword, int max_size, int (*custom_isspace)(char));
92
88
-PARSER *parser_init(RRDHOST *host, void *user, void *input, void *output, PARSER_INPUT_TYPE flags, void *ssl);
93
+PARSER *parser_init(RRDHOST *host, void *user, FILE *fp_input, FILE *fp_output, int fd, PARSER_INPUT_TYPE flags, void *ssl);
94
int parser_add_keyword(PARSER *working_parser, char *keyword, keyword_function func);
95
int parser_next(PARSER *working_parser);
96
int parser_action(PARSER *working_parser, char *input);
streaming/compression.c
+95
-141
@@ -5,6 +5,7 @@
5
6
#define STREAM_COMPRESSION_MSG "STREAM_COMPRESSION"
7
8
+// signature MUST end with a newline
9
#define SIGNATURE ((uint32_t)('z' | 0x80) | (0x80 << 8) | (0x80 << 16) | ('\n' << 24))
10
#define SIGNATURE_MASK ((uint32_t)0xff | (0x80 << 8) | (0x80 << 16) | (0xff << 24))
11
#define SIGNATURE_SIZE 4
@@ -29,7 +30,7 @@ static void lz4_compressor_reset(struct compressor_state *state)
30
if (state->data) {
31
if (state->data->stream) {
32
LZ4_resetStream_fast(state->data->stream);
32
- info("%s: Compressor Reset", STREAM_COMPRESSION_MSG);
33
+ internal_error(true, "%s: compressor reset", STREAM_COMPRESSION_MSG);
34
}
35
state->data->input_ring_buffer_pos = 0;
36
}
@@ -139,11 +140,12 @@ struct compressor_state *create_compressor()
140
/*
141
* LZ4 streaming API decompressor specific data
142
*/
142
-struct decompressor_data {
143
- LZ4_streamDecode_t *stream;
144
- char *stream_buffer;
145
- size_t stream_buffer_size;
146
- size_t stream_buffer_pos;
143
+struct decompressor_stream {
144
+ LZ4_streamDecode_t *lz4_stream;
145
+ char *buffer;
146
+ size_t size;
147
+ size_t write_at;
148
+ size_t read_at;
149
};
150
151
/*
@@ -151,12 +153,12 @@ struct decompressor_data {
153
*/
154
static void lz4_decompressor_reset(struct decompressor_state *state)
155
{
154
- if (state->data) {
155
- if (state->data->stream)
156
- LZ4_setStreamDecode(state->data->stream, NULL, 0);
157
- state->data->stream_buffer_pos = 0;
158
- state->buffer_len = 0;
159
- state->out_buffer_len = 0;
156
+ if (state->stream) {
157
+ if (state->stream->lz4_stream)
158
+ LZ4_setStreamDecode(state->stream->lz4_stream, NULL, 0);
159
+
160
+ state->stream->write_at = 0;
161
+ state->stream->read_at = 0;
162
}
163
}
164
@@ -167,177 +169,129 @@ static void lz4_decompressor_destroy(struct decompressor_state **state)
169
{
170
if (state && *state) {
171
struct decompressor_state *s = *state;
170
- if (s->data) {
172
+ if (s->stream) {
173
debug(D_STREAM, "%s: Destroying decompressor.", STREAM_COMPRESSION_MSG);
172
- if (s->data->stream)
173
- LZ4_freeStreamDecode(s->data->stream);
174
- freez(s->data->stream_buffer);
175
- freez(s->data);
174
+ if (s->stream->lz4_stream)
175
+ LZ4_freeStreamDecode(s->stream->lz4_stream);
176
+ freez(s->stream->buffer);
177
+ freez(s->stream);
178
}
177
- freez(s->buffer);
179
freez(s);
180
*state = NULL;
181
}
182
}
183
183
-static size_t decode_compress_header(const char *data, size_t data_size)
184
-{
185
- if (!data || !data_size)
184
+static size_t decode_compress_header(const char *data, size_t data_size) {
185
+ if (unlikely(!data || !data_size))
186
return 0;
187
- if (data_size < SIGNATURE_SIZE)
187
+
188
+ if (unlikely(data_size != SIGNATURE_SIZE))
189
return 0;
190
+
191
uint32_t sign = *(uint32_t *)data;
190
- if ((sign & SIGNATURE_MASK) != SIGNATURE)
192
+ if (unlikely((sign & SIGNATURE_MASK) != SIGNATURE))
193
return 0;
194
+
195
size_t length = ((sign >> 8) & 0x7f) | ((sign >> 9) & (0x7f << 7));
196
return length;
197
}
198
196
-/*
197
- * Check input data for the compression header
198
- * Return the size of compressed data or 0 for uncompressed data
199
- */
200
-size_t is_compressed_data(const char *data, size_t data_size)
201
-{
202
- return decode_compress_header(data, data_size);
203
-}
204
-
199
/*
200
* Start the collection of compressed data in an internal buffer
201
* Return the size of compressed data or 0 for uncompressed data
202
*/
209
-static size_t lz4_decompressor_start(struct decompressor_state *state, const char *header, size_t header_size)
210
-{
211
- size_t length = decode_compress_header(header, header_size);
212
- if (!length)
213
- return 0;
214
-
215
- if (!state->buffer) {
216
- state->buffer = mallocz(length);
217
- state->buffer_size = length;
218
- } else if (state->buffer_size < length) {
219
- state->buffer = reallocz(state->buffer, length);
220
- state->buffer_size = length;
221
- }
222
- state->buffer_len = length;
223
- state->buffer_pos = 0;
224
- state->out_buffer_pos = 0;
225
- state->out_buffer_len = 0;
226
- return length;
227
-}
228
-
229
-/*
230
- * Add a chunk of compressed data to the internal buffer
231
- * Return the current size of compressed data or 0 for error
232
- */
233
-static size_t lz4_decompressor_put(struct decompressor_state *state, const char *data, size_t size)
234
-{
235
- if (!state || !size || !data)
236
- return 0;
237
- if (!state->buffer)
238
- fatal("STREAM: No decompressor buffer allocated");
239
-
240
- if (state->buffer_pos + size > state->buffer_len) {
241
- error("STREAM: Decompressor buffer overflow %lu + %lu > %lu",
242
- (long unsigned int) state->buffer_pos, (long unsigned int) size,
243
- (long unsigned int) state->buffer_len);
244
- size = state->buffer_len - state->buffer_pos;
245
- }
246
- memcpy(state->buffer + state->buffer_pos, data, size);
247
- state->buffer_pos += size;
248
- return state->buffer_pos;
249
-}
203
+static size_t lz4_decompressor_start(struct decompressor_state *state __maybe_unused, const char *header, size_t header_size) {
204
+ if(unlikely(state->stream->read_at != state->stream->write_at))
205
+ fatal("%s: asked to decompress new data, while there are unread data in the decompression buffer!"
206
+ , STREAM_COMPRESSION_MSG);
207
251
-static size_t saving_percent(size_t comp_len, size_t src_len)
252
-{
253
- if (comp_len > src_len)
254
- comp_len = src_len;
255
- if (!src_len)
256
- return 0;
257
- return 100 - comp_len * 100 / src_len;
208
+ return decode_compress_header(header, header_size);
209
}
210
211
/*
212
* Decompress the compressed data in the internal buffer
213
* Return the size of uncompressed data or 0 for error
214
*/
264
-static size_t lz4_decompressor_decompress(struct decompressor_state *state)
265
-{
266
- if (!state)
267
- return 0;
268
- if (!state->buffer) {
269
- error("%s: No decompressor buffer allocated", STREAM_COMPRESSION_MSG);
215
+static size_t lz4_decompressor_decompress(struct decompressor_state *state, const char *compressed_data, size_t compressed_size) {
216
+ if (unlikely(!state || !compressed_data || !compressed_size))
217
return 0;
218
+
219
+ if(unlikely(state->stream->read_at != state->stream->write_at))
220
+ fatal("%s: asked to decompress new data, while there are unread data in the decompression buffer!"
221
+ , STREAM_COMPRESSION_MSG);
222
+
223
+ if (unlikely(state->stream->write_at >= state->stream->size / 2)) {
224
+ state->stream->write_at = 0;
225
+ state->stream->read_at = 0;
226
}
272
-
273
- long int decompressed_size = LZ4_decompress_safe_continue(state->data->stream, state->buffer,
274
- state->data->stream_buffer + state->data->stream_buffer_pos,
275
- state->buffer_len, state->data->stream_buffer_size - state->data->stream_buffer_pos);
276
- if (decompressed_size < 0) {
277
- error("%s: Decompressor error %ld", STREAM_COMPRESSION_MSG, decompressed_size);
227
+
228
+ long int decompressed_size = LZ4_decompress_safe_continue(
229
+ state->stream->lz4_stream
230
+ , compressed_data
231
+ , state->stream->buffer + state->stream->write_at
232
+ , (int)compressed_size
233
+ , (int)(state->stream->size - state->stream->write_at)
234
+ );
235
+
236
+ if (unlikely(decompressed_size < 0)) {
237
+ error("%s: decompressor returned negative decompressed bytes: %ld", STREAM_COMPRESSION_MSG, decompressed_size);
238
return 0;
239
}
240
281
- state->out_buffer = state->data->stream_buffer + state->data->stream_buffer_pos;
282
- state->data->stream_buffer_pos += decompressed_size;
283
- if (state->data->stream_buffer_pos >= state->data->stream_buffer_size - COMPRESSION_MAX_MSG_SIZE)
284
- state->data->stream_buffer_pos = 0;
285
- state->out_buffer_len = decompressed_size;
286
- state->out_buffer_pos = 0;
241
+ if(unlikely(decompressed_size + state->stream->write_at > state->stream->size))
242
+ fatal("%s: decompressor overflown the stream_buffer. size: %zu, pos: %zu, added: %ld, exceeding the buffer by %zu"
243
+ , STREAM_COMPRESSION_MSG
244
+ , state->stream->size
245
+ , state->stream->write_at
246
+ , decompressed_size
247
+ , state->stream->write_at + decompressed_size - state->stream->size
248
+ );
249
288
- // Some compression statistics
289
- size_t old_avg_saving = saving_percent(state->total_compressed, state->total_uncompressed);
290
- size_t old_avg_size = state->packet_count ? state->total_uncompressed / state->packet_count : 0;
250
+ state->stream->write_at += decompressed_size;
251
292
- state->total_compressed += state->buffer_len + SIGNATURE_SIZE;
252
+ // statistics
253
+ state->total_compressed += compressed_size + SIGNATURE_SIZE;
254
state->total_uncompressed += decompressed_size;
255
state->packet_count++;
256
296
- size_t saving = saving_percent(state->buffer_len, decompressed_size);
297
- size_t avg_saving = saving_percent(state->total_compressed, state->total_uncompressed);
298
- size_t avg_size = state->total_uncompressed / state->packet_count;
299
-
300
- (void)saving;
301
-
302
- if (old_avg_saving != avg_saving || old_avg_size != avg_size){
303
- debug(D_STREAM, "%s: Saving: %lu%% (avg. %lu%%), avg.size: %lu", STREAM_COMPRESSION_MSG,
304
- (long unsigned int) saving, (long unsigned int) avg_saving, (long unsigned int) avg_size);
305
- }
257
return decompressed_size;
258
}
259
260
/*
261
* Return the size of uncompressed data left in the internal buffer or 0 for error
262
*/
312
-static size_t lz4_decompressor_decompressed_bytes_in_buffer(struct decompressor_state *state)
313
-{
314
- return state->out_buffer_len ?
315
- state->out_buffer_len - state->out_buffer_pos : 0;
263
+static size_t lz4_decompressor_decompressed_bytes_in_buffer(struct decompressor_state *state) {
264
+ if(unlikely(state->stream->read_at > state->stream->write_at))
265
+ fatal("%s: invalid read/write stream positions"
266
+ , STREAM_COMPRESSION_MSG);
267
+
268
+ return state->stream->write_at - state->stream->read_at;
269
}
270
271
/*
272
* Fill the buffer provided with uncompressed data from the internal buffer
273
* Return the size of uncompressed data copied or 0 for error
274
*/
322
-static size_t lz4_decompressor_get(struct decompressor_state *state, char *data, size_t size)
323
-{
324
- if (!state || !size || !data)
275
+static size_t lz4_decompressor_get(struct decompressor_state *state, char *dst, size_t size) {
276
+ if (unlikely(!state || !size || !dst))
277
return 0;
326
- if (!state->out_buffer)
327
- fatal("%s: No decompressor output buffer allocated", STREAM_COMPRESSION_MSG);
328
- if (state->out_buffer_pos + size > state->out_buffer_len)
329
- size = state->out_buffer_len - state->out_buffer_pos;
330
-
331
- char *p = state->out_buffer + state->out_buffer_pos, *endp = p + size, *last_lf = NULL;
332
- for (; p < endp; ++p)
333
- if (*p == '\n' || *p == 0)
334
- last_lf = p;
335
- if (last_lf)
336
- size = last_lf + 1 - (state->out_buffer + state->out_buffer_pos);
337
-
338
- memcpy(data, state->out_buffer + state->out_buffer_pos, size);
339
- state->out_buffer_pos += size;
340
- return size;
278
+
279
+ size_t remaining = lz4_decompressor_decompressed_bytes_in_buffer(state);
280
+ if(unlikely(!remaining))
281
+ return 0;
282
+
283
+ size_t bytes_to_return = size;
284
+ if(bytes_to_return > remaining)
285
+ bytes_to_return = remaining;
286
+
287
+ memcpy(dst, state->stream->buffer + state->stream->read_at, bytes_to_return);
288
+ state->stream->read_at += bytes_to_return;
289
+
290
+ if(unlikely(state->stream->read_at > state->stream->write_at))
291
+ fatal("%s: invalid read/write stream positions"
292
+ , STREAM_COMPRESSION_MSG);
293
+
294
+ return bytes_to_return;
295
}
296
297
/*
@@ -347,20 +301,20 @@ static size_t lz4_decompressor_get(struct decompressor_state *state, char *data,
301
struct decompressor_state *create_decompressor()
302
{
303
struct decompressor_state *state = callocz(1, sizeof(struct decompressor_state));
304
+ state->signature_size = SIGNATURE_SIZE;
305
state->reset = lz4_decompressor_reset;
306
state->start = lz4_decompressor_start;
352
- state->put = lz4_decompressor_put;
307
state->decompress = lz4_decompressor_decompress;
308
state->get = lz4_decompressor_get;
309
state->decompressed_bytes_in_buffer = lz4_decompressor_decompressed_bytes_in_buffer;
310
state->destroy = lz4_decompressor_destroy;
311
358
- state->data = callocz(1, sizeof(struct decompressor_data));
359
- fatal_assert(state->data);
360
- state->data->stream = LZ4_createStreamDecode();
361
- state->data->stream_buffer_size = LZ4_decoderRingBufferSize(COMPRESSION_MAX_MSG_SIZE);
362
- state->data->stream_buffer = mallocz(state->data->stream_buffer_size);
363
- fatal_assert(state->data->stream_buffer);
312
+ state->stream = callocz(1, sizeof(struct decompressor_stream));
313
+ fatal_assert(state->stream);
314
+ state->stream->lz4_stream = LZ4_createStreamDecode();
315
+ state->stream->size = LZ4_decoderRingBufferSize(COMPRESSION_MAX_MSG_SIZE) * 2;
316
+ state->stream->buffer = mallocz(state->stream->size);
317
+ fatal_assert(state->stream->buffer);
318
state->reset(state);
319
debug(D_STREAM, "%s: Initialize streaming decompression!", STREAM_COMPRESSION_MSG);
320
return state;
streaming/receiver.c
+221
-207
@@ -3,7 +3,12 @@
3
#include "rrdpush.h"
4
#include "parser/parser.h"
5
6
+// IMPORTANT: to add workers, you have to edit WORKER_PARSER_FIRST_JOB accordingly
7
#define WORKER_RECEIVER_JOB_BYTES_READ (WORKER_PARSER_FIRST_JOB - 1)
8
+#define WORKER_RECEIVER_JOB_BYTES_UNCOMPRESSED (WORKER_PARSER_FIRST_JOB - 2)
9
+
10
+// this has to be the same at parser.h
11
+#define WORKER_RECEIVER_JOB_REPLICATION_COMPLETION (WORKER_PARSER_FIRST_JOB - 3)
12
13
#if WORKER_PARSER_FIRST_JOB < 1
14
#error The define WORKER_PARSER_FIRST_JOB needs to be at least 1
@@ -110,185 +115,182 @@ PARSER_RC streaming_claimed_id(char **words, size_t num_words, void *user)
115
return PARSER_RC_OK;
116
}
117
118
+static int read_stream(struct receiver_state *r, char* buffer, size_t size) {
119
+ if(unlikely(!size)) {
120
+ internal_error(true, "%s() asked to read zero bytes", __FUNCTION__);
121
+ return 0;
122
+ }
123
114
-#ifndef ENABLE_COMPRESSION
115
-/* The receiver socket is blocking, perform a single read into a buffer so that we can reassemble lines for parsing.
116
- */
117
-static int receiver_read(struct receiver_state *r, FILE *fp) {
124
#ifdef ENABLE_HTTPS
119
- if (r->ssl.conn && !r->ssl.flags) {
120
- ERR_clear_error();
121
- int desired = sizeof(r->read_buffer) - r->read_len - 1;
122
- int ret = SSL_read(r->ssl.conn, r->read_buffer + r->read_len, desired);
123
- if (ret > 0 ) {
124
- r->read_len += ret;
125
- worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, ret);
126
- return 0;
127
- }
128
- // Don't treat SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE differently on blocking socket
129
- u_long err;
130
- char buf[256];
131
- while ((err = ERR_get_error()) != 0) {
132
- ERR_error_string_n(err, buf, sizeof(buf));
133
- error("STREAM %s [receive from %s] ssl error: %s", r->hostname, r->client_ip, buf);
134
- }
135
- return 1;
136
- }
125
+ if (r->ssl.conn && r->ssl.flags == NETDATA_SSL_HANDSHAKE_COMPLETE)
126
+ return (int)netdata_ssl_read(r->ssl.conn, buffer, size);
127
#endif
138
- if (!fgets(r->read_buffer, sizeof(r->read_buffer), fp))
139
- return 1;
140
- r->read_len = strlen(r->read_buffer);
141
- worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, r->read_len);
142
- return 0;
143
-}
144
-#else
145
-/*
146
- * The receiver socket is blocking, perform a single read into a buffer so that we can reassemble lines for parsing.
147
- * if SSL encryption is on, then use SSL API for reading stream data.
148
- * Use line oriented fgets() in buffer from receiver_state is provided.
149
- * In other cases use fread to read binary data from socket.
150
- * Return zero on success and the number of bytes were read using pointer in the last argument.
151
- */
152
-static int read_stream(struct receiver_state *r, FILE *fp, char* buffer, size_t size, int* ret) {
153
- if (!ret)
154
- return 1;
155
- *ret = 0;
156
-#ifdef ENABLE_HTTPS
157
- if (r->ssl.conn && !r->ssl.flags) {
158
- ERR_clear_error();
159
- if (buffer != r->read_buffer + r->read_len) {
160
- *ret = SSL_read(r->ssl.conn, buffer, size);
161
- if (*ret > 0 )
162
- return 0;
163
- } else {
164
- // we need to receive data with LF to parse compression header
165
- size_t ofs = 0;
166
- int res = 0;
167
- errno = 0;
168
- while (ofs < size) {
169
- do {
170
- res = SSL_read(r->ssl.conn, buffer + ofs, 1);
171
- // When either SSL_ERROR_SYSCALL (OpenSSL < 3.0) or SSL_ERROR_SSL(OpenSSL > 3.0) happens,
172
- // the connection was lost https://www.openssl.org/docs/man3.0/man3/SSL_get_error.html,
173
- // without the test we will have an infinite loop https://github.com/netdata/netdata/issues/13092
174
- int local_ssl_err = SSL_get_error(r->ssl.conn, res);
175
- if (local_ssl_err == SSL_ERROR_SYSCALL || local_ssl_err == SSL_ERROR_SSL) {
176
- error("The SSL connection has error SSL_ERROR_SYSCALL(%d) and system is registering errno = %d",
177
- local_ssl_err, errno);
178
- return 1;
179
- }
180
- } while (res == 0);
181
-
182
- if (res < 0)
183
- break;
184
- if (buffer[ofs] == '\n')
185
- break;
186
- ofs += res;
187
- }
188
- if (res > 0) {
189
- ofs += res;
190
- *ret = ofs;
191
- buffer[ofs] = 0;
192
- return 0;
193
- }
194
- }
195
- // Don't treat SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE differently on blocking socket
196
- u_long err;
197
- char buf[256];
198
- while ((err = ERR_get_error()) != 0) {
199
- ERR_error_string_n(err, buf, sizeof(buf));
200
- error("STREAM %s [receive from %s] ssl error: %s", r->hostname, r->client_ip, buf);
201
- }
202
- return 1;
128
+
129
+ ssize_t bytes_read = read(r->fd, buffer, size);
130
+ if(bytes_read == 0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINPROGRESS)) {
131
+ error("STREAM: %s(): timeout while waiting for data on socket!", __FUNCTION__);
132
+ bytes_read = -3;
133
}
204
-#endif
205
- if (buffer != r->read_buffer + r->read_len) {
206
- // read to external buffer
207
- *ret = fread(buffer, 1, size, fp);
208
- if (!*ret)
209
- return 1;
210
- } else {
211
- if (!fgets(r->read_buffer, sizeof(r->read_buffer), fp))
212
- return 1;
213
- *ret = strlen(r->read_buffer);
134
+ else if (bytes_read == 0) {
135
+ error("STREAM: %s(): EOF while reading data from socket!", __FUNCTION__);
136
+ bytes_read = -1;
137
}
215
- return 0;
138
+ else if (bytes_read < 0) {
139
+ error("STREAM: %s() failed to read from socket!", __FUNCTION__);
140
+ bytes_read = -2;
141
+ }
142
+
143
+// do {
144
+// bytes_read = (int) fread(buffer, 1, size, fp);
145
+// if (unlikely(bytes_read <= 0)) {
146
+// if(feof(fp)) {
147
+// internal_error(true, "%s(): fread() failed with EOF", __FUNCTION__);
148
+// bytes_read = -2;
149
+// }
150
+// else if(ferror(fp)) {
151
+// internal_error(true, "%s(): fread() failed with ERROR", __FUNCTION__);
152
+// bytes_read = -3;
153
+// }
154
+// else bytes_read = 0;
155
+// }
156
+// else
157
+// worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, bytes_read);
158
+// } while(bytes_read == 0);
159
+
160
+ return (int)bytes_read;
161
}
162
218
-/*
219
- * Get the next line of data for parsing.
220
- * Return data from the decompressor buffer if available.
221
- * Otherwise read next line from the socket and check for compression header.
222
- * Return the line was read If no compression header was found.
223
- * Otherwise read the entire block of compressed data, decompress it
224
- * and return it in receiver_state buffer.
225
- * Return zero on success.
226
- */
227
-static int receiver_read(struct receiver_state *r, FILE *fp) {
228
- // check any decompressed data present
229
- if (r->decompressor && r->decompressor->decompressed_bytes_in_buffer(r->decompressor)) {
230
- size_t available = sizeof(r->read_buffer) - r->read_len;
163
+static bool receiver_read_uncompressed(struct receiver_state *r) {
164
+#ifdef NETDATA_INTERNAL_CHECKS
165
+ if(r->read_buffer[r->read_len] != '\0')
166
+ fatal("%s(): read_buffer does not start with zero", __FUNCTION__ );
167
+#endif
168
+
169
+ int bytes_read = read_stream(r, r->read_buffer + r->read_len, sizeof(r->read_buffer) - r->read_len - 1);
170
+ if(unlikely(bytes_read <= 0))
171
+ return false;
172
+
173
+ worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, (NETDATA_DOUBLE)bytes_read);
174
+ worker_set_metric(WORKER_RECEIVER_JOB_BYTES_UNCOMPRESSED, (NETDATA_DOUBLE)bytes_read);
175
+
176
+ r->read_len += bytes_read;
177
+ r->read_buffer[r->read_len] = '\0';
178
+
179
+ return true;
180
+}
181
+
182
+#ifdef ENABLE_COMPRESSION
183
+static bool receiver_read_compressed(struct receiver_state *r) {
184
+
185
+#ifdef NETDATA_INTERNAL_CHECKS
186
+ if(r->read_buffer[r->read_len] != '\0')
187
+ fatal("%s: read_buffer does not start with zero #2", __FUNCTION__ );
188
+#endif
189
+
190
+ // first use any available uncompressed data
191
+ if (r->decompressor->decompressed_bytes_in_buffer(r->decompressor)) {
192
+ size_t available = sizeof(r->read_buffer) - r->read_len - 1;
193
if (available) {
194
size_t len = r->decompressor->get(r->decompressor, r->read_buffer + r->read_len, available);
195
if (!len) {
234
- internal_error(true, "decompressor returned zero length");
235
- return 1;
196
+ internal_error(true, "decompressor returned zero length #1");
197
+ return false;
198
}
199
238
- r->read_len += len;
200
+ r->read_len += (int)len;
201
+ r->read_buffer[r->read_len] = '\0';
202
}
240
- return 0;
203
+ else
204
+ internal_error(true, "The line to read is too big! Already have %d bytes in read_buffer.", r->read_len);
205
+
206
+ return true;
207
}
208
243
- int ret = 0;
244
- if (read_stream(r, fp, r->read_buffer + r->read_len, sizeof(r->read_buffer) - r->read_len - 1, &ret)) {
245
- internal_error(true, "read_stream() failed (1).");
246
- return 1;
209
+ // no decompressed data available
210
+ // read the compression signature of the next block
211
+
212
+ if(unlikely(r->read_len + r->decompressor->signature_size > sizeof(r->read_buffer) - 1)) {
213
+ internal_error(true, "The last incomplete line does not leave enough room for the next compression header! Already have %d bytes in read_buffer.", r->read_len);
214
+ return false;
215
}
216
249
- worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, ret);
217
+ // read the compression signature from the stream
218
+ // we have to do a loop here, because read_stream() may return less than the data we need
219
+ int bytes_read = 0;
220
+ do {
221
+ int ret = read_stream(r, r->read_buffer + r->read_len + bytes_read, r->decompressor->signature_size - bytes_read);
222
+ if (unlikely(ret <= 0))
223
+ return false;
224
251
- if (!is_compressed_data(r->read_buffer, ret)) {
252
- r->read_len += ret;
253
- return 0;
225
+ bytes_read += ret;
226
+ } while(unlikely(bytes_read < (int)r->decompressor->signature_size));
227
+
228
+ worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, (NETDATA_DOUBLE)bytes_read);
229
+
230
+ if(unlikely(bytes_read != (int)r->decompressor->signature_size))
231
+ fatal("read %d bytes, but expected compression signature of size %zu", bytes_read, r->decompressor->signature_size);
232
+
233
+ size_t compressed_message_size = r->decompressor->start(r->decompressor, r->read_buffer + r->read_len, bytes_read);
234
+ if (unlikely(!compressed_message_size)) {
235
+ internal_error(true, "multiplexed uncompressed data in compressed stream!");
236
+ r->read_len += bytes_read;
237
+ r->read_buffer[r->read_len] = '\0';
238
+ return true;
239
+ }
240
+
241
+ if(unlikely(compressed_message_size > COMPRESSION_MAX_MSG_SIZE)) {
242
+ error("received a compressed message of %zu bytes, which is bigger than the max compressed message size supported of %zu. Ignoring message.",
243
+ compressed_message_size, (size_t)COMPRESSION_MAX_MSG_SIZE);
244
+ return false;
245
}
246
256
- if (unlikely(!r->decompressor))
257
- r->decompressor = create_decompressor();
258
-
259
- size_t bytes_to_read = r->decompressor->start(r->decompressor, r->read_buffer, ret);
247
+ // delete compression header from our read buffer
248
+ r->read_buffer[r->read_len] = '\0';
249
261
- // Read the entire block of compressed data because
262
- // we're unable to decompress incomplete block
263
- char compressed[bytes_to_read];
250
+ // Read the entire compressed block of compressed data
251
+ char compressed[compressed_message_size];
252
+ size_t compressed_bytes_read = 0;
253
do {
265
- if (read_stream(r, fp, compressed, bytes_to_read, &ret)) {
266
- internal_error(true, "read_stream() failed (2).");
267
- return 1;
254
+ size_t start = compressed_bytes_read;
255
+ size_t remaining = compressed_message_size - start;
256
+
257
+ int last_read_bytes = read_stream(r, &compressed[start], remaining);
258
+ if (unlikely(last_read_bytes <= 0)) {
259
+ internal_error(true, "read_stream() failed #2, with code %d", last_read_bytes);
260
+ return false;
261
}
262
270
- worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, ret);
263
+ compressed_bytes_read += last_read_bytes;
264
272
- // Send input data to decompressor
273
- if (ret)
274
- r->decompressor->put(r->decompressor, compressed, ret);
265
+ } while(unlikely(compressed_message_size > compressed_bytes_read));
266
276
- bytes_to_read -= ret;
277
- } while (bytes_to_read > 0);
267
+ worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, (NETDATA_DOUBLE)compressed_bytes_read);
268
279
- // Decompress
280
- size_t bytes_to_parse = r->decompressor->decompress(r->decompressor);
269
+ // decompress the compressed block
270
+ size_t bytes_to_parse = r->decompressor->decompress(r->decompressor, compressed, compressed_bytes_read);
271
if (!bytes_to_parse) {
272
internal_error(true, "no bytes to parse.");
283
- return 1;
273
+ return false;
274
}
275
286
- // Fill read buffer with decompressed data
287
- r->read_len = r->decompressor->get(r->decompressor, r->read_buffer, sizeof(r->read_buffer));
288
- return 0;
289
-}
276
+ worker_set_metric(WORKER_RECEIVER_JOB_BYTES_UNCOMPRESSED, (NETDATA_DOUBLE)bytes_to_parse);
277
291
-#endif
278
+ // fill read buffer with decompressed data
279
+ size_t len = (int)r->decompressor->get(r->decompressor, r->read_buffer + r->read_len, sizeof(r->read_buffer) - r->read_len - 1);
280
+ if (!len) {
281
+ internal_error(true, "decompressor returned zero length #2");
282
+ return false;
283
+ }
284
+ r->read_len += (int)len;
285
+ r->read_buffer[r->read_len] = '\0';
286
+
287
+ return true;
288
+}
289
+#else // !ENABLE_COMPRESSION
290
+static bool receiver_read_compressed(struct receiver_state *r) {
291
+ return receiver_read_uncompressed(r);
292
+}
293
+#endif // ENABLE_COMPRESSION
294
295
/* Produce a full line if one exists, statefully return where we start next time.
296
* When we hit the end of the buffer with a partial line move it to the beginning for the next fill.
@@ -302,7 +304,10 @@ static char *receiver_next_line(struct receiver_state *r, char *buffer, size_t b
304
char *de = &buffer[buffer_length - 2];
305
306
if(ss >= se) {
307
+ *ds = '\0';
308
+ *pos = 0;
309
r->read_len = 0;
310
+ r->read_buffer[r->read_len] = '\0';
311
return NULL;
312
}
313
@@ -333,6 +338,9 @@ static char *receiver_next_line(struct receiver_state *r, char *buffer, size_t b
338
// move everything to the beginning
339
memmove(r->read_buffer, &r->read_buffer[start], r->read_len - start);
340
r->read_len -= (int)start;
341
+ r->read_buffer[r->read_len] = '\0';
342
+ *ds = '\0';
343
+ *pos = 0;
344
return NULL;
345
}
346
@@ -342,7 +350,7 @@ static void streaming_parser_thread_cleanup(void *ptr) {
350
parser_destroy(parser);
351
}
352
345
-static size_t streaming_parser(struct receiver_state *rpt, struct plugind *cd, FILE *fp_in, FILE *fp_out, void *ssl) {
353
+static size_t streaming_parser(struct receiver_state *rpt, struct plugind *cd, int fd, void *ssl) {
354
size_t result;
355
356
PARSER_USER_OBJECT user = {
@@ -353,7 +361,7 @@ static size_t streaming_parser(struct receiver_state *rpt, struct plugind *cd, F
361
.trust_durations = 1
362
};
363
356
- PARSER *parser = parser_init(rpt->host, &user, fp_in, fp_out, PARSER_INPUT_SPLIT, ssl);
364
+ PARSER *parser = parser_init(rpt->host, &user, NULL, NULL, fd, PARSER_INPUT_SPLIT, ssl);
365
366
rrd_collector_started();
367
@@ -365,36 +373,56 @@ static size_t streaming_parser(struct receiver_state *rpt, struct plugind *cd, F
373
374
user.parser = parser;
375
376
+ bool compressed_connection = false;
377
#ifdef ENABLE_COMPRESSION
369
- if (rpt->decompressor)
370
- rpt->decompressor->reset(rpt->decompressor);
378
+ if(stream_has_capability(rpt, STREAM_CAP_COMPRESSION)) {
379
+ compressed_connection = true;
380
+
381
+ if (!rpt->decompressor)
382
+ rpt->decompressor = create_decompressor();
383
+ else
384
+ rpt->decompressor->reset(rpt->decompressor);
385
+ }
386
#endif
387
373
- char buffer[PLUGINSD_LINE_MAX + 2];
374
- do {
375
- if(receiver_read(rpt, fp_in)) break;
388
+ rpt->read_buffer[0] = '\0';
389
+ rpt->read_len = 0;
390
377
- size_t pos = 0;
378
- while(receiver_next_line(rpt, buffer, PLUGINSD_LINE_MAX + 2, &pos)) {
379
- if(unlikely(netdata_exit)) {
380
- internal_error(true, "exiting...");
381
- goto done;
382
- }
383
- if(unlikely(rpt->shutdown)) {
384
- internal_error(true, "parser shutdown...");
385
- goto done;
386
- }
387
- if (unlikely(parser_action(parser, buffer))) {
388
- internal_error(true, "parser_action() failed...");
389
- goto done;
390
- }
391
+ size_t read_buffer_start = 0;
392
+ char buffer[PLUGINSD_LINE_MAX + 2] = "";
393
+ while(!netdata_exit) {
394
+ if(!receiver_next_line(rpt, buffer, PLUGINSD_LINE_MAX + 2, &read_buffer_start)) {
395
+ bool have_new_data;
396
+ if(compressed_connection)
397
+ have_new_data = receiver_read_compressed(rpt);
398
+ else
399
+ have_new_data = receiver_read_uncompressed(rpt);
400
+
401
+ if(!have_new_data)
402
+ break;
403
+
404
+ rpt->last_msg_t = now_realtime_sec();
405
+ continue;
406
}
407
393
- rpt->last_msg_t = now_realtime_sec();
408
+ if(unlikely(netdata_exit)) {
409
+ internal_error(true, "exiting...");
410
+ goto done;
411
+ }
412
+ if(unlikely(rpt->shutdown)) {
413
+ internal_error(true, "parser shutdown...");
414
+ goto done;
415
+ }
416
+
417
+ if (unlikely(parser_action(parser, buffer))) {
418
+ internal_error(true, "parser_action() failed on keyword '%s'.", buffer);
419
+ break;
420
+ }
421
}
395
- while(!netdata_exit);
422
423
done:
424
+ internal_error(true, "Streaming receiver thread stopping...");
425
+
426
result = user.count;
427
428
// free parser with the pop function
@@ -644,42 +672,11 @@ static int rrdpush_receive(struct receiver_state *rpt)
672
error("STREAM %s [receive from [%s]:%s]: cannot remove the non-blocking flag from socket %d", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port, rpt->fd);
673
674
struct timeval timeout;
647
- timeout.tv_sec = 120;
675
+ timeout.tv_sec = 600;
676
timeout.tv_usec = 0;
677
if (unlikely(setsockopt(rpt->fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout) != 0))
678
error("STREAM %s [receive from [%s]:%s]: cannot set timeout for socket %d", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port, rpt->fd);
679
652
- // convert the socket to a FILE *
653
- // It seems that the same FILE * cannot be used for both reading and writing.
654
- // (reads and writes seem to interfere with each other, with undefined results).
655
-
656
- int fd_in = rpt->fd;
657
- int fd_out = fcntl(rpt->fd, F_DUPFD_CLOEXEC, 0);
658
- if(fd_out == -1) {
659
- error("STREAM %s [receive from [%s]:%s]: failed to duplicate FD %d.", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port, rpt->fd);
660
- log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rrdhost_hostname(rpt->host), "FAILED - SOCKET ERROR");
661
- close(fd_in);
662
- return 0;
663
- }
664
-
665
- FILE *fp_out = fdopen(fd_out, "w");
666
- if(!fp_out) {
667
- error("STREAM %s [receive from [%s]:%s]: failed to get a FILE pointer for fd_out %d.", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port, rpt->fd);
668
- log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rrdhost_hostname(rpt->host), "FAILED - SOCKET ERROR");
669
- close(fd_in);
670
- close(fd_out);
671
- return 0;
672
- }
673
-
674
- FILE *fp_in = fdopen(fd_in, "r");
675
- if(!fp_in) {
676
- error("STREAM %s [receive from [%s]:%s]: failed to get a FILE pointer for fd_in %d.", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port, rpt->fd);
677
- log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rrdhost_hostname(rpt->host), "FAILED - SOCKET ERROR");
678
- close(fd_in);
679
- fclose(fp_out);
680
- return 0;
681
- }
682
-
680
rrdhost_wrlock(rpt->host);
681
/* if(rpt->host->connected_senders > 0) {
682
rrdhost_unlock(rpt->host);
@@ -724,12 +721,19 @@ static int rrdpush_receive(struct receiver_state *rpt)
721
722
rrdhost_set_is_parent_label(++localhost->senders_count);
723
727
- rrdcontext_host_child_connected(rpt->host);
724
+ if(stream_has_capability(rpt->host->receiver, STREAM_CAP_REPLICATION)) {
725
+ RRDSET *st;
726
+ rrdset_foreach_read(st, rpt->host) {
727
+ rrdset_flag_clear(st, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS | RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED);
728
+ }
729
+ rrdset_foreach_done(st);
730
+ }
731
732
+ rrdcontext_host_child_connected(rpt->host);
733
734
rrdhost_flag_clear(rpt->host, RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED);
735
732
- size_t count = streaming_parser(rpt, &cd, fp_in, fp_out,
736
+ size_t count = streaming_parser(rpt, &cd, rpt->fd,
737
#ifdef ENABLE_HTTPS
738
(rpt->ssl.conn) ? &rpt->ssl : NULL
739
#else
@@ -746,6 +750,15 @@ static int rrdpush_receive(struct receiver_state *rpt)
750
error("STREAM %s [receive from [%s]:%s]: disconnected (completed %zu updates).",
751
rpt->hostname, rpt->client_ip, rpt->client_port, count);
752
753
+ if(stream_has_capability(rpt->host->receiver, STREAM_CAP_REPLICATION)) {
754
+ RRDSET *st;
755
+ rrdset_foreach_read(st, rpt->host) {
756
+ rrdset_flag_clear(st, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS);
757
+ rrdset_flag_set(st, RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED);
758
+ }
759
+ rrdset_foreach_done(st);
760
+ }
761
+
762
rrdcontext_host_child_disconnected(rpt->host);
763
764
#ifdef ENABLE_ACLK
@@ -779,8 +792,7 @@ static int rrdpush_receive(struct receiver_state *rpt)
792
}
793
794
// cleanup
782
- fclose(fp_in);
783
- fclose(fp_out);
795
+ close(rpt->fd);
796
return (int)count;
797
}
798
@@ -791,7 +803,9 @@ void *rrdpush_receiver_thread(void *ptr) {
803
info("STREAM %s [%s]:%s: receive thread created (task id %d)", rpt->hostname, rpt->client_ip, rpt->client_port, gettid());
804
805
worker_register("STREAMRCV");
794
- worker_register_job_custom_metric(WORKER_RECEIVER_JOB_BYTES_READ, "received bytes", "bytes/s", WORKER_METRIC_INCREMENTAL);
806
+ worker_register_job_custom_metric(WORKER_RECEIVER_JOB_BYTES_READ, "received bytes", "bytes/s", WORKER_METRIC_INCREMENT);
807
+ worker_register_job_custom_metric(WORKER_RECEIVER_JOB_BYTES_UNCOMPRESSED, "uncompressed bytes", "bytes/s", WORKER_METRIC_INCREMENT);
808
+ worker_register_job_custom_metric(WORKER_RECEIVER_JOB_REPLICATION_COMPLETION, "replication completion", "%", WORKER_METRIC_ABSOLUTE);
809
rrdpush_receive(rpt);
810
worker_unregister();
811
streaming/replication.c
+485
-16
@@ -1,6 +1,7 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
#include "replication.h"
4
+#include "Judy.h"
5
6
static time_t replicate_chart_timeframe(BUFFER *wb, RRDSET *st, time_t after, time_t before, bool enable_streaming) {
7
size_t dimensions = rrdset_number_of_dimensions(st);
@@ -13,6 +14,7 @@ static time_t replicate_chart_timeframe(BUFFER *wb, RRDSET *st, time_t after, ti
14
RRDDIM *rd;
15
struct storage_engine_query_handle handle;
16
STORAGE_POINT sp;
17
+ bool enabled;
18
} data[dimensions];
19
20
memset(data, 0, sizeof(data));
@@ -33,27 +35,35 @@ static time_t replicate_chart_timeframe(BUFFER *wb, RRDSET *st, time_t after, ti
35
if (rd_dfe.counter >= dimensions)
36
break;
37
36
- data[rd_dfe.counter].dict = rd_dfe.dict;
37
- data[rd_dfe.counter].rda = dictionary_acquired_item_dup(rd_dfe.dict, rd_dfe.item);
38
- data[rd_dfe.counter].rd = rd;
38
+ if(rd->exposed) {
39
+ data[rd_dfe.counter].dict = rd_dfe.dict;
40
+ data[rd_dfe.counter].rda = dictionary_acquired_item_dup(rd_dfe.dict, rd_dfe.item);
41
+ data[rd_dfe.counter].rd = rd;
42
40
- ops->init(rd->tiers[0]->db_metric_handle, &data[rd_dfe.counter].handle, after, before);
43
+ ops->init(rd->tiers[0]->db_metric_handle, &data[rd_dfe.counter].handle, after, before);
44
+
45
+ data[rd_dfe.counter].enabled = true;
46
+ }
47
+ else
48
+ data[rd_dfe.counter].enabled = false;
49
}
50
rrddim_foreach_done(rd);
51
}
52
45
- time_t now = after, actual_after = 0, actual_before = 0;
53
+ time_t now = after + 1, actual_after = 0, actual_before = 0; (void)actual_before;
54
while(now <= before) {
55
time_t min_start_time = 0, min_end_time = 0;
56
for (size_t i = 0; i < dimensions && data[i].rd; i++) {
57
+ if(!data[i].enabled) continue;
58
+
59
// fetch the first valid point for the dimension
60
int max_skip = 100;
61
while(data[i].sp.end_time < now && !ops->is_finished(&data[i].handle) && max_skip-- > 0)
62
data[i].sp = ops->next_metric(&data[i].handle);
63
54
- if(max_skip <= 0)
55
- error("REPLAY: host '%s', chart '%s', dimension '%s': db does not advance the query beyond time %llu",
56
- rrdhost_hostname(st->rrdhost), rrdset_id(st), rrddim_id(data[i].rd), (unsigned long long)now);
64
+ internal_error(max_skip <= 0,
65
+ "REPLAY: host '%s', chart '%s', dimension '%s': db does not advance the query beyond time %llu",
66
+ rrdhost_hostname(st->rrdhost), rrdset_id(st), rrddim_id(data[i].rd), (unsigned long long) now);
67
68
if(data[i].sp.end_time < now)
69
continue;
@@ -68,6 +78,17 @@ static time_t replicate_chart_timeframe(BUFFER *wb, RRDSET *st, time_t after, ti
78
}
79
}
80
81
+ time_t wall_clock_time = now_realtime_sec();
82
+ if(min_start_time > wall_clock_time + 1 || min_end_time > wall_clock_time + 1) {
83
+ internal_error(true,
84
+ "REPLAY: host '%s', chart '%s': db provided future start time %llu or end time %llu (now is %llu)",
85
+ rrdhost_hostname(st->rrdhost), rrdset_id(st),
86
+ (unsigned long long)min_start_time,
87
+ (unsigned long long)min_end_time,
88
+ (unsigned long long)wall_clock_time);
89
+ break;
90
+ }
91
+
92
if(min_end_time < now) {
93
internal_error(true,
94
"REPLAY: host '%s', chart '%s': no data on any dimension beyond time %llu",
@@ -85,14 +106,18 @@ static time_t replicate_chart_timeframe(BUFFER *wb, RRDSET *st, time_t after, ti
106
else
107
actual_before = min_end_time;
108
88
- buffer_sprintf(wb, PLUGINSD_KEYWORD_REPLAY_BEGIN " '' %llu %llu\n"
109
+ buffer_sprintf(wb, PLUGINSD_KEYWORD_REPLAY_BEGIN " '' %llu %llu %llu\n"
110
, (unsigned long long)min_start_time
90
- , (unsigned long long)min_end_time);
111
+ , (unsigned long long)min_end_time
112
+ , (unsigned long long)wall_clock_time
113
+ );
114
115
// output the replay values for this time
116
for (size_t i = 0; i < dimensions && data[i].rd; i++) {
117
+ if(!data[i].enabled) continue;
118
+
119
if(data[i].sp.start_time <= min_end_time && data[i].sp.end_time >= min_end_time)
95
- buffer_sprintf(wb, PLUGINSD_KEYWORD_REPLAY_SET " \"%s\" " NETDATA_DOUBLE_FORMAT_AUTO " \"%s\"\n",
120
+ buffer_sprintf(wb, PLUGINSD_KEYWORD_REPLAY_SET " \"%s\" " NETDATA_DOUBLE_FORMAT " \"%s\"\n",
121
rrddim_id(data[i].rd), data[i].sp.sum, data[i].sp.flags & SN_FLAG_RESET ? "R" : "");
122
else
123
buffer_sprintf(wb, PLUGINSD_KEYWORD_REPLAY_SET " \"%s\" NAN \"E\"\n",
@@ -123,6 +148,8 @@ static time_t replicate_chart_timeframe(BUFFER *wb, RRDSET *st, time_t after, ti
148
// release all the dictionary items acquired
149
// finalize the queries
150
for(size_t i = 0; i < dimensions && data[i].rda ;i++) {
151
+ if(!data[i].enabled) continue;
152
+
153
ops->finalize(&data[i].handle);
154
dictionary_acquired_item_release(data[i].dict, data[i].rda);
155
}
@@ -133,7 +160,9 @@ static time_t replicate_chart_timeframe(BUFFER *wb, RRDSET *st, time_t after, ti
160
static void replicate_chart_collection_state(BUFFER *wb, RRDSET *st) {
161
RRDDIM *rd;
162
rrddim_foreach_read(rd, st) {
136
- buffer_sprintf(wb, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE " \"%s\" %llu %lld " NETDATA_DOUBLE_FORMAT_AUTO " " NETDATA_DOUBLE_FORMAT_AUTO "\n",
163
+ if(!rd->exposed) continue;
164
+
165
+ buffer_sprintf(wb, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE " \"%s\" %llu %lld " NETDATA_DOUBLE_FORMAT " " NETDATA_DOUBLE_FORMAT "\n",
166
rrddim_id(rd),
167
(usec_t)rd->last_collected_time.tv_sec * USEC_PER_SEC + (usec_t)rd->last_collected_time.tv_usec,
168
rd->last_collected_value,
@@ -233,6 +262,9 @@ bool replicate_chart_response(RRDHOST *host, RRDSET *st, bool start_streaming, t
262
263
static bool send_replay_chart_cmd(send_command callback, void *callback_data, RRDSET *st, bool start_streaming, time_t after, time_t before) {
264
265
+ if(st->rrdhost->receiver && (!st->rrdhost->receiver->replication_first_time_t || after < st->rrdhost->receiver->replication_first_time_t))
266
+ st->rrdhost->receiver->replication_first_time_t = after;
267
+
268
#ifdef NETDATA_INTERNAL_CHECKS
269
if(after && before) {
270
char after_buf[LOG_DATE_LENGTH + 1], before_buf[LOG_DATE_LENGTH + 1];
@@ -252,6 +284,18 @@ static bool send_replay_chart_cmd(send_command callback, void *callback_data, RR
284
}
285
#endif
286
287
+#ifdef NETDATA_INTERNAL_CHECKS
288
+ internal_error(
289
+ st->replay.after != 0 || st->replay.before != 0,
290
+ "REPLAY: host '%s', chart '%s': sending replication request, while there is another inflight",
291
+ rrdhost_hostname(st->rrdhost), rrdset_id(st)
292
+ );
293
+
294
+ st->replay.start_streaming = start_streaming;
295
+ st->replay.after = after;
296
+ st->replay.before = before;
297
+#endif
298
+
299
debug(D_REPLICATION, PLUGINSD_KEYWORD_REPLAY_CHART " \"%s\" \"%s\" %llu %llu\n",
300
rrdset_id(st), start_streaming ? "true" : "false", (unsigned long long)after, (unsigned long long)before);
301
@@ -262,7 +306,7 @@ static bool send_replay_chart_cmd(send_command callback, void *callback_data, RR
306
307
int ret = callback(buffer, callback_data);
308
if (ret < 0) {
265
- error("failed to send replay request to child (ret=%d)", ret);
309
+ error("REPLICATION: failed to send replication request to child (error %d)", ret);
310
return false;
311
}
312
@@ -277,7 +321,7 @@ bool replicate_chart_request(send_command callback, void *callback_data, RRDHOST
321
322
// if replication is disabled, send an empty replication request
323
// asking no data
280
- if (!host->rrdpush_enable_replication) {
324
+ if (unlikely(!rrdhost_option_check(host, RRDHOST_OPTION_REPLICATION))) {
325
internal_error(true,
326
"REPLAY: host '%s', chart '%s': sending empty replication request because replication is disabled",
327
rrdhost_hostname(host), rrdset_id(st));
@@ -325,8 +369,7 @@ bool replicate_chart_request(send_command callback, void *callback_data, RRDHOST
369
last_entry_local = now;
370
}
371
328
- // should never happen but it if does, start streaming without asking
329
- // for any data
372
+ // should never happen but if it does, start streaming without asking for any data
373
if (last_entry_local > last_entry_child) {
374
error("REPLAY: host '%s', chart '%s': sending empty replication request because our last entry (%llu) in later than the child one (%llu)",
375
rrdhost_hostname(host), rrdset_id(st), (unsigned long long)last_entry_local, (unsigned long long)last_entry_child);
@@ -350,3 +393,429 @@ bool replicate_chart_request(send_command callback, void *callback_data, RRDHOST
393
394
return send_replay_chart_cmd(callback, callback_data, st, start_streaming, first_entry_wanted, last_entry_wanted);
395
}
396
+
397
+// ----------------------------------------------------------------------------
398
+
399
+static size_t sender_buffer_used_percent(struct sender_state *s) {
400
+ netdata_mutex_lock(&s->mutex);
401
+ size_t available = cbuffer_available_size_unsafe(s->host->sender->buffer);
402
+ netdata_mutex_unlock(&s->mutex);
403
+
404
+ return (s->host->sender->buffer->max_size - available) * 100 / s->host->sender->buffer->max_size;
405
+}
406
+
407
+
408
+// ----------------------------------------------------------------------------
409
+// replication thread
410
+
411
+// replication request in sender DICTIONARY
412
+// used for de-duplicating the requests
413
+struct replication_request {
414
+ struct sender_state *sender;
415
+ usec_t sender_last_flush_ut;
416
+ STRING *chart_id;
417
+ time_t after; // key for sorting (JudyL)
418
+ time_t before;
419
+ bool start_streaming;
420
+ bool found;
421
+};
422
+
423
+// replication sort entry in JudyL array
424
+// used for sorting all requests, across all nodes
425
+struct replication_sort_entry {
426
+ struct replication_request req;
427
+
428
+ const void *unique_id; // used as a key to identify the sort entry - we never access its contents
429
+ bool executed;
430
+ struct replication_sort_entry *next;
431
+};
432
+
433
+// the global variables for the replication thread
434
+static struct replication_thread {
435
+ netdata_mutex_t mutex;
436
+
437
+ size_t added;
438
+ size_t removed;
439
+ time_t first_time_t;
440
+ size_t requests_count;
441
+ struct replication_request *requests;
442
+
443
+ Pvoid_t JudyL_array;
444
+} rep = {
445
+ .mutex = NETDATA_MUTEX_INITIALIZER,
446
+ .added = 0,
447
+ .removed = 0,
448
+ .first_time_t = 0,
449
+ .requests_count = 0,
450
+ .requests = NULL,
451
+ .JudyL_array = NULL,
452
+};
453
+
454
+static __thread int replication_recursive_mutex_recursions = 0;
455
+
456
+static void replication_recursive_lock() {
457
+ if(++replication_recursive_mutex_recursions == 1)
458
+ netdata_mutex_lock(&rep.mutex);
459
+
460
+#ifdef NETDATA_INTERNAL_CHECKS
461
+ if(replication_recursive_mutex_recursions < 0 || replication_recursive_mutex_recursions > 2)
462
+ fatal("REPLICATION: recursions is %d", replication_recursive_mutex_recursions);
463
+#endif
464
+}
465
+
466
+static void replication_recursive_unlock() {
467
+ if(--replication_recursive_mutex_recursions == 0)
468
+ netdata_mutex_unlock(&rep.mutex);
469
+
470
+#ifdef NETDATA_INTERNAL_CHECKS
471
+ if(replication_recursive_mutex_recursions < 0 || replication_recursive_mutex_recursions > 2)
472
+ fatal("REPLICATION: recursions is %d", replication_recursive_mutex_recursions);
473
+#endif
474
+}
475
+
476
+// ----------------------------------------------------------------------------
477
+// replication sort entry management
478
+
479
+static struct replication_sort_entry *replication_sort_entry_create(struct replication_request *r, const void *unique_id) {
480
+ struct replication_sort_entry *t = mallocz(sizeof(struct replication_sort_entry));
481
+
482
+ // copy the request
483
+ t->req = *r;
484
+ t->req.chart_id = string_dup(r->chart_id);
485
+
486
+
487
+ t->unique_id = unique_id;
488
+ t->executed = false;
489
+ t->next = NULL;
490
+ return t;
491
+}
492
+
493
+static void replication_sort_entry_destroy(struct replication_sort_entry *t) {
494
+ string_freez(t->req.chart_id);
495
+ freez(t);
496
+}
497
+
498
+static struct replication_sort_entry *replication_sort_entry_add(struct replication_request *r, const void *unique_id) {
499
+ struct replication_sort_entry *t = replication_sort_entry_create(r, unique_id);
500
+
501
+ replication_recursive_lock();
502
+
503
+ rep.added++;
504
+
505
+ Pvoid_t *PValue;
506
+
507
+ PValue = JudyLGet(rep.JudyL_array, (Word_t) r->after, PJE0);
508
+ if(!PValue)
509
+ PValue = JudyLIns(&rep.JudyL_array, (Word_t) r->after, PJE0);
510
+
511
+ t->next = *PValue;
512
+ *PValue = t;
513
+
514
+ if(!rep.first_time_t || r->after < rep.first_time_t)
515
+ rep.first_time_t = r->after;
516
+
517
+ replication_recursive_unlock();
518
+
519
+ return t;
520
+}
521
+
522
+static void replication_sort_entry_del(struct sender_state *sender, STRING *chart_id, time_t after, const DICTIONARY_ITEM *item) {
523
+ Pvoid_t *PValue;
524
+ struct replication_sort_entry *to_delete = NULL;
525
+
526
+ replication_recursive_lock();
527
+
528
+ rep.removed++;
529
+
530
+ PValue = JudyLGet(rep.JudyL_array, after, PJE0);
531
+ if(PValue) {
532
+ struct replication_sort_entry *t = *PValue;
533
+ t->executed = true; // make sure we don't get it again
534
+
535
+ if(!t->next) {
536
+ // we are alone here, delete the judy entry
537
+
538
+ if(t->unique_id != item)
539
+ fatal("Item to delete is not matching host '%s', chart '%s', time %ld.",
540
+ rrdhost_hostname(sender->host), string2str(chart_id), after);
541
+
542
+ to_delete = t;
543
+ JudyLDel(&rep.JudyL_array, after, PJE0);
544
+ }
545
+ else {
546
+ // find our entry in the linked list
547
+
548
+ struct replication_sort_entry *t_old = NULL;
549
+ do {
550
+ if(t->unique_id == item) {
551
+ to_delete = t;
552
+
553
+ if(t_old)
554
+ t_old->next = t->next;
555
+ else
556
+ *PValue = t->next;
557
+
558
+ break;
559
+ }
560
+
561
+ t_old = t;
562
+ t = t->next;
563
+
564
+ } while(t);
565
+ }
566
+ }
567
+
568
+ if(!to_delete)
569
+ fatal("Cannot find sort entry to delete for host '%s', chart '%s', time %ld.",
570
+ rrdhost_hostname(sender->host), string2str(chart_id), after);
571
+
572
+ replication_recursive_unlock();
573
+
574
+ replication_sort_entry_destroy(to_delete);
575
+}
576
+
577
+static struct replication_request replication_request_get_first_available() {
578
+ struct replication_sort_entry *found = NULL;
579
+ Pvoid_t *PValue;
580
+ Word_t Index;
581
+
582
+ replication_recursive_lock();
583
+
584
+ rep.requests_count = JudyLCount(rep.JudyL_array, 0, 0xFFFFFFFF, PJE0);
585
+ if(!rep.requests_count) {
586
+ replication_recursive_unlock();
587
+ return (struct replication_request){ .found = false };
588
+ }
589
+
590
+ Index = 0;
591
+ PValue = JudyLFirst(rep.JudyL_array, &Index, PJE0);
592
+ while(!found && PValue) {
593
+ struct replication_sort_entry *t;
594
+
595
+ for(t = *PValue; t ;t = t->next) {
596
+ if(!t->executed
597
+ && sender_buffer_used_percent(t->req.sender) <= 10
598
+ && t->req.sender_last_flush_ut == __atomic_load_n(&t->req.sender->last_flush_time_ut, __ATOMIC_SEQ_CST)
599
+ ) {
600
+ found = t;
601
+ found->executed = true;
602
+ break;
603
+ }
604
+ }
605
+
606
+ if(!found)
607
+ PValue = JudyLNext(rep.JudyL_array, &Index, PJE0);
608
+ }
609
+
610
+ // copy the values we need, while we have the lock
611
+ struct replication_request ret;
612
+
613
+ if(found) {
614
+ ret = found->req;
615
+ ret.chart_id = string_dup(ret.chart_id);
616
+ ret.found = true;
617
+ }
618
+ else
619
+ ret.found = false;
620
+
621
+ replication_recursive_unlock();
622
+
623
+ return ret;
624
+}
625
+
626
+// ----------------------------------------------------------------------------
627
+// replication request management
628
+
629
+static void replication_request_react_callback(const DICTIONARY_ITEM *item, void *value __maybe_unused, void *sender_state __maybe_unused) {
630
+ struct sender_state *s = sender_state; (void)s;
631
+ struct replication_request *r = value;
632
+
633
+ // IMPORTANT:
634
+ // We use the react instead of the insert callback
635
+ // because we want the item to be atomically visible
636
+ // to our replication thread, immediately after.
637
+
638
+ // If we put this at the insert callback, the item is not guaranteed
639
+ // to be atomically visible to others, so the replication thread
640
+ // may see the replication sort entry, but fail to find the dictionary item
641
+ // related to it.
642
+
643
+ replication_sort_entry_add(r, item);
644
+ __atomic_fetch_add(&r->sender->replication_pending_requests, 1, __ATOMIC_SEQ_CST);
645
+}
646
+
647
+static bool replication_request_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *old_value, void *new_value, void *sender_state) {
648
+ struct sender_state *s = sender_state; (void)s;
649
+ struct replication_request *r = old_value; (void)r;
650
+ struct replication_request *r_new = new_value;
651
+
652
+ internal_error(
653
+ true,
654
+ "STREAM %s [send to %s]: ignoring duplicate replication command received for chart '%s' (existing from %llu to %llu [%s], new from %llu to %llu [%s])",
655
+ rrdhost_hostname(s->host), s->connected_to, dictionary_acquired_item_name(item),
656
+ (unsigned long long)r->after, (unsigned long long)r->before, r->start_streaming ? "true" : "false",
657
+ (unsigned long long)r_new->after, (unsigned long long)r_new->before, r_new->start_streaming ? "true" : "false");
658
+
659
+ string_freez(r_new->chart_id);
660
+
661
+ return false;
662
+}
663
+
664
+static void replication_request_delete_callback(const DICTIONARY_ITEM *item, void *value, void *sender_state __maybe_unused) {
665
+ struct replication_request *r = value;
666
+
667
+ replication_sort_entry_del(r->sender, r->chart_id, r->after, item);
668
+
669
+ string_freez(r->chart_id);
670
+ __atomic_fetch_sub(&r->sender->replication_pending_requests, 1, __ATOMIC_SEQ_CST);
671
+}
672
+
673
+
674
+// ----------------------------------------------------------------------------
675
+// public API
676
+
677
+void replication_add_request(struct sender_state *sender, const char *chart_id, time_t after, time_t before, bool start_streaming) {
678
+ struct replication_request tmp = {
679
+ .sender = sender,
680
+ .chart_id = string_strdupz(chart_id),
681
+ .after = after,
682
+ .before = before,
683
+ .start_streaming = start_streaming,
684
+ .sender_last_flush_ut = __atomic_load_n(&sender->last_flush_time_ut, __ATOMIC_SEQ_CST),
685
+ };
686
+
687
+ dictionary_set(sender->replication_requests, chart_id, &tmp, sizeof(struct replication_request));
688
+}
689
+
690
+void replication_flush_sender(struct sender_state *sender) {
691
+ // allow the dictionary destructor to go faster on locks
692
+ replication_recursive_lock();
693
+ dictionary_flush(sender->replication_requests);
694
+ replication_recursive_unlock();
695
+}
696
+
697
+void replication_init_sender(struct sender_state *sender) {
698
+ sender->replication_requests = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);
699
+ dictionary_register_react_callback(sender->replication_requests, replication_request_react_callback, sender);
700
+ dictionary_register_conflict_callback(sender->replication_requests, replication_request_conflict_callback, sender);
701
+ dictionary_register_delete_callback(sender->replication_requests, replication_request_delete_callback, sender);
702
+}
703
+
704
+void replication_cleanup_sender(struct sender_state *sender) {
705
+ // allow the dictionary destructor to go faster on locks
706
+ replication_recursive_lock();
707
+ dictionary_destroy(sender->replication_requests);
708
+ replication_recursive_unlock();
709
+}
710
+
711
+// ----------------------------------------------------------------------------
712
+// replication thread
713
+
714
+static void replication_main_cleanup(void *ptr) {
715
+ struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
716
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
717
+
718
+ // custom code
719
+ worker_unregister();
720
+
721
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
722
+}
723
+
724
+#define WORKER_JOB_ITERATION 1
725
+#define WORKER_JOB_REPLAYING 2
726
+#define WORKER_JOB_CUSTOM_METRIC_PENDING_REQUESTS 3
727
+#define WORKER_JOB_CUSTOM_METRIC_COMPLETION 4
728
+#define WORKER_JOB_CUSTOM_METRIC_ADDED 5
729
+#define WORKER_JOB_CUSTOM_METRIC_DONE 6
730
+
731
+void *replication_thread_main(void *ptr __maybe_unused) {
732
+ netdata_thread_cleanup_push(replication_main_cleanup, ptr);
733
+
734
+ worker_register("REPLICATION");
735
+
736
+ worker_register_job_name(WORKER_JOB_ITERATION, "iteration");
737
+ worker_register_job_name(WORKER_JOB_REPLAYING, "replaying");
738
+
739
+ worker_register_job_custom_metric(WORKER_JOB_CUSTOM_METRIC_PENDING_REQUESTS, "pending requests", "requests", WORKER_METRIC_ABSOLUTE);
740
+ worker_register_job_custom_metric(WORKER_JOB_CUSTOM_METRIC_COMPLETION, "completion", "%", WORKER_METRIC_ABSOLUTE);
741
+ worker_register_job_custom_metric(WORKER_JOB_CUSTOM_METRIC_ADDED, "added requests", "requests/s", WORKER_METRIC_INCREMENTAL_TOTAL);
742
+ worker_register_job_custom_metric(WORKER_JOB_CUSTOM_METRIC_DONE, "finished requests", "requests/s", WORKER_METRIC_INCREMENTAL_TOTAL);
743
+
744
+ while(!netdata_exit) {
745
+ worker_is_busy(WORKER_JOB_ITERATION);
746
+
747
+ // this call also updates our statistics
748
+ struct replication_request r = replication_request_get_first_available();
749
+
750
+ if(r.found) {
751
+ // delete the request from the dictionary
752
+ dictionary_del(r.sender->replication_requests, string2str(r.chart_id));
753
+ }
754
+
755
+ worker_set_metric(WORKER_JOB_CUSTOM_METRIC_PENDING_REQUESTS, (NETDATA_DOUBLE)rep.requests_count);
756
+ worker_set_metric(WORKER_JOB_CUSTOM_METRIC_ADDED, (NETDATA_DOUBLE)rep.added);
757
+ worker_set_metric(WORKER_JOB_CUSTOM_METRIC_DONE, (NETDATA_DOUBLE)rep.removed);
758
+
759
+ if(!r.found && !rep.requests_count) {
760
+ worker_set_metric(WORKER_JOB_CUSTOM_METRIC_COMPLETION, 100.0);
761
+ worker_is_idle();
762
+ sleep_usec(1000 * USEC_PER_MS);
763
+ continue;
764
+ }
765
+
766
+ if(!r.found) {
767
+ worker_is_idle();
768
+ sleep_usec(1 * USEC_PER_MS);
769
+ continue;
770
+ }
771
+
772
+ RRDSET *st = rrdset_find(r.sender->host, string2str(r.chart_id));
773
+ if(!st) {
774
+ internal_error(true, "REPLAY: chart '%s' not found on host '%s'",
775
+ string2str(r.chart_id), rrdhost_hostname(r.sender->host));
776
+
777
+ continue;
778
+ }
779
+
780
+ worker_is_busy(WORKER_JOB_REPLAYING);
781
+
782
+ time_t latest_first_time_t = r.after;
783
+
784
+ if(r.after < r.sender->replication_first_time || !r.sender->replication_first_time)
785
+ r.sender->replication_first_time = r.after;
786
+
787
+ if(r.before < r.sender->replication_min_time || !r.sender->replication_min_time)
788
+ r.sender->replication_min_time = r.before;
789
+
790
+ netdata_thread_disable_cancelability();
791
+
792
+ // send the replication data
793
+ bool start_streaming = replicate_chart_response(st->rrdhost, st,
794
+ r.start_streaming, r.after, r.before);
795
+
796
+ netdata_thread_enable_cancelability();
797
+
798
+ if(start_streaming && r.sender_last_flush_ut == __atomic_load_n(&r.sender->last_flush_time_ut, __ATOMIC_SEQ_CST)) {
799
+ __atomic_fetch_add(&r.sender->receiving_metrics, 1, __ATOMIC_SEQ_CST);
800
+
801
+ // enable normal streaming if we have to
802
+ // but only if the sender buffer has not been flushed since we started
803
+
804
+ debug(D_REPLICATION, "Enabling metric streaming for chart %s.%s",
805
+ rrdhost_hostname(r.sender->host), rrdset_id(st));
806
+
807
+ rrdset_flag_set(st, RRDSET_FLAG_SENDER_REPLICATION_FINISHED);
808
+ }
809
+
810
+ // statistics
811
+ {
812
+ time_t now = now_realtime_sec();
813
+ time_t total = now - rep.first_time_t;
814
+ time_t done = latest_first_time_t - rep.first_time_t;
815
+ worker_set_metric(WORKER_JOB_CUSTOM_METRIC_COMPLETION, (NETDATA_DOUBLE)done * 100.0 / (NETDATA_DOUBLE)total);
816
+ }
817
+ }
818
+
819
+ netdata_thread_cleanup_pop(1);
820
+ return NULL;
821
+}
streaming/replication.h
+5
@@ -14,4 +14,9 @@ bool replicate_chart_request(send_command callback, void *callback_data,
14
time_t first_entry_child, time_t last_entry_child,
15
time_t response_first_start_time, time_t response_last_end_time);
16
17
+void replication_init_sender(struct sender_state *sender);
18
+void replication_cleanup_sender(struct sender_state *sender);
19
+void replication_flush_sender(struct sender_state *sender);
20
+void replication_add_request(struct sender_state *sender, const char *chart_id, time_t after, time_t before, bool start_streaming);
21
+
22
#endif /* REPLICATION_H */
streaming/rrdpush.c
+2
-1
@@ -940,7 +940,8 @@ static void stream_capabilities_to_string(BUFFER *wb, STREAM_CAPABILITIES caps)
940
if(caps & STREAM_CAP_CLABELS) buffer_strcat(wb, "CLABELS ");
941
if(caps & STREAM_CAP_COMPRESSION) buffer_strcat(wb, "COMPRESSION ");
942
if(caps & STREAM_CAP_FUNCTIONS) buffer_strcat(wb, "FUNCTIONS ");
943
- if(caps & STREAM_CAP_REPLICATION) buffer_strcat(wb, "REPLICATION");
943
+ if(caps & STREAM_CAP_REPLICATION) buffer_strcat(wb, "REPLICATION ");
944
+ if(caps & STREAM_CAP_BINARY) buffer_strcat(wb, "BINARY ");
945
}
946
947
void log_receiver_capabilities(struct receiver_state *rpt) {
streaming/rrdpush.h
+18
-15
@@ -7,7 +7,6 @@
7
#include "libnetdata/libnetdata.h"
8
#include "web/server/web_client.h"
9
#include "daemon/common.h"
10
-#include "replication.h"
10
11
#define CONNECTED_TO_SIZE 100
12
@@ -36,6 +35,7 @@ typedef enum {
35
STREAM_CAP_COMPRESSION = (1 << 10), // lz4 compression supported
36
STREAM_CAP_FUNCTIONS = (1 << 11), // plugin functions supported
37
STREAM_CAP_REPLICATION = (1 << 12), // replication supported
38
+ STREAM_CAP_BINARY = (1 << 13), // streaming supports binary data
39
40
// this must be signed int, so don't use the last bit
41
// needed for negotiating errors between parent and child
@@ -45,12 +45,12 @@ typedef enum {
45
#define STREAM_HAS_COMPRESSION STREAM_CAP_COMPRESSION
46
#else
47
#define STREAM_HAS_COMPRESSION 0
48
-#endif //ENABLE_COMPRESSION
48
+#endif // ENABLE_COMPRESSION
49
50
#define STREAM_OUR_CAPABILITIES ( \
51
STREAM_CAP_V1 | STREAM_CAP_V2 | STREAM_CAP_VN | STREAM_CAP_VCAPS | \
52
STREAM_CAP_HLABELS | STREAM_CAP_CLAIM | STREAM_CAP_CLABELS | \
53
- STREAM_HAS_COMPRESSION | STREAM_CAP_FUNCTIONS | STREAM_CAP_REPLICATION )
53
+ STREAM_HAS_COMPRESSION | STREAM_CAP_FUNCTIONS | STREAM_CAP_REPLICATION | STREAM_CAP_BINARY )
54
55
#define stream_has_capability(rpt, capability) ((rpt) && ((rpt)->capabilities & (capability)))
56
@@ -107,21 +107,14 @@ struct compressor_state {
107
};
108
109
struct decompressor_state {
110
- char *buffer;
111
- size_t buffer_size;
112
- size_t buffer_len;
113
- size_t buffer_pos;
114
- char *out_buffer;
115
- size_t out_buffer_len;
116
- size_t out_buffer_pos;
110
+ size_t signature_size;
111
size_t total_compressed;
112
size_t total_uncompressed;
113
size_t packet_count;
120
- struct decompressor_data *data; // Decompression API specific data
114
+ struct decompressor_stream *stream; // Decompression API specific data
115
void (*reset)(struct decompressor_state *state);
116
size_t (*start)(struct decompressor_state *state, const char *header, size_t header_size);
123
- size_t (*put)(struct decompressor_state *state, const char *data, size_t size);
124
- size_t (*decompress)(struct decompressor_state *state);
117
+ size_t (*decompress)(struct decompressor_state *state, const char *compressed_data, size_t compressed_size);
118
size_t (*decompressed_bytes_in_buffer)(struct decompressor_state *state);
119
size_t (*get)(struct decompressor_state *state, char *data, size_t size);
120
void (*destroy)(struct decompressor_state **state);
@@ -149,7 +142,7 @@ struct sender_state {
142
size_t sent_bytes;
143
size_t sent_bytes_on_this_connection;
144
size_t send_attempts;
152
- time_t last_sent_t;
145
+ time_t last_traffic_seen_t;
146
size_t not_connected_loops;
147
// Metrics are collected asynchronously by collector threads calling rrdset_done_push(). This can also trigger
148
// the lazy creation of the sender thread - both cases (buffer access and thread creation) are guarded here.
@@ -170,6 +163,12 @@ struct sender_state {
163
#endif
164
165
DICTIONARY *replication_requests;
166
+ size_t replication_pending_requests;
167
+ time_t replication_first_time;
168
+ time_t replication_min_time;
169
+
170
+ usec_t last_flush_time_ut;
171
+ size_t receiving_metrics;
172
};
173
174
struct receiver_state {
@@ -204,6 +203,8 @@ struct receiver_state {
203
unsigned int rrdpush_compression;
204
struct decompressor_state *decompressor;
205
#endif
206
+
207
+ time_t replication_first_time_t;
208
};
209
210
struct rrdpush_destinations {
@@ -233,6 +234,7 @@ void rrdpush_destinations_init(RRDHOST *host);
234
void rrdpush_destinations_free(RRDHOST *host);
235
236
void sender_init(RRDHOST *host);
237
+
238
BUFFER *sender_start(struct sender_state *s);
239
void sender_commit(struct sender_state *s, BUFFER *wb);
240
void sender_cancel(struct sender_state *s);
@@ -264,7 +266,6 @@ void rrdpush_signal_sender_to_wake_up(struct sender_state *s);
266
#ifdef ENABLE_COMPRESSION
267
struct compressor_state *create_compressor();
268
struct decompressor_state *create_decompressor();
267
-size_t is_compressed_data(const char *data, size_t data_size);
269
#endif
270
271
void log_receiver_capabilities(struct receiver_state *rpt);
@@ -272,4 +273,6 @@ void log_sender_capabilities(struct sender_state *s);
273
STREAM_CAPABILITIES convert_stream_version_to_capabilities(int32_t version);
274
int32_t stream_capabilities_to_vn(uint32_t caps);
275
276
+#include "replication.h"
277
+
278
#endif //NETDATA_RRDPUSH_H
streaming/sender.c
+86
-171
@@ -21,9 +21,11 @@
21
#define WORKER_SENDER_JOB_BUFFER_RATIO 15
22
#define WORKER_SENDER_JOB_BYTES_RECEIVED 16
23
#define WORKER_SENDER_JOB_BYTES_SENT 17
24
+#define WORKER_SENDER_JOB_REPLAY_REQUEST 18
25
+#define WORKER_SENDER_JOB_FUNCTION_REQUEST 19
26
25
-#if WORKER_UTILIZATION_MAX_JOB_TYPES < 18
26
-#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 18
27
+#if WORKER_UTILIZATION_MAX_JOB_TYPES < 20
28
+#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 20
29
#endif
30
31
extern struct config stream_config;
@@ -31,12 +33,6 @@ extern int netdata_use_ssl_on_stream;
33
extern char *netdata_ssl_ca_path;
34
extern char *netdata_ssl_ca_file;
35
34
-struct replication_request {
35
- bool start_streaming;
36
- time_t after;
37
- time_t before;
38
-};
39
-
36
static __thread BUFFER *sender_thread_buffer = NULL;
37
static __thread bool sender_thread_buffer_used = false;
38
@@ -81,6 +77,8 @@ static inline void deactivate_compression(struct sender_state *s) {
77
}
78
#endif
79
80
+#define SENDER_BUFFER_ADAPT_TO_TIMES_MAX_SIZE 3
81
+
82
// Collector thread finishing a transmission
83
void sender_commit(struct sender_state *s, BUFFER *wb) {
84
@@ -100,43 +98,49 @@ void sender_commit(struct sender_state *s, BUFFER *wb) {
98
99
netdata_mutex_lock(&s->mutex);
100
103
- if(unlikely(s->host->sender->buffer->max_size < (buffer_strlen(wb) + 1) * 2)) {
104
- error("STREAM %s [send to %s]: max buffer size of %zu is too small for data of size %zu. Increasing the max buffer size to twice the max data size.",
105
- rrdhost_hostname(s->host), s->connected_to, s->host->sender->buffer->max_size, buffer_strlen(wb) + 1);
101
+ if(unlikely(s->host->sender->buffer->max_size < (src_len + 1) * SENDER_BUFFER_ADAPT_TO_TIMES_MAX_SIZE)) {
102
+ info("STREAM %s [send to %s]: max buffer size of %zu is too small for a data message of size %zu. Increasing the max buffer size to %d times the max data message size.",
103
+ rrdhost_hostname(s->host), s->connected_to, s->host->sender->buffer->max_size, buffer_strlen(wb) + 1, SENDER_BUFFER_ADAPT_TO_TIMES_MAX_SIZE);
104
107
- s->host->sender->buffer->max_size = (buffer_strlen(wb) + 1) * 2;
105
+ s->host->sender->buffer->max_size = (src_len + 1) * SENDER_BUFFER_ADAPT_TO_TIMES_MAX_SIZE;
106
}
107
108
#ifdef ENABLE_COMPRESSION
111
- if (s->flags & SENDER_FLAG_COMPRESSION && s->compressor) {
109
+ if (stream_has_capability(s, STREAM_CAP_COMPRESSION) && s->compressor) {
110
while(src_len) {
111
size_t size_to_compress = src_len;
112
115
- if(size_to_compress > COMPRESSION_MAX_MSG_SIZE) {
116
- // we need to find the last newline
117
- // so that the decompressor will have a whole line to work with
118
-
119
- const char *t = &src[COMPRESSION_MAX_MSG_SIZE - 1];
120
- while(t-- > src)
121
- if(*t == '\n')
122
- break;
123
-
124
- if(t == src)
113
+ if(unlikely(size_to_compress > COMPRESSION_MAX_MSG_SIZE)) {
114
+ if (stream_has_capability(s, STREAM_CAP_BINARY))
115
size_to_compress = COMPRESSION_MAX_MSG_SIZE;
126
- else
127
- size_to_compress = t - src + 1;
116
+ else {
117
+ if (size_to_compress > COMPRESSION_MAX_MSG_SIZE) {
118
+ // we need to find the last newline
119
+ // so that the decompressor will have a whole line to work with
120
+
121
+ const char *t = &src[COMPRESSION_MAX_MSG_SIZE];
122
+ while (--t >= src)
123
+ if (unlikely(*t == '\n'))
124
+ break;
125
+
126
+ if (t <= src) {
127
+ size_to_compress = COMPRESSION_MAX_MSG_SIZE;
128
+ } else
129
+ size_to_compress = t - src + 1;
130
+ }
131
+ }
132
}
133
134
char *dst;
135
size_t dst_len = s->compressor->compress(s->compressor, src, size_to_compress, &dst);
136
if (!dst_len) {
133
- error("STREAM %s [send to %s]: compression failed. Resetting compressor and re-trying",
137
+ error("STREAM %s [send to %s]: COMPRESSION failed. Resetting compressor and re-trying",
138
rrdhost_hostname(s->host), s->connected_to);
139
140
s->compressor->reset(s->compressor);
141
dst_len = s->compressor->compress(s->compressor, src, size_to_compress, &dst);
142
if(!dst_len) {
139
- error("STREAM %s [send to %s]: compression failed again. Deactivating compression",
143
+ error("STREAM %s [send to %s]: COMPRESSION failed again. Deactivating compression",
144
rrdhost_hostname(s->host), s->connected_to);
145
146
deactivate_compression(s);
@@ -255,13 +259,17 @@ static void rrdpush_sender_thread_reset_all_charts(RRDHOST *host) {
259
}
260
261
static inline void rrdpush_sender_thread_data_flush(RRDHOST *host) {
262
+ __atomic_store_n(&host->sender->last_flush_time_ut, now_realtime_usec(), __ATOMIC_SEQ_CST);
263
+
264
netdata_mutex_lock(&host->sender->mutex);
265
cbuffer_flush(host->sender->buffer);
266
netdata_mutex_unlock(&host->sender->mutex);
267
268
rrdpush_sender_thread_reset_all_charts(host);
269
rrdpush_sender_thread_send_custom_host_variables(host);
264
- dictionary_flush(host->sender->replication_requests);
270
+ replication_flush_sender(host->sender);
271
+
272
+ __atomic_store_n(&host->sender->receiving_metrics, 0, __ATOMIC_SEQ_CST);
273
}
274
275
void rrdpush_encode_variable(stream_encoded_t *se, RRDHOST *host)
@@ -343,7 +351,7 @@ struct {
351
.dynamic = false,
352
.error = "remote server rejected this stream, the host we are trying to stream is already streamed to it",
353
.worker_job_id = WORKER_SENDER_JOB_DISCONNECT_BAD_HANDSHAKE,
346
- .postpone_reconnect_seconds = 1 * 60, // 1 minute
354
+ .postpone_reconnect_seconds = 2 * 60, // 2 minutes
355
},
356
{
357
.response = START_STREAMING_ERROR_NOT_PERMITTED,
@@ -468,7 +476,7 @@ static bool rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_p
476
477
#ifdef ENABLE_COMPRESSION
478
// If we don't want compression, remove it from our capabilities
471
- if(!(s->flags & SENDER_FLAG_COMPRESSION) && stream_has_capability(s, STREAM_CAP_COMPRESSION))
479
+ if(!(s->flags & SENDER_FLAG_COMPRESSION))
480
s->capabilities &= ~STREAM_CAP_COMPRESSION;
481
#endif // ENABLE_COMPRESSION
482
@@ -664,20 +672,12 @@ static bool rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_p
672
return false;
673
674
#ifdef ENABLE_COMPRESSION
667
- // if the stream does not have compression capability,
668
- // shut it down for us too.
669
- // FIXME - this means that if there are multiple parents and one of them does not support compression
670
- // we are going to shut it down for all of them eventually...
671
- if(!stream_has_capability(s, STREAM_CAP_COMPRESSION))
672
- s->flags &= ~SENDER_FLAG_COMPRESSION;
673
-
674
- if(s->flags & SENDER_FLAG_COMPRESSION) {
675
- if(s->compressor)
675
+ if(stream_has_capability(s, STREAM_CAP_COMPRESSION)) {
676
+ if(!s->compressor)
677
+ s->compressor = create_compressor();
678
+ else
679
s->compressor->reset(s->compressor);
680
}
678
- else
679
- info("STREAM %s [send to %s]: compression is disabled on this connection.", rrdhost_hostname(host), s->connected_to);
680
-
681
#endif //ENABLE_COMPRESSION
682
683
log_sender_capabilities(s);
@@ -698,8 +698,6 @@ static bool attempt_to_connect(struct sender_state *state)
698
state->send_attempts = 0;
699
700
if(rrdpush_sender_thread_connect_to_parent(state->host, state->default_port, state->timeout, state)) {
701
- state->last_sent_t = now_monotonic_sec();
702
-
701
// reset the buffer, to properly send charts and metrics
702
rrdpush_sender_thread_data_flush(state->host);
703
@@ -748,7 +746,7 @@ static ssize_t attempt_to_send(struct sender_state *s) {
746
#ifdef ENABLE_HTTPS
747
SSL *conn = s->host->sender->ssl.conn ;
748
if(conn && s->host->sender->ssl.flags == NETDATA_SSL_HANDSHAKE_COMPLETE)
751
- ret = SSL_write(conn, chunk, outstanding);
749
+ ret = netdata_ssl_write(conn, chunk, outstanding);
750
else
751
ret = send(s->rrdpush_sender_socket, chunk, outstanding, MSG_DONTWAIT);
752
#else
@@ -760,7 +758,6 @@ static ssize_t attempt_to_send(struct sender_state *s) {
758
s->sent_bytes_on_this_connection += ret;
759
s->sent_bytes += ret;
760
debug(D_STREAM, "STREAM %s [send to %s]: Sent %zd bytes", rrdhost_hostname(s->host), s->connected_to, ret);
763
- s->last_sent_t = now_monotonic_sec();
761
}
762
else if (ret == -1 && (errno == EAGAIN || errno == EINTR || errno == EWOULDBLOCK))
763
debug(D_STREAM, "STREAM %s [send to %s]: unavailable after polling POLLOUT", rrdhost_hostname(s->host), s->connected_to);
@@ -783,24 +780,14 @@ static ssize_t attempt_read(struct sender_state *s) {
780
781
#ifdef ENABLE_HTTPS
782
if (s->host->sender->ssl.conn && s->host->sender->ssl.flags == NETDATA_SSL_HANDSHAKE_COMPLETE) {
786
- ERR_clear_error();
787
- int desired = sizeof(s->read_buffer) - s->read_len - 1;
788
- ret = SSL_read(s->host->sender->ssl.conn, s->read_buffer, desired);
783
+ size_t desired = sizeof(s->read_buffer) - s->read_len - 1;
784
+ ret = netdata_ssl_read(s->host->sender->ssl.conn, s->read_buffer, desired);
785
if (ret > 0 ) {
790
- s->read_len += ret;
786
+ s->read_len += (int)ret;
787
return ret;
788
}
793
- int sslerrno = SSL_get_error(s->host->sender->ssl.conn, desired);
794
- if (sslerrno == SSL_ERROR_WANT_READ || sslerrno == SSL_ERROR_WANT_WRITE)
795
- return ret;
789
790
worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SSL_ERROR);
798
- u_long err;
799
- char buf[256];
800
- while ((err = ERR_get_error()) != 0) {
801
- ERR_error_string_n(err, buf, sizeof(buf));
802
- error("STREAM %s [send to %s] SSL error: %s", rrdhost_hostname(s->host), s->connected_to, buf);
803
- }
791
rrdpush_sender_thread_close_socket(s->host);
792
return ret;
793
}
@@ -865,6 +852,8 @@ void stream_execute_function_callback(BUFFER *func_wb, int code, void *data) {
852
853
// This is just a placeholder until the gap filling state machine is inserted
854
void execute_commands(struct sender_state *s) {
855
+ worker_is_busy(WORKER_SENDER_JOB_EXECUTE);
856
+
857
char *start = s->read_buffer, *end = &s->read_buffer[s->read_len], *newline;
858
*end = 0;
859
while( start < end && (newline = strchr(start, '\n')) ) {
@@ -881,6 +870,8 @@ void execute_commands(struct sender_state *s) {
870
const char *keyword = get_word(words, num_words, 0);
871
872
if(keyword && strcmp(keyword, PLUGINSD_KEYWORD_FUNCTION) == 0) {
873
+ worker_is_busy(WORKER_SENDER_JOB_FUNCTION_REQUEST);
874
+
875
char *transaction = get_word(words, num_words, 1);
876
char *timeout_s = get_word(words, num_words, 2);
877
char *function = get_word(words, num_words, 3);
@@ -909,7 +900,10 @@ void execute_commands(struct sender_state *s) {
900
stream_execute_function_callback(wb, code, tmp);
901
}
902
}
912
- } else if (keyword && strcmp(keyword, PLUGINSD_KEYWORD_REPLAY_CHART) == 0) {
903
+ }
904
+ else if (keyword && strcmp(keyword, PLUGINSD_KEYWORD_REPLAY_CHART) == 0) {
905
+ worker_is_busy(WORKER_SENDER_JOB_REPLAY_REQUEST);
906
+
907
const char *chart_id = get_word(words, num_words, 1);
908
const char *start_streaming = get_word(words, num_words, 2);
909
const char *after = get_word(words, num_words, 3);
@@ -924,18 +918,20 @@ void execute_commands(struct sender_state *s) {
918
start_streaming ? start_streaming : "(unset)",
919
after ? after : "(unset)",
920
before ? before : "(unset)");
927
- } else {
928
- struct replication_request tmp = {
929
- .start_streaming = !strcmp(start_streaming, "true"),
930
- .after = strtoll(after, NULL, 0),
931
- .before = strtoll(before, NULL, 0),
932
- };
933
- dictionary_set(s->replication_requests, chart_id, &tmp, sizeof(struct replication_request));
921
}
935
- } else {
922
+ else {
923
+ replication_add_request(s, chart_id,
924
+ strtoll(after, NULL, 0),
925
+ strtoll(before, NULL, 0),
926
+ !strcmp(start_streaming, "true")
927
+ );
928
+ }
929
+ }
930
+ else {
931
error("STREAM %s [send to %s] received unknown command over connection: %s", rrdhost_hostname(s->host), s->connected_to, words[0]?words[0]:"(unset)");
932
}
933
934
+ worker_is_busy(WORKER_SENDER_JOB_EXECUTE);
935
start = newline + 1;
936
}
937
if (start < end) {
@@ -1037,42 +1033,6 @@ static void rrdpush_sender_thread_cleanup_callback(void *ptr) {
1033
freez(data);
1034
}
1035
1040
-static void replication_request_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value __maybe_unused, void *sender_state __maybe_unused) {
1041
- ;
1042
-}
1043
-static bool replication_request_conflict_callback(const DICTIONARY_ITEM *item, void *old_value, void *new_value, void *sender_state) {
1044
- struct sender_state *s = sender_state;
1045
- struct replication_request *rr = old_value;
1046
- struct replication_request *rr_new = new_value;
1047
-
1048
- error("STREAM %s [send to %s]: duplicate replication command received for chart '%s' (existing from %llu to %llu [%s], new from %llu to %llu [%s])",
1049
- rrdhost_hostname(s->host), s->connected_to, dictionary_acquired_item_name(item),
1050
- (unsigned long long)rr->after, (unsigned long long)rr->before, rr->start_streaming?"true":"false",
1051
- (unsigned long long)rr_new->after, (unsigned long long)rr_new->before, rr_new->start_streaming?"true":"false");
1052
-
1053
- bool updated = false;
1054
-
1055
- if(rr_new->after < rr->after) {
1056
- rr->after = rr_new->after;
1057
- updated = true;
1058
- }
1059
-
1060
- if(rr_new->before > rr->before) {
1061
- rr->before = rr_new->before;
1062
- updated = true;
1063
- }
1064
-
1065
- if(rr_new->start_streaming != rr->start_streaming) {
1066
- rr->start_streaming = true;
1067
- updated = true;
1068
- }
1069
-
1070
- return updated;
1071
-}
1072
-static void replication_request_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value __maybe_unused, void *sender_state __maybe_unused) {
1073
- ;
1074
-}
1075
-
1036
void sender_init(RRDHOST *host)
1037
{
1038
if (host->sender)
@@ -1092,63 +1052,12 @@ void sender_init(RRDHOST *host)
1052
host->sender->flags |= SENDER_FLAG_COMPRESSION;
1053
host->sender->compressor = create_compressor();
1054
}
1055
+ else
1056
+ host->sender->flags &= ~SENDER_FLAG_COMPRESSION;
1057
#endif
1058
1059
netdata_mutex_init(&host->sender->mutex);
1098
-
1099
- host->sender->replication_requests = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
1100
- dictionary_register_insert_callback(host->sender->replication_requests, replication_request_insert_callback, host->sender);
1101
- dictionary_register_conflict_callback(host->sender->replication_requests, replication_request_conflict_callback, host->sender);
1102
- dictionary_register_delete_callback(host->sender->replication_requests, replication_request_delete_callback, host->sender);
1103
-}
1104
-
1105
-static size_t sender_buffer_used_percent(struct sender_state *s) {
1106
- netdata_mutex_lock(&s->mutex);
1107
- size_t available = cbuffer_available_size_unsafe(s->host->sender->buffer);
1108
- netdata_mutex_unlock(&s->mutex);
1109
-
1110
- return (s->host->sender->buffer->max_size - available) * 100 / s->host->sender->buffer->max_size;
1111
-}
1112
-
1113
-static void process_replication_requests(struct sender_state *s) {
1114
- if(dictionary_entries(s->replication_requests) == 0)
1115
- return;
1116
-
1117
- struct replication_request *rr;
1118
- dfe_start_write(s->replication_requests, rr) {
1119
- size_t used_percent = sender_buffer_used_percent(s);
1120
- if(used_percent > 50) break;
1121
-
1122
- // delete it from the dictionary
1123
- // the current item is referenced - it will not go away until the next iteration of the dfe loop
1124
- dictionary_del(s->replication_requests, rr_dfe.name);
1125
-
1126
- // find the chart
1127
- RRDSET *st = rrdset_find(s->host, rr_dfe.name);
1128
- if(unlikely(!st)) {
1129
- internal_error(true,
1130
- "STREAM %s [send to %s]: cannot find chart '%s' to satisfy pending replication command."
1131
- , rrdhost_hostname(s->host), s->connected_to, rr_dfe.name);
1132
- continue;
1133
- }
1134
-
1135
- netdata_thread_disable_cancelability();
1136
-
1137
- // send the replication data
1138
- bool start_streaming = replicate_chart_response(st->rrdhost, st,
1139
- rr->start_streaming, rr->after, rr->before);
1140
-
1141
- netdata_thread_enable_cancelability();
1142
-
1143
- // enable normal streaming if we have to
1144
- if (start_streaming) {
1145
- debug(D_REPLICATION, "Enabling metric streaming for chart %s.%s",
1146
- rrdhost_hostname(s->host), rrdset_id(st));
1147
-
1148
- rrdset_flag_set(st, RRDSET_FLAG_SENDER_REPLICATION_FINISHED);
1149
- }
1150
- }
1151
- dfe_done(rr);
1060
+ replication_init_sender(host->sender);
1061
}
1062
1063
void *rrdpush_sender_thread(void *ptr) {
@@ -1171,9 +1080,12 @@ void *rrdpush_sender_thread(void *ptr) {
1080
worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_NO_COMPRESSION, "disconnect no compression");
1081
worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_BAD_HANDSHAKE, "disconnect bad handshake");
1082
1083
+ worker_register_job_name(WORKER_SENDER_JOB_REPLAY_REQUEST, "replay request");
1084
+ worker_register_job_name(WORKER_SENDER_JOB_FUNCTION_REQUEST, "function");
1085
+
1086
worker_register_job_custom_metric(WORKER_SENDER_JOB_BUFFER_RATIO, "used buffer ratio", "%", WORKER_METRIC_ABSOLUTE);
1175
- worker_register_job_custom_metric(WORKER_SENDER_JOB_BYTES_RECEIVED, "bytes received", "bytes/s", WORKER_METRIC_INCREMENTAL);
1176
- worker_register_job_custom_metric(WORKER_SENDER_JOB_BYTES_SENT, "bytes sent", "bytes/s", WORKER_METRIC_INCREMENTAL);
1087
+ worker_register_job_custom_metric(WORKER_SENDER_JOB_BYTES_RECEIVED, "bytes received", "bytes/s", WORKER_METRIC_INCREMENT);
1088
+ worker_register_job_custom_metric(WORKER_SENDER_JOB_BYTES_SENT, "bytes sent", "bytes/s", WORKER_METRIC_INCREMENT);
1089
1090
struct sender_state *s = ptr;
1091
s->tid = gettid();
@@ -1196,7 +1108,7 @@ void *rrdpush_sender_thread(void *ptr) {
1108
info("STREAM %s [send]: thread created (task id %d)", rrdhost_hostname(s->host), s->tid);
1109
1110
s->timeout = (int)appconfig_get_number(
1199
- &stream_config, CONFIG_SECTION_STREAM, "timeout seconds", 60);
1111
+ &stream_config, CONFIG_SECTION_STREAM, "timeout seconds", 600);
1112
1113
s->default_port = (int)appconfig_get_number(
1114
&stream_config, CONFIG_SECTION_STREAM, "default port", 19999);
@@ -1255,6 +1167,7 @@ void *rrdpush_sender_thread(void *ptr) {
1167
if(unlikely(!attempt_to_connect(s)))
1168
continue;
1169
1170
+ s->last_traffic_seen_t = now_monotonic_sec();
1171
rrdpush_claimed_id(s->host);
1172
rrdpush_send_host_labels(s->host);
1173
@@ -1265,7 +1178,9 @@ void *rrdpush_sender_thread(void *ptr) {
1178
}
1179
1180
// If the TCP window never opened then something is wrong, restart connection
1268
- if(unlikely(now_monotonic_sec() - s->last_sent_t > s->timeout)) {
1181
+ if(unlikely(now_monotonic_sec() - s->last_traffic_seen_t > s->timeout &&
1182
+ __atomic_load_n(&s->replication_pending_requests, __ATOMIC_SEQ_CST) == 0) &&
1183
+ __atomic_load_n(&s->receiving_metrics, __ATOMIC_SEQ_CST) != 0) {
1184
worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT);
1185
error("STREAM %s [send to %s]: could not send metrics for %d seconds - closing connection - we have sent %zu bytes on this connection via %zu send attempts.", rrdhost_hostname(s->host), s->connected_to, s->timeout, s->sent_bytes_on_this_connection, s->send_attempts);
1186
rrdpush_sender_thread_close_socket(s->host);
@@ -1342,8 +1257,10 @@ void *rrdpush_sender_thread(void *ptr) {
1257
if(likely(outstanding && (fds[Socket].revents & POLLOUT))) {
1258
worker_is_busy(WORKER_SENDER_JOB_SOCKET_SEND);
1259
ssize_t bytes = attempt_to_send(s);
1345
- if(bytes > 0)
1346
- worker_set_metric(WORKER_SENDER_JOB_BYTES_SENT, bytes);
1260
+ if(bytes > 0) {
1261
+ s->last_traffic_seen_t = now_monotonic_sec();
1262
+ worker_set_metric(WORKER_SENDER_JOB_BYTES_SENT, (NETDATA_DOUBLE)bytes);
1263
+ }
1264
}
1265
1266
// If the collector woke us up then empty the pipe to remove the signal
@@ -1359,16 +1276,14 @@ void *rrdpush_sender_thread(void *ptr) {
1276
if (fds[Socket].revents & POLLIN) {
1277
worker_is_busy(WORKER_SENDER_JOB_SOCKET_RECEIVE);
1278
ssize_t bytes = attempt_read(s);
1362
- if(bytes > 0)
1363
- worker_set_metric(WORKER_SENDER_JOB_BYTES_RECEIVED, bytes);
1279
+ if(bytes > 0) {
1280
+ s->last_traffic_seen_t = now_monotonic_sec();
1281
+ worker_set_metric(WORKER_SENDER_JOB_BYTES_RECEIVED, (NETDATA_DOUBLE)bytes);
1282
+ }
1283
}
1284
1366
- if(unlikely(s->read_len)) {
1367
- worker_is_busy(WORKER_SENDER_JOB_EXECUTE);
1285
+ if(unlikely(s->read_len))
1286
execute_commands(s);
1369
- }
1370
-
1371
- process_replication_requests(s);
1287
1288
if(unlikely(fds[Collector].revents & (POLLERR|POLLHUP|POLLNVAL))) {
1289
char *error = NULL;
web/api/queries/query.c
+105
-84
@@ -718,16 +718,22 @@ static size_t query_metric_first_working_tier(QUERY_METRIC *qm) {
718
return 0;
719
}
720
721
-static long query_plan_points_coverage_weight(time_t db_first_t, time_t db_last_t, time_t db_update_every, time_t after_wanted, time_t before_wanted, size_t points_wanted) {
721
+static long query_plan_points_coverage_weight(time_t db_first_t, time_t db_last_t, time_t db_update_every, time_t after_wanted, time_t before_wanted, size_t points_wanted, size_t tier __maybe_unused) {
722
if(db_first_t == 0 || db_last_t == 0 || db_update_every == 0)
723
return -LONG_MAX;
724
725
time_t common_first_t = MAX(db_first_t, after_wanted);
726
time_t common_last_t = MIN(db_last_t, before_wanted);
727
728
+ long time_coverage = (common_last_t - common_first_t) * 1000000 / (before_wanted - after_wanted);
729
+ size_t points_wanted_in_coverage = points_wanted * time_coverage / 1000000;
730
+
731
long points_available = (common_last_t - common_first_t) / db_update_every;
729
- long points_delta = (long)(points_available - points_wanted);
730
- long points_coverage = (points_delta < 0) ? (long)(points_available * 1000 / points_wanted): 1000;
732
+ long points_delta = (long)(points_available - points_wanted_in_coverage);
733
+ long points_coverage = (points_delta < 0) ? (long)(points_available * time_coverage / points_wanted_in_coverage) : time_coverage;
734
+
735
+ // a way to benefit higher tiers
736
+ // points_coverage += (long)tier * 10000;
737
738
if(points_available <= 0)
739
return -LONG_MAX;
@@ -757,7 +763,7 @@ static size_t query_metric_best_tier_for_timeframe(QUERY_METRIC *qm, time_t afte
763
continue;
764
}
765
760
- weight[tier] = query_plan_points_coverage_weight(first_t, last_t, update_every, after_wanted, before_wanted, points_wanted);
766
+ weight[tier] = query_plan_points_coverage_weight(first_t, last_t, update_every, after_wanted, before_wanted, points_wanted, tier);
767
}
768
769
size_t best_tier = 0;
@@ -813,7 +819,7 @@ static size_t rrddim_find_best_tier_for_timeframe(QUERY_TARGET *qt, time_t after
819
common_update_every = MIN(update_every, common_update_every);
820
}
821
816
- weight[tier] = query_plan_points_coverage_weight(common_first_t, common_last_t, common_update_every, after_wanted, before_wanted, points_wanted);
822
+ weight[tier] = query_plan_points_coverage_weight(common_first_t, common_last_t, common_update_every, after_wanted, before_wanted, points_wanted, tier);
823
}
824
825
size_t best_tier = 0;
@@ -1094,8 +1100,11 @@ static bool query_plan(QUERY_ENGINE_OPS *ops, time_t after_wanted, time_t before
1100
qsort(&ops->plan.data, ops->plan.entries, sizeof(QUERY_PLAN_ENTRY), compare_query_plan_entries_on_start_time);
1101
1102
// make sure it has the whole timeframe we need
1097
- ops->plan.data[0].after = after_wanted;
1098
- ops->plan.data[ops->plan.entries - 1].before = before_wanted;
1103
+ if(ops->plan.data[0].after < after_wanted)
1104
+ ops->plan.data[0].after = after_wanted;
1105
+
1106
+ if(ops->plan.data[ops->plan.entries - 1].before > before_wanted)
1107
+ ops->plan.data[ops->plan.entries - 1].before = before_wanted;
1108
1109
//buffer_sprintf(wb, ": FINAL STEPS %zu", ops->plan.entries);
1110
@@ -1191,15 +1200,18 @@ static inline void rrd2rrdr_do_dimension(RRDR *r, size_t dim_id_in_rrdr) {
1200
time_t now_start_time = after_wanted - ops.query_granularity;
1201
time_t now_end_time = after_wanted + ops.view_update_every - ops.query_granularity;
1202
1203
+ size_t db_points_read_since_plan_switch = 0; (void)db_points_read_since_plan_switch;
1204
+
1205
// The main loop, based on the query granularity we need
1206
for( ; points_added < points_wanted ; now_start_time = now_end_time, now_end_time += ops.view_update_every) {
1207
1197
- if(query_plan_should_switch_plan(ops, now_end_time))
1208
+ if(unlikely(query_plan_should_switch_plan(ops, now_end_time))) {
1209
query_planer_next_plan(&ops, now_end_time, new_point.end_time);
1210
+ db_points_read_since_plan_switch = 0;
1211
+ }
1212
1213
// read all the points of the db, prior to the time we need (now_end_time)
1214
1202
-
1215
size_t count_same_end_time = 0;
1216
while(count_same_end_time < 100) {
1217
if(likely(count_same_end_time == 0)) {
@@ -1223,6 +1235,7 @@ static inline void rrd2rrdr_do_dimension(RRDR *r, size_t dim_id_in_rrdr) {
1235
1236
// fetch the new point
1237
{
1238
+ db_points_read_since_plan_switch++;
1239
STORAGE_POINT sp = ops.next_metric(&ops.handle);
1240
1241
ops.db_points_read_per_tier[ops.tier]++;
@@ -1280,9 +1293,10 @@ static inline void rrd2rrdr_do_dimension(RRDR *r, size_t dim_id_in_rrdr) {
1293
1294
// check if the db is advancing the query
1295
if(unlikely(new_point.end_time <= last1_point.end_time)) {
1283
- internal_error(true, "QUERY: '%s', dimension '%s' next_metric() returned point %zu from %ld to %ld, before the last point %zu end time %ld, now is %ld to %ld",
1296
+ internal_error(db_points_read_since_plan_switch > 1,
1297
+ "QUERY: '%s', dimension '%s' next_metric() returned point %zu from %ld to %ld, before the last point %zu from %ld to %ld, now is %ld to %ld",
1298
qt->id, string2str(qm->dimension.id), new_point.id, new_point.start_time, new_point.end_time,
1285
- last1_point.id, last1_point.end_time, now_start_time, now_end_time);
1299
+ last1_point.id, last1_point.start_time, last1_point.end_time, now_start_time, now_end_time);
1300
1301
count_same_end_time++;
1302
continue;
@@ -1350,35 +1364,35 @@ static inline void rrd2rrdr_do_dimension(RRDR *r, size_t dim_id_in_rrdr) {
1364
current_point = new_point;
1365
query_interpolate_point(current_point, last1_point, now_end_time);
1366
1353
- internal_error(current_point.id > 0
1354
- && last1_point.id == 0
1355
- && current_point.end_time > after_wanted
1356
- && current_point.end_time > now_end_time,
1357
- "QUERY: '%s', dimension '%s', after %ld, before %ld, view update every %ld,"
1358
- " query granularity %ld, interpolating point %zu (from %ld to %ld) at %ld,"
1359
- " but we could really favor by having last_point1 in this query.",
1360
- qt->id, string2str(qm->dimension.id),
1361
- after_wanted, before_wanted,
1362
- ops.view_update_every, ops.query_granularity,
1363
- current_point.id, current_point.start_time, current_point.end_time,
1364
- now_end_time);
1367
+// internal_error(current_point.id > 0
1368
+// && last1_point.id == 0
1369
+// && current_point.end_time > after_wanted
1370
+// && current_point.end_time > now_end_time,
1371
+// "QUERY: '%s', dimension '%s', after %ld, before %ld, view update every %ld,"
1372
+// " query granularity %ld, interpolating point %zu (from %ld to %ld) at %ld,"
1373
+// " but we could really favor by having last_point1 in this query.",
1374
+// qt->id, string2str(qm->dimension.id),
1375
+// after_wanted, before_wanted,
1376
+// ops.view_update_every, ops.query_granularity,
1377
+// current_point.id, current_point.start_time, current_point.end_time,
1378
+// now_end_time);
1379
}
1380
else if(likely(now_end_time <= last1_point.end_time)) {
1381
// our LAST point is still valid
1382
current_point = last1_point;
1383
query_interpolate_point(current_point, last2_point, now_end_time);
1384
1371
- internal_error(current_point.id > 0
1372
- && last2_point.id == 0
1373
- && current_point.end_time > after_wanted
1374
- && current_point.end_time > now_end_time,
1375
- "QUERY: '%s', dimension '%s', after %ld, before %ld, view update every %ld,"
1376
- " query granularity %ld, interpolating point %zu (from %ld to %ld) at %ld,"
1377
- " but we could really favor by having last_point2 in this query.",
1378
- qt->id, string2str(qm->dimension.id),
1379
- after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,
1380
- current_point.id, current_point.start_time, current_point.end_time,
1381
- now_end_time);
1385
+// internal_error(current_point.id > 0
1386
+// && last2_point.id == 0
1387
+// && current_point.end_time > after_wanted
1388
+// && current_point.end_time > now_end_time,
1389
+// "QUERY: '%s', dimension '%s', after %ld, before %ld, view update every %ld,"
1390
+// " query granularity %ld, interpolating point %zu (from %ld to %ld) at %ld,"
1391
+// " but we could really favor by having last_point2 in this query.",
1392
+// qt->id, string2str(qm->dimension.id),
1393
+// after_wanted, before_wanted, ops.view_update_every, ops.query_granularity,
1394
+// current_point.id, current_point.start_time, current_point.end_time,
1395
+// now_end_time);
1396
}
1397
else {
1398
// a GAP, we don't have a value this time
@@ -1647,6 +1661,21 @@ bool rrdr_relative_window_to_absolute(time_t *after, time_t *before) {
1661
after_requested -= delta;
1662
}
1663
1664
+ time_t absolute_minimum_time = now - (10 * 365 * 86400);
1665
+ time_t absolute_maximum_time = now + (1 * 365 * 86400);
1666
+
1667
+ if (after_requested < absolute_minimum_time && !unittest_running)
1668
+ after_requested = absolute_minimum_time;
1669
+
1670
+ if (after_requested > absolute_maximum_time && !unittest_running)
1671
+ after_requested = absolute_maximum_time;
1672
+
1673
+ if (before_requested < absolute_minimum_time && !unittest_running)
1674
+ before_requested = absolute_minimum_time;
1675
+
1676
+ if (before_requested > absolute_maximum_time && !unittest_running)
1677
+ before_requested = absolute_maximum_time;
1678
+
1679
*before = before_requested;
1680
*after = after_requested;
1681
@@ -1836,6 +1865,11 @@ bool query_target_calculate_window(QUERY_TARGET *qt) {
1865
query_debug_log(":max points_wanted %zu", points_wanted);
1866
}
1867
1868
+ if(points_wanted > 86400 && !unittest_running) {
1869
+ points_wanted = 86400;
1870
+ query_debug_log(":absolute max points_wanted %zu", points_wanted);
1871
+ }
1872
+
1873
// calculate the desired grouping of source data points
1874
size_t group = points_available / points_wanted;
1875
if (group == 0) group = 1;
@@ -1971,37 +2005,23 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2005
return NULL;
2006
}
2007
1974
- time_t timeout = qt->request.timeout;
1975
- time_t resampling_time_requested = qt->request.resampling_time;
1976
- time_t after_requested = qt->request.after;
1977
- time_t before_requested = qt->request.before;
1978
- size_t points_requested = qt->request.points;
1979
-
1980
- RRDR_OPTIONS options = qt->window.options;
1981
- size_t points_wanted = qt->window.points;
1982
- time_t after_wanted = qt->window.after;
1983
- time_t before_wanted = qt->window.before;
1984
- bool relative_period_requested = qt->window.relative;
1985
- bool aligned = qt->window.aligned;
1986
- RRDR_GROUPING group_method = qt->window.group_method;
1987
- size_t group = qt->window.group;
1988
- size_t resampling_group = qt->window.resampling_group;
1989
- time_t query_granularity = qt->window.query_granularity;
2008
+ // qt.window members are the WANTED ones.
2009
+ // qt.request members are the REQUESTED ones.
2010
2011
RRDR *r = rrdr_create(owa, qt);
2012
if(unlikely(!r)) {
2013
internal_error(true, "QUERY: cannot create RRDR for %s, after=%ld, before=%ld, points=%zu",
1994
- qt->id, after_wanted, before_wanted, points_wanted);
2014
+ qt->id, qt->window.after, qt->window.before, qt->window.points);
2015
return NULL;
2016
}
2017
1998
- if(unlikely(!r->d || !points_wanted)) {
2018
+ if(unlikely(!r->d || !qt->window.points)) {
2019
internal_error(true, "QUERY: returning empty RRDR (no dimensions in RRDSET) for %s, after=%ld, before=%ld, points=%zu",
2000
- qt->id, after_wanted, before_wanted, points_wanted);
2020
+ qt->id, qt->window.after, qt->window.before, qt->window.points);
2021
return r;
2022
}
2023
2004
- if(relative_period_requested)
2024
+ if(qt->window.relative)
2025
r->result_options |= RRDR_RESULT_OPTION_RELATIVE;
2026
else
2027
r->result_options |= RRDR_RESULT_OPTION_ABSOLUTE;
@@ -2034,7 +2054,8 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2054
long dimensions_used = 0, dimensions_nonzero = 0;
2055
struct timeval query_start_time;
2056
struct timeval query_current_time;
2037
- if (timeout) now_realtime_timeval(&query_start_time);
2057
+ if (qt->request.timeout)
2058
+ now_realtime_timeval(&query_start_time);
2059
2060
for(size_t c = 0, max = qt->query.used; c < max ; c++) {
2061
// set the query target dimension options to rrdr
@@ -2046,7 +2067,7 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2067
r->internal.grouping_reset(r);
2068
2069
rrd2rrdr_do_dimension(r, c);
2049
- if (timeout)
2070
+ if (qt->request.timeout)
2071
now_realtime_timeval(&query_current_time);
2072
2073
if(r->od[c] & RRDR_DIMENSION_NONZERO)
@@ -2082,9 +2103,9 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2103
}
2104
2105
dimensions_used++;
2085
- if (timeout && ((NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0) > (NETDATA_DOUBLE)timeout) {
2106
+ if (qt->request.timeout && ((NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0) > (NETDATA_DOUBLE)qt->request.timeout) {
2107
log_access("QUERY CANCELED RUNTIME EXCEEDED %0.2f ms (LIMIT %lld ms)",
2087
- (NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0, (long long)timeout);
2108
+ (NETDATA_DOUBLE)dt_usec(&query_start_time, &query_current_time) / 1000.0, (long long)qt->request.timeout);
2109
r->result_options |= RRDR_RESULT_OPTION_CANCEL;
2110
break;
2111
}
@@ -2093,44 +2114,44 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2114
#ifdef NETDATA_INTERNAL_CHECKS
2115
if (dimensions_used) {
2116
if(r->internal.log)
2096
- rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
2097
- after_wanted, after_requested, before_wanted, before_requested,
2098
- points_requested, points_wanted, /*after_slot, before_slot,*/
2117
+ rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
2118
+ qt->window.after, qt->request.after, qt->window.before, qt->request.before,
2119
+ qt->request.points, qt->window.points, /*after_slot, before_slot,*/
2120
r->internal.log);
2121
2101
- if(r->rows != points_wanted)
2102
- rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
2103
- after_wanted, after_requested, before_wanted, before_requested,
2104
- points_requested, points_wanted, /*after_slot, before_slot,*/
2122
+ if(r->rows != qt->window.points)
2123
+ rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
2124
+ qt->window.after, qt->request.after, qt->window.before, qt->request.before,
2125
+ qt->request.points, qt->window.points, /*after_slot, before_slot,*/
2126
"got 'points' is not wanted 'points'");
2127
2107
- if(aligned && (r->before % (group * query_granularity)) != 0)
2108
- rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
2109
- after_wanted, after_requested, before_wanted,before_wanted,
2110
- points_requested, points_wanted, /*after_slot, before_slot,*/
2128
+ if(qt->window.aligned && (r->before % (qt->window.group * qt->window.query_granularity)) != 0)
2129
+ rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
2130
+ qt->window.after, qt->request.after, qt->window.before,qt->request.before,
2131
+ qt->request.points, qt->window.points, /*after_slot, before_slot,*/
2132
"'before' is not aligned but alignment is required");
2133
2134
// 'after' should not be aligned, since we start inside the first group
2114
- //if(aligned && (r->after % group) != 0)
2115
- // rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group, after_wanted, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required");
2135
+ //if(qt->window.aligned && (r->after % group) != 0)
2136
+ // rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group, qt->window.after, after_requested, before_wanted, before_requested, points_requested, points_wanted, after_slot, before_slot, "'after' is not aligned but alignment is required");
2137
2117
- if(r->before != before_wanted)
2118
- rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
2119
- after_wanted, after_requested, before_wanted, before_requested,
2120
- points_requested, points_wanted, /*after_slot, before_slot,*/
2138
+ if(r->before != qt->window.before)
2139
+ rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
2140
+ qt->window.after, qt->request.after, qt->window.before, qt->request.before,
2141
+ qt->request.points, qt->window.points, /*after_slot, before_slot,*/
2142
"chart is not aligned to requested 'before'");
2143
2123
- if(r->before != before_wanted)
2124
- rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
2125
- after_wanted, after_requested, before_wanted, before_requested,
2126
- points_requested, points_wanted, /*after_slot, before_slot,*/
2144
+ if(r->before != qt->window.before)
2145
+ rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
2146
+ qt->window.after, qt->request.after, qt->window.before, qt->request.before,
2147
+ qt->request.points, qt->window.points, /*after_slot, before_slot,*/
2148
"got 'before' is not wanted 'before'");
2149
2150
// reported 'after' varies, depending on group
2130
- if(r->after != after_wanted)
2131
- rrd2rrdr_log_request_response_metadata(r, options, group_method, aligned, group, resampling_time_requested, resampling_group,
2132
- after_wanted, after_requested, before_wanted, before_requested,
2133
- points_requested, points_wanted, /*after_slot, before_slot,*/
2151
+ if(r->after != qt->window.after)
2152
+ rrd2rrdr_log_request_response_metadata(r, qt->window.options, qt->window.group_method, qt->window.aligned, qt->window.group, qt->request.resampling_time, qt->window.resampling_group,
2153
+ qt->window.after, qt->request.after, qt->window.before, qt->request.before,
2154
+ qt->request.points, qt->window.points, /*after_slot, before_slot,*/
2155
"got 'after' is not wanted 'after'");
2156
2157
}
@@ -2140,7 +2161,7 @@ RRDR *rrd2rrdr(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2161
r->internal.grouping_free(r);
2162
2163
// when all the dimensions are zero, we should return all of them
2143
- if(unlikely(options & RRDR_OPTION_NONZERO && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) {
2164
+ if(unlikely((qt->window.options & RRDR_OPTION_NONZERO) && !dimensions_nonzero && !(r->result_options & RRDR_RESULT_OPTION_CANCEL))) {
2165
// all the dimensions are zero
2166
// mark them as NONZERO to send them all
2167
for(size_t c = 0, max = qt->query.used; c < max ; c++) {
web/api/web_api_v1.c
+1
-1
@@ -1226,7 +1226,7 @@ inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb)
1226
#ifdef ENABLE_COMPRESSION
1227
if(host->sender){
1228
buffer_strcat(wb, "\t\"stream-compression\": ");
1229
- buffer_strcat(wb, (host->sender->flags & SENDER_FLAG_COMPRESSION) ? "true" : "false");
1229
+ buffer_strcat(wb, stream_has_capability(host->sender, STREAM_CAP_COMPRESSION) ? "true" : "false");
1230
buffer_strcat(wb, ",\n");
1231
}else{
1232
buffer_strcat(wb, "\t\"stream-compression\": null,\n");
web/server/web_client.c
+3
-3
@@ -1056,7 +1056,7 @@ static inline ssize_t web_client_send_data(struct web_client *w,const void *buf,
1056
#ifdef ENABLE_HTTPS
1057
if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) {
1058
if ( ( w->ssl.conn ) && ( !w->ssl.flags ) ){
1059
- bytes = SSL_write(w->ssl.conn,buf, len) ;
1059
+ bytes = netdata_ssl_write(w->ssl.conn, buf, len) ;
1060
} else {
1061
bytes = send(w->ofd,buf, len , flags);
1062
}
@@ -1213,7 +1213,7 @@ static inline void web_client_send_http_header(struct web_client *w) {
1213
#ifdef ENABLE_HTTPS
1214
if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) {
1215
if ( ( w->ssl.conn ) && ( !w->ssl.flags ) ){
1216
- while((bytes = SSL_write(w->ssl.conn, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output))) < 0) {
1216
+ while((bytes = netdata_ssl_write(w->ssl.conn, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output))) < 0) {
1217
count++;
1218
if(count > 100 || (errno != EAGAIN && errno != EWOULDBLOCK)) {
1219
error("Cannot send HTTPS headers to web client.");
@@ -1909,7 +1909,7 @@ ssize_t web_client_receive(struct web_client *w)
1909
#ifdef ENABLE_HTTPS
1910
if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) {
1911
if ( ( w->ssl.conn ) && (!w->ssl.flags)) {
1912
- bytes = SSL_read(w->ssl.conn, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1));
1912
+ bytes = netdata_ssl_read(w->ssl.conn, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1));
1913
}else {
1914
bytes = recv(w->ifd, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1), MSG_DONTWAIT);
1915
}