Remove Chart/Dim based communication (#13650)
Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>
Timotej S committed
Sep 27, 2022 at 18:31 UTC
f89f8845252370c70403c1f3badb26518aa63182
30 files changed
+51
-2688
CMakeLists.txt
-12
@@ -752,8 +752,6 @@ set(RRD_PLUGIN_FILES
752
database/sqlite/sqlite_health.h
753
database/sqlite/sqlite_aclk_node.c
754
database/sqlite/sqlite_aclk_node.h
755
- database/sqlite/sqlite_aclk_chart.c
756
- database/sqlite/sqlite_aclk_chart.h
755
database/sqlite/sqlite_aclk_alert.c
756
database/sqlite/sqlite_aclk_alert.h
757
database/sqlite/sqlite3.c
@@ -894,8 +892,6 @@ set(ACLK_FILES
892
aclk/aclk_rx_msgs.h
893
aclk/https_client.c
894
aclk/https_client.h
897
- aclk/aclk_charts_api.c
898
- aclk/aclk_charts_api.h
895
aclk/aclk_alarm_api.c
896
aclk/aclk_alarm_api.h
897
aclk/aclk_contexts_api.c
@@ -922,10 +918,6 @@ set(ACLK_FILES
918
aclk/schema-wrappers/node_connection.h
919
aclk/schema-wrappers/node_creation.cc
920
aclk/schema-wrappers/node_creation.h
925
- aclk/schema-wrappers/chart_stream.cc
926
- aclk/schema-wrappers/chart_stream.h
927
- aclk/schema-wrappers/chart_config.cc
928
- aclk/schema-wrappers/chart_config.h
921
aclk/schema-wrappers/alarm_stream.cc
922
aclk/schema-wrappers/alarm_stream.h
923
aclk/schema-wrappers/alarm_config.cc
@@ -1247,10 +1239,6 @@ set(ACLK_PROTO_DEFS
1239
aclk/aclk-schemas/proto/agent/v1/connection.proto
1240
aclk/aclk-schemas/proto/alarm/v1/config.proto
1241
aclk/aclk-schemas/proto/alarm/v1/stream.proto
1250
- aclk/aclk-schemas/proto/chart/v1/config.proto
1251
- aclk/aclk-schemas/proto/chart/v1/dimension.proto
1252
- aclk/aclk-schemas/proto/chart/v1/instance.proto
1253
- aclk/aclk-schemas/proto/chart/v1/stream.proto
1242
aclk/aclk-schemas/proto/nodeinstance/connection/v1/connection.proto
1243
aclk/aclk-schemas/proto/nodeinstance/create/v1/creation.proto
1244
aclk/aclk-schemas/proto/nodeinstance/info/v1/info.proto
Makefile.am
-36
@@ -467,8 +467,6 @@ RRD_PLUGIN_FILES = \
467
database/sqlite/sqlite_health.h \
468
database/sqlite/sqlite_aclk_node.c \
469
database/sqlite/sqlite_aclk_node.h \
470
- database/sqlite/sqlite_aclk_chart.c \
471
- database/sqlite/sqlite_aclk_chart.h \
470
database/sqlite/sqlite_aclk_alert.c \
471
database/sqlite/sqlite_aclk_alert.h \
472
database/sqlite/sqlite3.c \
@@ -710,8 +708,6 @@ ACLK_FILES = \
708
mqtt_websockets/c-rbuf/src/ringbuffer_internal.h \
709
mqtt_websockets/MQTT-C/src/mqtt.c \
710
mqtt_websockets/MQTT-C/include/mqtt.h \
713
- aclk/aclk_charts_api.c \
714
- aclk/aclk_charts_api.h \
711
aclk/aclk_alarm_api.c \
712
aclk/aclk_alarm_api.h \
713
aclk/aclk_contexts_api.c \
@@ -722,10 +718,6 @@ ACLK_FILES = \
718
aclk/schema-wrappers/node_connection.h \
719
aclk/schema-wrappers/node_creation.cc \
720
aclk/schema-wrappers/node_creation.h \
725
- aclk/schema-wrappers/chart_stream.cc \
726
- aclk/schema-wrappers/chart_stream.h \
727
- aclk/schema-wrappers/chart_config.cc \
728
- aclk/schema-wrappers/chart_config.h \
721
aclk/schema-wrappers/alarm_stream.cc \
722
aclk/schema-wrappers/alarm_stream.h \
723
aclk/schema-wrappers/alarm_config.cc \
@@ -753,10 +745,6 @@ ACLK_PROTO_DEFINITIONS = \
745
aclk/aclk-schemas/proto/agent/v1/connection.proto \
746
aclk/aclk-schemas/proto/alarm/v1/config.proto \
747
aclk/aclk-schemas/proto/alarm/v1/stream.proto \
756
- aclk/aclk-schemas/proto/chart/v1/config.proto \
757
- aclk/aclk-schemas/proto/chart/v1/dimension.proto \
758
- aclk/aclk-schemas/proto/chart/v1/instance.proto \
759
- aclk/aclk-schemas/proto/chart/v1/stream.proto \
748
aclk/aclk-schemas/proto/nodeinstance/connection/v1/connection.proto \
749
aclk/aclk-schemas/proto/nodeinstance/create/v1/creation.proto \
750
aclk/aclk-schemas/proto/nodeinstance/info/v1/info.proto \
@@ -772,14 +760,6 @@ ACLK_PROTO_BUILT_FILES = aclk/aclk-schemas/proto/agent/v1/connection.pb.cc \
760
aclk/aclk-schemas/proto/nodeinstance/connection/v1/connection.pb.h \
761
aclk/aclk-schemas/proto/nodeinstance/create/v1/creation.pb.cc \
762
aclk/aclk-schemas/proto/nodeinstance/create/v1/creation.pb.h \
775
- aclk/aclk-schemas/proto/chart/v1/stream.pb.cc \
776
- aclk/aclk-schemas/proto/chart/v1/stream.pb.h \
777
- aclk/aclk-schemas/proto/chart/v1/instance.pb.cc \
778
- aclk/aclk-schemas/proto/chart/v1/instance.pb.h \
779
- aclk/aclk-schemas/proto/chart/v1/dimension.pb.cc \
780
- aclk/aclk-schemas/proto/chart/v1/dimension.pb.h \
781
- aclk/aclk-schemas/proto/chart/v1/config.pb.cc \
782
- aclk/aclk-schemas/proto/chart/v1/config.pb.h \
763
aclk/aclk-schemas/proto/aclk/v1/lib.pb.cc \
764
aclk/aclk-schemas/proto/aclk/v1/lib.pb.h \
765
aclk/aclk-schemas/proto/agent/v1/disconnect.pb.cc \
@@ -812,22 +792,6 @@ aclk/aclk-schemas/proto/nodeinstance/create/v1/creation.pb.cc \
792
aclk/aclk-schemas/proto/nodeinstance/create/v1/creation.pb.h: aclk/aclk-schemas/proto/nodeinstance/create/v1/creation.proto
793
$(PROTOC) -I=aclk/aclk-schemas --cpp_out=$(builddir)/aclk/aclk-schemas $^
794
815
-aclk/aclk-schemas/proto/chart/v1/stream.pb.cc \
816
-aclk/aclk-schemas/proto/chart/v1/stream.pb.h: aclk/aclk-schemas/proto/chart/v1/stream.proto
817
- $(PROTOC) -I=aclk/aclk-schemas --cpp_out=$(builddir)/aclk/aclk-schemas $^
818
-
819
-aclk/aclk-schemas/proto/chart/v1/instance.pb.cc \
820
-aclk/aclk-schemas/proto/chart/v1/instance.pb.h: aclk/aclk-schemas/proto/chart/v1/instance.proto
821
- $(PROTOC) -I=aclk/aclk-schemas --cpp_out=$(builddir)/aclk/aclk-schemas $^
822
-
823
-aclk/aclk-schemas/proto/chart/v1/dimension.pb.cc \
824
-aclk/aclk-schemas/proto/chart/v1/dimension.pb.h: aclk/aclk-schemas/proto/chart/v1/dimension.proto
825
- $(PROTOC) -I=aclk/aclk-schemas --cpp_out=$(builddir)/aclk/aclk-schemas $^
826
-
827
-aclk/aclk-schemas/proto/chart/v1/config.pb.cc \
828
-aclk/aclk-schemas/proto/chart/v1/config.pb.h: aclk/aclk-schemas/proto/chart/v1/config.proto
829
- $(PROTOC) -I=aclk/aclk-schemas --cpp_out=$(builddir)/aclk/aclk-schemas $^
830
-
795
aclk/aclk-schemas/proto/aclk/v1/lib.pb.cc \
796
aclk/aclk-schemas/proto/aclk/v1/lib.pb.h: aclk/aclk-schemas/proto/aclk/v1/lib.proto
797
$(PROTOC) -I=aclk/aclk-schemas --cpp_out=$(builddir)/aclk/aclk-schemas $^
aclk/aclk.c
+2
-80
@@ -788,7 +788,7 @@ void aclk_host_state_update(RRDHOST *host, int cmd)
788
{ .name = "proto", .version = 1, .enabled = 1 },
789
{ .name = "ml", .version = ml_capable(localhost), .enabled = ml_enabled(host) },
790
{ .name = "mc", .version = enable_metric_correlations ? metric_correlations_version : 0, .enabled = enable_metric_correlations },
791
- { .name = "ctx", .version = 1, .enabled = rrdcontext_enabled },
791
+ { .name = "ctx", .version = 1, .enabled = 1 },
792
{ .name = NULL, .version = 0, .enabled = 0 }
793
};
794
node_state_update.capabilities = caps;
@@ -834,7 +834,7 @@ void aclk_send_node_instances()
834
{ .name = "proto", .version = 1, .enabled = 1 },
835
{ .name = "ml", .version = ml_capable(localhost), .enabled = host ? ml_enabled(host) : 0 },
836
{ .name = "mc", .version = enable_metric_correlations ? metric_correlations_version : 0, .enabled = enable_metric_correlations },
837
- { .name = "ctx", .version = 1, .enabled = rrdcontext_enabled },
837
+ { .name = "ctx", .version = 1, .enabled = 1 },
838
{ .name = NULL, .version = 0, .enabled = 0 }
839
};
840
node_state_update.capabilities = caps;
@@ -905,38 +905,6 @@ static void fill_alert_status_for_host(BUFFER *wb, RRDHOST *host)
905
status.last_submitted_sequence_id
906
);
907
}
908
-
909
-static void fill_chart_status_for_host(BUFFER *wb, RRDHOST *host)
910
-{
911
- struct aclk_chart_sync_stats *stats = aclk_get_chart_sync_stats(host);
912
- if (!stats) {
913
- buffer_strcat(wb, "\n\t\tFailed to get alert streaming status for this host");
914
- return;
915
- }
916
- buffer_sprintf(wb,
917
- "\n\t\tUpdates: %d"
918
- "\n\t\tBatch ID: %"PRIu64
919
- "\n\t\tMin Seq ID: %"PRIu64
920
- "\n\t\tMax Seq ID: %"PRIu64
921
- "\n\t\tPending Min Seq ID: %"PRIu64
922
- "\n\t\tPending Max Seq ID: %"PRIu64
923
- "\n\t\tSent Min Seq ID: %"PRIu64
924
- "\n\t\tSent Max Seq ID: %"PRIu64
925
- "\n\t\tAcked Min Seq ID: %"PRIu64
926
- "\n\t\tAcked Max Seq ID: %"PRIu64,
927
- stats->updates,
928
- stats->batch_id,
929
- stats->min_seqid,
930
- stats->max_seqid,
931
- stats->min_seqid_pend,
932
- stats->max_seqid_pend,
933
- stats->min_seqid_sent,
934
- stats->max_seqid_sent,
935
- stats->min_seqid_ack,
936
- stats->max_seqid_ack
937
- );
938
- freez(stats);
939
-}
908
#endif /* ENABLE_ACLK */
909
910
char *aclk_state(void)
@@ -1018,9 +986,6 @@ char *aclk_state(void)
986
987
buffer_strcat(wb, "\n\tAlert Streaming Status:");
988
fill_alert_status_for_host(wb, host);
1021
-
1022
- buffer_strcat(wb, "\n\tChart Streaming Status:");
1023
- fill_chart_status_for_host(wb, host);
989
}
990
rrd_unlock();
991
}
@@ -1058,45 +1023,6 @@ static void fill_alert_status_for_host_json(json_object *obj, RRDHOST *host)
1023
json_object_object_add(obj, "last-submitted-seq-id", tmp);
1024
}
1025
1061
-static void fill_chart_status_for_host_json(json_object *obj, RRDHOST *host)
1062
-{
1063
- struct aclk_chart_sync_stats *stats = aclk_get_chart_sync_stats(host);
1064
- if (!stats)
1065
- return;
1066
-
1067
- json_object *tmp = json_object_new_int(stats->updates);
1068
- json_object_object_add(obj, "updates", tmp);
1069
-
1070
- tmp = json_object_new_int(stats->batch_id);
1071
- json_object_object_add(obj, "batch-id", tmp);
1072
-
1073
- tmp = json_object_new_int(stats->min_seqid);
1074
- json_object_object_add(obj, "min-seq-id", tmp);
1075
-
1076
- tmp = json_object_new_int(stats->max_seqid);
1077
- json_object_object_add(obj, "max-seq-id", tmp);
1078
-
1079
- tmp = json_object_new_int(stats->min_seqid_pend);
1080
- json_object_object_add(obj, "pending-min-seq-id", tmp);
1081
-
1082
- tmp = json_object_new_int(stats->max_seqid_pend);
1083
- json_object_object_add(obj, "pending-max-seq-id", tmp);
1084
-
1085
- tmp = json_object_new_int(stats->min_seqid_sent);
1086
- json_object_object_add(obj, "sent-min-seq-id", tmp);
1087
-
1088
- tmp = json_object_new_int(stats->max_seqid_sent);
1089
- json_object_object_add(obj, "sent-max-seq-id", tmp);
1090
-
1091
- tmp = json_object_new_int(stats->min_seqid_ack);
1092
- json_object_object_add(obj, "acked-min-seq-id", tmp);
1093
-
1094
- tmp = json_object_new_int(stats->max_seqid_ack);
1095
- json_object_object_add(obj, "acked-max-seq-id", tmp);
1096
-
1097
- freez(stats);
1098
-}
1099
-
1026
static json_object *timestamp_to_json(const time_t *t)
1027
{
1028
struct tm *tmptr, tmbuf;
@@ -1215,10 +1141,6 @@ char *aclk_state_json(void)
1141
fill_alert_status_for_host_json(tmp, host);
1142
json_object_object_add(nodeinstance, "alert-sync-status", tmp);
1143
1218
- tmp = json_object_new_object();
1219
- fill_chart_status_for_host_json(tmp, host);
1220
- json_object_object_add(nodeinstance, "chart-sync-status", tmp);
1221
-
1144
json_object_array_add(grp, nodeinstance);
1145
}
1146
rrd_unlock();
aclk/aclk_charts_api.c
deleted
-77
@@ -1,77 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-#include "aclk_charts_api.h"
3
-
4
-#include "aclk_query_queue.h"
5
-
6
-#define CHART_DIM_UPDATE_NAME "ChartsAndDimensionsUpdated"
7
-
8
-void aclk_chart_inst_update(char **payloads, size_t *payload_sizes, struct aclk_message_position *new_positions)
9
-{
10
- aclk_query_t query = aclk_query_new(CHART_DIMS_UPDATE);
11
- query->data.bin_payload.payload = generate_charts_updated(&query->data.bin_payload.size, payloads, payload_sizes, new_positions);
12
- query->data.bin_payload.msg_name = CHART_DIM_UPDATE_NAME;
13
- QUEUE_IF_PAYLOAD_PRESENT(query);
14
-}
15
-
16
-void aclk_chart_dim_update(char **payloads, size_t *payload_sizes, struct aclk_message_position *new_positions)
17
-{
18
- aclk_query_t query = aclk_query_new(CHART_DIMS_UPDATE);
19
- query->data.bin_payload.topic = ACLK_TOPICID_CHART_DIMS;
20
- query->data.bin_payload.payload = generate_chart_dimensions_updated(&query->data.bin_payload.size, payloads, payload_sizes, new_positions);
21
- query->data.bin_payload.msg_name = CHART_DIM_UPDATE_NAME;
22
- QUEUE_IF_PAYLOAD_PRESENT(query);
23
-}
24
-
25
-void aclk_chart_inst_and_dim_update(char **payloads, size_t *payload_sizes, int *is_dim, struct aclk_message_position *new_positions, uint64_t batch_id)
26
-{
27
- aclk_query_t query = aclk_query_new(CHART_DIMS_UPDATE);
28
- query->data.bin_payload.topic = ACLK_TOPICID_CHART_DIMS;
29
- query->data.bin_payload.payload = generate_charts_and_dimensions_updated(&query->data.bin_payload.size, payloads, payload_sizes, is_dim, new_positions, batch_id);
30
- query->data.bin_payload.msg_name = CHART_DIM_UPDATE_NAME;
31
- QUEUE_IF_PAYLOAD_PRESENT(query);
32
-}
33
-
34
-void aclk_chart_config_updated(struct chart_config_updated *config_list, int list_size)
35
-{
36
- aclk_query_t query = aclk_query_new(CHART_CONFIG_UPDATED);
37
- query->data.bin_payload.topic = ACLK_TOPICID_CHART_CONFIGS_UPDATED;
38
- query->data.bin_payload.payload = generate_chart_configs_updated(&query->data.bin_payload.size, config_list, list_size);
39
- query->data.bin_payload.msg_name = "ChartConfigsUpdated";
40
- QUEUE_IF_PAYLOAD_PRESENT(query);
41
-}
42
-
43
-void aclk_chart_reset(chart_reset_t reset)
44
-{
45
- aclk_query_t query = aclk_query_new(CHART_RESET);
46
- query->data.bin_payload.topic = ACLK_TOPICID_CHART_RESET;
47
- query->data.bin_payload.payload = generate_reset_chart_messages(&query->data.bin_payload.size, reset);
48
- query->data.bin_payload.msg_name = "ResetChartMessages";
49
- QUEUE_IF_PAYLOAD_PRESENT(query);
50
-}
51
-
52
-void aclk_retention_updated(struct retention_updated *data)
53
-{
54
- aclk_query_t query = aclk_query_new(RETENTION_UPDATED);
55
- query->data.bin_payload.topic = ACLK_TOPICID_RETENTION_UPDATED;
56
- query->data.bin_payload.payload = generate_retention_updated(&query->data.bin_payload.size, data);
57
- query->data.bin_payload.msg_name = "RetentionUpdated";
58
- QUEUE_IF_PAYLOAD_PRESENT(query);
59
-}
60
-
61
-void aclk_update_node_info(struct update_node_info *info)
62
-{
63
- aclk_query_t query = aclk_query_new(UPDATE_NODE_INFO);
64
- query->data.bin_payload.topic = ACLK_TOPICID_NODE_INFO;
65
- query->data.bin_payload.payload = generate_update_node_info_message(&query->data.bin_payload.size, info);
66
- query->data.bin_payload.msg_name = "UpdateNodeInfo";
67
- QUEUE_IF_PAYLOAD_PRESENT(query);
68
-}
69
-
70
-void aclk_update_node_collectors(struct update_node_collectors *collectors)
71
-{
72
- aclk_query_t query = aclk_query_new(UPDATE_NODE_COLLECTORS);
73
- query->data.bin_payload.topic = ACLK_TOPICID_NODE_COLLECTORS;
74
- query->data.bin_payload.payload = generate_update_node_collectors_message(&query->data.bin_payload.size, collectors);
75
- query->data.bin_payload.msg_name = "UpdateNodeCollectors";
76
- QUEUE_IF_PAYLOAD_PRESENT(query);
77
-}
aclk/aclk_charts_api.h
deleted
-22
@@ -1,22 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-#ifndef ACLK_CHARTS_H
3
-#define ACLK_CHARTS_H
4
-
5
-#include "../daemon/common.h"
6
-#include "schema-wrappers/schema_wrappers.h"
7
-
8
-void aclk_chart_inst_update(char **payloads, size_t *payload_sizes, struct aclk_message_position *new_positions);
9
-void aclk_chart_dim_update(char **payloads, size_t *payload_sizes, struct aclk_message_position *new_positions);
10
-void aclk_chart_inst_and_dim_update(char **payloads, size_t *payload_sizes, int *is_dim, struct aclk_message_position *new_positions, uint64_t batch_id);
11
-
12
-void aclk_chart_config_updated(struct chart_config_updated *config_list, int list_size);
13
-
14
-void aclk_chart_reset(chart_reset_t reset);
15
-
16
-void aclk_retention_updated(struct retention_updated *data);
17
-
18
-void aclk_update_node_info(struct update_node_info *info);
19
-
20
-void aclk_update_node_collectors(struct update_node_collectors *collectors);
21
-
22
-#endif /* ACLK_CHARTS_H */
aclk/aclk_contexts_api.c
+18
@@ -21,3 +21,21 @@ void aclk_send_contexts_updated(contexts_updated_t data)
21
query->data.bin_payload.msg_name = "ContextsUpdated";
22
QUEUE_IF_PAYLOAD_PRESENT(query);
23
}
24
+
25
+void aclk_update_node_collectors(struct update_node_collectors *collectors)
26
+{
27
+ aclk_query_t query = aclk_query_new(UPDATE_NODE_COLLECTORS);
28
+ query->data.bin_payload.topic = ACLK_TOPICID_NODE_COLLECTORS;
29
+ query->data.bin_payload.payload = generate_update_node_collectors_message(&query->data.bin_payload.size, collectors);
30
+ query->data.bin_payload.msg_name = "UpdateNodeCollectors";
31
+ QUEUE_IF_PAYLOAD_PRESENT(query);
32
+}
33
+
34
+void aclk_update_node_info(struct update_node_info *info)
35
+{
36
+ aclk_query_t query = aclk_query_new(UPDATE_NODE_INFO);
37
+ query->data.bin_payload.topic = ACLK_TOPICID_NODE_INFO;
38
+ query->data.bin_payload.payload = generate_update_node_info_message(&query->data.bin_payload.size, info);
39
+ query->data.bin_payload.msg_name = "UpdateNodeInfo";
40
+ QUEUE_IF_PAYLOAD_PRESENT(query);
41
+}
aclk/aclk_contexts_api.h
+2
@@ -7,6 +7,8 @@
7
8
void aclk_send_contexts_snapshot(contexts_snapshot_t data);
9
void aclk_send_contexts_updated(contexts_updated_t data);
10
+void aclk_update_node_collectors(struct update_node_collectors *collectors);
11
+void aclk_update_node_info(struct update_node_info *info);
12
13
#endif /* ACLK_CONTEXTS_API_H */
14
aclk/aclk_otp.c
+1
-4
@@ -844,10 +844,7 @@ int aclk_get_env(aclk_env_t *env, const char* aclk_hostname, int aclk_port) {
844
return 1;
845
}
846
847
- if (rrdcontext_enabled)
848
- buffer_sprintf(buf, "/api/v1/env?v=%s&cap=proto,ctx&claim_id=%s", &(VERSION[1]) /* skip 'v' at beginning */, agent_id);
849
- else
850
- buffer_sprintf(buf, "/api/v1/env?v=%s&cap=proto&claim_id=%s", &(VERSION[1]) /* skip 'v' at beginning */, agent_id);
847
+ buffer_sprintf(buf, "/api/v1/env?v=%s&cap=proto,ctx&claim_id=%s", &(VERSION[1]) /* skip 'v' at beginning */, agent_id);
848
849
freez(agent_id);
850
aclk/aclk_rx_msgs.c
+10
-29
@@ -293,7 +293,7 @@ int create_node_instance_result(const char *msg, size_t msg_len)
293
{ .name = "proto", .version = 1, .enabled = 1 },
294
{ .name = "ml", .version = ml_capable(localhost), .enabled = host ? ml_enabled(host) : 0 },
295
{ .name = "mc", .version = enable_metric_correlations ? metric_correlations_version : 0, .enabled = enable_metric_correlations },
296
- { .name = "ctx", .version = 1, .enabled = rrdcontext_enabled },
296
+ { .name = "ctx", .version = 1, .enabled = 1 },
297
{ .name = NULL, .version = 0, .enabled = 0 }
298
};
299
node_state_update.capabilities = caps;
@@ -322,44 +322,25 @@ int send_node_instances(const char *msg, size_t msg_len)
322
323
int stream_charts_and_dimensions(const char *msg, size_t msg_len)
324
{
325
- aclk_ctx_based = 0;
326
- stream_charts_and_dims_t res = parse_stream_charts_and_dims(msg, msg_len);
327
- if (!res.claim_id || !res.node_id) {
328
- error("Error parsing StreamChartsAndDimensions msg");
329
- freez(res.claim_id);
330
- freez(res.node_id);
331
- return 1;
332
- }
333
- chart_batch_id = res.batch_id;
334
- aclk_start_streaming(res.node_id, res.seq_id, res.seq_id_created_at.tv_sec, res.batch_id);
335
- freez(res.claim_id);
336
- freez(res.node_id);
325
+ UNUSED(msg);
326
+ UNUSED(msg_len);
327
+ error_report("Received obsolete StreamChartsAndDimensions msg");
328
return 0;
329
}
330
331
int charts_and_dimensions_ack(const char *msg, size_t msg_len)
332
{
342
- chart_and_dim_ack_t res = parse_chart_and_dimensions_ack(msg, msg_len);
343
- if (!res.claim_id || !res.node_id) {
344
- error("Error parsing StreamChartsAndDimensions msg");
345
- freez(res.claim_id);
346
- freez(res.node_id);
347
- return 1;
348
- }
349
- aclk_ack_chart_sequence_id(res.node_id, res.last_seq_id);
350
- freez(res.claim_id);
351
- freez(res.node_id);
333
+ UNUSED(msg);
334
+ UNUSED(msg_len);
335
+ error_report("Received obsolete StreamChartsAndDimensionsAck msg");
336
return 0;
337
}
338
339
int update_chart_configs(const char *msg, size_t msg_len)
340
{
357
- struct update_chart_config res = parse_update_chart_config(msg, msg_len);
358
- if (!res.claim_id || !res.node_id || !res.hashes)
359
- error("Error parsing UpdateChartConfigs msg");
360
- else
361
- aclk_get_chart_config(res.hashes);
362
- destroy_update_chart_config(&res);
341
+ UNUSED(msg);
342
+ UNUSED(msg_len);
343
+ error_report("Received obsolete UpdateChartConfigs msg");
344
return 0;
345
}
346
aclk/aclk_tx_msgs.c
+1
-1
@@ -225,7 +225,7 @@ uint16_t aclk_send_agent_connection_update(mqtt_wss_client client, int reachable
225
{ .name = "ml", .version = 1, .enabled = ml_enabled(localhost) },
226
#endif
227
{ .name = "mc", .version = enable_metric_correlations ? metric_correlations_version : 0, .enabled = enable_metric_correlations },
228
- { .name = "ctx", .version = 1, .enabled = rrdcontext_enabled },
228
+ { .name = "ctx", .version = 1, .enabled = 1 },
229
{ .name = NULL, .version = 0, .enabled = 0 }
230
};
231
aclk/schema-wrappers/chart_config.cc
deleted
-105
@@ -1,105 +0,0 @@
1
-#include "chart_config.h"
2
-
3
-#include "proto/chart/v1/config.pb.h"
4
-
5
-#include "libnetdata/libnetdata.h"
6
-
7
-#include "schema_wrapper_utils.h"
8
-
9
-void destroy_update_chart_config(struct update_chart_config *cfg)
10
-{
11
- freez(cfg->claim_id);
12
- freez(cfg->node_id);
13
- freez(cfg->hashes);
14
-}
15
-
16
-void destroy_chart_config_updated(struct chart_config_updated *cfg)
17
-{
18
- freez(cfg->type);
19
- freez(cfg->family);
20
- freez(cfg->context);
21
- freez(cfg->title);
22
- freez(cfg->plugin);
23
- freez(cfg->module);
24
- freez(cfg->units);
25
- freez(cfg->config_hash);
26
-}
27
-
28
-struct update_chart_config parse_update_chart_config(const char *data, size_t len)
29
-{
30
- chart::v1::UpdateChartConfigs cfgs;
31
- update_chart_config res;
32
- memset(&res, 0, sizeof(res));
33
-
34
- if (!cfgs.ParseFromArray(data, len))
35
- return res;
36
-
37
- res.claim_id = strdupz(cfgs.claim_id().c_str());
38
- res.node_id = strdupz(cfgs.node_id().c_str());
39
-
40
- // to not do bazillion tiny allocations for individual strings
41
- // we calculate how much memory we will need for all of them
42
- // and allocate at once
43
- int hash_count = cfgs.config_hashes_size();
44
- size_t total_strlen = 0;
45
- for (int i = 0; i < hash_count; i++)
46
- total_strlen += cfgs.config_hashes(i).length();
47
- total_strlen += hash_count; //null bytes
48
-
49
- res.hashes = (char**)callocz( 1,
50
- (hash_count+1) * sizeof(char*) + //char * array incl. terminating NULL at the end
51
- total_strlen //strings themselves incl. 1 null byte each
52
- );
53
-
54
- char* dest = ((char*)res.hashes) + (hash_count + 1 /* NULL ptr */) * sizeof(char *);
55
- // now copy them strings
56
- // null bytes handled by callocz
57
- for (int i = 0; i < hash_count; i++) {
58
- strcpy(dest, cfgs.config_hashes(i).c_str());
59
- res.hashes[i] = dest;
60
- dest += strlen(dest) + 1 /* end string null */;
61
- }
62
-
63
- return res;
64
-}
65
-
66
-char *generate_chart_configs_updated(size_t *len, const struct chart_config_updated *config_list, int list_size)
67
-{
68
- chart::v1::ChartConfigsUpdated configs;
69
- for (int i = 0; i < list_size; i++) {
70
- chart::v1::ChartConfigUpdated *config = configs.add_configs();
71
- config->set_type(config_list[i].type);
72
- if (config_list[i].family)
73
- config->set_family(config_list[i].family);
74
- config->set_context(config_list[i].context);
75
- config->set_title(config_list[i].title);
76
- config->set_priority(config_list[i].priority);
77
- config->set_plugin(config_list[i].plugin);
78
-
79
- if (config_list[i].module)
80
- config->set_module(config_list[i].module);
81
-
82
- switch (config_list[i].chart_type) {
83
- case RRDSET_TYPE_LINE:
84
- config->set_chart_type(chart::v1::LINE);
85
- break;
86
- case RRDSET_TYPE_AREA:
87
- config->set_chart_type(chart::v1::AREA);
88
- break;
89
- case RRDSET_TYPE_STACKED:
90
- config->set_chart_type(chart::v1::STACKED);
91
- break;
92
- default:
93
- return NULL;
94
- }
95
-
96
- config->set_units(config_list[i].units);
97
- config->set_config_hash(config_list[i].config_hash);
98
- }
99
-
100
- *len = PROTO_COMPAT_MSG_SIZE(configs);
101
- char *bin = (char*)mallocz(*len);
102
- configs.SerializeToArray(bin, *len);
103
-
104
- return bin;
105
-}
aclk/schema-wrappers/chart_config.h
deleted
-50
@@ -1,50 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-#ifndef ACLK_SCHEMA_WRAPPER_CHART_CONFIG_H
4
-#define ACLK_SCHEMA_WRAPPER_CHART_CONFIG_H
5
-
6
-#include <stdlib.h>
7
-
8
-#include "database/rrd.h"
9
-
10
-#ifdef __cplusplus
11
-extern "C" {
12
-#endif
13
-
14
-struct update_chart_config {
15
- char *claim_id;
16
- char *node_id;
17
- char **hashes;
18
-};
19
-
20
-enum chart_config_chart_type {
21
- LINE,
22
- AREA,
23
- STACKED
24
-};
25
-
26
-struct chart_config_updated {
27
- char *type;
28
- char *family;
29
- char *context;
30
- char *title;
31
- uint64_t priority;
32
- char *plugin;
33
- char *module;
34
- RRDSET_TYPE chart_type;
35
- char *units;
36
- char *config_hash;
37
-};
38
-
39
-void destroy_update_chart_config(struct update_chart_config *cfg);
40
-void destroy_chart_config_updated(struct chart_config_updated *cfg);
41
-
42
-struct update_chart_config parse_update_chart_config(const char *data, size_t len);
43
-
44
-char *generate_chart_configs_updated(size_t *len, const struct chart_config_updated *config_list, int list_size);
45
-
46
-#ifdef __cplusplus
47
-}
48
-#endif
49
-
50
-#endif /* ACLK_SCHEMA_WRAPPER_CHART_CONFIG_H */
aclk/schema-wrappers/chart_stream.cc
deleted
-337
@@ -1,337 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-#include "aclk/aclk_util.h"
4
-
5
-#include "proto/chart/v1/stream.pb.h"
6
-#include "chart_stream.h"
7
-
8
-#include "schema_wrapper_utils.h"
9
-
10
-#include <sys/time.h>
11
-#include <stdlib.h>
12
-
13
-stream_charts_and_dims_t parse_stream_charts_and_dims(const char *data, size_t len)
14
-{
15
- chart::v1::StreamChartsAndDimensions msg;
16
- stream_charts_and_dims_t res;
17
- memset(&res, 0, sizeof(res));
18
-
19
- if (!msg.ParseFromArray(data, len))
20
- return res;
21
-
22
- res.node_id = strdup(msg.node_id().c_str());
23
- res.claim_id = strdup(msg.claim_id().c_str());
24
- res.seq_id = msg.sequence_id();
25
- res.batch_id = msg.batch_id();
26
- set_timeval_from_google_timestamp(msg.seq_id_created_at(), &res.seq_id_created_at);
27
-
28
- return res;
29
-}
30
-
31
-chart_and_dim_ack_t parse_chart_and_dimensions_ack(const char *data, size_t len)
32
-{
33
- chart::v1::ChartsAndDimensionsAck msg;
34
- chart_and_dim_ack_t res = { .claim_id = NULL, .node_id = NULL, .last_seq_id = 0 };
35
-
36
- if (!msg.ParseFromArray(data, len))
37
- return res;
38
-
39
- res.node_id = strdup(msg.node_id().c_str());
40
- res.claim_id = strdup(msg.claim_id().c_str());
41
- res.last_seq_id = msg.last_sequence_id();
42
-
43
- return res;
44
-}
45
-
46
-char *generate_reset_chart_messages(size_t *len, chart_reset_t reset)
47
-{
48
- chart::v1::ResetChartMessages msg;
49
-
50
- msg.set_claim_id(reset.claim_id);
51
- msg.set_node_id(reset.node_id);
52
- switch (reset.reason) {
53
- case DB_EMPTY:
54
- msg.set_reason(chart::v1::ResetReason::DB_EMPTY);
55
- break;
56
- case SEQ_ID_NOT_EXISTS:
57
- msg.set_reason(chart::v1::ResetReason::SEQ_ID_NOT_EXISTS);
58
- break;
59
- case TIMESTAMP_MISMATCH:
60
- msg.set_reason(chart::v1::ResetReason::TIMESTAMP_MISMATCH);
61
- break;
62
- default:
63
- return NULL;
64
- }
65
-
66
- *len = PROTO_COMPAT_MSG_SIZE(msg);
67
- char *bin = (char*)malloc(*len);
68
- if (bin)
69
- msg.SerializeToArray(bin, *len);
70
-
71
- return bin;
72
-}
73
-
74
-void chart_instance_updated_destroy(struct chart_instance_updated *instance)
75
-{
76
- freez((char*)instance->id);
77
- freez((char*)instance->claim_id);
78
-
79
- rrdlabels_destroy(instance->chart_labels);
80
-
81
- freez((char*)instance->config_hash);
82
-}
83
-
84
-static int set_chart_instance_updated(chart::v1::ChartInstanceUpdated *chart, const struct chart_instance_updated *update)
85
-{
86
- google::protobuf::Map<std::string, std::string> *map;
87
- aclk_lib::v1::ACLKMessagePosition *pos;
88
-
89
- chart->set_id(update->id);
90
- chart->set_claim_id(update->claim_id);
91
- chart->set_node_id(update->node_id);
92
- chart->set_name(update->name);
93
-
94
- map = chart->mutable_chart_labels();
95
- rrdlabels_walkthrough_read(update->chart_labels, label_add_to_map_callback, map);
96
-
97
- switch (update->memory_mode) {
98
- case RRD_MEMORY_MODE_NONE:
99
- chart->set_memory_mode(chart::v1::NONE);
100
- break;
101
- case RRD_MEMORY_MODE_RAM:
102
- chart->set_memory_mode(chart::v1::RAM);
103
- break;
104
- case RRD_MEMORY_MODE_MAP:
105
- chart->set_memory_mode(chart::v1::MAP);
106
- break;
107
- case RRD_MEMORY_MODE_SAVE:
108
- chart->set_memory_mode(chart::v1::SAVE);
109
- break;
110
- case RRD_MEMORY_MODE_ALLOC:
111
- chart->set_memory_mode(chart::v1::ALLOC);
112
- break;
113
- case RRD_MEMORY_MODE_DBENGINE:
114
- chart->set_memory_mode(chart::v1::DB_ENGINE);
115
- break;
116
- default:
117
- return 1;
118
- break;
119
- }
120
-
121
- chart->set_update_every_interval(update->update_every);
122
- chart->set_config_hash(update->config_hash);
123
-
124
- pos = chart->mutable_position();
125
- pos->set_sequence_id(update->position.sequence_id);
126
- pos->set_previous_sequence_id(update->position.previous_sequence_id);
127
- set_google_timestamp_from_timeval(update->position.seq_id_creation_time, pos->mutable_seq_id_created_at());
128
-
129
- return 0;
130
-}
131
-
132
-static int set_chart_dim_updated(chart::v1::ChartDimensionUpdated *dim, const struct chart_dimension_updated *c_dim)
133
-{
134
- aclk_lib::v1::ACLKMessagePosition *pos;
135
-
136
- dim->set_id(c_dim->id);
137
- dim->set_chart_id(c_dim->chart_id);
138
- dim->set_node_id(c_dim->node_id);
139
- dim->set_claim_id(c_dim->claim_id);
140
- dim->set_name(c_dim->name);
141
-
142
- set_google_timestamp_from_timeval(c_dim->created_at, dim->mutable_created_at());
143
- set_google_timestamp_from_timeval(c_dim->last_timestamp, dim->mutable_last_timestamp());
144
-
145
- pos = dim->mutable_position();
146
- pos->set_sequence_id(c_dim->position.sequence_id);
147
- pos->set_previous_sequence_id(c_dim->position.previous_sequence_id);
148
- set_google_timestamp_from_timeval(c_dim->position.seq_id_creation_time, pos->mutable_seq_id_created_at());
149
-
150
- return 0;
151
-}
152
-
153
-char *generate_charts_and_dimensions_updated(size_t *len, char **payloads, size_t *payload_sizes, int *is_dim, struct aclk_message_position *new_positions, uint64_t batch_id)
154
-{
155
- chart::v1::ChartsAndDimensionsUpdated msg;
156
- chart::v1::ChartInstanceUpdated db_chart;
157
- chart::v1::ChartDimensionUpdated db_dim;
158
- aclk_lib::v1::ACLKMessagePosition *pos;
159
-
160
- msg.set_batch_id(batch_id);
161
-
162
- for (int i = 0; payloads[i]; i++) {
163
- if (is_dim[i]) {
164
- if (!db_dim.ParseFromArray(payloads[i], payload_sizes[i])) {
165
- error("[ACLK] Could not parse chart::v1::chart_dimension_updated");
166
- return NULL;
167
- }
168
-
169
- pos = db_dim.mutable_position();
170
- pos->set_sequence_id(new_positions[i].sequence_id);
171
- pos->set_previous_sequence_id(new_positions[i].previous_sequence_id);
172
- set_google_timestamp_from_timeval(new_positions[i].seq_id_creation_time, pos->mutable_seq_id_created_at());
173
-
174
- chart::v1::ChartDimensionUpdated *dim = msg.add_dimensions();
175
- *dim = db_dim;
176
- } else {
177
- if (!db_chart.ParseFromArray(payloads[i], payload_sizes[i])) {
178
- error("[ACLK] Could not parse chart::v1::ChartInstanceUpdated");
179
- return NULL;
180
- }
181
-
182
- pos = db_chart.mutable_position();
183
- pos->set_sequence_id(new_positions[i].sequence_id);
184
- pos->set_previous_sequence_id(new_positions[i].previous_sequence_id);
185
- set_google_timestamp_from_timeval(new_positions[i].seq_id_creation_time, pos->mutable_seq_id_created_at());
186
-
187
- chart::v1::ChartInstanceUpdated *chart = msg.add_charts();
188
- *chart = db_chart;
189
- }
190
- }
191
-
192
- *len = PROTO_COMPAT_MSG_SIZE(msg);
193
- char *bin = (char*)mallocz(*len);
194
- msg.SerializeToArray(bin, *len);
195
-
196
- return bin;
197
-}
198
-
199
-char *generate_charts_updated(size_t *len, char **payloads, size_t *payload_sizes, struct aclk_message_position *new_positions)
200
-{
201
- chart::v1::ChartsAndDimensionsUpdated msg;
202
-
203
- msg.set_batch_id(chart_batch_id);
204
-
205
- for (int i = 0; payloads[i]; i++) {
206
- chart::v1::ChartInstanceUpdated db_msg;
207
- chart::v1::ChartInstanceUpdated *chart;
208
- aclk_lib::v1::ACLKMessagePosition *pos;
209
-
210
- if (!db_msg.ParseFromArray(payloads[i], payload_sizes[i])) {
211
- error("[ACLK] Could not parse chart::v1::ChartInstanceUpdated");
212
- return NULL;
213
- }
214
-
215
- pos = db_msg.mutable_position();
216
- pos->set_sequence_id(new_positions[i].sequence_id);
217
- pos->set_previous_sequence_id(new_positions[i].previous_sequence_id);
218
- set_google_timestamp_from_timeval(new_positions[i].seq_id_creation_time, pos->mutable_seq_id_created_at());
219
-
220
- chart = msg.add_charts();
221
- *chart = db_msg;
222
- }
223
-
224
- *len = PROTO_COMPAT_MSG_SIZE(msg);
225
- char *bin = (char*)mallocz(*len);
226
- msg.SerializeToArray(bin, *len);
227
-
228
- return bin;
229
-}
230
-
231
-char *generate_chart_dimensions_updated(size_t *len, char **payloads, size_t *payload_sizes, struct aclk_message_position *new_positions)
232
-{
233
- chart::v1::ChartsAndDimensionsUpdated msg;
234
-
235
- msg.set_batch_id(chart_batch_id);
236
-
237
- for (int i = 0; payloads[i]; i++) {
238
- chart::v1::ChartDimensionUpdated db_msg;
239
- chart::v1::ChartDimensionUpdated *dim;
240
- aclk_lib::v1::ACLKMessagePosition *pos;
241
-
242
- if (!db_msg.ParseFromArray(payloads[i], payload_sizes[i])) {
243
- error("[ACLK] Could not parse chart::v1::chart_dimension_updated");
244
- return NULL;
245
- }
246
-
247
- pos = db_msg.mutable_position();
248
- pos->set_sequence_id(new_positions[i].sequence_id);
249
- pos->set_previous_sequence_id(new_positions[i].previous_sequence_id);
250
- set_google_timestamp_from_timeval(new_positions[i].seq_id_creation_time, pos->mutable_seq_id_created_at());
251
-
252
- dim = msg.add_dimensions();
253
- *dim = db_msg;
254
- }
255
-
256
- *len = PROTO_COMPAT_MSG_SIZE(msg);
257
- char *bin = (char*)mallocz(*len);
258
- msg.SerializeToArray(bin, *len);
259
-
260
- return bin;
261
-}
262
-
263
-char *generate_chart_instance_updated(size_t *len, const struct chart_instance_updated *update)
264
-{
265
- chart::v1::ChartInstanceUpdated *chart = new chart::v1::ChartInstanceUpdated();
266
-
267
- if (set_chart_instance_updated(chart, update))
268
- return NULL;
269
-
270
- *len = PROTO_COMPAT_MSG_SIZE_PTR(chart);
271
- char *bin = (char*)mallocz(*len);
272
- chart->SerializeToArray(bin, *len);
273
-
274
- delete chart;
275
- return bin;
276
-}
277
-
278
-char *generate_chart_dimension_updated(size_t *len, const struct chart_dimension_updated *dim)
279
-{
280
- chart::v1::ChartDimensionUpdated *proto_dim = new chart::v1::ChartDimensionUpdated();
281
-
282
- if (set_chart_dim_updated(proto_dim, dim))
283
- return NULL;
284
-
285
- *len = PROTO_COMPAT_MSG_SIZE_PTR(proto_dim);
286
- char *bin = (char*)mallocz(*len);
287
- proto_dim->SerializeToArray(bin, *len);
288
-
289
- delete proto_dim;
290
- return bin;
291
-}
292
-
293
-using namespace google::protobuf;
294
-
295
-char *generate_retention_updated(size_t *len, struct retention_updated *data)
296
-{
297
- chart::v1::RetentionUpdated msg;
298
-
299
- msg.set_claim_id(data->claim_id);
300
- msg.set_node_id(data->node_id);
301
-
302
- switch (data->memory_mode) {
303
- case RRD_MEMORY_MODE_NONE:
304
- msg.set_memory_mode(chart::v1::NONE);
305
- break;
306
- case RRD_MEMORY_MODE_RAM:
307
- msg.set_memory_mode(chart::v1::RAM);
308
- break;
309
- case RRD_MEMORY_MODE_MAP:
310
- msg.set_memory_mode(chart::v1::MAP);
311
- break;
312
- case RRD_MEMORY_MODE_SAVE:
313
- msg.set_memory_mode(chart::v1::SAVE);
314
- break;
315
- case RRD_MEMORY_MODE_ALLOC:
316
- msg.set_memory_mode(chart::v1::ALLOC);
317
- break;
318
- case RRD_MEMORY_MODE_DBENGINE:
319
- msg.set_memory_mode(chart::v1::DB_ENGINE);
320
- break;
321
- default:
322
- return NULL;
323
- }
324
-
325
- for (int i = 0; i < data->interval_duration_count; i++) {
326
- Map<uint32, uint32> *map = msg.mutable_interval_durations();
327
- map->insert({data->interval_durations[i].update_every, data->interval_durations[i].retention});
328
- }
329
-
330
- set_google_timestamp_from_timeval(data->rotation_timestamp, msg.mutable_rotation_timestamp());
331
-
332
- *len = PROTO_COMPAT_MSG_SIZE(msg);
333
- char *bin = (char*)mallocz(*len);
334
- msg.SerializeToArray(bin, *len);
335
-
336
- return bin;
337
-}
aclk/schema-wrappers/chart_stream.h
deleted
-121
@@ -1,121 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-#ifndef ACLK_SCHEMA_WRAPPER_CHART_STREAM_H
4
-#define ACLK_SCHEMA_WRAPPER_CHART_STREAM_H
5
-
6
-#ifdef __cplusplus
7
-extern "C" {
8
-#endif
9
-
10
-#include "database/rrd.h"
11
-
12
-typedef struct {
13
- char* claim_id;
14
- char* node_id;
15
-
16
- uint64_t seq_id;
17
- uint64_t batch_id;
18
-
19
- struct timeval seq_id_created_at;
20
-} stream_charts_and_dims_t;
21
-
22
-stream_charts_and_dims_t parse_stream_charts_and_dims(const char *data, size_t len);
23
-
24
-typedef struct {
25
- char* claim_id;
26
- char* node_id;
27
-
28
- uint64_t last_seq_id;
29
-} chart_and_dim_ack_t;
30
-
31
-chart_and_dim_ack_t parse_chart_and_dimensions_ack(const char *data, size_t len);
32
-
33
-enum chart_reset_reason {
34
- DB_EMPTY,
35
- SEQ_ID_NOT_EXISTS,
36
- TIMESTAMP_MISMATCH
37
-};
38
-
39
-typedef struct {
40
- char *claim_id;
41
- char *node_id;
42
-
43
- enum chart_reset_reason reason;
44
-} chart_reset_t;
45
-
46
-char *generate_reset_chart_messages(size_t *len, const chart_reset_t reset);
47
-
48
-struct aclk_message_position {
49
- uint64_t sequence_id;
50
- struct timeval seq_id_creation_time;
51
- uint64_t previous_sequence_id;
52
-};
53
-
54
-struct chart_instance_updated {
55
- const char *id;
56
- const char *claim_id;
57
- const char *node_id;
58
- const char *name;
59
-
60
- DICTIONARY *chart_labels;
61
-
62
- RRD_MEMORY_MODE memory_mode;
63
-
64
- uint32_t update_every;
65
- const char * config_hash;
66
-
67
- struct aclk_message_position position;
68
-};
69
-
70
-void chart_instance_updated_destroy(struct chart_instance_updated *instance);
71
-
72
-struct chart_dimension_updated {
73
- const char *id;
74
- const char *chart_id;
75
- const char *node_id;
76
- const char *claim_id;
77
- const char *name;
78
- struct timeval created_at;
79
- struct timeval last_timestamp;
80
- struct aclk_message_position position;
81
-};
82
-
83
-typedef struct {
84
- struct chart_instance_updated *charts;
85
- uint16_t chart_count;
86
-
87
- struct chart_dimension_updated *dims;
88
- uint16_t dim_count;
89
-
90
- uint64_t batch_id;
91
-} charts_and_dims_updated_t;
92
-
93
-struct interval_duration {
94
- uint32_t update_every;
95
- uint32_t retention;
96
-};
97
-
98
-struct retention_updated {
99
- char *claim_id;
100
- char *node_id;
101
-
102
- RRD_MEMORY_MODE memory_mode;
103
-
104
- struct interval_duration *interval_durations;
105
- int interval_duration_count;
106
-
107
- struct timeval rotation_timestamp;
108
-};
109
-
110
-char *generate_charts_and_dimensions_updated(size_t *len, char **payloads, size_t *payload_sizes, int *is_dim, struct aclk_message_position *new_positions, uint64_t batch_id);
111
-char *generate_charts_updated(size_t *len, char **payloads, size_t *payload_sizes, struct aclk_message_position *new_positions);
112
-char *generate_chart_instance_updated(size_t *len, const struct chart_instance_updated *update);
113
-char *generate_chart_dimensions_updated(size_t *len, char **payloads, size_t *payload_sizes, struct aclk_message_position *new_positions);
114
-char *generate_chart_dimension_updated(size_t *len, const struct chart_dimension_updated *dim);
115
-char *generate_retention_updated(size_t *len, struct retention_updated *data);
116
-
117
-#ifdef __cplusplus
118
-}
119
-#endif
120
-
121
-#endif /* ACLK_SCHEMA_WRAPPER_CHART_STREAM_H */
aclk/schema-wrappers/proto_2_json.cc
-16
@@ -4,8 +4,6 @@
4
#include "proto/alarm/v1/config.pb.h"
5
#include "proto/alarm/v1/stream.pb.h"
6
#include "proto/aclk/v1/lib.pb.h"
7
-#include "proto/chart/v1/config.pb.h"
8
-#include "proto/chart/v1/stream.pb.h"
7
#include "proto/agent/v1/connection.pb.h"
8
#include "proto/agent/v1/disconnect.pb.h"
9
#include "proto/nodeinstance/connection/v1/connection.pb.h"
@@ -29,14 +27,6 @@ static google::protobuf::Message *msg_name_to_protomsg(const char *msgname)
27
return new nodeinstance::v1::UpdateNodeInstanceConnection;
28
if (!strcmp(msgname, "CreateNodeInstance"))
29
return new nodeinstance::create::v1::CreateNodeInstance;
32
- if (!strcmp(msgname, "ChartsAndDimensionsUpdated"))
33
- return new chart::v1::ChartsAndDimensionsUpdated;
34
- if (!strcmp(msgname, "ChartConfigsUpdated"))
35
- return new chart::v1::ChartConfigsUpdated;
36
- if (!strcmp(msgname, "ResetChartMessages"))
37
- return new chart::v1::ResetChartMessages;
38
- if (!strcmp(msgname, "RetentionUpdated"))
39
- return new chart::v1::RetentionUpdated;
30
if (!strcmp(msgname, "UpdateNodeInfo"))
31
return new nodeinstance::info::v1::UpdateNodeInfo;
32
if (!strcmp(msgname, "AlarmLogHealth"))
@@ -59,12 +49,6 @@ static google::protobuf::Message *msg_name_to_protomsg(const char *msgname)
49
return new nodeinstance::create::v1::CreateNodeInstanceResult;
50
if (!strcmp(msgname, "SendNodeInstances"))
51
return new agent::v1::SendNodeInstances;
62
- if (!strcmp(msgname, "StreamChartsAndDimensions"))
63
- return new chart::v1::StreamChartsAndDimensions;
64
- if (!strcmp(msgname, "ChartsAndDimensionsAck"))
65
- return new chart::v1::ChartsAndDimensionsAck;
66
- if (!strcmp(msgname, "UpdateChartConfigs"))
67
- return new chart::v1::UpdateChartConfigs;
52
if (!strcmp(msgname, "StartAlarmStreaming"))
53
return new alarms::v1::StartAlarmStreaming;
54
if (!strcmp(msgname, "SendAlarmLogHealth"))
aclk/schema-wrappers/schema_wrappers.h
-2
@@ -8,8 +8,6 @@
8
#include "connection.h"
9
#include "node_connection.h"
10
#include "node_creation.h"
11
-#include "chart_config.h"
12
-#include "chart_stream.h"
11
#include "alarm_config.h"
12
#include "alarm_stream.h"
13
#include "node_info.h"
daemon/main.c
-7
@@ -735,12 +735,6 @@ static void get_netdata_configured_variables() {
735
config_set_number(CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above);
736
}
737
738
- // --------------------------------------------------------------------
739
- // rrdcontext
740
-
741
- rrdcontext_enabled = config_get_boolean(CONFIG_SECTION_CLOUD, "rrdcontexts", rrdcontext_enabled);
742
-
743
-
738
// --------------------------------------------------------------------
739
// get various system parameters
740
@@ -987,7 +981,6 @@ int main(int argc, char **argv) {
981
// No call to load the config file on this code-path
982
post_conf_load(&user);
983
get_netdata_configured_variables();
990
- rrdcontext_enabled = CONFIG_BOOLEAN_NO;
984
default_rrd_update_every = 1;
985
default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;
986
default_health_enabled = 0;
daemon/service.c
-7
@@ -61,9 +61,6 @@ static void svc_rrddim_obsolete_to_archive(RRDDIM *rd) {
61
}
62
else {
63
/* Do not delete this dimension */
64
-#ifdef ENABLE_ACLK
65
- queue_dimension_to_aclk(rd, calc_dimension_liveness(rd, now_realtime_sec()));
66
-#endif
64
return;
65
}
66
}
@@ -139,10 +136,6 @@ static void svc_rrdhost_cleanup_obsolete_charts(RRDHOST *host) {
136
rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS);
137
svc_rrdset_archive_obsolete_dimensions(st, false);
138
}
142
-#ifdef ENABLE_ACLK
143
- else
144
- sql_check_chart_liveness(st);
145
-#endif
139
}
140
rrdset_foreach_done(st);
141
}
daemon/unit_test.c
-36
@@ -1550,42 +1550,6 @@ int test_sqlite(void) {
1550
BUFFER *sql = buffer_create(ACLK_SYNC_QUERY_SIZE);
1551
char *uuid_str = "0000_000";
1552
1553
- buffer_sprintf(sql, TABLE_ACLK_CHART, uuid_str);
1554
- rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), 0, 0, NULL);
1555
- buffer_flush(sql);
1556
- if (rc != SQLITE_OK)
1557
- goto error;
1558
-
1559
- buffer_sprintf(sql, TABLE_ACLK_CHART_PAYLOAD, uuid_str);
1560
- rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), 0, 0, NULL);
1561
- buffer_flush(sql);
1562
- if (rc != SQLITE_OK)
1563
- goto error;
1564
-
1565
- buffer_sprintf(sql, TABLE_ACLK_CHART_LATEST, uuid_str);
1566
- rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), 0, 0, NULL);
1567
- if (rc != SQLITE_OK)
1568
- goto error;
1569
- buffer_flush(sql);
1570
-
1571
- buffer_sprintf(sql, INDEX_ACLK_CHART, uuid_str, uuid_str);
1572
- rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), 0, 0, NULL);
1573
- if (rc != SQLITE_OK)
1574
- goto error;
1575
- buffer_flush(sql);
1576
-
1577
- buffer_sprintf(sql, INDEX_ACLK_CHART_LATEST, uuid_str, uuid_str);
1578
- rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), 0, 0, NULL);
1579
- if (rc != SQLITE_OK)
1580
- goto error;
1581
- buffer_flush(sql);
1582
-
1583
- buffer_sprintf(sql, TRIGGER_ACLK_CHART_PAYLOAD, uuid_str, uuid_str, uuid_str);
1584
- rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), 0, 0, NULL);
1585
- if (rc != SQLITE_OK)
1586
- goto error;
1587
- buffer_flush(sql);
1588
-
1553
buffer_sprintf(sql, TABLE_ACLK_ALERT, uuid_str);
1554
rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), 0, 0, NULL);
1555
if (rc != SQLITE_OK)
database/engine/rrdengine.c
-1
@@ -922,7 +922,6 @@ static void after_delete_old_data(struct rrdengine_worker_config* wc)
922
wc->now_deleting_files = NULL;
923
924
wc->cleanup_thread_deleting_files = 0;
925
- aclk_data_rotated();
925
rrdcontext_db_rotation();
926
927
/* interrupt event loop */
database/rrd.h
+1
-2
@@ -491,7 +491,7 @@ typedef enum rrdset_flags {
491
// No new values have been collected for this chart since agent start, or it was marked RRDSET_FLAG_OBSOLETE at
492
// least rrdset_free_obsolete_time seconds ago.
493
RRDSET_FLAG_ARCHIVED = (1 << 15),
494
- RRDSET_FLAG_ACLK = (1 << 16),
494
+// RRDSET_FLAG_ACLK = (1 << 16), // not used anymore
495
RRDSET_FLAG_PENDING_FOREACH_ALARMS = (1 << 17), // contains dims with uninitialized foreach alarms
496
RRDSET_FLAG_ANOMALY_DETECTION = (1 << 18), // flag to identify anomaly detection charts.
497
RRDSET_FLAG_INDEXED_ID = (1 << 19), // the rrdset is indexed by its id
@@ -1289,7 +1289,6 @@ extern int get_tier_grouping(int tier);
1289
#include "sqlite/sqlite_functions.h"
1290
#include "sqlite/sqlite_context.h"
1291
#include "sqlite/sqlite_aclk.h"
1292
-#include "sqlite/sqlite_aclk_chart.h"
1292
#include "sqlite/sqlite_aclk_alert.h"
1293
#include "sqlite/sqlite_aclk_node.h"
1294
#include "sqlite/sqlite_health.h"
database/rrdcontext.c
-58
@@ -6,10 +6,6 @@
6
#include "aclk/aclk_contexts_api.h"
7
#include "aclk/aclk.h"
8
9
-int rrdcontext_enabled = CONFIG_BOOLEAN_YES;
10
-
11
-// #define LOG_POST_PROCESSING_QUEUE_INSERTIONS 1
12
-
9
#define MESSAGES_PER_BUNDLE_TO_SEND_TO_HUB_PER_HOST 5000
10
#define FULL_RETENTION_SCAN_DELAY_AFTER_DB_ROTATION_SECS 120
11
#define RRDCONTEXT_WORKER_THREAD_HEARTBEAT_USEC (1000 * USEC_PER_MS)
@@ -834,9 +830,6 @@ static void rrdinstance_react_callback(const DICTIONARY_ITEM *item __maybe_unuse
830
}
831
832
void rrdinstances_create_in_rrdcontext(RRDCONTEXT *rc) {
837
- if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
838
- return;
839
-
833
if(unlikely(!rc || rc->rrdinstances)) return;
834
835
rc->rrdinstances = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);
@@ -1286,9 +1279,6 @@ static void rrdcontext_post_processing_queue_delete_callback(const DICTIONARY_IT
1279
}
1280
1281
void rrdhost_create_rrdcontexts(RRDHOST *host) {
1289
- if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
1290
- return;
1291
-
1282
if(unlikely(!host)) return;
1283
if(likely(host->rrdctx)) return;
1284
@@ -1349,102 +1339,61 @@ void rrdhost_destroy_rrdcontexts(RRDHOST *host) {
1339
// public API
1340
1341
void rrdcontext_updated_rrddim(RRDDIM *rd) {
1352
- if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
1353
- return;
1354
-
1342
rrdmetric_from_rrddim(rd);
1343
}
1344
1345
void rrdcontext_removed_rrddim(RRDDIM *rd) {
1359
- if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
1360
- return;
1361
-
1346
rrdmetric_rrddim_is_freed(rd);
1347
}
1348
1349
void rrdcontext_updated_rrddim_algorithm(RRDDIM *rd) {
1366
- if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
1367
- return;
1368
-
1350
rrdmetric_updated_rrddim_flags(rd);
1351
}
1352
1353
void rrdcontext_updated_rrddim_multiplier(RRDDIM *rd) {
1373
- if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
1374
- return;
1375
-
1354
rrdmetric_updated_rrddim_flags(rd);
1355
}
1356
1357
void rrdcontext_updated_rrddim_divisor(RRDDIM *rd) {
1380
- if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
1381
- return;
1382
-
1358
rrdmetric_updated_rrddim_flags(rd);
1359
}
1360
1361
void rrdcontext_updated_rrddim_flags(RRDDIM *rd) {
1387
- if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
1388
- return;
1389
-
1362
rrdmetric_updated_rrddim_flags(rd);
1363
}
1364
1365
void rrdcontext_collected_rrddim(RRDDIM *rd) {
1394
- if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
1395
- return;
1396
-
1366
rrdmetric_collected_rrddim(rd);
1367
}
1368
1369
void rrdcontext_updated_rrdset(RRDSET *st) {
1401
- if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
1402
- return;
1403
-
1370
rrdinstance_from_rrdset(st);
1371
}
1372
1373
void rrdcontext_removed_rrdset(RRDSET *st) {
1408
- if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
1409
- return;
1410
-
1374
rrdinstance_rrdset_is_freed(st);
1375
}
1376
1377
void rrdcontext_updated_rrdset_name(RRDSET *st) {
1415
- if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
1416
- return;
1417
-
1378
rrdinstance_updated_rrdset_name(st);
1379
}
1380
1381
void rrdcontext_updated_rrdset_flags(RRDSET *st) {
1422
- if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
1423
- return;
1424
-
1382
rrdinstance_updated_rrdset_flags(st);
1383
}
1384
1385
void rrdcontext_collected_rrdset(RRDSET *st) {
1429
- if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
1430
- return;
1431
-
1386
rrdinstance_collected_rrdset(st);
1387
}
1388
1389
void rrdcontext_host_child_connected(RRDHOST *host) {
1390
(void)host;
1391
1438
- if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
1439
- return;
1440
-
1392
// no need to do anything here
1393
;
1394
}
1395
1396
void rrdcontext_host_child_disconnected(RRDHOST *host) {
1446
- if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
1447
- return;
1397
1398
rrdcontext_recalculate_host_retention(host, RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD, false);
1399
}
@@ -2142,9 +2091,6 @@ static void rrdcontext_load_context_callback(VERSIONED_CONTEXT_DATA *ctx_data, v
2091
}
2092
2093
void rrdhost_load_rrdcontext_data(RRDHOST *host) {
2145
- if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
2146
- return;
2147
-
2094
if(host->rrdctx) return;
2095
2096
rrdhost_create_rrdcontexts(host);
@@ -3044,9 +2990,6 @@ static void rrdcontext_main_cleanup(void *ptr) {
2990
void *rrdcontext_main(void *ptr) {
2991
netdata_thread_cleanup_push(rrdcontext_main_cleanup, ptr);
2992
3047
- if(unlikely(rrdcontext_enabled == CONFIG_BOOLEAN_NO))
3048
- goto exit;
3049
-
2993
worker_register("RRDCONTEXT");
2994
worker_register_job_name(WORKER_JOB_HOSTS, "hosts");
2995
worker_register_job_name(WORKER_JOB_CHECK, "dedup checks");
@@ -3108,7 +3051,6 @@ void *rrdcontext_main(void *ptr) {
3051
worker_set_metric(WORKER_JOB_PP_QUEUE_SIZE, (NETDATA_DOUBLE)pp_queued_contexts_for_all_hosts);
3052
}
3053
3111
-exit:
3054
netdata_thread_cleanup_pop(1);
3055
return NULL;
3056
}
database/rrdcontext.h
-2
@@ -85,8 +85,6 @@ extern void rrdcontext_hub_stop_streaming_command(void *cmd);
85
// ----------------------------------------------------------------------------
86
// public API for threads
87
88
-extern int rrdcontext_enabled;
89
-
88
extern void rrdcontext_db_rotation(void);
89
extern void *rrdcontext_main(void *);
90
database/rrddim.c
+2
-3
@@ -281,9 +281,6 @@ static void rrddim_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo
281
if(ctr->react_action == RRDDIM_REACT_UPDATED) {
282
debug(D_METADATALOG, "DIMENSION [%s] metadata updated", rrddim_id(rd));
283
(void)sql_store_dimension(&rd->metric_uuid, &rd->rrdset->chart_uuid, rrddim_id(rd), rrddim_name(rd), rd->multiplier, rd->divisor, rd->algorithm);
284
-#ifdef ENABLE_ACLK
285
- queue_dimension_to_aclk(rd, calc_dimension_liveness(rd, now_realtime_sec()));
286
-#endif
284
285
// the chart needs to be updated to the parent
286
rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
@@ -431,6 +428,8 @@ time_t rrddim_first_entry_t(RRDDIM *rd) {
428
// last collected time : Dimension is not live
429
430
#ifdef ENABLE_ACLK
431
+#define RRDSET_MINIMUM_DIM_LIVE_MULTIPLIER (3)
432
+#define RRDSET_MINIMUM_DIM_OFFLINE_MULTIPLIER (30)
433
time_t calc_dimension_liveness(RRDDIM *rd, time_t now)
434
{
435
time_t last_updated = rd->last_collected_time.tv_sec;
database/rrdset.c
-23
@@ -331,9 +331,6 @@ static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
331
ctr->react_action |= RRDSET_REACT_CHART_ARCHIVED_TO_LIVE;
332
}
333
334
- if(ctr->react_action)
335
- rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
336
-
334
rrdset_update_permanent_labels(st);
335
336
rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
@@ -1355,20 +1352,6 @@ void rrdset_done(RRDSET *st) {
1352
1353
netdata_thread_disable_cancelability();
1354
1358
-#ifdef ENABLE_ACLK
1359
- time_t mark = now_realtime_sec();
1360
- bool rrdset_flag_aclk = rrdset_flag_check(st, RRDSET_FLAG_ACLK);
1361
- bool rrdset_is_ar = rrdset_is_ar_chart(st);
1362
-
1363
- if (likely(!rrdset_is_ar)) {
1364
- if (unlikely(!rrdset_flag_aclk)) {
1365
- if (likely(rrdset_number_of_dimensions(st) && st->counter_done && !queue_chart_to_aclk(st))) {
1366
- rrdset_flag_set(st, RRDSET_FLAG_ACLK);
1367
- }
1368
- }
1369
- }
1370
-#endif
1371
-
1355
if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE))) {
1356
error("Chart '%s' has the OBSOLETE flag set, but it is collected.", rrdset_id(st));
1357
rrdset_isnot_obsolete(st);
@@ -1805,12 +1788,6 @@ after_second_database_work:
1788
rd = rda->rd;
1789
if(unlikely(!rd)) continue;
1790
1808
-#ifdef ENABLE_ACLK
1809
- if (likely(!rrdset_is_ar)) {
1810
- if (!rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN) && likely(rrdset_flag_aclk))
1811
- queue_dimension_to_aclk(rd, calc_dimension_liveness(rd, mark));
1812
- }
1813
-#endif
1791
if(unlikely(!rd->updated))
1792
continue;
1793
database/sqlite/sqlite_aclk.c
+12
-229
@@ -3,7 +3,6 @@
3
#include "sqlite_functions.h"
4
#include "sqlite_aclk.h"
5
6
-#include "sqlite_aclk_chart.h"
6
#include "sqlite_aclk_node.h"
7
8
#ifdef ENABLE_ACLK
@@ -34,28 +33,6 @@ const char *aclk_sync_config[] = {
33
34
uv_mutex_t aclk_async_lock;
35
struct aclk_database_worker_config *aclk_thread_head = NULL;
37
-int retention_running = 0;
38
-
39
-#ifdef ENABLE_ACLK
40
-static void stop_retention_run()
41
-{
42
- uv_mutex_lock(&aclk_async_lock);
43
- retention_running = 0;
44
- uv_mutex_unlock(&aclk_async_lock);
45
-}
46
-
47
-static int request_retention_run()
48
-{
49
- int rc = 0;
50
- uv_mutex_lock(&aclk_async_lock);
51
- if (unlikely(retention_running))
52
- rc = 1;
53
- else
54
- retention_running = 1;
55
- uv_mutex_unlock(&aclk_async_lock);
56
- return rc;
57
-}
58
-#endif
36
37
int claimed()
38
{
@@ -378,15 +355,13 @@ void sql_aclk_sync_init(void)
355
info("SQLite aclk sync initialization completed");
356
fatal_assert(0 == uv_mutex_init(&aclk_async_lock));
357
381
- if (likely(rrdcontext_enabled == CONFIG_BOOLEAN_YES)) {
382
- rc = sqlite3_exec_monitored(db_meta, "SELECT host_id, hostname, registry_hostname, update_every, os, "
383
- "timezone, tags, hops, memory_mode, abbrev_timezone, utc_offset, program_name, "
384
- "program_version, entries, health_enabled FROM host WHERE hops >0;",
385
- create_host_callback, NULL, &err_msg);
386
- if (rc != SQLITE_OK) {
387
- error_report("SQLite error when loading archived hosts, rc = %d (%s)", rc, err_msg);
388
- sqlite3_free(err_msg);
389
- }
358
+ rc = sqlite3_exec_monitored(db_meta, "SELECT host_id, hostname, registry_hostname, update_every, os, "
359
+ "timezone, tags, hops, memory_mode, abbrev_timezone, utc_offset, program_name, "
360
+ "program_version, entries, health_enabled FROM host WHERE hops >0;",
361
+ create_host_callback, NULL, &err_msg);
362
+ if (rc != SQLITE_OK) {
363
+ error_report("SQLite error when loading archived hosts, rc = %d (%s)", rc, err_msg);
364
+ sqlite3_free(err_msg);
365
}
366
367
rc = sqlite3_exec_monitored(db_meta, "SELECT ni.host_id, ni.node_id FROM host h, node_instance ni WHERE "
@@ -429,30 +404,6 @@ static void timer_cb(uv_timer_t* handle)
404
}
405
406
if (aclk_connected) {
432
- if (wc->rotation_after && wc->rotation_after < now) {
433
- cmd.opcode = ACLK_DATABASE_UPD_RETENTION;
434
- if (!aclk_database_enq_cmd_noblock(wc, &cmd))
435
- wc->rotation_after = now + ACLK_DATABASE_ROTATION_INTERVAL;
436
- }
437
-
438
- if (wc->chart_updates && !wc->chart_pending && wc->chart_payload_count) {
439
- cmd.opcode = ACLK_DATABASE_PUSH_CHART;
440
- cmd.count = ACLK_MAX_CHART_BATCH;
441
- cmd.param1 = ACLK_MAX_CHART_BATCH_COUNT;
442
- if (!aclk_database_enq_cmd_noblock(wc, &cmd)) {
443
- if (wc->retry_count)
444
- info("Queued chart/dimension payload command %s, retry count = %u", wc->host_guid, wc->retry_count);
445
- wc->chart_pending = 1;
446
- wc->retry_count = 0;
447
- } else {
448
- wc->retry_count++;
449
- if (wc->retry_count % 100 == 0)
450
- error_report("Failed to queue chart/dimension payload command %s, retry count = %u",
451
- wc->host_guid,
452
- wc->retry_count);
453
- }
454
- }
455
-
407
if (wc->alert_updates && !wc->pause_alert_updates) {
408
cmd.opcode = ACLK_DATABASE_PUSH_ALERT;
409
cmd.count = ACLK_MAX_ALERT_UPDATES;
@@ -462,52 +413,12 @@ static void timer_cb(uv_timer_t* handle)
413
#endif
414
}
415
465
-
466
-#ifdef ENABLE_ACLK
467
-void after_send_retention(uv_work_t *req, int status)
468
-{
469
- struct aclk_database_worker_config *wc = req->data;
470
- (void)status;
471
- stop_retention_run();
472
- wc->retention_running = 0;
473
-
474
- struct aclk_database_cmd cmd;
475
- memset(&cmd, 0, sizeof(cmd));
476
- cmd.opcode = ACLK_DATABASE_DIM_DELETION;
477
- if (aclk_database_enq_cmd_noblock(wc, &cmd))
478
- info("Failed to queue a dimension deletion message");
479
-
480
- cmd.opcode = ACLK_DATABASE_NODE_INFO;
481
- if (aclk_database_enq_cmd_noblock(wc, &cmd))
482
- info("Failed to queue a node update info message");
483
-}
484
-
485
-
486
-static void send_retention(uv_work_t *req)
487
-{
488
- struct aclk_database_worker_config *wc = req->data;
489
-
490
- if (unlikely(wc->is_shutting_down))
491
- return;
492
-
493
- aclk_update_retention(wc);
494
-}
495
-#endif
496
-
416
#define MAX_CMD_BATCH_SIZE (256)
417
418
void aclk_database_worker(void *arg)
419
{
420
worker_register("ACLKSYNC");
421
worker_register_job_name(ACLK_DATABASE_NOOP, "noop");
503
- worker_register_job_name(ACLK_DATABASE_ADD_CHART, "chart add");
504
- worker_register_job_name(ACLK_DATABASE_ADD_DIMENSION, "dimension add");
505
- worker_register_job_name(ACLK_DATABASE_PUSH_CHART, "chart push");
506
- worker_register_job_name(ACLK_DATABASE_PUSH_CHART_CONFIG, "chart conf push");
507
- worker_register_job_name(ACLK_DATABASE_RESET_CHART, "chart reset");
508
- worker_register_job_name(ACLK_DATABASE_CHART_ACK, "chart ack");
509
- worker_register_job_name(ACLK_DATABASE_UPD_RETENTION, "retention check");
510
- worker_register_job_name(ACLK_DATABASE_DIM_DELETION, "dimension delete");
422
worker_register_job_name(ACLK_DATABASE_ORPHAN_HOST, "node orphan");
423
worker_register_job_name(ACLK_DATABASE_ALARM_HEALTH_LOG, "alert log");
424
worker_register_job_name(ACLK_DATABASE_CLEANUP, "cleanup");
@@ -568,17 +479,9 @@ void aclk_database_worker(void *arg)
479
info("Starting ACLK sync thread for host %s -- scratch area %lu bytes", wc->host_guid, (unsigned long int) sizeof(*wc));
480
481
memset(&cmd, 0, sizeof(cmd));
571
-#ifdef ENABLE_ACLK
572
- uv_work_t retention_work;
573
- sql_get_last_chart_sequence(wc);
574
- wc->chart_payload_count = sql_get_pending_count(wc);
575
- if (!wc->chart_payload_count)
576
- info("%s: No pending charts and dimensions detected during startup", wc->host_guid);
577
-#endif
482
483
wc->startup_time = now_realtime_sec();
484
wc->cleanup_after = wc->startup_time + ACLK_DATABASE_CLEANUP_FIRST;
581
- wc->rotation_after = wc->startup_time + ACLK_DATABASE_ROTATION_DELAY;
485
486
debug(D_ACLK_SYNC,"Node %s reports pending message count = %u", wc->node_id, wc->chart_payload_count);
487
@@ -613,6 +516,9 @@ void aclk_database_worker(void *arg)
516
sql_maint_aclk_sync_database(wc, cmd);
517
if (wc->host == localhost)
518
sql_check_aclk_table_list(wc);
519
+ cmd.opcode = ACLK_DATABASE_NODE_INFO;
520
+ cmd.completion = NULL;
521
+ (void) aclk_database_enq_cmd_noblock(wc, &cmd);
522
break;
523
524
case ACLK_DATABASE_DELETE_HOST:
@@ -620,33 +526,6 @@ void aclk_database_worker(void *arg)
526
sql_delete_aclk_table_list(wc, cmd);
527
break;
528
623
-// CHART / DIMENSION OPERATIONS
624
-#ifdef ENABLE_ACLK
625
- case ACLK_DATABASE_ADD_CHART:
626
- debug(D_ACLK_SYNC, "Adding chart event for %s", wc->host_guid);
627
- aclk_add_chart_event(wc, cmd);
628
- break;
629
- case ACLK_DATABASE_ADD_DIMENSION:
630
- debug(D_ACLK_SYNC, "Adding dimension event for %s", wc->host_guid);
631
- aclk_add_dimension_event(wc, cmd);
632
- break;
633
- case ACLK_DATABASE_PUSH_CHART:
634
- debug(D_ACLK_SYNC, "Pushing chart info to the cloud for node %s", wc->host_guid);
635
- aclk_send_chart_event(wc, cmd);
636
- break;
637
- case ACLK_DATABASE_PUSH_CHART_CONFIG:
638
- debug(D_ACLK_SYNC, "Pushing chart config info to the cloud for node %s", wc->host_guid);
639
- aclk_send_chart_config(wc, cmd);
640
- break;
641
- case ACLK_DATABASE_CHART_ACK:
642
- debug(D_ACLK_SYNC, "ACK chart SEQ for %s to %"PRIu64, wc->uuid_str, (uint64_t) cmd.param1);
643
- aclk_receive_chart_ack(wc, cmd);
644
- break;
645
- case ACLK_DATABASE_RESET_CHART:
646
- debug(D_ACLK_SYNC, "RESET chart SEQ for %s to %"PRIu64, wc->uuid_str, (uint64_t) cmd.param1);
647
- aclk_receive_chart_reset(wc, cmd);
648
- break;
649
-#endif
529
// ALERTS
530
case ACLK_DATABASE_PUSH_ALERT_CONFIG:
531
debug(D_ACLK_SYNC,"Pushing chart config info to the cloud for %s", wc->host_guid);
@@ -679,27 +558,6 @@ void aclk_database_worker(void *arg)
558
sql_build_node_collectors(wc);
559
break;
560
#ifdef ENABLE_ACLK
682
- case ACLK_DATABASE_DIM_DELETION:
683
- debug(D_ACLK_SYNC,"Sending dimension deletion information %s", wc->uuid_str);
684
- aclk_process_dimension_deletion(wc, cmd);
685
- break;
686
- case ACLK_DATABASE_UPD_RETENTION:
687
- if (unlikely(wc->retention_running))
688
- break;
689
-
690
- if (unlikely(request_retention_run())) {
691
- wc->rotation_after = now_realtime_sec() + ACLK_DATABASE_RETENTION_RETRY;
692
- break;
693
- }
694
-
695
- debug(D_ACLK_SYNC,"Sending retention info for %s", wc->uuid_str);
696
- retention_work.data = wc;
697
- wc->retention_running = 1;
698
- if (unlikely(uv_queue_work(loop, &retention_work, send_retention, after_send_retention))) {
699
- wc->retention_running = 0;
700
- stop_retention_run();
701
- }
702
- break;
561
562
// NODE_INSTANCE DETECTION
563
case ACLK_DATABASE_ORPHAN_HOST:
@@ -809,30 +667,6 @@ void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id)
667
668
BUFFER *sql = buffer_create(ACLK_SYNC_QUERY_SIZE);
669
812
- buffer_sprintf(sql, TABLE_ACLK_CHART, uuid_str);
813
- db_execute(buffer_tostring(sql));
814
- buffer_flush(sql);
815
-
816
- buffer_sprintf(sql, TABLE_ACLK_CHART_PAYLOAD, uuid_str);
817
- db_execute(buffer_tostring(sql));
818
- buffer_flush(sql);
819
-
820
- buffer_sprintf(sql, TABLE_ACLK_CHART_LATEST, uuid_str);
821
- db_execute(buffer_tostring(sql));
822
- buffer_flush(sql);
823
-
824
- buffer_sprintf(sql, INDEX_ACLK_CHART, uuid_str, uuid_str);
825
- db_execute(buffer_tostring(sql));
826
- buffer_flush(sql);
827
-
828
- buffer_sprintf(sql, INDEX_ACLK_CHART_LATEST, uuid_str, uuid_str);
829
- db_execute(buffer_tostring(sql));
830
- buffer_flush(sql);
831
-
832
- buffer_sprintf(sql, TRIGGER_ACLK_CHART_PAYLOAD, uuid_str, uuid_str, uuid_str);
833
- db_execute(buffer_tostring(sql));
834
- buffer_flush(sql);
835
-
670
buffer_sprintf(sql, TABLE_ACLK_ALERT, uuid_str);
671
db_execute(buffer_tostring(sql));
672
buffer_flush(sql);
@@ -857,9 +691,9 @@ void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id)
691
wc->host = host;
692
strcpy(wc->uuid_str, uuid_str);
693
strcpy(wc->host_guid, host_guid);
860
- wc->chart_updates = 0;
694
+// wc->chart_updates = 0;
695
wc->alert_updates = 0;
862
- wc->retry_count = 0;
696
+// wc->retry_count = 0;
697
aclk_database_init_cmd_queue(wc);
698
aclk_add_worker_thread(wc);
699
fatal_assert(0 == uv_thread_create(&(wc->thread), aclk_database_worker, wc));
@@ -879,31 +713,9 @@ void sql_maint_aclk_sync_database(struct aclk_database_worker_config *wc, struct
713
714
BUFFER *sql = buffer_create(ACLK_SYNC_QUERY_SIZE);
715
882
- buffer_sprintf(sql,"DELETE FROM aclk_chart_%s WHERE date_submitted IS NOT NULL AND "
883
- "CAST(date_updated AS INT) < unixepoch()-%d;", wc->uuid_str, ACLK_DELETE_ACK_INTERNAL);
884
- db_execute(buffer_tostring(sql));
885
- buffer_flush(sql);
886
-
887
- buffer_sprintf(sql,"DELETE FROM aclk_chart_payload_%s WHERE unique_id NOT IN "
888
- "(SELECT unique_id FROM aclk_chart_%s) AND unique_id NOT IN (SELECT unique_id FROM aclk_chart_latest_%s);",
889
- wc->uuid_str, wc->uuid_str, wc->uuid_str);
890
- db_execute(buffer_tostring(sql));
891
- buffer_flush(sql);
892
-
716
buffer_sprintf(sql,"DELETE FROM aclk_alert_%s WHERE date_submitted IS NOT NULL AND "
717
"CAST(date_cloud_ack AS INT) < unixepoch()-%d;", wc->uuid_str, ACLK_DELETE_ACK_ALERTS_INTERNAL);
718
db_execute(buffer_tostring(sql));
896
- buffer_flush(sql);
897
-
898
- buffer_sprintf(sql,"UPDATE aclk_chart_%s SET status = NULL, date_submitted=unixepoch() WHERE "
899
- "date_submitted IS NULL AND CAST(date_created AS INT) < unixepoch()-%d;", wc->uuid_str, ACLK_AUTO_MARK_SUBMIT_INTERVAL);
900
- db_execute(buffer_tostring(sql));
901
- buffer_flush(sql);
902
-
903
- buffer_sprintf(sql,"UPDATE aclk_chart_%s SET date_updated = unixepoch() WHERE date_updated IS NULL"
904
- " AND date_submitted IS NOT NULL AND CAST(date_submitted AS INT) < unixepoch()-%d;",
905
- wc->uuid_str, ACLK_AUTO_MARK_UPDATED_INTERVAL);
906
- db_execute(buffer_tostring(sql));
719
720
buffer_free(sql);
721
return;
@@ -1032,32 +844,3 @@ void sql_check_aclk_table_list(struct aclk_database_worker_config *wc)
844
845
return;
846
}
1035
-
1036
-void aclk_data_rotated(void)
1037
-{
1038
-#ifdef ENABLE_ACLK
1039
-
1040
- if (!aclk_connected)
1041
- return;
1042
-
1043
- time_t next_rotation_time = now_realtime_sec()+ACLK_DATABASE_ROTATION_DELAY;
1044
- rrd_rdlock();
1045
- RRDHOST *this_host;
1046
- rrdhost_foreach_read(this_host) {
1047
- struct aclk_database_worker_config *wc = this_host->dbsync_worker;
1048
- if (wc)
1049
- wc->rotation_after = next_rotation_time;
1050
- }
1051
- rrd_unlock();
1052
-
1053
- struct aclk_database_worker_config *tmp = aclk_thread_head;
1054
-
1055
- uv_mutex_lock(&aclk_async_lock);
1056
- while (tmp) {
1057
- tmp->rotation_after = next_rotation_time;
1058
- tmp = tmp->next;
1059
- }
1060
- uv_mutex_unlock(&aclk_async_lock);
1061
-#endif
1062
- return;
1063
-}
database/sqlite/sqlite_aclk.h
-49
@@ -5,8 +5,6 @@
5
6
#include "sqlite3.h"
7
8
-// TODO: To be added
9
-#include "../../aclk/schema-wrappers/chart_stream.h"
8
9
#ifndef ACLK_MAX_CHART_BATCH
10
#define ACLK_MAX_CHART_BATCH (200)
@@ -16,14 +14,8 @@
14
#endif
15
#define ACLK_MAX_ALERT_UPDATES (5)
16
#define ACLK_DATABASE_CLEANUP_FIRST (60)
19
-#define ACLK_DATABASE_ROTATION_DELAY (180)
20
-#define ACLK_DATABASE_RETENTION_RETRY (60)
17
#define ACLK_DATABASE_CLEANUP_INTERVAL (3600)
22
-#define ACLK_DATABASE_ROTATION_INTERVAL (3600)
23
-#define ACLK_DELETE_ACK_INTERNAL (600)
18
#define ACLK_DELETE_ACK_ALERTS_INTERNAL (86400)
25
-#define ACLK_AUTO_MARK_SUBMIT_INTERVAL (3600)
26
-#define ACLK_AUTO_MARK_UPDATED_INTERVAL (1800)
19
#define ACLK_SYNC_QUERY_SIZE 512
20
21
struct aclk_completion {
@@ -87,44 +79,14 @@ static inline char *get_str_from_uuid(uuid_t *uuid)
79
return strdupz(uuid_str);
80
}
81
90
-#define TABLE_ACLK_CHART "CREATE TABLE IF NOT EXISTS aclk_chart_%s (sequence_id INTEGER PRIMARY KEY, " \
91
- "date_created, date_updated, date_submitted, status, uuid, type, unique_id, " \
92
- "update_count default 1, unique(uuid, status));"
93
-
94
-#define TABLE_ACLK_CHART_PAYLOAD "CREATE TABLE IF NOT EXISTS aclk_chart_payload_%s (unique_id BLOB PRIMARY KEY, " \
95
- "uuid, claim_id, type, date_created, payload);"
96
-
97
-#define TABLE_ACLK_CHART_LATEST "CREATE TABLE IF NOT EXISTS aclk_chart_latest_%s (uuid BLOB PRIMARY KEY, " \
98
- "unique_id, date_submitted);"
99
-
100
-#define TRIGGER_ACLK_CHART_PAYLOAD "CREATE TRIGGER IF NOT EXISTS aclk_tr_chart_payload_%s " \
101
- "after insert on aclk_chart_payload_%s " \
102
- "begin insert into aclk_chart_%s (uuid, unique_id, type, status, date_created) values " \
103
- " (new.uuid, new.unique_id, new.type, 'pending', unixepoch()) on conflict(uuid, status) " \
104
- " do update set unique_id = new.unique_id, update_count = update_count + 1; " \
105
- "end;"
106
-
82
#define TABLE_ACLK_ALERT "CREATE TABLE IF NOT EXISTS aclk_alert_%s (sequence_id INTEGER PRIMARY KEY, " \
83
"alert_unique_id, date_created, date_submitted, date_cloud_ack, " \
84
"unique(alert_unique_id));"
85
111
-#define INDEX_ACLK_CHART "CREATE INDEX IF NOT EXISTS aclk_chart_index_%s ON aclk_chart_%s (unique_id);"
112
-
113
-#define INDEX_ACLK_CHART_LATEST "CREATE INDEX IF NOT EXISTS aclk_chart_latest_index_%s ON aclk_chart_latest_%s (unique_id);"
114
-
86
#define INDEX_ACLK_ALERT "CREATE INDEX IF NOT EXISTS aclk_alert_index_%s ON aclk_alert_%s (alert_unique_id);"
116
-
87
enum aclk_database_opcode {
88
ACLK_DATABASE_NOOP = 0,
89
120
- ACLK_DATABASE_ADD_CHART,
121
- ACLK_DATABASE_ADD_DIMENSION,
122
- ACLK_DATABASE_PUSH_CHART,
123
- ACLK_DATABASE_PUSH_CHART_CONFIG,
124
- ACLK_DATABASE_RESET_CHART,
125
- ACLK_DATABASE_CHART_ACK,
126
- ACLK_DATABASE_UPD_RETENTION,
127
- ACLK_DATABASE_DIM_DELETION,
90
ACLK_DATABASE_ORPHAN_HOST,
91
ACLK_DATABASE_ALARM_HEALTH_LOG,
92
ACLK_DATABASE_CLEANUP,
@@ -172,12 +134,8 @@ struct aclk_database_worker_config {
134
char node_id[GUID_LEN + 1];
135
char host_guid[GUID_LEN + 1];
136
char *hostname; // hostname to avoid constant lookups
175
- uint64_t chart_sequence_id; // last chart_sequence_id
176
- time_t chart_timestamp; // last chart timestamp
137
time_t cleanup_after; // Start a cleanup after this timestamp
138
time_t startup_time; // When the sync thread started
179
- time_t rotation_after;
180
- uint64_t batch_id; // batch id to use
139
uint64_t alerts_batch_id; // batch id for alerts to use
140
uint64_t alerts_start_seq_id; // cloud has asked to start streaming from
141
uint64_t alert_sequence_id; // last alert sequence_id
@@ -193,15 +151,9 @@ struct aclk_database_worker_config {
151
uv_cond_t cmd_cond;
152
volatile unsigned queue_size;
153
struct aclk_database_cmdqueue cmd_queue;
196
- uint32_t retry_count;
197
- int chart_updates;
154
int alert_updates;
199
- time_t batch_created;
155
int node_info_send;
156
time_t node_collectors_send;
202
- int chart_pending;
203
- int chart_reset_count;
204
- int retention_running;
157
volatile unsigned is_shutting_down;
158
volatile unsigned is_orphan;
159
struct aclk_database_worker_config *next;
@@ -236,7 +188,6 @@ extern int aclk_database_enq_cmd_noblock(struct aclk_database_worker_config *wc,
188
extern void aclk_database_enq_cmd(struct aclk_database_worker_config *wc, struct aclk_database_cmd *cmd);
189
extern void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id);
190
int aclk_worker_enq_cmd(char *node_id, struct aclk_database_cmd *cmd);
239
-void aclk_data_rotated(void);
191
void sql_aclk_sync_init(void);
192
void sql_check_aclk_table_list(struct aclk_database_worker_config *wc);
193
void sql_delete_aclk_table_list(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
database/sqlite/sqlite_aclk_chart.c
deleted
-1304
@@ -1,1304 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-#include "sqlite_functions.h"
4
-#include "sqlite_aclk_chart.h"
5
-
6
-#ifdef ENABLE_ACLK
7
-#include "../../aclk/aclk_charts_api.h"
8
-#include "../../aclk/aclk.h"
9
-
10
-static inline int
11
-sql_queue_chart_payload(struct aclk_database_worker_config *wc, void *data, enum aclk_database_opcode opcode)
12
-{
13
- int rc;
14
- if (unlikely(!wc))
15
- return 1;
16
-
17
- struct aclk_database_cmd cmd;
18
- memset(&cmd, 0, sizeof(cmd));
19
- cmd.opcode = opcode;
20
- cmd.data = data;
21
- rc = aclk_database_enq_cmd_noblock(wc, &cmd);
22
- return rc;
23
-}
24
-
25
-static time_t payload_sent(char *uuid_str, uuid_t *uuid, void *payload, size_t payload_size)
26
-{
27
- static __thread sqlite3_stmt *res = NULL;
28
- int rc;
29
- time_t send_status = 0;
30
-
31
- if (unlikely(!res)) {
32
- char sql[ACLK_SYNC_QUERY_SIZE];
33
- snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, "SELECT acl.date_submitted FROM aclk_chart_latest_%s acl, aclk_chart_payload_%s acp "
34
- "WHERE acl.unique_id = acp.unique_id AND acl.uuid = @uuid AND acp.payload = @payload;",
35
- uuid_str, uuid_str);
36
- rc = prepare_statement(db_meta, sql, &res);
37
- if (rc != SQLITE_OK) {
38
- error_report("Failed to prepare statement to check payload data on %s", sql);
39
- return 0;
40
- }
41
- }
42
-
43
- rc = sqlite3_bind_blob(res, 1, uuid, sizeof(*uuid), SQLITE_STATIC);
44
- if (unlikely(rc != SQLITE_OK))
45
- goto bind_fail;
46
-
47
- rc = sqlite3_bind_blob(res, 2, payload, payload_size, SQLITE_STATIC);
48
- if (unlikely(rc != SQLITE_OK))
49
- goto bind_fail;
50
-
51
- while (sqlite3_step_monitored(res) == SQLITE_ROW) {
52
- send_status = (time_t) sqlite3_column_int64(res, 0);
53
- }
54
-
55
-bind_fail:
56
- if (unlikely(sqlite3_reset(res) != SQLITE_OK))
57
- error_report("Failed to reset statement in check payload, rc = %d", rc);
58
- return send_status;
59
-}
60
-
61
-static int aclk_add_chart_payload(
62
- struct aclk_database_worker_config *wc,
63
- uuid_t *uuid,
64
- char *claim_id,
65
- ACLK_PAYLOAD_TYPE payload_type,
66
- void *payload,
67
- size_t payload_size,
68
- time_t *send_status,
69
- int check_sent)
70
-{
71
- static __thread sqlite3_stmt *res_chart = NULL;
72
- int rc;
73
- time_t date_submitted;
74
-
75
- if (unlikely(!payload))
76
- return 0;
77
-
78
- if (check_sent) {
79
- date_submitted = payload_sent(wc->uuid_str, uuid, payload, payload_size);
80
- if (send_status)
81
- *send_status = date_submitted;
82
- if (date_submitted)
83
- return 0;
84
- }
85
-
86
- if (unlikely(!res_chart)) {
87
- char sql[ACLK_SYNC_QUERY_SIZE];
88
- snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1,
89
- "INSERT INTO aclk_chart_payload_%s (unique_id, uuid, claim_id, date_created, type, payload) " \
90
- "VALUES (@unique_id, @uuid, @claim_id, unixepoch(), @type, @payload);", wc->uuid_str);
91
- rc = prepare_statement(db_meta, sql, &res_chart);
92
- if (rc != SQLITE_OK) {
93
- error_report("Failed to prepare statement to store chart payload data");
94
- return 1;
95
- }
96
- }
97
-
98
- uuid_t unique_uuid;
99
- uuid_generate(unique_uuid);
100
-
101
- uuid_t claim_uuid;
102
- if (uuid_parse(claim_id, claim_uuid))
103
- return 1;
104
-
105
- rc = sqlite3_bind_blob(res_chart, 1, &unique_uuid, sizeof(unique_uuid), SQLITE_STATIC);
106
- if (unlikely(rc != SQLITE_OK))
107
- goto bind_fail;
108
-
109
- rc = sqlite3_bind_blob(res_chart, 2, uuid, sizeof(*uuid), SQLITE_STATIC);
110
- if (unlikely(rc != SQLITE_OK))
111
- goto bind_fail;
112
-
113
- rc = sqlite3_bind_blob(res_chart, 3, &claim_uuid, sizeof(claim_uuid), SQLITE_STATIC);
114
- if (unlikely(rc != SQLITE_OK))
115
- goto bind_fail;
116
-
117
- rc = sqlite3_bind_int(res_chart, 4, payload_type);
118
- if (unlikely(rc != SQLITE_OK))
119
- goto bind_fail;
120
-
121
- rc = sqlite3_bind_blob(res_chart, 5, payload, payload_size, SQLITE_STATIC);
122
- if (unlikely(rc != SQLITE_OK))
123
- goto bind_fail;
124
-
125
- rc = execute_insert(res_chart);
126
- if (unlikely(rc != SQLITE_DONE))
127
- error_report("Failed store chart payload event, rc = %d", rc);
128
- else {
129
- wc->chart_payload_count++;
130
- time_t now = now_realtime_sec();
131
- if (wc->rotation_after > now && wc->rotation_after < now + ACLK_DATABASE_ROTATION_DELAY)
132
- wc->rotation_after = now + ACLK_DATABASE_ROTATION_DELAY;
133
- }
134
-
135
-bind_fail:
136
- if (unlikely(sqlite3_reset(res_chart) != SQLITE_OK))
137
- error_report("Failed to reset statement in store chart payload, rc = %d", rc);
138
- return (rc != SQLITE_DONE);
139
-}
140
-
141
-int aclk_add_chart_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
142
-{
143
- int rc = 0;
144
- CHECK_SQLITE_CONNECTION(db_meta);
145
-
146
- char *claim_id = get_agent_claimid();
147
-
148
- RRDSET *st = cmd.data;
149
-
150
- if (likely(claim_id)) {
151
- struct chart_instance_updated chart_payload;
152
- memset(&chart_payload, 0, sizeof(chart_payload));
153
- chart_payload.config_hash = get_str_from_uuid(&st->hash_uuid);
154
- chart_payload.update_every = st->update_every;
155
- chart_payload.memory_mode = st->rrd_memory_mode;
156
- chart_payload.name = (char *)rrdset_name(st);
157
- chart_payload.node_id = wc->node_id;
158
- chart_payload.claim_id = claim_id;
159
- chart_payload.id = strdupz(rrdset_id(st));
160
-
161
- chart_payload.chart_labels = rrdlabels_create();
162
- rrdlabels_copy(chart_payload.chart_labels, st->rrdlabels);
163
-
164
- size_t size;
165
- char *payload = generate_chart_instance_updated(&size, &chart_payload);
166
- if (likely(payload))
167
- rc = aclk_add_chart_payload(wc, &st->chart_uuid, claim_id, ACLK_PAYLOAD_CHART, (void *) payload, size, NULL, 1);
168
- freez(payload);
169
- chart_instance_updated_destroy(&chart_payload);
170
- }
171
- return rc;
172
-}
173
-
174
-static inline int aclk_upd_dimension_event(struct aclk_database_worker_config *wc, char *claim_id, uuid_t *dim_uuid,
175
- const char *dim_id, const char *dim_name, const char *chart_type_id, time_t first_time, time_t last_time,
176
- time_t *send_status)
177
-{
178
- int rc = 0;
179
- size_t size;
180
-
181
- if (unlikely(!dim_uuid || !dim_id || !dim_name || !chart_type_id))
182
- return 0;
183
-
184
- struct chart_dimension_updated dim_payload;
185
- memset(&dim_payload, 0, sizeof(dim_payload));
186
-
187
-#ifdef NETDATA_INTERNAL_CHECKS
188
- if (!first_time)
189
- info("Host %s (node %s) deleting dimension id=[%s] name=[%s] chart=[%s]",
190
- wc->host_guid, wc->node_id, dim_id, dim_name, chart_type_id);
191
- if (last_time)
192
- info("Host %s (node %s) stopped collecting dimension id=[%s] name=[%s] chart=[%s] %ld seconds ago at %ld",
193
- wc->host_guid, wc->node_id, dim_id, dim_name, chart_type_id, now_realtime_sec() - last_time, last_time);
194
-#endif
195
-
196
- dim_payload.node_id = wc->node_id;
197
- dim_payload.claim_id = claim_id;
198
- dim_payload.name = dim_name;
199
- dim_payload.id = dim_id;
200
- dim_payload.chart_id = chart_type_id;
201
- dim_payload.created_at.tv_sec = first_time;
202
- dim_payload.last_timestamp.tv_sec = last_time;
203
- char *payload = generate_chart_dimension_updated(&size, &dim_payload);
204
- if (likely(payload))
205
- rc = aclk_add_chart_payload(wc, dim_uuid, claim_id, ACLK_PAYLOAD_DIMENSION, (void *)payload, size, send_status, 1);
206
- freez(payload);
207
- return rc;
208
-}
209
-
210
-void aclk_process_dimension_deletion(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
211
-{
212
- int rc = 0;
213
- sqlite3_stmt *res = NULL;
214
-
215
- if (!aclk_connected)
216
- return;
217
-
218
- if (unlikely(!db_meta))
219
- return;
220
-
221
- uuid_t host_id;
222
- if (uuid_parse(wc->host_guid, host_id))
223
- return;
224
-
225
- char *claim_id = get_agent_claimid();
226
- if (!claim_id)
227
- return;
228
-
229
- rc = sqlite3_prepare_v2(
230
- db_meta,
231
- "DELETE FROM dimension_delete where host_id = @host_id "
232
- "RETURNING dimension_id, dimension_name, chart_type_id, dim_id LIMIT 10;",
233
- -1,
234
- &res,
235
- 0);
236
-
237
- if (rc != SQLITE_OK) {
238
- error_report("Failed to prepare statement when trying to delete dimension deletes");
239
- freez(claim_id);
240
- return;
241
- }
242
-
243
- rc = sqlite3_bind_blob(res, 1, &host_id, sizeof(host_id), SQLITE_STATIC);
244
- if (unlikely(rc != SQLITE_OK))
245
- goto bind_fail;
246
-
247
- unsigned count = 0;
248
- while (sqlite3_step_monitored(res) == SQLITE_ROW) {
249
- (void) aclk_upd_dimension_event(
250
- wc,
251
- claim_id,
252
- (uuid_t *)sqlite3_column_text(res, 3),
253
- (const char *)sqlite3_column_text(res, 0),
254
- (const char *)sqlite3_column_text(res, 1),
255
- (const char *)sqlite3_column_text(res, 2),
256
- 0,
257
- 0,
258
- NULL);
259
- count++;
260
- }
261
-
262
- if (count) {
263
- memset(&cmd, 0, sizeof(cmd));
264
- cmd.opcode = ACLK_DATABASE_DIM_DELETION;
265
- if (aclk_database_enq_cmd_noblock(wc, &cmd))
266
- info("Failed to queue a dimension deletion message");
267
- }
268
-
269
-bind_fail:
270
- rc = sqlite3_finalize(res);
271
- if (unlikely(rc != SQLITE_OK))
272
- error_report("Failed to finalize statement when adding dimension deletion events, rc = %d", rc);
273
- freez(claim_id);
274
- return;
275
-}
276
-
277
-int aclk_add_dimension_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
278
-{
279
- int rc = 1;
280
- CHECK_SQLITE_CONNECTION(db_meta);
281
-
282
- struct aclk_chart_dimension_data *aclk_cd_data = cmd.data;
283
-
284
- char *claim_id = get_agent_claimid();
285
- if (!claim_id)
286
- goto cleanup;
287
-
288
- rc = aclk_add_chart_payload(wc, &aclk_cd_data->uuid, claim_id, ACLK_PAYLOAD_DIMENSION,
289
- (void *) aclk_cd_data->payload, aclk_cd_data->payload_size, NULL, aclk_cd_data->check_payload);
290
-
291
- freez(claim_id);
292
-cleanup:
293
- freez(aclk_cd_data->payload);
294
- freez(aclk_cd_data);
295
- return rc;
296
-}
297
-
298
-void aclk_send_chart_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
299
-{
300
- int rc;
301
-
302
- wc->chart_pending = 0;
303
- if (unlikely(!wc->chart_updates)) {
304
- log_access(
305
- "ACLK STA [%s (%s)]: Ignoring chart push event, updates have been turned off for this node.",
306
- wc->node_id,
307
- wc->host ? rrdhost_hostname(wc->host) : "N/A");
308
- return;
309
- }
310
-
311
- char *claim_id = get_agent_claimid();
312
- if (unlikely(!claim_id))
313
- return;
314
-
315
- uuid_t claim_uuid;
316
- if (uuid_parse(claim_id, claim_uuid))
317
- return;
318
-
319
- int limit = cmd.count > 0 ? cmd.count : 1;
320
-
321
- uint64_t first_sequence;
322
- uint64_t last_sequence;
323
- time_t last_timestamp = 0;
324
-
325
- char sql[ACLK_SYNC_QUERY_SIZE];
326
- static __thread sqlite3_stmt *res = NULL;
327
-
328
- if (unlikely(!res)) {
329
- snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1,"SELECT ac.sequence_id, acp.payload, ac.date_created, ac.type, ac.uuid " \
330
- "FROM aclk_chart_%s ac, aclk_chart_payload_%s acp " \
331
- "WHERE ac.date_submitted IS NULL AND ac.unique_id = acp.unique_id AND ac.update_count > 0 " \
332
- "AND acp.claim_id = @claim_id ORDER BY ac.sequence_id ASC LIMIT %d;", wc->uuid_str, wc->uuid_str, limit);
333
- rc = prepare_statement(db_meta, sql, &res);
334
- if (rc != SQLITE_OK) {
335
- error_report("Failed to prepare statement when trying to send a chart update via ACLK");
336
- freez(claim_id);
337
- return;
338
- }
339
- }
340
-
341
- rc = sqlite3_bind_blob(res, 1, claim_uuid, sizeof(claim_uuid), SQLITE_STATIC);
342
- if (unlikely(rc != SQLITE_OK))
343
- goto bind_fail;
344
-
345
- char **payload_list = callocz(limit + 1, sizeof(char *));
346
- size_t *payload_list_size = callocz(limit + 1, sizeof(size_t));
347
- size_t *payload_list_max_size = callocz(limit + 1, sizeof(size_t));
348
- struct aclk_message_position *position_list = callocz(limit + 1, sizeof(*position_list));
349
- int *is_dim = callocz(limit + 1, sizeof(*is_dim));
350
-
351
- int loop = cmd.param1;
352
-
353
- uint64_t start_sequence_id = wc->chart_sequence_id;
354
-
355
- while (loop > 0) {
356
- uint64_t previous_sequence_id = wc->chart_sequence_id;
357
- int count = 0;
358
- first_sequence = 0;
359
- last_sequence = 0;
360
- while (count < limit && sqlite3_step_monitored(res) == SQLITE_ROW) {
361
- size_t payload_size = sqlite3_column_bytes(res, 1);
362
- if (payload_list_max_size[count] < payload_size) {
363
- freez(payload_list[count]);
364
- payload_list_max_size[count] = payload_size;
365
- payload_list[count] = mallocz(payload_size);
366
- }
367
- payload_list_size[count] = payload_size;
368
- memcpy(payload_list[count], sqlite3_column_blob(res, 1), payload_size);
369
- position_list[count].sequence_id = (uint64_t)sqlite3_column_int64(res, 0);
370
- position_list[count].previous_sequence_id = previous_sequence_id;
371
- position_list[count].seq_id_creation_time.tv_sec = sqlite3_column_int64(res, 2);
372
- position_list[count].seq_id_creation_time.tv_usec = 0;
373
- if (!first_sequence)
374
- first_sequence = position_list[count].sequence_id;
375
- last_sequence = position_list[count].sequence_id;
376
- last_timestamp = position_list[count].seq_id_creation_time.tv_sec;
377
- previous_sequence_id = last_sequence;
378
- is_dim[count] = sqlite3_column_int(res, 3) > 0;
379
- count++;
380
- if (wc->chart_payload_count)
381
- wc->chart_payload_count--;
382
- }
383
- freez(payload_list[count]);
384
- payload_list_max_size[count] = 0;
385
- payload_list[count] = NULL;
386
-
387
- rc = sqlite3_reset(res);
388
- if (unlikely(rc != SQLITE_OK))
389
- error_report("Failed to reset statement when pushing chart events, rc = %d", rc);
390
-
391
- if (likely(first_sequence)) {
392
-
393
- db_lock();
394
- snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, "UPDATE aclk_chart_%s SET status = NULL, date_submitted=unixepoch() "
395
- "WHERE date_submitted IS NULL AND sequence_id BETWEEN %" PRIu64 " AND %" PRIu64 ";",
396
- wc->uuid_str, first_sequence, last_sequence);
397
- db_execute(sql);
398
- snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, "INSERT OR REPLACE INTO aclk_chart_latest_%s (uuid, unique_id, date_submitted) "
399
- " SELECT uuid, unique_id, date_submitted FROM aclk_chart_%s s "
400
- " WHERE date_submitted IS NOT NULL AND sequence_id BETWEEN %" PRIu64 " AND %" PRIu64
401
- " ;",
402
- wc->uuid_str, wc->uuid_str, first_sequence, last_sequence);
403
- db_execute(sql);
404
- db_unlock();
405
-
406
- aclk_chart_inst_and_dim_update(payload_list, payload_list_size, is_dim, position_list, wc->batch_id);
407
- log_access(
408
- "ACLK RES [%s (%s)]: CHARTS SENT from %" PRIu64 " to %" PRIu64 " batch=%" PRIu64,
409
- wc->node_id,
410
- wc->hostname ? wc->hostname : "N/A",
411
- first_sequence,
412
- last_sequence,
413
- wc->batch_id);
414
- wc->chart_sequence_id = last_sequence;
415
- wc->chart_timestamp = last_timestamp;
416
- } else
417
- break;
418
- --loop;
419
- }
420
-
421
- if (start_sequence_id != wc->chart_sequence_id) {
422
- time_t now = now_realtime_sec();
423
- if (wc->rotation_after > now && wc->rotation_after < now + ACLK_DATABASE_ROTATION_DELAY)
424
- wc->rotation_after = now + ACLK_DATABASE_ROTATION_DELAY;
425
- } else {
426
- wc->chart_payload_count = sql_get_pending_count(wc);
427
- if (!wc->chart_payload_count)
428
- log_access(
429
- "ACLK STA [%s (%s)]: Sync of charts and dimensions done in %ld seconds.",
430
- wc->node_id,
431
- wc->hostname ? wc->hostname : "N/A",
432
- now_realtime_sec() - wc->startup_time);
433
- }
434
-
435
- for (int i = 0; i <= limit; ++i)
436
- freez(payload_list[i]);
437
-
438
- freez(payload_list);
439
- freez(payload_list_size);
440
- freez(payload_list_max_size);
441
- freez(position_list);
442
- freez(is_dim);
443
-
444
-bind_fail:
445
- rc = sqlite3_reset(res);
446
- if (unlikely(rc != SQLITE_OK))
447
- error_report("Failed to reset statement when pushing chart events, rc = %d", rc);
448
-
449
- freez(claim_id);
450
- return;
451
-}
452
-
453
-// Push one chart config to the cloud
454
-int aclk_send_chart_config(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
455
-{
456
- UNUSED(wc);
457
-
458
- CHECK_SQLITE_CONNECTION(db_meta);
459
-
460
- sqlite3_stmt *res = NULL;
461
- int rc = 0;
462
-
463
- char *hash_id = (char *) cmd.data_param;
464
-
465
- uuid_t hash_uuid;
466
- rc = uuid_parse(hash_id, hash_uuid);
467
-
468
- if (unlikely(rc)) {
469
- freez((char *) cmd.data_param);
470
- return 1;
471
- }
472
-
473
- BUFFER *sql = buffer_create(1024);
474
- buffer_sprintf(sql, "SELECT type, family, context, title, priority, plugin, module, unit, chart_type " \
475
- "FROM chart_hash WHERE hash_id = @hash_id;");
476
-
477
- rc = sqlite3_prepare_v2(db_meta, buffer_tostring(sql), -1, &res, 0);
478
- if (rc != SQLITE_OK) {
479
- error_report("Failed to prepare statement when trying to fetch a chart hash configuration");
480
- goto fail;
481
- }
482
-
483
- rc = sqlite3_bind_blob(res, 1, &hash_uuid , sizeof(hash_uuid), SQLITE_STATIC);
484
- if (unlikely(rc != SQLITE_OK))
485
- goto bind_fail;
486
-
487
- struct chart_config_updated chart_config;
488
- chart_config.config_hash = NULL;
489
-
490
- while (sqlite3_step_monitored(res) == SQLITE_ROW) {
491
- chart_config.type = strdupz((char *)sqlite3_column_text(res, 0));
492
- chart_config.family = strdupz((char *)sqlite3_column_text(res, 1));
493
- chart_config.context = strdupz((char *)sqlite3_column_text(res, 2));
494
- chart_config.title = strdupz((char *)sqlite3_column_text(res, 3));
495
- chart_config.priority = sqlite3_column_int64(res, 4);
496
- chart_config.plugin = strdupz((char *)sqlite3_column_text(res, 5));
497
- chart_config.module = sqlite3_column_bytes(res, 6) > 0 ? strdupz((char *)sqlite3_column_text(res, 6)) : NULL;
498
- chart_config.chart_type = (RRDSET_TYPE) sqlite3_column_int(res,8);
499
- chart_config.units = strdupz((char *)sqlite3_column_text(res, 7));
500
- chart_config.config_hash = strdupz(hash_id);
501
- }
502
-
503
- if (likely(chart_config.config_hash)) {
504
- log_access(
505
- "ACLK REQ [%s (%s)]: Sending chart config for %s.",
506
- wc->node_id,
507
- wc->host ? rrdhost_hostname(wc->host) : "N/A",
508
- hash_id);
509
- aclk_chart_config_updated(&chart_config, 1);
510
- destroy_chart_config_updated(&chart_config);
511
- } else
512
- log_access(
513
- "ACLK STA [%s (%s)]: Chart config for %s not found.",
514
- wc->node_id,
515
- wc->host ? rrdhost_hostname(wc->host) : "N/A",
516
- hash_id);
517
-
518
-bind_fail:
519
- rc = sqlite3_finalize(res);
520
- if (unlikely(rc != SQLITE_OK))
521
- error_report("Failed to reset statement when pushing chart config hash, rc = %d", rc);
522
-fail:
523
- freez((char *)cmd.data_param);
524
- buffer_free(sql);
525
- return rc;
526
-}
527
-
528
-void aclk_receive_chart_ack(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
529
-{
530
- int rc;
531
- sqlite3_stmt *res = NULL;
532
-
533
- char sql[ACLK_SYNC_QUERY_SIZE];
534
-
535
- snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1,"UPDATE aclk_chart_%s SET date_updated=unixepoch() WHERE sequence_id <= @sequence_id "
536
- "AND date_submitted IS NOT NULL AND date_updated IS NULL;", wc->uuid_str);
537
-
538
- rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
539
- if (rc != SQLITE_OK) {
540
- error_report("Failed to prepare statement to ack chart sequence ids");
541
- return;
542
- }
543
-
544
- rc = sqlite3_bind_int64(res, 1, (uint64_t) cmd.param1);
545
- if (unlikely(rc != SQLITE_OK))
546
- goto bind_fail;
547
-
548
- rc = execute_insert(res);
549
- if (rc != SQLITE_DONE)
550
- error_report("Failed to ACK sequence id, rc = %d", rc);
551
- else
552
- log_access(
553
- "ACLK STA [%s (%s)]: CHARTS ACKNOWLEDGED IN THE DATABASE UP TO %" PRIu64,
554
- wc->node_id,
555
- wc->host ? rrdhost_hostname(wc->host) : "N/A",
556
- cmd.param1);
557
-
558
-bind_fail:
559
- if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
560
- error_report("Failed to finalize statement to ACK older sequence ids, rc = %d", rc);
561
- return;
562
-}
563
-
564
-void aclk_receive_chart_reset(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
565
-{
566
- BUFFER *sql = buffer_create(1024);
567
- buffer_sprintf(
568
- sql,
569
- "UPDATE aclk_chart_%s SET status = NULL, date_submitted = NULL WHERE sequence_id >= %" PRIu64 ";",
570
- wc->uuid_str,
571
- cmd.param1);
572
- db_execute(buffer_tostring(sql));
573
- if (cmd.param1 == 1) {
574
- buffer_flush(sql);
575
- log_access("ACLK REQ [%s (%s)]: Received chart full resync.", wc->node_id, wc->hostname ? wc->hostname: "N/A");
576
- buffer_sprintf(sql, "DELETE FROM aclk_chart_payload_%s; DELETE FROM aclk_chart_%s; " \
577
- "DELETE FROM aclk_chart_latest_%s;", wc->uuid_str, wc->uuid_str, wc->uuid_str);
578
- db_lock();
579
-
580
- db_execute("BEGIN TRANSACTION;");
581
- db_execute(buffer_tostring(sql));
582
- db_execute("COMMIT TRANSACTION;");
583
-
584
- db_unlock();
585
- wc->chart_sequence_id = 0;
586
- wc->chart_timestamp = 0;
587
- wc->chart_payload_count = 0;
588
-
589
- RRDHOST *host = wc->host;
590
- if (likely(host)) {
591
- RRDSET *st;
592
- rrdset_foreach_read(st, host) {
593
- rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
594
- RRDDIM *rd;
595
- rrddim_foreach_read(rd, st) {
596
- rrddim_flag_clear(rd, RRDDIM_FLAG_ACLK);
597
- rd->aclk_live_status = (rd->aclk_live_status == 0);
598
- }
599
- rrddim_foreach_done(rd);
600
- }
601
- rrdset_foreach_done(st);
602
- }
603
- else
604
- error_report("ACLK synchronization thread for %s is not linked to HOST", wc->host_guid);
605
-
606
- }
607
- else {
608
- log_access(
609
- "ACLK STA [%s (%s)]: RESTARTING CHART SYNC FROM SEQUENCE %" PRIu64,
610
- wc->node_id,
611
- wc->hostname ? wc->hostname : "N/A",
612
- cmd.param1);
613
- wc->chart_payload_count = sql_get_pending_count(wc);
614
- sql_get_last_chart_sequence(wc);
615
- }
616
- buffer_free(sql);
617
- wc->chart_updates = 1;
618
- return;
619
-}
620
-
621
-//
622
-// Functions called directly from ACLK threads and will queue commands
623
-//
624
-void aclk_get_chart_config(char **hash_id)
625
-{
626
- struct aclk_database_worker_config *wc = (struct aclk_database_worker_config *)localhost->dbsync_worker;
627
-
628
- if (unlikely(!wc || !hash_id))
629
- return;
630
-
631
- struct aclk_database_cmd cmd;
632
- memset(&cmd, 0, sizeof(cmd));
633
- cmd.opcode = ACLK_DATABASE_PUSH_CHART_CONFIG;
634
- for (int i = 0; hash_id[i]; ++i) {
635
- // TODO: Verify that we have a valid hash_id
636
- log_access(
637
- "ACLK REQ [%s (%s)]: Request %d for chart config with hash %s received.",
638
- wc->node_id,
639
- wc->host ? rrdhost_hostname(wc->host) : "N/A",
640
- i,
641
- hash_id[i]);
642
- cmd.data_param = (void *)strdupz(hash_id[i]);
643
- aclk_database_enq_cmd(wc, &cmd);
644
- }
645
- return;
646
-}
647
-
648
-// Send a command to a node_id
649
-// Need to discover the thread that will handle the request
650
-// if thread not in active hosts, then try to find in the queue
651
-static void aclk_submit_param_command(char *node_id, enum aclk_database_opcode aclk_command, uint64_t param)
652
-{
653
- if (unlikely(!node_id))
654
- return;
655
-
656
- struct aclk_database_worker_config *wc = NULL;
657
- struct aclk_database_cmd cmd;
658
- memset(&cmd, 0, sizeof(cmd));
659
- cmd.opcode = aclk_command;
660
- cmd.param1 = param;
661
-
662
- RRDHOST *host = find_host_by_node_id(node_id);
663
- if (likely(host))
664
- wc = (struct aclk_database_worker_config *)host->dbsync_worker;
665
-
666
- if (wc)
667
- aclk_database_enq_cmd(wc, &cmd);
668
- else {
669
- if (aclk_worker_enq_cmd(node_id, &cmd))
670
- log_access("ACLK STA [%s (N/A)]: ACLK synchronization thread is not active.", node_id);
671
- }
672
- return;
673
-}
674
-
675
-void aclk_ack_chart_sequence_id(char *node_id, uint64_t last_sequence_id)
676
-{
677
- if (unlikely(!node_id))
678
- return;
679
-
680
- char *hostname = get_hostname_by_node_id(node_id);
681
- log_access("ACLK REQ [%s (%s)]: CHARTS ACKNOWLEDGED upto %" PRIu64, node_id, hostname ? hostname : "N/A",
682
- last_sequence_id);
683
- freez(hostname);
684
- aclk_submit_param_command(node_id, ACLK_DATABASE_CHART_ACK, last_sequence_id);
685
- return;
686
-}
687
-
688
-// Start streaming charts / dimensions for node_id
689
-void aclk_start_streaming(char *node_id, uint64_t sequence_id, time_t created_at, uint64_t batch_id)
690
-{
691
- UNUSED(created_at);
692
- if (unlikely(!node_id))
693
- return;
694
-
695
- uuid_t node_uuid;
696
- if (uuid_parse(node_id, node_uuid)) {
697
- log_access("ACLK REQ [%s (N/A)]: CHARTS STREAM ignored, invalid node id", node_id);
698
- return;
699
- }
700
-
701
- struct aclk_database_worker_config *wc = find_inactive_wc_by_node_id(node_id);
702
- rrd_rdlock();
703
- RRDHOST *host;
704
- rrdhost_foreach_read(host) {
705
- if (wc || (host->node_id && !(uuid_compare(*host->node_id, node_uuid)))) {
706
- rrd_unlock();
707
- if (!wc)
708
- wc = (struct aclk_database_worker_config *)host->dbsync_worker ?
709
- (struct aclk_database_worker_config *)host->dbsync_worker :
710
- (struct aclk_database_worker_config *)find_inactive_wc_by_node_id(node_id);
711
- if (likely(wc)) {
712
- wc->chart_reset_count++;
713
- __sync_synchronize();
714
- wc->chart_updates = 0;
715
- wc->batch_id = batch_id;
716
- __sync_synchronize();
717
- wc->batch_created = now_realtime_sec();
718
- log_access(
719
- "ACLK REQ [%s (%s)]: CHARTS STREAM from %"PRIu64" (LOCAL %"PRIu64") t=%ld resets=%d" ,
720
- wc->node_id,
721
- wc->hostname ? wc->hostname : "N/A",
722
- sequence_id + 1,
723
- wc->chart_sequence_id,
724
- wc->chart_timestamp,
725
- wc->chart_reset_count);
726
- if (sequence_id > wc->chart_sequence_id || wc->chart_reset_count > 10) {
727
- log_access(
728
- "ACLK RES [%s (%s)]: CHARTS FULL RESYNC REQUEST "
729
- "remote_seq=%" PRIu64 " local_seq=%" PRIu64 " resets=%d ",
730
- wc->node_id,
731
- wc->hostname ? wc->hostname : "N/A",
732
- sequence_id,
733
- wc->chart_sequence_id,
734
- wc->chart_reset_count);
735
-
736
- chart_reset_t chart_reset;
737
- chart_reset.claim_id = get_agent_claimid();
738
- if (chart_reset.claim_id) {
739
- chart_reset.node_id = node_id;
740
- chart_reset.reason = SEQ_ID_NOT_EXISTS;
741
- aclk_chart_reset(chart_reset);
742
- freez(chart_reset.claim_id);
743
- wc->chart_reset_count = -1;
744
- }
745
- } else {
746
- struct aclk_database_cmd cmd;
747
- memset(&cmd, 0, sizeof(cmd));
748
- // TODO: handle timestamp
749
- if (sequence_id < wc->chart_sequence_id ||
750
- !sequence_id) { // || created_at != wc->chart_timestamp) {
751
- log_access(
752
- "ACLK REQ [%s (%s)]: CHART RESET from %" PRIu64 " t=%ld batch=%" PRIu64,
753
- wc->node_id,
754
- wc->hostname ? wc->hostname : "N/A",
755
- sequence_id + 1,
756
- wc->chart_timestamp,
757
- wc->batch_id);
758
- cmd.opcode = ACLK_DATABASE_RESET_CHART;
759
- cmd.param1 = sequence_id + 1;
760
- cmd.completion = NULL;
761
- aclk_database_enq_cmd(wc, &cmd);
762
- } else {
763
- wc->chart_reset_count = 0;
764
- wc->chart_updates = 1;
765
- }
766
- }
767
- } else {
768
- log_access("ACLK STA [%s (%s)]: ACLK synchronization thread is not active.", node_id, wc->hostname ? wc->hostname : "N/A");
769
- }
770
- return;
771
- }
772
- }
773
- rrd_unlock();
774
-}
775
-
776
-#define SQL_SELECT_HOST_MEMORY_MODE "SELECT memory_mode FROM chart WHERE host_id = @host_id LIMIT 1;"
777
-
778
-static RRD_MEMORY_MODE sql_get_host_memory_mode(uuid_t *host_id)
779
-{
780
- int rc;
781
-
782
- RRD_MEMORY_MODE memory_mode = RRD_MEMORY_MODE_RAM;
783
- sqlite3_stmt *res = NULL;
784
-
785
- rc = sqlite3_prepare_v2(db_meta, SQL_SELECT_HOST_MEMORY_MODE, -1, &res, 0);
786
-
787
- if (unlikely(rc != SQLITE_OK)) {
788
- error_report("Failed to prepare statement to read host memory mode");
789
- return memory_mode;
790
- }
791
-
792
- rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);
793
- if (unlikely(rc != SQLITE_OK)) {
794
- error_report("Failed to bind host parameter to fetch host memory mode");
795
- goto failed;
796
- }
797
-
798
- while (sqlite3_step_monitored(res) == SQLITE_ROW) {
799
- memory_mode = (RRD_MEMORY_MODE)sqlite3_column_int(res, 0);
800
- }
801
-
802
-failed:
803
- rc = sqlite3_finalize(res);
804
- if (unlikely(rc != SQLITE_OK))
805
- error_report("Failed to finalize the prepared statement when reading host memory mode");
806
- return memory_mode;
807
-}
808
-
809
-#define SELECT_HOST_DIMENSION_LIST \
810
- "SELECT d.dim_id, c.update_every, c.type||'.'||c.id, d.id, d.name FROM chart c, dimension d " \
811
- "WHERE d.chart_id = c.chart_id AND c.host_id = @host_id ORDER BY c.update_every ASC;"
812
-
813
-#define SELECT_HOST_CHART_LIST \
814
- "SELECT distinct h.host_id, c.update_every, c.type||'.'||c.id FROM chart c, host h " \
815
- "WHERE c.host_id = h.host_id AND c.host_id = @host_id ORDER BY c.update_every ASC;"
816
-
817
-void aclk_update_retention(struct aclk_database_worker_config *wc)
818
-{
819
- int rc;
820
-
821
- if (!aclk_connected)
822
- return;
823
-
824
- if (wc->host && rrdhost_flag_check(wc->host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS)) {
825
- internal_error(true, "Skipping aclk_update_retention for host %s because context streaming is enabled", rrdhost_hostname(wc->host));
826
- return;
827
- }
828
-
829
- char *claim_id = get_agent_claimid();
830
- if (unlikely(!claim_id))
831
- return;
832
-
833
- sqlite3_stmt *res = NULL;
834
- RRD_MEMORY_MODE memory_mode;
835
-
836
- uuid_t host_uuid;
837
- rc = uuid_parse(wc->host_guid, host_uuid);
838
- if (unlikely(rc)) {
839
- freez(claim_id);
840
- return;
841
- }
842
-
843
- if (wc->host)
844
- memory_mode = wc->host->rrd_memory_mode;
845
- else
846
- memory_mode = sql_get_host_memory_mode(&host_uuid);
847
-
848
- if (memory_mode == RRD_MEMORY_MODE_DBENGINE)
849
- rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_DIMENSION_LIST, -1, &res, 0);
850
- else
851
- rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_CHART_LIST, -1, &res, 0);
852
-
853
- if (unlikely(rc != SQLITE_OK)) {
854
- error_report("Failed to prepare statement to fetch host dimensions");
855
- freez(claim_id);
856
- return;
857
- }
858
-
859
- rc = sqlite3_bind_blob(res, 1, &host_uuid, sizeof(host_uuid), SQLITE_STATIC);
860
- if (unlikely(rc != SQLITE_OK)) {
861
- error_report("Failed to bind host parameter to fetch host dimensions");
862
- goto failed;
863
- }
864
-
865
- time_t start_time = LONG_MAX;
866
- time_t first_entry_t;
867
- time_t last_entry_t;
868
- uint32_t update_every = 0;
869
- uint32_t dimension_update_count = 0;
870
- uint32_t total_checked = 0;
871
- uint32_t total_deleted= 0;
872
- uint32_t total_stopped= 0;
873
- time_t send_status;
874
-
875
- struct retention_updated rotate_data;
876
-
877
- memset(&rotate_data, 0, sizeof(rotate_data));
878
-
879
- int max_intervals = 32;
880
-
881
- rotate_data.interval_duration_count = 0;
882
- rotate_data.interval_durations = callocz(max_intervals, sizeof(*rotate_data.interval_durations));
883
-
884
- now_realtime_timeval(&rotate_data.rotation_timestamp);
885
- rotate_data.memory_mode = memory_mode;
886
- rotate_data.claim_id = claim_id;
887
- rotate_data.node_id = strdupz(wc->node_id);
888
-
889
- time_t now = now_realtime_sec();
890
- while (sqlite3_step_monitored(res) == SQLITE_ROW && dimension_update_count < ACLK_MAX_DIMENSION_CLEANUP) {
891
- if (unlikely(netdata_exit))
892
- break;
893
- if (!update_every || update_every != (uint32_t)sqlite3_column_int(res, 1)) {
894
- if (update_every) {
895
- debug(D_ACLK_SYNC, "Update %s for %u oldest time = %ld", wc->host_guid, update_every, start_time);
896
- if (start_time == LONG_MAX)
897
- rotate_data.interval_durations[rotate_data.interval_duration_count].retention = 0;
898
- else
899
- rotate_data.interval_durations[rotate_data.interval_duration_count].retention =
900
- rotate_data.rotation_timestamp.tv_sec - start_time;
901
- rotate_data.interval_duration_count++;
902
- }
903
- update_every = (uint32_t)sqlite3_column_int(res, 1);
904
- rotate_data.interval_durations[rotate_data.interval_duration_count].update_every = update_every;
905
- start_time = LONG_MAX;
906
- }
907
-#ifdef ENABLE_DBENGINE
908
- if (memory_mode == RRD_MEMORY_MODE_DBENGINE)
909
- rc =
910
- rrdeng_metric_latest_time_by_uuid((uuid_t *)sqlite3_column_blob(res, 0), &first_entry_t, &last_entry_t, 0);
911
- else
912
-#endif
913
- {
914
- if (wc->host) {
915
- RRDSET *st = NULL;
916
- rc = (st = rrdset_find(wc->host, (const char *)sqlite3_column_text(res, 2))) ? 0 : 1;
917
- if (!rc) {
918
- first_entry_t = rrdset_first_entry_t(st);
919
- last_entry_t = rrdset_last_entry_t(st);
920
- }
921
- } else {
922
- rc = 0;
923
- first_entry_t = rotate_data.rotation_timestamp.tv_sec;
924
- }
925
- }
926
-
927
- if (likely(!rc && first_entry_t))
928
- start_time = MIN(start_time, first_entry_t);
929
-
930
- if (memory_mode == RRD_MEMORY_MODE_DBENGINE && wc->chart_updates && (dimension_update_count < ACLK_MAX_DIMENSION_CLEANUP)) {
931
- int live = ((uint32_t)(now - last_entry_t) < (RRDSET_MINIMUM_DIM_LIVE_MULTIPLIER * update_every));
932
- if (rc) {
933
- first_entry_t = 0;
934
- last_entry_t = 0;
935
- live = 0;
936
- }
937
- if (!wc->host || !first_entry_t) {
938
- if (!first_entry_t) {
939
- delete_dimension_uuid((uuid_t *)sqlite3_column_blob(res, 0));
940
- total_deleted++;
941
- dimension_update_count++;
942
- }
943
- else {
944
- (void)aclk_upd_dimension_event(
945
- wc,
946
- claim_id,
947
- (uuid_t *)sqlite3_column_blob(res, 0),
948
- (const char *)(const char *)sqlite3_column_text(res, 3),
949
- (const char *)(const char *)sqlite3_column_text(res, 4),
950
- (const char *)(const char *)sqlite3_column_text(res, 2),
951
- first_entry_t,
952
- live ? 0 : last_entry_t,
953
- &send_status);
954
-
955
- if (!send_status) {
956
- if (last_entry_t)
957
- total_stopped++;
958
- dimension_update_count++;
959
- }
960
- }
961
- }
962
- }
963
- total_checked++;
964
- }
965
- if (update_every) {
966
- debug(D_ACLK_SYNC, "Update %s for %u oldest time = %ld", wc->host_guid, update_every, start_time);
967
- if (start_time == LONG_MAX)
968
- rotate_data.interval_durations[rotate_data.interval_duration_count].retention = 0;
969
- else
970
- rotate_data.interval_durations[rotate_data.interval_duration_count].retention =
971
- rotate_data.rotation_timestamp.tv_sec - start_time;
972
- rotate_data.interval_duration_count++;
973
- }
974
-
975
- if (dimension_update_count < ACLK_MAX_DIMENSION_CLEANUP && !netdata_exit)
976
- log_access("ACLK STA [%s (%s)]: UPDATES %d RETENTION MESSAGE SENT. CHECKED %u DIMENSIONS. %u DELETED, %u STOPPED COLLECTING",
977
- wc->node_id, wc->hostname ? wc->hostname : "N/A", wc->chart_updates, total_checked, total_deleted, total_stopped);
978
- else
979
- log_access("ACLK STA [%s (%s)]: UPDATES %d RETENTION MESSAGE NOT SENT. CHECKED %u DIMENSIONS. %u DELETED, %u STOPPED COLLECTING",
980
- wc->node_id, wc->hostname ? wc->hostname : "N/A", wc->chart_updates, total_checked, total_deleted, total_stopped);
981
-
982
-#ifdef NETDATA_INTERNAL_CHECKS
983
- info("Retention update for %s (chart updates = %d)", wc->host_guid, wc->chart_updates);
984
- for (int i = 0; i < rotate_data.interval_duration_count; ++i)
985
- info(
986
- "Update for host %s (node %s) for %u Retention = %u",
987
- wc->host_guid,
988
- wc->node_id,
989
- rotate_data.interval_durations[i].update_every,
990
- rotate_data.interval_durations[i].retention);
991
-#endif
992
- if (dimension_update_count < ACLK_MAX_DIMENSION_CLEANUP && !netdata_exit)
993
- aclk_retention_updated(&rotate_data);
994
- freez(rotate_data.node_id);
995
- freez(rotate_data.interval_durations);
996
-
997
-failed:
998
- freez(claim_id);
999
- rc = sqlite3_finalize(res);
1000
- if (unlikely(rc != SQLITE_OK))
1001
- error_report("Failed to finalize the prepared statement when reading host dimensions");
1002
- return;
1003
-}
1004
-
1005
-uint32_t sql_get_pending_count(struct aclk_database_worker_config *wc)
1006
-{
1007
- char sql[ACLK_SYNC_QUERY_SIZE];
1008
- static __thread sqlite3_stmt *res = NULL;
1009
-
1010
- snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, "SELECT count(1) FROM aclk_chart_%s ac WHERE ac.date_submitted IS NULL;", wc->uuid_str);
1011
-
1012
- int rc;
1013
- uint32_t chart_payload_count = 0;
1014
- if (unlikely(!res)) {
1015
- rc = prepare_statement(db_meta, sql, &res);
1016
- if (rc != SQLITE_OK) {
1017
- error_report("Failed to prepare statement to count pending messages");
1018
- return 0;
1019
- }
1020
- }
1021
- while (sqlite3_step_monitored(res) == SQLITE_ROW)
1022
- chart_payload_count = (uint32_t) sqlite3_column_int(res, 0);
1023
-
1024
- rc = sqlite3_reset(res);
1025
- if (unlikely(rc != SQLITE_OK))
1026
- error_report("Failed to reset statement when fetching pending messages, rc = %d", rc);
1027
-
1028
- return chart_payload_count;
1029
-}
1030
-
1031
-void sql_get_last_chart_sequence(struct aclk_database_worker_config *wc)
1032
-{
1033
- char sql[ACLK_SYNC_QUERY_SIZE];
1034
-
1035
- snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, "SELECT ac.sequence_id, ac.date_created FROM aclk_chart_%s ac " \
1036
- "WHERE ac.date_submitted IS NOT NULL ORDER BY ac.sequence_id DESC LIMIT 1;", wc->uuid_str);
1037
-
1038
- int rc;
1039
- sqlite3_stmt *res = NULL;
1040
- rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
1041
- if (rc != SQLITE_OK) {
1042
- error_report("Failed to prepare statement to find last chart sequence id");
1043
- return;
1044
- }
1045
-
1046
- wc->chart_sequence_id = 0;
1047
- wc->chart_timestamp = 0;
1048
- while (sqlite3_step_monitored(res) == SQLITE_ROW) {
1049
- wc->chart_sequence_id = (uint64_t)sqlite3_column_int64(res, 0);
1050
- wc->chart_timestamp = (time_t)sqlite3_column_int64(res, 1);
1051
- }
1052
-
1053
- debug(D_ACLK_SYNC, "Node %s reports last sequence_id=%" PRIu64, wc->node_id, wc->chart_sequence_id);
1054
-
1055
- rc = sqlite3_finalize(res);
1056
- if (unlikely(rc != SQLITE_OK))
1057
- error_report("Failed to reset statement when fetching chart sequence info, rc = %d", rc);
1058
-
1059
- return;
1060
-}
1061
-
1062
-void queue_dimension_to_aclk(RRDDIM *rd, time_t last_updated)
1063
-{
1064
- RRDHOST *host = rd->rrdset->rrdhost;
1065
- if (likely(rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS)))
1066
- return;
1067
-
1068
- int live = !last_updated;
1069
-
1070
- if (likely(rd->aclk_live_status == live))
1071
- return;
1072
-
1073
- time_t created_at = rd->tiers[0]->query_ops.oldest_time(rd->tiers[0]->db_metric_handle);
1074
-
1075
- if (unlikely(!created_at && rd->updated))
1076
- created_at = rd->last_collected_time.tv_sec;
1077
-
1078
- rd->aclk_live_status = live;
1079
-
1080
- struct aclk_database_worker_config *wc = rd->rrdset->rrdhost->dbsync_worker;
1081
- if (unlikely(!wc))
1082
- return;
1083
-
1084
- char *claim_id = get_agent_claimid();
1085
- if (unlikely(!claim_id))
1086
- return;
1087
-
1088
- struct chart_dimension_updated dim_payload;
1089
- memset(&dim_payload, 0, sizeof(dim_payload));
1090
- dim_payload.node_id = wc->node_id;
1091
- dim_payload.claim_id = claim_id;
1092
- dim_payload.name = rrddim_name(rd);
1093
- dim_payload.id = rrddim_id(rd);
1094
- dim_payload.chart_id = rrdset_id(rd->rrdset);
1095
- dim_payload.created_at.tv_sec = created_at;
1096
- dim_payload.last_timestamp.tv_sec = last_updated;
1097
-
1098
- size_t size = 0;
1099
- char *payload = generate_chart_dimension_updated(&size, &dim_payload);
1100
-
1101
- freez(claim_id);
1102
- if (unlikely(!payload))
1103
- return;
1104
-
1105
- struct aclk_chart_dimension_data *aclk_cd_data = mallocz(sizeof(*aclk_cd_data));
1106
- uuid_copy(aclk_cd_data->uuid, rd->metric_uuid);
1107
- aclk_cd_data->payload = payload;
1108
- aclk_cd_data->payload_size = size;
1109
- aclk_cd_data->check_payload = 1;
1110
-
1111
- struct aclk_database_cmd cmd;
1112
- memset(&cmd, 0, sizeof(cmd));
1113
-
1114
- cmd.opcode = ACLK_DATABASE_ADD_DIMENSION;
1115
- cmd.data = aclk_cd_data;
1116
- int rc = aclk_database_enq_cmd_noblock(wc, &cmd);
1117
-
1118
- if (unlikely(rc)) {
1119
- freez(aclk_cd_data->payload);
1120
- freez(aclk_cd_data);
1121
- rd->aclk_live_status = !live;
1122
- }
1123
- return;
1124
-}
1125
-
1126
-void aclk_send_dimension_update(RRDDIM *rd)
1127
-{
1128
- char *claim_id = get_agent_claimid();
1129
- if (unlikely(!claim_id))
1130
- return;
1131
-
1132
- time_t first_entry_t = rrddim_first_entry_t(rd);
1133
- time_t last_entry_t = rrddim_last_entry_t(rd);
1134
-
1135
- time_t now = now_realtime_sec();
1136
- int live = ((now - rd->last_collected_time.tv_sec) < (RRDSET_MINIMUM_DIM_LIVE_MULTIPLIER * rd->update_every));
1137
-
1138
- if (!live || rd->aclk_live_status != live || !first_entry_t) {
1139
- (void)aclk_upd_dimension_event(
1140
- rd->rrdset->rrdhost->dbsync_worker,
1141
- claim_id,
1142
- &rd->metric_uuid,
1143
- rrddim_id(rd),
1144
- rrddim_name(rd),
1145
- rrdset_id(rd->rrdset),
1146
- first_entry_t,
1147
- live ? 0 : last_entry_t,
1148
- NULL);
1149
-
1150
- if (!first_entry_t)
1151
- debug(
1152
- D_ACLK_SYNC,
1153
- "%s: Update dimension chart=%s dim=%s live=%d (%ld, %ld)",
1154
- rrdhost_hostname(rd->rrdset->rrdhost),
1155
- rrdset_name(rd->rrdset),
1156
- rrddim_name(rd),
1157
- live,
1158
- first_entry_t,
1159
- last_entry_t);
1160
- else
1161
- debug(
1162
- D_ACLK_SYNC,
1163
- "%s: Update dimension chart=%s dim=%s live=%d (%ld, %ld) collected %ld seconds ago",
1164
- rrdhost_hostname(rd->rrdset->rrdhost),
1165
- rrdset_name(rd->rrdset),
1166
- rrddim_name(rd),
1167
- live,
1168
- first_entry_t,
1169
- last_entry_t,
1170
- now - last_entry_t);
1171
- rd->aclk_live_status = live;
1172
- }
1173
-
1174
- freez(claim_id);
1175
- return;
1176
-}
1177
-
1178
-#define SQL_SEQ_NULL(result, n) sqlite3_column_type(result, n) == SQLITE_NULL ? 0 : sqlite3_column_int64(result, n)
1179
-
1180
-struct aclk_chart_sync_stats *aclk_get_chart_sync_stats(RRDHOST *host)
1181
-{
1182
- struct aclk_chart_sync_stats *aclk_statistics = NULL;
1183
-
1184
- struct aclk_database_worker_config *wc = NULL;
1185
- wc = (struct aclk_database_worker_config *)host->dbsync_worker;
1186
- if (!wc)
1187
- return NULL;
1188
-
1189
- aclk_statistics = callocz(1, sizeof(struct aclk_chart_sync_stats));
1190
-
1191
- aclk_statistics->updates = wc->chart_updates;
1192
- aclk_statistics->batch_id = wc->batch_id;
1193
-
1194
- char host_uuid_fixed[GUID_LEN + 1];
1195
-
1196
- strncpy(host_uuid_fixed, host->machine_guid, GUID_LEN);
1197
- host_uuid_fixed[GUID_LEN] = 0;
1198
-
1199
- host_uuid_fixed[8] = '_';
1200
- host_uuid_fixed[13] = '_';
1201
- host_uuid_fixed[18] = '_';
1202
- host_uuid_fixed[23] = '_';
1203
-
1204
- sqlite3_stmt *res = NULL;
1205
- BUFFER *sql = buffer_create(1024);
1206
- buffer_sprintf(sql, "SELECT min(sequence_id), max(sequence_id), 0 FROM aclk_chart_%s;", host_uuid_fixed);
1207
- buffer_sprintf(sql, "SELECT min(sequence_id), max(sequence_id), 0 FROM aclk_chart_%s WHERE date_submitted IS NULL;", host_uuid_fixed);
1208
- buffer_sprintf(sql, "SELECT min(sequence_id), max(sequence_id), 0 FROM aclk_chart_%s WHERE date_submitted IS NOT NULL;", host_uuid_fixed);
1209
- buffer_sprintf(sql, "SELECT min(sequence_id), max(sequence_id), 0 FROM aclk_chart_%s WHERE date_updated IS NOT NULL;", host_uuid_fixed);
1210
- buffer_sprintf(sql, "SELECT max(date_created), max(date_submitted), max(date_updated), 0 FROM aclk_chart_%s;", host_uuid_fixed);
1211
-
1212
- int rc = sqlite3_prepare_v2(db_meta, buffer_tostring(sql), -1, &res, 0);
1213
- if (rc != SQLITE_OK) {
1214
- buffer_free(sql);
1215
- freez(aclk_statistics);
1216
- return NULL;
1217
- }
1218
-
1219
- rc = sqlite3_step_monitored(res);
1220
- if (rc == SQLITE_ROW) {
1221
- aclk_statistics->min_seqid = SQL_SEQ_NULL(res, 0);
1222
- aclk_statistics->max_seqid = SQL_SEQ_NULL(res, 1);
1223
- }
1224
-
1225
- rc = sqlite3_step_monitored(res);
1226
- if (rc == SQLITE_ROW) {
1227
- aclk_statistics->min_seqid_pend = SQL_SEQ_NULL(res, 0);
1228
- aclk_statistics->max_seqid_pend = SQL_SEQ_NULL(res, 1);
1229
- }
1230
-
1231
- rc = sqlite3_step_monitored(res);
1232
- if (rc == SQLITE_ROW) {
1233
- aclk_statistics->min_seqid_sent = SQL_SEQ_NULL(res, 0);
1234
- aclk_statistics->max_seqid_sent = SQL_SEQ_NULL(res, 1);
1235
- }
1236
-
1237
- rc = sqlite3_step_monitored(res);
1238
- if (rc == SQLITE_ROW) {
1239
- aclk_statistics->min_seqid_ack = SQL_SEQ_NULL(res, 0);
1240
- aclk_statistics->max_seqid_ack = SQL_SEQ_NULL(res, 1);
1241
- }
1242
-
1243
- rc = sqlite3_step_monitored(res);
1244
- if (rc == SQLITE_ROW) {
1245
- aclk_statistics->min_seqid_ack = SQL_SEQ_NULL(res, 0);
1246
- aclk_statistics->max_seqid_ack = SQL_SEQ_NULL(res, 1);
1247
- }
1248
-
1249
- rc = sqlite3_step_monitored(res);
1250
- if (rc == SQLITE_ROW) {
1251
- aclk_statistics->max_date_created = (time_t) SQL_SEQ_NULL(res, 0);
1252
- aclk_statistics->max_date_submitted = (time_t) SQL_SEQ_NULL(res, 1);
1253
- aclk_statistics->max_date_ack = (time_t) SQL_SEQ_NULL(res, 2);
1254
- }
1255
-
1256
- rc = sqlite3_finalize(res);
1257
- if (unlikely(rc != SQLITE_OK))
1258
- error_report("Failed to finalize statement when fetching aclk sync statistics, rc = %d", rc);
1259
-
1260
- buffer_free(sql);
1261
- return aclk_statistics;
1262
-}
1263
-
1264
-void sql_check_chart_liveness(RRDSET *st) {
1265
- RRDDIM *rd;
1266
-
1267
- if (unlikely(rrdset_is_ar_chart(st)))
1268
- return;
1269
-
1270
- if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_ACLK)))
1271
- return;
1272
-
1273
- if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_ACLK))) {
1274
- if (likely(rrdset_number_of_dimensions(st) && st->counter_done && !queue_chart_to_aclk(st))) {
1275
- debug(D_ACLK_SYNC,"Check chart liveness [%s] submit chart definition", rrdset_name(st));
1276
- rrdset_flag_set(st, RRDSET_FLAG_ACLK);
1277
- }
1278
- }
1279
- else
1280
- debug(D_ACLK_SYNC,"Check chart liveness [%s] chart definition already submitted", rrdset_name(st));
1281
-
1282
- time_t mark = now_realtime_sec();
1283
-
1284
- debug(D_ACLK_SYNC,"Check chart liveness [%s] scanning dimensions", rrdset_name(st));
1285
- rrddim_foreach_read(rd, st) {
1286
- if (!rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN))
1287
- queue_dimension_to_aclk(rd, calc_dimension_liveness(rd, mark));
1288
- }
1289
- rrddim_foreach_done(rd);
1290
-}
1291
-
1292
-// ST is read locked
1293
-int queue_chart_to_aclk(RRDSET *st)
1294
-{
1295
- RRDHOST *host = st->rrdhost;
1296
-
1297
- if (likely(rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_CONTEXTS)))
1298
- return 0;
1299
-
1300
- return sql_queue_chart_payload((struct aclk_database_worker_config *) st->rrdhost->dbsync_worker,
1301
- st, ACLK_DATABASE_ADD_CHART);
1302
-}
1303
-
1304
-#endif //ENABLE_ACLK
database/sqlite/sqlite_aclk_chart.h
deleted
-71
@@ -1,71 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-#ifndef NETDATA_SQLITE_ACLK_CHART_H
4
-#define NETDATA_SQLITE_ACLK_CHART_H
5
-
6
-
7
-typedef enum payload_type {
8
- ACLK_PAYLOAD_CHART,
9
- ACLK_PAYLOAD_DIMENSION,
10
- ACLK_PAYLOAD_DIMENSION_ROTATED
11
-} ACLK_PAYLOAD_TYPE;
12
-
13
-extern sqlite3 *db_meta;
14
-
15
-#ifndef RRDSET_MINIMUM_DIM_LIVE_MULTIPLIER
16
-#define RRDSET_MINIMUM_DIM_LIVE_MULTIPLIER (3)
17
-#endif
18
-
19
-#ifndef RRDSET_MINIMUM_DIM_OFFLINE_MULTIPLIER
20
-#define RRDSET_MINIMUM_DIM_OFFLINE_MULTIPLIER (30)
21
-#endif
22
-
23
-#ifndef ACLK_MAX_DIMENSION_CLEANUP
24
-#define ACLK_MAX_DIMENSION_CLEANUP (500)
25
-#endif
26
-
27
-struct aclk_chart_dimension_data {
28
- uuid_t uuid;
29
- char *payload;
30
- size_t payload_size;
31
- uint8_t check_payload;
32
-};
33
-
34
-struct aclk_chart_sync_stats {
35
- int updates;
36
- uint64_t batch_id;
37
- uint64_t min_seqid;
38
- uint64_t max_seqid;
39
- uint64_t min_seqid_pend;
40
- uint64_t max_seqid_pend;
41
- uint64_t min_seqid_sent;
42
- uint64_t max_seqid_sent;
43
- uint64_t min_seqid_ack;
44
- uint64_t max_seqid_ack;
45
- time_t max_date_created;
46
- time_t max_date_submitted;
47
- time_t max_date_ack;
48
-};
49
-
50
-extern int queue_chart_to_aclk(RRDSET *st);
51
-extern void queue_dimension_to_aclk(RRDDIM *rd, time_t last_updated);
52
-extern void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id);
53
-int aclk_add_chart_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
54
-int aclk_add_dimension_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
55
-int aclk_send_chart_config(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
56
-void aclk_ack_chart_sequence_id(char *node_id, uint64_t last_sequence_id);
57
-void aclk_get_chart_config(char **hash_id_list);
58
-void aclk_send_chart_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
59
-void aclk_start_streaming(char *node_id, uint64_t seq_id, time_t created_at, uint64_t batch_id);
60
-void sql_chart_deduplicate(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
61
-void sql_check_rotation_state(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
62
-void sql_get_last_chart_sequence(struct aclk_database_worker_config *wc);
63
-void aclk_receive_chart_reset(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
64
-void aclk_receive_chart_ack(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
65
-void aclk_process_dimension_deletion(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
66
-uint32_t sql_get_pending_count(struct aclk_database_worker_config *wc);
67
-void aclk_send_dimension_update(RRDDIM *rd);
68
-struct aclk_chart_sync_stats *aclk_get_chart_sync_stats(RRDHOST *host);
69
-void sql_check_chart_liveness(RRDSET *st);
70
-void aclk_update_retention(struct aclk_database_worker_config *wc);
71
-#endif //NETDATA_SQLITE_ACLK_CHART_H
database/sqlite/sqlite_aclk_node.c
+2
-4
@@ -3,9 +3,7 @@
3
#include "sqlite_functions.h"
4
#include "sqlite_aclk_node.h"
5
6
-#ifdef ENABLE_ACLK
7
-#include "../../aclk/aclk_charts_api.h"
8
-#endif
6
+#include "../../aclk/aclk_contexts_api.h"
7
8
#ifdef ENABLE_ACLK
9
DICTIONARY *collectors_from_charts(RRDHOST *host, DICTIONARY *dict) {
@@ -77,7 +75,7 @@ void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_dat
75
{ .name = "proto", .version = 1, .enabled = 1 },
76
{ .name = "ml", .version = ml_capable(localhost), .enabled = ml_enabled(wc->host) },
77
{ .name = "mc", .version = enable_metric_correlations ? metric_correlations_version : 0, .enabled = enable_metric_correlations },
80
- { .name = "ctx", .version = 1, .enabled = rrdcontext_enabled},
78
+ { .name = "ctx", .version = 1, .enabled = 1 },
79
{ .name = NULL, .version = 0, .enabled = 0 }
80
};
81
node_info.node_instance_capabilities = instance_caps;