@cryptotaxi247 / netdata-1 / commits / 12f16063f

Enable additional functionality for the new cloud architecture (#11579)

Stelios Fragkakis committed Oct 6, 2021 at 20:55 UTC 12f16063f51e51a8e3c0e0ae727a634258b95219
20 files changed +721 -204
aclk/aclk.c
+3 -3
@@ -973,7 +973,7 @@ void ng_aclk_host_state_update(RRDHOST *host, int cmd)
973 rrdhost_aclk_state_lock(localhost);
974 create_query->data.node_creation.claim_id = strdupz(localhost->aclk_state.claimed_id);
975 rrdhost_aclk_state_unlock(localhost);
976 - create_query->data.node_creation.hops = 1; //TODO - real hop count instead of hardcoded
976 + create_query->data.node_creation.hops = (uint32_t) host->system_info->hops;
977 create_query->data.node_creation.hostname = strdupz(host->hostname);
978 create_query->data.node_creation.machine_guid = strdupz(host->machine_guid);
979 aclk_queue_query(create_query);
@@ -981,7 +981,7 @@ void ng_aclk_host_state_update(RRDHOST *host, int cmd)
981 }
982
983 aclk_query_t query = aclk_query_new(NODE_STATE_UPDATE);
984 - query->data.node_update.hops = 1; //TODO - real hop count instead of hardcoded
984 + query->data.node_update.hops = (uint32_t) host->system_info->hops;
985 rrdhost_aclk_state_lock(localhost);
986 query->data.node_update.claim_id = strdupz(localhost->aclk_state.claimed_id);
987 rrdhost_aclk_state_unlock(localhost);
@@ -1020,7 +1020,7 @@ void aclk_send_node_instances()
1020 rrdhost_aclk_state_lock(localhost);
1021 create_query->data.node_creation.claim_id = strdupz(localhost->aclk_state.claimed_id);
1022 rrdhost_aclk_state_unlock(localhost);
1023 - create_query->data.node_creation.hops = uuid_compare(list->host_id, localhost->host_uuid) ? 1 : 0; // TODO - when streaming supports hops
1023 + create_query->data.node_creation.hops = list->hops;
1024 create_query->data.node_creation.hostname = list->hostname;
1025 create_query->data.node_creation.machine_guid = mallocz(UUID_STR_LEN);
1026 uuid_unparse_lower(list->host_id, (char*)create_query->data.node_creation.machine_guid);
aclk/aclk_charts_api.c
+4
@@ -16,6 +16,7 @@ void aclk_chart_inst_update(char **payloads, size_t *payload_sizes, struct aclk_
16 void aclk_chart_dim_update(char **payloads, size_t *payload_sizes, struct aclk_message_position *new_positions)
17 {
18 aclk_query_t query = aclk_query_new(CHART_DIMS_UPDATE);
19 + query->data.bin_payload.topic = ACLK_TOPICID_CHART_DIMS;
20 query->data.bin_payload.payload = generate_chart_dimensions_updated(&query->data.bin_payload.size, payloads, payload_sizes, new_positions);
21 query->data.bin_payload.msg_name = CHART_DIM_UPDATE_NAME;
22 QUEUE_IF_PAYLOAD_PRESENT(query);
@@ -24,6 +25,7 @@ void aclk_chart_dim_update(char **payloads, size_t *payload_sizes, struct aclk_m
25 void aclk_chart_inst_and_dim_update(char **payloads, size_t *payload_sizes, int *is_dim, struct aclk_message_position *new_positions, uint64_t batch_id)
26 {
27 aclk_query_t query = aclk_query_new(CHART_DIMS_UPDATE);
28 + query->data.bin_payload.topic = ACLK_TOPICID_CHART_DIMS;
29 query->data.bin_payload.payload = generate_charts_and_dimensions_updated(&query->data.bin_payload.size, payloads, payload_sizes, is_dim, new_positions, batch_id);
30 query->data.bin_payload.msg_name = CHART_DIM_UPDATE_NAME;
31 QUEUE_IF_PAYLOAD_PRESENT(query);
@@ -32,6 +34,7 @@ void aclk_chart_inst_and_dim_update(char **payloads, size_t *payload_sizes, int
34 void aclk_chart_config_updated(struct chart_config_updated *config_list, int list_size)
35 {
36 aclk_query_t query = aclk_query_new(CHART_CONFIG_UPDATED);
37 + query->data.bin_payload.topic = ACLK_TOPICID_CHART_CONFIGS_UPDATED;
38 query->data.bin_payload.payload = generate_chart_configs_updated(&query->data.bin_payload.size, config_list, list_size);
39 query->data.bin_payload.msg_name = "ChartConfigsUpdated";
40 QUEUE_IF_PAYLOAD_PRESENT(query);
@@ -40,6 +43,7 @@ void aclk_chart_config_updated(struct chart_config_updated *config_list, int lis
43 void aclk_chart_reset(chart_reset_t reset)
44 {
45 aclk_query_t query = aclk_query_new(CHART_RESET);
46 + query->data.bin_payload.topic = ACLK_TOPICID_CHART_RESET;
47 query->data.bin_payload.payload = generate_reset_chart_messages(&query->data.bin_payload.size, reset);
48 query->data.bin_payload.msg_name = "ResetChartMessages";
49 QUEUE_IF_PAYLOAD_PRESENT(query);
aclk/aclk_rx_msgs.c
+67
@@ -330,6 +330,73 @@ void aclk_handle_new_cloud_msg(const char *message_type, const char *msg, size_t
330 return;
331 }
332
333 + if (!strcmp(message_type, "StreamChartsAndDimensions")) {
334 + stream_charts_and_dims_t res = parse_stream_charts_and_dims(msg, msg_len);
335 + if (!res.claim_id || !res.node_id) {
336 + error("Error parsing StreamChartsAndDimensions msg");
337 + freez(res.claim_id);
338 + freez(res.node_id);
339 + return;
340 + }
341 + chart_batch_id = res.batch_id;
342 + aclk_start_streaming(res.node_id, res.seq_id, res.seq_id_created_at.tv_sec, res.batch_id);
343 + freez(res.claim_id);
344 + freez(res.node_id);
345 + return;
346 + }
347 + if (!strcmp(message_type, "ChartsAndDimensionsAck")) {
348 + chart_and_dim_ack_t res = parse_chart_and_dimensions_ack(msg, msg_len);
349 + if (!res.claim_id || !res.node_id) {
350 + error("Error parsing StreamChartsAndDimensions msg");
351 + freez(res.claim_id);
352 + freez(res.node_id);
353 + return;
354 + }
355 + aclk_ack_chart_sequence_id(res.node_id, res.last_seq_id);
356 + freez(res.claim_id);
357 + freez(res.node_id);
358 + return;
359 + }
360 + if (!strcmp(message_type, "UpdateChartConfigs")) {
361 + struct update_chart_config res = parse_update_chart_config(msg, msg_len);
362 + if (!res.claim_id || !res.node_id || !res.hashes)
363 + error("Error parsing UpdateChartConfigs msg");
364 + else
365 + aclk_get_chart_config(res.hashes);
366 + destroy_update_chart_config(&res);
367 + return;
368 + }
369 + if (!strcmp(message_type, "StartAlarmStreaming")) {
370 + struct start_alarm_streaming res = parse_start_alarm_streaming(msg, msg_len);
371 + if (!res.node_id || !res.batch_id) {
372 + error("Error parsing StartAlarmStreaming");
373 + freez(res.node_id);
374 + return;
375 + }
376 + aclk_start_alert_streaming(res.node_id, res.batch_id, res.start_seq_id);
377 + freez(res.node_id);
378 + return;
379 + }
380 + if (!strcmp(message_type, "SendAlarmLogHealth")) {
381 + char *node_id = parse_send_alarm_log_health(msg, msg_len);
382 + if (!node_id) {
383 + error("Error parsing SendAlarmLogHealth");
384 + return;
385 + }
386 + aclk_send_alarm_health_log(node_id);
387 + freez(node_id);
388 + return;
389 + }
390 + if (!strcmp(message_type, "SendAlarmConfiguration")) {
391 + char *config_hash = parse_send_alarm_configuration(msg, msg_len);
392 + if (!config_hash || !*config_hash) {
393 + error("Error parsing SendAlarmConfiguration");
394 + return;
395 + }
396 + aclk_send_alarm_configuration(config_hash);
397 + freez(config_hash);
398 + return;
399 + }
400 error ("Unknown new cloud arch message type received \"%s\"", message_type);
401 }
402 #endif
aclk/schema-wrappers/alarm_stream.cc
+2 -2
@@ -97,8 +97,8 @@ static alarmstream::v1::AlarmStatus aclk_alarm_status_to_proto(enum aclk_alarm_s
97
98 void destroy_alarm_log_entry(struct alarm_log_entry *entry)
99 {
100 - freez(entry->node_id);
101 - freez(entry->claim_id);
100 + //freez(entry->node_id);
101 + //freez(entry->claim_id);
102
103 freez(entry->chart);
104 freez(entry->name);
aclk/schema-wrappers/chart_stream.cc
-2
@@ -75,8 +75,6 @@ void chart_instance_updated_destroy(struct chart_instance_updated *instance)
75 {
76 freez((char*)instance->id);
77 freez((char*)instance->claim_id);
78 - freez((char*)instance->node_id);
79 - freez((char*)instance->name);
78
79 free_label_list(instance->label_head);
80
database/engine/rrdengine.c
+1
@@ -861,6 +861,7 @@ static void after_delete_old_data(struct rrdengine_worker_config* wc)
861 wc->now_deleting_files = NULL;
862
863 wc->cleanup_thread_deleting_files = 0;
864 + aclk_data_rotated();
865
866 /* interrupt event loop */
867 uv_stop(wc->loop);
database/rrd.h
+5
@@ -50,6 +50,7 @@ struct context_param {
50 uint8_t flags;
51 };
52
53 +#define RRDSET_MINIMUM_LIVE_COUNT 3
54 #define META_CHART_UPDATED 1
55 #define META_PLUGIN_UPDATED 2
56 #define META_MODULE_UPDATED 4
@@ -1359,4 +1360,8 @@ extern void set_host_properties(
1360 #endif
1361 #include "sqlite/sqlite_functions.h"
1362 #include "sqlite/sqlite_aclk.h"
1363 +#include "sqlite/sqlite_aclk_chart.h"
1364 +#include "sqlite/sqlite_aclk_alert.h"
1365 +#include "sqlite/sqlite_aclk_node.h"
1366 +#include "sqlite/sqlite_health.h"
1367 #endif /* NETDATA_RRD_H */
database/rrddim.c
+10 -7
@@ -210,7 +210,7 @@ void rrdcalc_link_to_rrddim(RRDDIM *rd, RRDSET *st, RRDHOST *host) {
210 }
211 }
212 #ifdef ENABLE_ACLK
213 - rrdset_flag_set(st, RRDSET_FLAG_ACLK);
213 + rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
214 #endif
215 }
216
@@ -387,6 +387,9 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
387 rd->last_collected_time.tv_usec = 0;
388 rd->rrdset = st;
389 rd->state = mallocz(sizeof(*rd->state));
390 +#ifdef ENABLE_ACLK
391 + rd->state->aclk_live_status = -1;
392 +#endif
393 (void) find_dimension_uuid(st, rd, &(rd->state->metric_uuid));
394 if(memory_mode == RRD_MEMORY_MODE_DBENGINE) {
395 #ifdef ENABLE_DBENGINE
@@ -453,7 +456,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
456
457 rrdset_unlock(st);
458 #ifdef ENABLE_ACLK
456 - rrdset_flag_set(st, RRDSET_FLAG_ACLK);
459 + rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
460 #endif
461 return(rd);
462 }
@@ -521,7 +524,7 @@ void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated)
524 }
525 #ifdef ENABLE_ACLK
526 if (db_rotated || RRD_MEMORY_MODE_DBENGINE != rrd_memory_mode)
524 - rrdset_flag_set(st, RRDSET_FLAG_ACLK);
527 + rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
528 #endif
529 }
530
@@ -542,7 +545,7 @@ int rrddim_hide(RRDSET *st, const char *id) {
545
546 rrddim_flag_set(rd, RRDDIM_FLAG_HIDDEN);
547 #ifdef ENABLE_ACLK
545 - rrdset_flag_set(st, RRDSET_FLAG_ACLK);
548 + rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
549 #endif
550 return 0;
551 }
@@ -559,7 +562,7 @@ int rrddim_unhide(RRDSET *st, const char *id) {
562
563 rrddim_flag_clear(rd, RRDDIM_FLAG_HIDDEN);
564 #ifdef ENABLE_ACLK
562 - rrdset_flag_set(st, RRDSET_FLAG_ACLK);
565 + rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
566 #endif
567 return 0;
568 }
@@ -574,7 +577,7 @@ inline void rrddim_is_obsolete(RRDSET *st, RRDDIM *rd) {
577 rrddim_flag_set(rd, RRDDIM_FLAG_OBSOLETE);
578 rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS);
579 #ifdef ENABLE_ACLK
577 - rrdset_flag_set(st, RRDSET_FLAG_ACLK);
580 + rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
581 #endif
582 }
583
@@ -583,7 +586,7 @@ inline void rrddim_isnot_obsolete(RRDSET *st __maybe_unused, RRDDIM *rd) {
586
587 rrddim_flag_clear(rd, RRDDIM_FLAG_OBSOLETE);
588 #ifdef ENABLE_ACLK
586 - rrdset_flag_set(st, RRDSET_FLAG_ACLK);
589 + rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
590 #endif
591 }
592
database/rrdhost.c
+2
@@ -746,7 +746,9 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
746 fatal("Failed to initialize dbengine");
747 }
748 #endif
749 +#ifdef ACLK_NEWARCH_DEVMODE
750 sql_aclk_sync_init();
751 +#endif
752 rrd_unlock();
753
754 web_client_api_v1_management_init();
database/rrdset.c
+28 -4
@@ -649,7 +649,7 @@ RRDSET *rrdset_create_custom(
649 aclk_add_collector(host, st->plugin_name, st->module_name);
650 }
651 }
652 - rrdset_flag_set(st, RRDSET_FLAG_ACLK);
652 + rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
653 }
654 #endif
655 freez(old_plugin);
@@ -935,12 +935,13 @@ RRDSET *rrdset_create_custom(
935 update_chart_metadata(st->chart_uuid, st, id, name);
936
937 store_active_chart(st->chart_uuid);
938 + compute_chart_hash(st);
939
940 rrdhost_unlock(host);
941 #ifdef ENABLE_ACLK
942 if (netdata_cloud_setting)
943 aclk_add_collector(host, plugin, module);
943 - rrdset_flag_set(st, RRDSET_FLAG_ACLK);
944 + rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
945 #endif
946 return(st);
947 }
@@ -1377,10 +1378,19 @@ void rrdset_done(RRDSET *st) {
1378 rrdset_rdlock(st);
1379
1380 #ifdef ENABLE_ACLK
1380 - if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_ACLK))) {
1381 - rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
1381 + #ifdef ENABLE_NEW_CLOUD_PROTOCOL
1382 + if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_ACLK))) {
1383 + if (st->counter_done >= RRDSET_MINIMUM_LIVE_COUNT) {
1384 + if (likely(!sql_queue_chart_to_aclk(st)))
1385 + rrdset_flag_set(st, RRDSET_FLAG_ACLK);
1386 + }
1387 + }
1388 + #else
1389 + if (unlikely(!rrdset_flag_check(st, RRDSET_FLAG_ACLK))) {
1390 + rrdset_flag_set(st, RRDSET_FLAG_ACLK);
1391 aclk_update_chart(st->rrdhost, st->id, 1);
1392 }
1393 + #endif
1394 #endif
1395
1396 if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE))) {
@@ -1783,9 +1793,23 @@ after_first_database_work:
1793 after_second_database_work:
1794 st->last_collected_total = st->collected_total;
1795
1796 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL
1797 + time_t mark = now_realtime_sec();
1798 +#endif
1799 rrddim_foreach_read(rd, st) {
1800 if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
1801 continue;
1802 +
1803 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL
1804 + int live = ((mark - rd->last_collected_time.tv_sec) < (RRDSET_MINIMUM_LIVE_COUNT * rd->update_every));
1805 + if (unlikely(live != rd->state->aclk_live_status)) {
1806 + if (likely(rrdset_flag_check(st, RRDSET_FLAG_ACLK))) {
1807 + if (likely(!sql_queue_dimension_to_aclk(rd))) {
1808 + rd->state->aclk_live_status = live;
1809 + }
1810 + }
1811 + }
1812 +#endif
1813 if(unlikely(!rd->updated))
1814 continue;
1815
database/sqlite/sqlite_aclk.c
+68 -76
@@ -3,17 +3,17 @@
3 #include "sqlite_functions.h"
4 #include "sqlite_aclk.h"
5
6 -// TODO: To be added
6 #include "sqlite_aclk_chart.h"
8 -//#include "sqlite_aclk_alert.h"
7 #include "sqlite_aclk_node.h"
8
9 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL
10 +#include "../../aclk/aclk.h"
11 +#endif
12 +
13 const char *aclk_sync_config[] = {
14 NULL,
15 };
16
15 -int aclk_architecture = 0;
16 -
17 uv_mutex_t aclk_async_lock;
18 struct aclk_database_worker_config *aclk_thread_head = NULL;
19
@@ -78,24 +78,6 @@ void aclk_database_init_cmd_queue(struct aclk_database_worker_config *wc)
78 fatal_assert(0 == uv_mutex_init(&wc->cmd_mutex));
79 }
80
81 -void aclk_database_enq_cmd_nowake(struct aclk_database_worker_config *wc, struct aclk_database_cmd *cmd)
82 -{
83 - unsigned queue_size;
84 -
85 - /* wait for free space in queue */
86 - uv_mutex_lock(&wc->cmd_mutex);
87 - while ((queue_size = wc->queue_size) == ACLK_DATABASE_CMD_Q_MAX_SIZE) {
88 - uv_cond_wait(&wc->cmd_cond, &wc->cmd_mutex);
89 - }
90 - fatal_assert(queue_size < ACLK_DATABASE_CMD_Q_MAX_SIZE);
91 - /* enqueue command */
92 - wc->cmd_queue.cmd_array[wc->cmd_queue.tail] = *cmd;
93 - wc->cmd_queue.tail = wc->cmd_queue.tail != ACLK_DATABASE_CMD_Q_MAX_SIZE - 1 ?
94 - wc->cmd_queue.tail + 1 : 0;
95 - wc->queue_size = queue_size + 1;
96 - uv_mutex_unlock(&wc->cmd_mutex);
97 -}
98 -
81 int aclk_database_enq_cmd_noblock(struct aclk_database_worker_config *wc, struct aclk_database_cmd *cmd)
82 {
83 unsigned queue_size;
@@ -207,7 +189,7 @@ int aclk_start_sync_thread(void *data, int argc, char **argv, char **column)
189
190 void sql_aclk_sync_init(void)
191 {
210 -#ifdef ACLK_NEWARCH_DEVMODE
192 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL
193 char *err_msg = NULL;
194 int rc;
195
@@ -251,35 +233,49 @@ static void async_cb(uv_async_t *handle)
233
234 static void timer_cb(uv_timer_t* handle)
235 {
254 - struct aclk_database_worker_config *wc = handle->data;
236 uv_stop(handle->loop);
237 uv_update_time(handle->loop);
238
239 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL
240 + struct aclk_database_worker_config *wc = handle->data;
241 struct aclk_database_cmd cmd;
242 memset(&cmd, 0, sizeof(cmd));
243 cmd.opcode = ACLK_DATABASE_TIMER;
244 aclk_database_enq_cmd_noblock(wc, &cmd);
245
263 - if (wc->cleanup_after && wc->cleanup_after < now_realtime_sec()) {
246 + time_t now = now_realtime_sec();
247 +
248 + if (wc->cleanup_after && wc->cleanup_after < now) {
249 cmd.opcode = ACLK_DATABASE_CLEANUP;
250 if (!aclk_database_enq_cmd_noblock(wc, &cmd))
251 wc->cleanup_after += ACLK_DATABASE_CLEANUP_INTERVAL;
252 }
253
269 - if (wc->chart_updates && !wc->chart_pending) {
270 - cmd.opcode = ACLK_DATABASE_PUSH_CHART;
271 - cmd.count = ACLK_MAX_CHART_BATCH;
272 - cmd.completion = NULL;
273 - cmd.param1 = ACLK_MAX_CHART_BATCH_COUNT;
274 - if (!aclk_database_enq_cmd_noblock(wc, &cmd))
275 - wc->chart_pending = 1;
276 - }
254 + if (aclk_use_new_cloud_arch && aclk_connected) {
255 + if (wc->rotation_after && wc->rotation_after < now) {
256 + cmd.opcode = ACLK_DATABASE_NODE_INFO;
257 + aclk_database_enq_cmd_noblock(wc, &cmd);
258
278 - if (wc->alert_updates) {
279 - cmd.opcode = ACLK_DATABASE_PUSH_ALERT;
280 - cmd.count = ACLK_MAX_ALERT_UPDATES;
281 - aclk_database_enq_cmd_noblock(wc, &cmd);
259 + cmd.opcode = ACLK_DATABASE_UPD_RETENTION;
260 + if (!aclk_database_enq_cmd_noblock(wc, &cmd))
261 + wc->rotation_after += ACLK_DATABASE_ROTATION_INTERVAL;
262 + }
263 +
264 + if (wc->chart_updates && !wc->chart_pending) {
265 + cmd.opcode = ACLK_DATABASE_PUSH_CHART;
266 + cmd.count = ACLK_MAX_CHART_BATCH;
267 + cmd.param1 = ACLK_MAX_CHART_BATCH_COUNT;
268 + if (!aclk_database_enq_cmd_noblock(wc, &cmd))
269 + wc->chart_pending = 1;
270 + }
271 +
272 + if (wc->alert_updates) {
273 + cmd.opcode = ACLK_DATABASE_PUSH_ALERT;
274 + cmd.count = ACLK_MAX_ALERT_UPDATES;
275 + aclk_database_enq_cmd_noblock(wc, &cmd);
276 + }
277 }
278 +#endif
279 }
280
281 #define MAX_CMD_BATCH_SIZE (256)
@@ -334,8 +330,14 @@ void aclk_database_worker(void *arg)
330 wc->node_info_send = (wc->host && !localhost);
331 aclk_add_worker_thread(wc);
332 info("Starting ACLK sync thread for host %s -- scratch area %lu bytes", wc->host_guid, sizeof(*wc));
337 -// TODO: To be added
338 -// sql_get_last_chart_sequence(wc, cmd);
333 +
334 + memset(&cmd, 0, sizeof(cmd));
335 + sql_get_last_chart_sequence(wc, cmd);
336 + wc->chart_updates = 0;
337 + wc->alert_updates = 0;
338 + wc->startup_time = now_realtime_sec();
339 + wc->cleanup_after = wc->startup_time + ACLK_DATABASE_CLEANUP_FIRST;
340 + wc->rotation_after = wc->startup_time + ACLK_DATABASE_ROTATION_DELAY;
341 while (likely(shutdown == 0)) {
342 uv_run(loop, UV_RUN_DEFAULT);
343
@@ -345,11 +347,8 @@ void aclk_database_worker(void *arg)
347 /* wait for commands */
348 cmd_batch_size = 0;
349 do {
348 - if (unlikely(cmd_batch_size >= MAX_CMD_BATCH_SIZE)) {
349 - info("DEBUG: %s Processed %u commands, current queue about %u",
350 - wc->uuid_str, cmd_batch_size, wc->queue_size);
350 + if (unlikely(cmd_batch_size >= MAX_CMD_BATCH_SIZE))
351 break;
352 - }
352 cmd = aclk_database_deq_cmd(wc);
353 opcode = cmd.opcode;
354 ++cmd_batch_size;
@@ -365,14 +364,6 @@ void aclk_database_worker(void *arg)
364 if (wc->host == localhost)
365 sql_check_aclk_table_list(wc);
366 break;
368 - case ACLK_DATABASE_CHECK:
369 - debug(D_ACLK_SYNC, "Checking database dimensions for %s", wc->host_guid);
370 -// sql_check_dimension_state(wc, cmd);
371 - break;
372 - case ACLK_DATABASE_CHECK_ROTATION:
373 - debug(D_ACLK_SYNC, "Checking database for rotation %s", wc->host_guid);
374 -// sql_check_rotation_state(wc, cmd);
375 - break;
367 case ACLK_DATABASE_DELETE_HOST:
368 debug(D_ACLK_SYNC,"Cleaning ACLK tables for %s", (char *) cmd.data);
369 sql_delete_aclk_table_list(wc, cmd);
@@ -407,19 +398,19 @@ void aclk_database_worker(void *arg)
398 // ALERTS
399 case ACLK_DATABASE_ADD_ALERT:
400 debug(D_ACLK_SYNC,"Adding alert event for %s", wc->host_guid);
410 -// aclk_add_alert_event(wc, cmd);
401 + aclk_add_alert_event(wc, cmd);
402 break;
403 case ACLK_DATABASE_PUSH_ALERT_CONFIG:
404 debug(D_ACLK_SYNC,"Pushing chart config info to the cloud for %s", wc->host_guid);
414 -// aclk_push_alert_config_event(wc, cmd);
405 + aclk_push_alert_config_event(wc, cmd);
406 break;
407 case ACLK_DATABASE_PUSH_ALERT:
408 debug(D_ACLK_SYNC, "Pushing alert info to the cloud for %s", wc->host_guid);
418 -// aclk_push_alert_event(wc, cmd);
409 + aclk_push_alert_event(wc, cmd);
410 break;
411 case ACLK_DATABASE_ALARM_HEALTH_LOG:
412 debug(D_ACLK_SYNC, "Pushing alarm health log to the cloud for %s", wc->host_guid);
422 -// aclk_push_alarm_health_log(wc, cmd);
413 + aclk_push_alarm_health_log(wc, cmd);
414 break;
415
416 // NODE OPERATIONS
@@ -427,8 +418,9 @@ void aclk_database_worker(void *arg)
418 debug(D_ACLK_SYNC,"Sending node info for %s", wc->uuid_str);
419 sql_build_node_info(wc, cmd);
420 break;
430 - case ACLK_DATABASE_UPD_STATS:
431 -// sql_update_metric_statistics(wc, cmd);
421 + case ACLK_DATABASE_UPD_RETENTION:
422 + debug(D_ACLK_SYNC,"Sending retention info for %s", wc->uuid_str);
423 + aclk_update_retention(wc, cmd);
424 break;
425
426 // NODE_INSTANCE DETECTION
@@ -446,7 +438,7 @@ void aclk_database_worker(void *arg)
438 }
439 }
440 }
449 - if (wc->node_info_send && wc->host && localhost && claimed()) {
441 + if (wc->node_info_send && wc->host && localhost && claimed() && aclk_connected) {
442 cmd.opcode = ACLK_DATABASE_NODE_INFO;
443 cmd.completion = NULL;
444 wc->node_info_send = aclk_database_enq_cmd_noblock(wc, &cmd);
@@ -471,7 +463,7 @@ void aclk_database_worker(void *arg)
463
464 /*
465 * uv_async_send after uv_close does not seem to crash in linux at the moment,
474 - * it is however undocumented behaviour and we need to be aware if this becomes
466 + * it is however undocumented behaviour we need to be aware if this becomes
467 * an issue in the future.
468 */
469 uv_close((uv_handle_t *)&wc->async, NULL);
@@ -509,13 +501,9 @@ error_after_loop_init:
501
502 // -------------------------------------------------------------
503
512 -void aclk_set_architecture(int mode)
513 -{
514 - aclk_architecture = mode;
515 -}
516 -
504 void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id)
505 {
506 +#ifdef ENABLE_ACLK
507 char uuid_str[GUID_LEN + 1];
508 char host_guid[GUID_LEN + 1];
509
@@ -568,15 +556,17 @@ void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id)
556 if (likely(host))
557 host->dbsync_worker = (void *) wc;
558 wc->host = host;
571 - wc->chart_updates = 0;
572 - wc->alert_updates = 0;
573 - wc->startup_time = now_realtime_sec();
574 - wc->cleanup_after = wc->startup_time + ACLK_DATABASE_CLEANUP_FIRST;
559 strcpy(wc->uuid_str, uuid_str);
560 strcpy(wc->host_guid, host_guid);
561 if (node_id && !uuid_is_null(*node_id))
562 uuid_unparse_lower(*node_id, wc->node_id);
563 fatal_assert(0 == uv_thread_create(&(wc->thread), aclk_database_worker, wc));
564 +#else
565 + UNUSED(host);
566 + UNUSED(host_uuid);
567 + UNUSED(node_id);
568 +#endif
569 + return;
570 }
571
572 void sql_maint_aclk_sync_database(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
@@ -722,19 +712,20 @@ void sql_check_aclk_table_list(struct aclk_database_worker_config *wc)
712 return;
713 }
714
725 -void aclk_data_rotated(RRDHOST *host)
715 +void aclk_data_rotated(void)
716 {
727 - UNUSED(host);
717 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL
718
729 - debug(D_ACLK_SYNC,"Processing data base rotation event");
730 - struct aclk_database_cmd cmd;
731 - memset(&cmd, 0, sizeof(cmd));
732 - cmd.opcode = ACLK_DATABASE_UPD_STATS;
719 + if (!aclk_use_new_cloud_arch || !aclk_connected)
720 + return;
721
722 + time_t next_rotation_time = now_realtime_sec()+ACLK_DATABASE_ROTATION_DELAY;
723 rrd_wrlock();
724 RRDHOST *this_host = localhost;
725 while (this_host) {
737 - aclk_database_enq_cmd((struct aclk_database_worker_config *)this_host->dbsync_worker, &cmd);
726 + struct aclk_database_worker_config *wc = this_host->dbsync_worker;
727 + if (wc)
728 + wc->rotation_after = next_rotation_time;
729 this_host = this_host->next;
730 }
731 rrd_unlock();
@@ -743,9 +734,10 @@ void aclk_data_rotated(RRDHOST *host)
734
735 uv_mutex_lock(&aclk_async_lock);
736 while (tmp) {
746 - aclk_database_enq_cmd(tmp, &cmd);
737 + tmp->rotation_after = next_rotation_time;
738 tmp = tmp->next;
739 }
740 uv_mutex_unlock(&aclk_async_lock);
741 +#endif
742 return;
743 }
database/sqlite/sqlite_aclk.h
+5 -11
@@ -15,9 +15,10 @@
15 #define ACLK_MAX_CHART_BATCH_COUNT (10)
16 #endif
17 #define ACLK_MAX_ALERT_UPDATES (5)
18 -#define ACLK_SYNC_RETRY_COUNT "10"
18 #define ACLK_DATABASE_CLEANUP_FIRST (60)
19 +#define ACLK_DATABASE_ROTATION_DELAY (60)
20 #define ACLK_DATABASE_CLEANUP_INTERVAL (3600)
21 +#define ACLK_DATABASE_ROTATION_INTERVAL (3600)
22 #define ACLK_DELETE_ACK_INTERNAL (600)
23 #define ACLK_SYNC_QUERY_SIZE 512
24
@@ -60,8 +61,6 @@ static inline void aclk_complete(struct aclk_completion *p)
61
62 extern uv_mutex_t aclk_async_lock;
63
63 -extern int aclk_architecture;
64 -
64 static inline void uuid_unparse_lower_fix(uuid_t *uuid, char *out)
65 {
66 uuid_unparse_lower(*uuid, out);
@@ -120,8 +119,6 @@ enum aclk_database_opcode {
119 ACLK_DATABASE_ADD_DIMENSION,
120 ACLK_DATABASE_ALARM_HEALTH_LOG,
121 ACLK_DATABASE_CHART_ACK,
123 - ACLK_DATABASE_CHECK,
124 - ACLK_DATABASE_CHECK_ROTATION,
122 ACLK_DATABASE_CLEANUP,
123 ACLK_DATABASE_DELETE_HOST,
124 ACLK_DATABASE_NODE_INFO,
@@ -130,11 +127,9 @@ enum aclk_database_opcode {
127 ACLK_DATABASE_PUSH_CHART,
128 ACLK_DATABASE_PUSH_CHART_CONFIG,
129 ACLK_DATABASE_RESET_CHART,
133 - ACLK_DATABASE_RESET_NODE,
130 ACLK_DATABASE_SHUTDOWN,
131 ACLK_DATABASE_TIMER,
136 - ACLK_DATABASE_UPD_STATS,
137 - ACLK_DATABASE_MAX_OPCODE
132 + ACLK_DATABASE_UPD_RETENTION
133 };
134
135 struct aclk_chart_payload_t {
@@ -170,6 +165,7 @@ struct aclk_database_worker_config {
165 time_t chart_timestamp; // last chart timestamp
166 time_t cleanup_after; // Start a cleanup after this timestamp
167 time_t startup_time; // When the sync thread started
168 + time_t rotation_after;
169 uint64_t batch_id; // batch id to use
170 uint64_t alerts_batch_id; // batch id for alerts to use
171 uint64_t alerts_start_seq_id; // cloud has asked to start streaming from
@@ -215,13 +211,11 @@ extern sqlite3 *db_meta;
211
212 extern int aclk_database_enq_cmd_noblock(struct aclk_database_worker_config *wc, struct aclk_database_cmd *cmd);
213 extern void aclk_database_enq_cmd(struct aclk_database_worker_config *wc, struct aclk_database_cmd *cmd);
218 -extern void aclk_set_architecture(int mode);
214 extern void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id);
215 int aclk_worker_enq_cmd(char *node_id, struct aclk_database_cmd *cmd);
221 -void aclk_data_rotated(RRDHOST *host);
216 +void aclk_data_rotated(void);
217 void sql_aclk_sync_init(void);
218 void sql_check_aclk_table_list(struct aclk_database_worker_config *wc);
219 void sql_delete_aclk_table_list(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
225 -void sql_drop_host_aclk_table_list(uuid_t *host_uuid);
220 void sql_maint_aclk_sync_database(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
221 #endif //NETDATA_SQLITE_ACLK_H
database/sqlite/sqlite_aclk_alert.c
+74 -38
@@ -3,7 +3,10 @@
3 #include "sqlite_functions.h"
4 #include "sqlite_aclk_alert.h"
5
6 +#ifdef ENABLE_ACLK
7 #include "../../aclk/aclk_alarm_api.h"
8 +#include "../../aclk/aclk.h"
9 +#endif
10
11 // will replace call to aclk_update_alarm in health/health_log.c
12 // and handle both cases
@@ -11,8 +14,14 @@ void sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae)
14 {
15 //check aclk architecture and handle old json alarm update to cloud
16 //include also the valid statuses for this case
14 - /* if (!aclk_architecture)
15 - aclk_update_alarm(host, ae); */
17 +#ifdef ENABLE_ACLK
18 + if (!aclk_use_new_cloud_arch) {
19 + if ((ae->new_status == RRDCALC_STATUS_WARNING || ae->new_status == RRDCALC_STATUS_CRITICAL) ||
20 + ((ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL))) {
21 + aclk_update_alarm(host, ae);
22 + }
23 + return;
24 + }
25
26 if (ae->flags & HEALTH_ENTRY_FLAG_ACLK_QUEUED)
27 return;
@@ -33,6 +42,10 @@ void sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae)
42 cmd.completion = NULL;
43 aclk_database_enq_cmd((struct aclk_database_worker_config *) host->dbsync_worker, &cmd);
44 ae->flags |= HEALTH_ENTRY_FLAG_ACLK_QUEUED;
45 +#else
46 + UNUSED(host);
47 + UNUSED(ae);
48 +#endif
49 return;
50 }
51
@@ -79,6 +92,7 @@ bind_fail:
92
93 int rrdcalc_status_to_proto_enum(RRDCALC_STATUS status)
94 {
95 +#ifdef ENABLE_ACLK
96 switch(status) {
97 case RRDCALC_STATUS_REMOVED:
98 return ALARM_STATUS_REMOVED;
@@ -98,6 +112,10 @@ int rrdcalc_status_to_proto_enum(RRDCALC_STATUS status)
112 default:
113 return ALARM_STATUS_UNKNOWN;
114 }
115 +#else
116 + UNUSED(status);
117 + return 1;
118 +#endif
119 }
120
121 void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
@@ -122,8 +140,8 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
140 if (wc->alerts_start_seq_id != 0) {
141 buffer_sprintf(
142 sql,
125 - "UPDATE aclk_alert_%s SET date_submitted = NULL, date_cloud_ack = NULL WHERE sequence_id >= %" PRIu64
126 - "; UPDATE aclk_alert_%s SET date_cloud_ack = strftime('%%s','now') WHERE sequence_id < %" PRIu64
143 + "UPDATE aclk_alert_%s SET date_submitted = NULL, date_cloud_ack = NULL WHERE sequence_id >= %"PRIu64
144 + "; UPDATE aclk_alert_%s SET date_cloud_ack = strftime('%%s','now') WHERE sequence_id < %"PRIu64
145 " and date_cloud_ack is null",
146 wc->uuid_str,
147 wc->alerts_start_seq_id,
@@ -163,7 +181,7 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
181 char old_value_string[100 + 1];
182 char new_value_string[100 + 1];
183
166 - alarm_log.node_id = strdupz(wc->node_id);
184 + alarm_log.node_id = wc->node_id;
185 alarm_log.claim_id = claim_id;
186
187 alarm_log.chart = strdupz((char *)sqlite3_column_text(res, 12));
@@ -179,10 +197,13 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
197
198 alarm_log.utc_offset = wc->host->utc_offset;
199 alarm_log.timezone = strdupz((char *)wc->host->abbrev_timezone);
182 - alarm_log.exec_path = sqlite3_column_bytes(res, 14) > 0 ? strdupz((char *)sqlite3_column_text(res, 14)) : strdupz((char *)wc->host->health_default_exec);
200 + alarm_log.exec_path = sqlite3_column_bytes(res, 14) > 0 ? strdupz((char *)sqlite3_column_text(res, 14)) :
201 + strdupz((char *)wc->host->health_default_exec);
202 alarm_log.conf_source = strdupz((char *)sqlite3_column_text(res, 16));
203
185 - char *edit_command = sqlite3_column_bytes(res, 16) > 0 ? health_edit_command_from_source((char *)sqlite3_column_text(res, 16)) : strdupz("UNKNOWN=0");
204 + char *edit_command = sqlite3_column_bytes(res, 16) > 0 ?
205 + health_edit_command_from_source((char *)sqlite3_column_text(res, 16)) :
206 + strdupz("UNKNOWN=0");
207 alarm_log.command = strdupz(edit_command);
208
209 alarm_log.duration = (time_t) sqlite3_column_int64(res, 6);
@@ -193,10 +214,23 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
214 alarm_log.delay_up_to_timestamp = (time_t) sqlite3_column_int64(res, 10);
215 alarm_log.last_repeat = (time_t) sqlite3_column_int64(res, 25);
216
196 - alarm_log.silenced = ( (sqlite3_column_int64(res, 8) & HEALTH_ENTRY_FLAG_SILENCED) || ( sqlite3_column_type(res, 15) != SQLITE_NULL && !strncmp((char *)sqlite3_column_text(res,15), "silent", 6)) ) ? 1 : 0;
217 + alarm_log.silenced = ((sqlite3_column_int64(res, 8) & HEALTH_ENTRY_FLAG_SILENCED) ||
218 + (sqlite3_column_type(res, 15) != SQLITE_NULL &&
219 + !strncmp((char *)sqlite3_column_text(res, 15), "silent", 6))) ?
220 + 1 :
221 + 0;
222 +
223 + alarm_log.value_string =
224 + sqlite3_column_type(res, 23) == SQLITE_NULL ?
225 + strdupz((char *)"-") :
226 + strdupz((char *)format_value_and_unit(
227 + new_value_string, 100, sqlite3_column_double(res, 23), (char *)sqlite3_column_text(res, 17), -1));
228
198 - alarm_log.value_string = sqlite3_column_type(res, 23) == SQLITE_NULL ? strdupz((char *)"-") : strdupz((char *)format_value_and_unit(new_value_string, 100, sqlite3_column_double(res, 23), (char *) sqlite3_column_text(res, 17), -1));
199 - alarm_log.old_value_string = sqlite3_column_type(res, 24) == SQLITE_NULL ? strdupz((char *)"-") : strdupz((char *)format_value_and_unit(old_value_string, 100, sqlite3_column_double(res, 24), (char *) sqlite3_column_text(res, 17), -1));
229 + alarm_log.old_value_string =
230 + sqlite3_column_type(res, 24) == SQLITE_NULL ?
231 + strdupz((char *)"-") :
232 + strdupz((char *)format_value_and_unit(
233 + old_value_string, 100, sqlite3_column_double(res, 24), (char *)sqlite3_column_text(res, 17), -1));
234
235 alarm_log.value = (calculated_number) sqlite3_column_double(res, 23);
236 alarm_log.old_value = (calculated_number) sqlite3_column_double(res, 24);
@@ -204,7 +238,6 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
238 alarm_log.updated = (sqlite3_column_int64(res, 8) & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0;
239 alarm_log.rendered_info = strdupz((char *)sqlite3_column_text(res, 18));
240
207 - info("DEBUG: %s pushing alert seq %" PRIu64 " - %" PRIu64"", wc->uuid_str, (uint64_t) sqlite3_column_int64(res, 0), (uint64_t) sqlite3_column_int64(res, 1));
241 aclk_send_alarm_log_entry(&alarm_log);
242
243 if (first_sequence_id == 0)
@@ -214,12 +247,14 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
247 destroy_alarm_log_entry(&alarm_log);
248 freez(edit_command);
249 }
217 - buffer_flush(sql);
250
219 - buffer_sprintf(sql, "UPDATE aclk_alert_%s SET date_submitted=strftime('%%s') "
220 - "WHERE date_submitted IS NULL AND sequence_id BETWEEN %" PRIu64 " AND %" PRIu64 ";",
221 - wc->uuid_str, first_sequence_id, last_sequence_id);
222 - db_execute(buffer_tostring(sql));
251 + if (first_sequence_id) {
252 + buffer_flush(sql);
253 + buffer_sprintf(sql, "UPDATE aclk_alert_%s SET date_submitted=strftime('%%s') "
254 + "WHERE date_submitted IS NULL AND sequence_id BETWEEN %" PRIu64 " AND %" PRIu64 ";",
255 + wc->uuid_str, first_sequence_id, last_sequence_id);
256 + db_execute(buffer_tostring(sql));
257 + }
258
259 rc = sqlite3_finalize(res);
260 if (unlikely(rc != SQLITE_OK))
@@ -278,12 +313,8 @@ void aclk_push_alarm_health_log(struct aclk_database_worker_config *wc, struct a
313 sqlite3_stmt *res = NULL;
314
315 //TODO: make this better: include info from health log too
281 - buffer_sprintf(sql, "select aa.sequence_id, aa.date_created, \
282 - (select laa.sequence_id from aclk_alert_%s laa \
283 - order by laa.sequence_id desc limit 1), \
284 - (select laa.date_created from aclk_alert_%s laa \
285 - order by laa.sequence_id desc limit 1) \
286 - from aclk_alert_%s aa order by aa.sequence_id asc limit 1;", wc->uuid_str, wc->uuid_str, wc->uuid_str);
316 + buffer_sprintf(sql, "SELECT MIN(sequence_id), MIN(date_created), MAX(sequence_id), MAX(date_created) " \
317 + "FROM aclk_alert_%s;", wc->uuid_str);
318
319 rc = sqlite3_prepare_v2(db_meta, buffer_tostring(sql), -1, &res, 0);
320 if (rc != SQLITE_OK) {
@@ -318,7 +349,7 @@ void aclk_push_alarm_health_log(struct aclk_database_worker_config *wc, struct a
349
350 struct alarm_log_health alarm_log;
351 alarm_log.claim_id = claim_id;
321 - alarm_log.node_id = strdupz(wc->node_id);
352 + alarm_log.node_id = wc->node_id;
353 alarm_log.log_entries = log_entries;
354 alarm_log.status = wc->alert_updates == 0 ? 2 : 1;
355
@@ -330,7 +361,6 @@ void aclk_push_alarm_health_log(struct aclk_database_worker_config *wc, struct a
361 if (unlikely(rc != SQLITE_OK))
362 error_report("Failed to reset statement to get health log statistics from the database, rc = %d", rc);
363
333 - freez((char *)alarm_log.node_id);
364 freez(claim_id);
365 buffer_free(sql);
366 #endif
@@ -359,6 +389,11 @@ void aclk_send_alarm_configuration(char *config_hash)
389 return;
390 }
391
392 +#define SQL_SELECT_ALERT_CONFIG "SELECT alarm, template, on_key, class, type, component, os, hosts, plugin," \
393 + "module, charts, families, lookup, every, units, green, red, calc, warn, crit, to_key, exec, delay, repeat, info," \
394 + "options, host_labels, p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after," \
395 + "p_db_lookup_before, p_update_every FROM alert_hash WHERE hash_id = @hash_id;"
396 +
397 int aclk_push_alert_config_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
398 {
399 UNUSED(wc);
@@ -372,20 +407,16 @@ int aclk_push_alert_config_event(struct aclk_database_worker_config *wc, struct
407 sqlite3_stmt *res = NULL;
408
409 char *config_hash = (char *) cmd.data_param;
375 - BUFFER *sql = buffer_create(1024);
376 - buffer_sprintf(
377 - sql,
378 - "SELECT alarm, template, on_key, class, type, component, os, hosts, plugin, module, charts, families, lookup, every, units, green, red, calc, warn, crit, to_key, exec, delay, repeat, info, options, host_labels, p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after, p_db_lookup_before, p_update_every FROM alert_hash WHERE hash_id = @hash_id;");
410
380 - rc = sqlite3_prepare_v2(db_meta, buffer_tostring(sql), -1, &res, 0);
411 + rc = sqlite3_prepare_v2(db_meta, SQL_SELECT_ALERT_CONFIG, -1, &res, 0);
412 if (rc != SQLITE_OK) {
382 - error_report("Failed to prepare statement when trying to fetch a chart hash configuration");
383 - goto fail;
413 + error_report("Failed to prepare statement when trying to fetch an alarm hash configuration");
414 + return 1;
415 }
416
417 uuid_t hash_uuid;
418 if (uuid_parse(config_hash, hash_uuid))
388 - goto fail;
419 + return 1;
420
421 rc = sqlite3_bind_blob(res, 1, &hash_uuid , sizeof(hash_uuid), SQLITE_STATIC);
422 if (unlikely(rc != SQLITE_OK))
@@ -463,16 +494,13 @@ int aclk_push_alert_config_event(struct aclk_database_worker_config *wc, struct
494 destroy_aclk_alarm_configuration(&alarm_config);
495 }
496 else
466 - info("DEBUG: Alert config for %s not found", config_hash);
497 + info("Alert config for %s not found", config_hash);
498
468 - bind_fail:
499 +bind_fail:
500 rc = sqlite3_finalize(res);
501 if (unlikely(rc != SQLITE_OK))
502 error_report("Failed to reset statement when pushing alarm config hash, rc = %d", rc);
503
473 - fail:
474 - buffer_free(sql);
475 -
504 return rc;
505 #endif
506 return 0;
@@ -497,6 +525,11 @@ void aclk_start_alert_streaming(char *node_id, uint64_t batch_id, uint64_t start
525 wc = (struct aclk_database_worker_config *)host->dbsync_worker;
526 rrd_unlock();
527
528 + if (unlikely(!host->health_enabled)) {
529 + info("Ignoring request to stream alert state changes, health is disabled for %s", host->machine_guid);
530 + return;
531 + }
532 +
533 if (likely(wc)) {
534 info("START streaming alerts for %s enabled with batch_id %"PRIu64" and start_seq_id %"PRIu64, node_id, batch_id, start_seq_id);
535 __sync_synchronize();
@@ -528,7 +561,10 @@ int sql_queue_removed_alerts_to_aclk(RRDHOST *host)
561 BUFFER *sql = buffer_create(1024);
562
563 buffer_sprintf(sql,"insert into aclk_alert_%s (alert_unique_id, date_created) " \
531 - "select unique_id alert_unique_id, strftime('%%s') date_created from health_log_%s where new_status = -2 and updated_by_id = 0 and unique_id not in (select alert_unique_id from aclk_alert_%s) order by unique_id asc on conflict (alert_unique_id) do nothing;", wc->uuid_str, wc->uuid_str, wc->uuid_str);
564 + "select unique_id alert_unique_id, strftime('%%s') date_created from health_log_%s " \
565 + "where new_status = -2 and updated_by_id = 0 and unique_id not in " \
566 + "(select alert_unique_id from aclk_alert_%s) order by unique_id asc " \
567 + "on conflict (alert_unique_id) do nothing;", wc->uuid_str, wc->uuid_str, wc->uuid_str);
568
569 db_execute(buffer_tostring(sql));
570
database/sqlite/sqlite_aclk_chart.c
+74 -45
@@ -3,8 +3,12 @@
3 #include "sqlite_functions.h"
4 #include "sqlite_aclk_chart.h"
5
6 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL
7 #include "../../aclk/aclk_charts_api.h"
8 +#include "../../aclk/aclk.h"
9 +#endif
10
11 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL
12 static inline int sql_queue_chart_payload(struct aclk_database_worker_config *wc,
13 void *data, enum aclk_database_opcode opcode)
14 {
@@ -117,6 +121,7 @@ bind_fail:
121 error_report("Failed to reset statement in store chart payload, rc = %d", rc);
122 return (rc != SQLITE_DONE);
123 }
124 +#endif
125
126 int aclk_add_chart_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
127 {
@@ -134,8 +139,8 @@ int aclk_add_chart_event(struct aclk_database_worker_config *wc, struct aclk_dat
139 chart_payload.config_hash = get_str_from_uuid(&st->state->hash_id);
140 chart_payload.update_every = st->update_every;
141 chart_payload.memory_mode = st->rrd_memory_mode;
137 - chart_payload.name = strdupz((char *)st->name);
138 - chart_payload.node_id = strdupz(wc->node_id);
142 + chart_payload.name = (char *)st->name;
143 + chart_payload.node_id = wc->node_id;
144 chart_payload.claim_id = claim_id;
145 chart_payload.id = strdupz(st->id);
146
@@ -186,12 +191,12 @@ int aclk_add_dimension_event(struct aclk_database_worker_config *wc, struct aclk
191 size_t size;
192
193 memset(&dim_payload, 0, sizeof(dim_payload));
189 - dim_payload.node_id = strdupz(wc->node_id);
194 + dim_payload.node_id = wc->node_id;
195 dim_payload.claim_id = claim_id;
191 - dim_payload.name = strdupz(rd->name);
192 - dim_payload.id = strdupz(rd->id);
196 + dim_payload.name = rd->name;
197 + dim_payload.id = rd->id;
198
194 - dim_payload.chart_id = strdupz(rd->rrdset->name);
199 + dim_payload.chart_id = rd->rrdset->name;
200 dim_payload.created_at.tv_sec = first_t;
201 if (unlikely(!live))
202 dim_payload.last_timestamp.tv_sec = last_t;
@@ -199,10 +204,6 @@ int aclk_add_dimension_event(struct aclk_database_worker_config *wc, struct aclk
204 char *payload = generate_chart_dimension_updated(&size, &dim_payload);
205 if (likely(payload))
206 rc = aclk_add_chart_payload(wc->uuid_str, &rd->state->metric_uuid, claim_id, ACLK_PAYLOAD_DIMENSION, (void *)payload, size);
202 - freez((char *)dim_payload.node_id);
203 - freez((char *)dim_payload.chart_id);
204 - freez((char *)dim_payload.name);
205 - freez((char *)dim_payload.id);
207 freez(payload);
208 freez(claim_id);
209 }
@@ -229,6 +230,10 @@ void aclk_send_chart_event(struct aclk_database_worker_config *wc, struct aclk_d
230 if (unlikely(!claim_id))
231 return;
232
233 + uuid_t claim_uuid;
234 + if (uuid_parse(claim_id, claim_uuid))
235 + return;
236 +
237 int limit = cmd.count > 0 ? cmd.count : 1;
238
239 uint64_t first_sequence;
@@ -252,7 +257,7 @@ void aclk_send_chart_event(struct aclk_database_worker_config *wc, struct aclk_d
257 return;
258 }
259
255 - rc = sqlite3_bind_text(res, 1, claim_id , -1, SQLITE_STATIC);
260 + rc = sqlite3_bind_blob(res, 1, claim_uuid , sizeof(claim_uuid), SQLITE_STATIC);
261 if (unlikely(rc != SQLITE_OK))
262 goto bind_fail;
263
@@ -404,7 +409,7 @@ int aclk_send_chart_config(struct aclk_database_worker_config *wc, struct aclk_d
409 destroy_chart_config_updated(&chart_config);
410 }
411 else
407 - info("DEBUG: Chart config for %s not found", hash_id);
412 + info("Chart config for %s not found", hash_id);
413
414 bind_fail:
415 rc = sqlite3_finalize(res);
@@ -455,6 +460,7 @@ void aclk_receive_chart_ack(struct aclk_database_worker_config *wc, struct aclk_
460
461 void aclk_receive_chart_reset(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
462 {
463 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL
464 BUFFER *sql = buffer_create(1024);
465 buffer_sprintf(sql, "UPDATE aclk_chart_%s SET status = NULL, date_submitted = NULL WHERE sequence_id >= %"PRIu64";",
466 wc->uuid_str, cmd.param1);
@@ -462,15 +468,18 @@ void aclk_receive_chart_reset(struct aclk_database_worker_config *wc, struct acl
468 if (cmd.param1 == 1) {
469 db_lock();
470 buffer_flush(sql);
465 - info("DEBUG: Deleting all data for %s", wc->uuid_str);
466 - buffer_sprintf(sql, "DELETE FROM aclk_chart_payload_%s; DELETE FROM aclk_chart_%s; DELETE FROM aclk_chart_latest_%s;",
467 - wc->uuid_str, wc->uuid_str, wc->uuid_str);
471 + info("Received full resync for %s", wc->uuid_str);
472 + buffer_sprintf(sql, "DELETE FROM aclk_chart_payload_%s; DELETE FROM aclk_chart_%s; " \
473 + "DELETE FROM aclk_chart_latest_%s;", wc->uuid_str, wc->uuid_str, wc->uuid_str);
474 +
475 + db_execute("BEGIN TRANSACTION;");
476 db_execute(buffer_tostring(sql));
477 + db_execute("COMMIT TRANSACTION;");
478 +
479 db_unlock();
480 wc->chart_sequence_id = 0;
481 wc->chart_timestamp = 0;
482
473 -#ifdef ENABLE_NEW_CLOUD_PROTOCOL
483 RRDHOST *host = wc->host;
484 rrdhost_rdlock(host);
485 RRDSET *st;
@@ -484,7 +493,6 @@ void aclk_receive_chart_reset(struct aclk_database_worker_config *wc, struct acl
493 rrdset_unlock(st);
494 }
495 rrdhost_unlock(host);
487 -#endif
496 }
497 else {
498 //sql_chart_deduplicate(wc, cmd);
@@ -492,6 +500,11 @@ void aclk_receive_chart_reset(struct aclk_database_worker_config *wc, struct acl
500 }
501 buffer_free(sql);
502 wc->chart_updates = 1;
503 +#else
504 + UNUSED(wc);
505 + UNUSED(cmd);
506 +#endif
507 +
508 return;
509 }
510
@@ -555,6 +568,7 @@ void aclk_ack_chart_sequence_id(char *node_id, uint64_t last_sequence_id)
568 return;
569 }
570
571 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL
572 void aclk_reset_chart_event(char *node_id, uint64_t last_sequence_id)
573 {
574 if (unlikely(!node_id))
@@ -564,21 +578,47 @@ void aclk_reset_chart_event(char *node_id, uint64_t last_sequence_id)
578 aclk_submit_param_command(node_id, ACLK_DATABASE_RESET_CHART, last_sequence_id);
579 return;
580 }
581 +#endif
582
583 // ST is read locked
584 int sql_queue_chart_to_aclk(RRDSET *st)
585 {
586 +#ifdef ENABLE_ACLK
587 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL
588 + if (!aclk_use_new_cloud_arch)
589 +#endif
590 + {
591 + rrdset_flag_clear(st, RRDSET_FLAG_ACLK);
592 + aclk_update_chart(st->rrdhost, st->id, 1);
593 + return 0;
594 + }
595 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL
596 return sql_queue_chart_payload((struct aclk_database_worker_config *) st->rrdhost->dbsync_worker,
597 st, ACLK_DATABASE_ADD_CHART);
598 +#else
599 + return 0;
600 +#endif
601 +#else
602 + UNUSED(st);
603 + return 0;
604 +#endif
605 }
606
607 int sql_queue_dimension_to_aclk(RRDDIM *rd)
608 {
609 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL
610 + if (!aclk_use_new_cloud_arch)
611 + return 0;
612 +
613 int rc = sql_queue_chart_payload((struct aclk_database_worker_config *) rd->rrdset->rrdhost->dbsync_worker,
614 rd, ACLK_DATABASE_ADD_DIMENSION);
615 if (likely(!rc))
616 rrddim_flag_set(rd, RRDDIM_FLAG_ACLK);
617 return rc;
618 +#else
619 + UNUSED(rd);
620 + return 0;
621 +#endif
622 }
623
624 void sql_chart_deduplicate(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
@@ -674,6 +714,7 @@ fail:
714 // Start streaming charts / dimensions for node_id
715 void aclk_start_streaming(char *node_id, uint64_t sequence_id, time_t created_at, uint64_t batch_id)
716 {
717 + UNUSED(created_at);
718 #ifdef ENABLE_NEW_CLOUD_PROTOCOL
719 if (unlikely(!node_id))
720 return;
@@ -692,52 +733,41 @@ void aclk_start_streaming(char *node_id, uint64_t sequence_id, time_t created_at
733 rrd_unlock();
734 wc = (struct aclk_database_worker_config *)host->dbsync_worker;
735 if (likely(wc)) {
695 - // if (unlikely(!wc->chart_updates)) {
696 - // struct aclk_database_cmd cmd;
697 - // cmd.opcode = ACLK_DATABASE_NODE_INFO;
698 - // cmd.completion = NULL;
699 - // aclk_database_enq_cmd(wc, &cmd);
700 - // }
701 -
736 wc->chart_reset_count++;
737 + __sync_synchronize();
738 wc->chart_updates = 0;
739 wc->batch_id = batch_id;
740 + __sync_synchronize();
741 wc->batch_created = now_realtime_sec();
706 - info("DEBUG: START streaming charts for %s (%s) enabled -- last streamed sequence %"PRIu64" t=%ld (reset count=%d)", node_id, wc->uuid_str,
707 - wc->chart_sequence_id, wc->chart_timestamp, wc->chart_reset_count);
708 - // If mismatch detected
742 if (sequence_id > wc->chart_sequence_id || wc->chart_reset_count > 10) {
710 - info("DEBUG: Full resync requested -- reset_count=%d", wc->chart_reset_count);
743 + debug(D_ACLK_SYNC,"Requesting full resync from the cloud -- reset_count=%d", wc->chart_reset_count);
744 chart_reset_t chart_reset;
712 - chart_reset.node_id = strdupz(node_id);
745 chart_reset.claim_id = is_agent_claimed();
714 - chart_reset.reason = SEQ_ID_NOT_EXISTS;
715 - aclk_chart_reset(chart_reset);
716 -// wc->chart_updates = 0;
717 - wc->chart_reset_count = -1;
746 + if (chart_reset.claim_id) {
747 + chart_reset.node_id = node_id;
748 + chart_reset.reason = SEQ_ID_NOT_EXISTS;
749 + aclk_chart_reset(chart_reset);
750 + freez(chart_reset.claim_id);
751 + wc->chart_reset_count = -1;
752 + }
753 return;
754 } else {
755 struct aclk_database_cmd cmd;
756 memset(&cmd, 0, sizeof(cmd));
757 // TODO: handle timestamp
723 -// if (!wc->chart_reset_count)
724 -// wc->chart_delay = now_realtime_sec() + 60;
725 -// else
726 -// wc->chart_delay = 0;
727 -
758 if (sequence_id < wc->chart_sequence_id) { // || created_at != wc->chart_timestamp) {
729 -// wc->chart_updates = 0;
730 - if (sequence_id)
731 - info("DEBUG: Synchonization mismatch detected");
732 - else
733 - info("DEBUG: Synchonization mismatch detected; full resync ACKed from the cloud");
759 cmd.opcode = ACLK_DATABASE_RESET_CHART;
760 cmd.param1 = sequence_id + 1;
761 cmd.completion = NULL;
762 aclk_database_enq_cmd(wc, &cmd);
763 }
739 - else
764 + else {
765 + debug(D_ACLK_SYNC,"START streaming charts for %s enabled -- last streamed sequence %"PRIu64 \
766 + " t=%ld (reset count=%d)", wc->host_guid, wc->chart_sequence_id,
767 + wc->chart_timestamp, wc->chart_reset_count);
768 + wc->chart_reset_count = 0;
769 wc->chart_updates = 1;
770 + }
771 }
772 }
773 else
@@ -750,7 +780,6 @@ void aclk_start_streaming(char *node_id, uint64_t sequence_id, time_t created_at
780 #else
781 UNUSED(node_id);
782 UNUSED(sequence_id);
753 - UNUSED(created_at);
783 UNUSED(batch_id);
784 #endif
785 return;
database/sqlite/sqlite_aclk_node.c
+160
@@ -5,6 +5,7 @@
5
6 #ifdef ENABLE_NEW_CLOUD_PROTOCOL
7 #include "../../aclk/aclk_charts_api.h"
8 +#include "../../aclk/aclk.h"
9 #endif
10
11 void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
@@ -14,6 +15,9 @@ void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_dat
15 #ifdef ENABLE_NEW_CLOUD_PROTOCOL
16 struct update_node_info node_info;
17
18 + if (!wc->host)
19 + return;
20 +
21 rrd_wrlock();
22 node_info.node_id = wc->node_id;
23 node_info.claim_id = is_agent_claimed();
@@ -59,3 +63,159 @@ void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_dat
63
64 return;
65 }
66 +#define SQL_SELECT_HOST_MEMORY_MODE "select memory_mode from chart where host_id = @host_id limit 1;"
67 +static RRD_MEMORY_MODE sql_get_host_memory_mode(uuid_t *host_id)
68 +{
69 + int rc;
70 +
71 + RRD_MEMORY_MODE memory_mode = RRD_MEMORY_MODE_RAM;
72 + sqlite3_stmt *res = NULL;
73 +
74 + rc = sqlite3_prepare_v2(db_meta, SQL_SELECT_HOST_MEMORY_MODE, -1, &res, 0);
75 +
76 + if (unlikely(rc != SQLITE_OK)) {
77 + error_report("Failed to prepare statement to read host memory mode");
78 + return memory_mode;
79 + }
80 +
81 + rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);
82 + if (unlikely(rc != SQLITE_OK)) {
83 + error_report("Failed to bind host parameter to fetch host memory mode");
84 + goto failed;
85 + }
86 +
87 + while (sqlite3_step(res) == SQLITE_ROW) {
88 + memory_mode = (RRD_MEMORY_MODE) sqlite3_column_int(res, 0);
89 + }
90 +
91 +failed:
92 + rc = sqlite3_finalize(res);
93 + if (unlikely(rc != SQLITE_OK))
94 + error_report("Failed to finalize the prepared statement when reading host memory mode");
95 + return memory_mode;
96 +}
97 +
98 +#define SELECT_HOST_DIMENSION_LIST "SELECT d.dim_id, c.update_every, c.type||'.'||c.id FROM chart c, dimension d, host h " \
99 + "WHERE d.chart_id = c.chart_id AND c.host_id = h.host_id AND c.host_id = @host_id ORDER BY c.update_every ASC;"
100 +
101 +#define SELECT_HOST_CHART_LIST "SELECT distinct h.host_id, c.update_every, c.type||'.'||c.id FROM chart c, host h " \
102 + "WHERE c.host_id = h.host_id AND c.host_id = @host_id ORDER BY c.update_every ASC;"
103 +
104 +void aclk_update_retention(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
105 +{
106 + UNUSED(cmd);
107 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL
108 + int rc;
109 +
110 + if (!aclk_use_new_cloud_arch || !aclk_connected)
111 + return;
112 +
113 + char *claim_id = is_agent_claimed();
114 + if (unlikely(!claim_id))
115 + return;
116 +
117 + sqlite3_stmt *res = NULL;
118 + RRD_MEMORY_MODE memory_mode;
119 +
120 + uuid_t host_uuid;
121 + rc = uuid_parse(wc->host_guid, host_uuid);
122 + if (unlikely(rc))
123 + return;
124 +
125 + if (wc->host)
126 + memory_mode = wc->host->rrd_memory_mode;
127 + else
128 + memory_mode = sql_get_host_memory_mode(&host_uuid);
129 +
130 + if (memory_mode == RRD_MEMORY_MODE_DBENGINE)
131 + rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_DIMENSION_LIST, -1, &res, 0);
132 + else
133 + rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_CHART_LIST, -1, &res, 0);
134 +
135 + if (unlikely(rc != SQLITE_OK)) {
136 + error_report("Failed to prepare statement to fetch host dimensions");
137 + freez(claim_id);
138 + return;
139 + }
140 +
141 + rc = sqlite3_bind_blob(res, 1, &host_uuid, sizeof(host_uuid), SQLITE_STATIC);
142 + if (unlikely(rc != SQLITE_OK)) {
143 + error_report("Failed to bind host parameter to fetch host dimensions");
144 + goto failed;
145 + }
146 +
147 + time_t start_time = LONG_MAX;
148 + time_t first_entry_t;
149 + uint32_t update_every = 0;
150 +
151 + struct retention_updated rotate_data;
152 +
153 + memset(&rotate_data, 0, sizeof(rotate_data));
154 +
155 + int max_intervals = 32;
156 +
157 + rotate_data.interval_duration_count = 0;
158 + rotate_data.interval_durations = callocz(max_intervals, sizeof(*rotate_data.interval_durations));
159 +
160 + now_realtime_timeval(&rotate_data.rotation_timestamp);
161 + rotate_data.memory_mode = memory_mode;
162 + rotate_data.claim_id = claim_id;
163 + rotate_data.node_id = strdupz(wc->node_id);
164 +
165 + while (sqlite3_step(res) == SQLITE_ROW) {
166 + if (!update_every || update_every != (uint32_t) sqlite3_column_int(res, 1)) {
167 + if (update_every) {
168 + debug(D_ACLK_SYNC,"Update %s for %u oldest time = %ld", wc->host_guid, update_every, start_time);
169 + rotate_data.interval_durations[rotate_data.interval_duration_count].retention = rotate_data.rotation_timestamp.tv_sec - start_time;
170 + rotate_data.interval_duration_count++;
171 + }
172 + update_every = (uint32_t) sqlite3_column_int(res, 1);
173 + rotate_data.interval_durations[rotate_data.interval_duration_count].update_every = update_every;
174 + start_time = LONG_MAX;
175 + }
176 +#ifdef ENABLE_DBENGINE
177 + time_t last_entry_t;
178 + if (memory_mode == RRD_MEMORY_MODE_DBENGINE)
179 + rc = rrdeng_metric_latest_time_by_uuid((uuid_t *)sqlite3_column_blob(res, 0), &first_entry_t, &last_entry_t);
180 + else
181 +#endif
182 + {
183 + if (wc->host) {
184 + RRDSET *st = NULL;
185 + rc = (st = rrdset_find(wc->host, (const char *)sqlite3_column_text(res, 2))) ? 0 : 1;
186 + if (!rc)
187 + first_entry_t = rrdset_first_entry_t(st);
188 + }
189 + else {
190 + rc = 0;
191 + first_entry_t = rotate_data.rotation_timestamp.tv_sec;
192 + }
193 + }
194 +
195 + if (likely(!rc && first_entry_t))
196 + start_time = MIN(start_time, first_entry_t);
197 + }
198 + if (update_every) {
199 + debug(D_ACLK_SYNC, "Update %s for %u oldest time = %ld", wc->host_guid, update_every, start_time);
200 + rotate_data.interval_durations[rotate_data.interval_duration_count].retention = rotate_data.rotation_timestamp.tv_sec - start_time;
201 + rotate_data.interval_duration_count++;
202 + }
203 +
204 + for (int i = 0; i < rotate_data.interval_duration_count; ++i) {
205 + debug(D_ACLK_SYNC,"%d --> Update %s for %u Retention = %u", i, wc->host_guid,
206 + rotate_data.interval_durations[i].update_every, rotate_data.interval_durations[i].retention);
207 + };
208 + aclk_retention_updated(&rotate_data);
209 + freez(rotate_data.node_id);
210 + freez(rotate_data.interval_durations);
211 +
212 +failed:
213 + freez(claim_id);
214 + rc = sqlite3_finalize(res);
215 + if (unlikely(rc != SQLITE_OK))
216 + error_report("Failed to finalize the prepared statement when reading host dimensions");
217 +#else
218 + UNUSED(wc);
219 +#endif
220 + return;
221 +}
\ No newline at end of file
database/sqlite/sqlite_aclk_node.h
+1
@@ -4,4 +4,5 @@
4 #define NETDATA_SQLITE_ACLK_NODE_H
5
6 void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
7 +void aclk_update_retention(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
8 #endif //NETDATA_SQLITE_ACLK_NODE_H
database/sqlite/sqlite_functions.c
+201 -7
@@ -26,6 +26,25 @@ const char *database_config[] = {
26 "red text, warn text, crit text, exec text, to_key text, info text, delay text, options text, "
27 "repeat text, host_labels text, p_db_lookup_dimensions text, p_db_lookup_method text, p_db_lookup_options int, "
28 "p_db_lookup_after int, p_db_lookup_before int, p_update_every int);",
29 +
30 + "CREATE TABLE IF NOT EXISTS chart_hash_map(chart_id blob , hash_id blob, UNIQUE (chart_id, hash_id));",
31 +
32 + "CREATE TABLE IF NOT EXISTS chart_hash(hash_id blob PRIMARY KEY,type text, id text, name text, "
33 + "family text, context text, title text, unit text, plugin text, "
34 + "module text, priority integer, chart_type, last_used);",
35 +
36 + "CREATE VIEW IF NOT EXISTS v_chart_hash as SELECT ch.*, chm.chart_id FROM chart_hash ch, chart_hash_map chm "
37 + "WHERE ch.hash_id = chm.hash_id;",
38 +
39 + "CREATE TRIGGER IF NOT EXISTS tr_v_chart_hash INSTEAD OF INSERT on v_chart_hash BEGIN "
40 + "INSERT INTO chart_hash (hash_id, type, id, name, family, context, title, unit, plugin, "
41 + "module, priority, chart_type, last_used) "
42 + "values (new.hash_id, new.type, new.id, new.name, new.family, new.context, new.title, new.unit, new.plugin, "
43 + "new.module, new.priority, new.chart_type, strftime('%s')) "
44 + "ON CONFLICT (hash_id) DO UPDATE SET last_used = strftime('%s'); "
45 + "INSERT INTO chart_hash_map (chart_id, hash_id) values (new.chart_id, new.hash_id) "
46 + "on conflict (chart_id, hash_id) do nothing; END; ",
47 +
48 "delete from chart_active;",
49 "delete from dimension_active;",
50 "delete from chart where chart_id not in (select chart_id from dimension);",
@@ -1331,6 +1350,174 @@ failed:
1350 return;
1351 }
1352
1353 +
1354 +/*
1355 + * Store a chart hash in the database
1356 + */
1357 +
1358 +#define SQL_STORE_CHART_HASH "insert into v_chart_hash (hash_id, type, id, " \
1359 + "name, family, context, title, unit, plugin, module, priority, chart_type, last_used, chart_id) " \
1360 + "values (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11, ?12, strftime('%s'), ?13);"
1361 +
1362 +int sql_store_chart_hash(
1363 + uuid_t *hash_id, uuid_t *chart_id, const char *type, const char *id, const char *name, const char *family,
1364 + const char *context, const char *title, const char *units, const char *plugin, const char *module, long priority,
1365 + RRDSET_TYPE chart_type)
1366 +{
1367 + static __thread sqlite3_stmt *res = NULL;
1368 + int rc, param = 0;
1369 +
1370 + if (unlikely(!db_meta)) {
1371 + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
1372 + return 0;
1373 + error_report("Database has not been initialized");
1374 + return 1;
1375 + }
1376 +
1377 + if (unlikely(!res)) {
1378 + rc = prepare_statement(db_meta, SQL_STORE_CHART_HASH, &res);
1379 + if (unlikely(rc != SQLITE_OK)) {
1380 + error_report("Failed to prepare statement to store chart, rc = %d", rc);
1381 + return 1;
1382 + }
1383 + }
1384 +
1385 + param++;
1386 + rc = sqlite3_bind_blob(res, 1, hash_id, sizeof(*hash_id), SQLITE_STATIC);
1387 + if (unlikely(rc != SQLITE_OK))
1388 + goto bind_fail;
1389 +
1390 + param++;
1391 + rc = sqlite3_bind_text(res, 2, type, -1, SQLITE_STATIC);
1392 + if (unlikely(rc != SQLITE_OK))
1393 + goto bind_fail;
1394 +
1395 + param++;
1396 + rc = sqlite3_bind_text(res, 3, id, -1, SQLITE_STATIC);
1397 + if (unlikely(rc != SQLITE_OK))
1398 + goto bind_fail;
1399 +
1400 + param++;
1401 + if (name && *name)
1402 + rc = sqlite3_bind_text(res, 4, name, -1, SQLITE_STATIC);
1403 + else
1404 + rc = sqlite3_bind_null(res, 4);
1405 + if (unlikely(rc != SQLITE_OK))
1406 + goto bind_fail;
1407 +
1408 + param++;
1409 + rc = sqlite3_bind_text(res, 5, family, -1, SQLITE_STATIC);
1410 + if (unlikely(rc != SQLITE_OK))
1411 + goto bind_fail;
1412 +
1413 + param++;
1414 + rc = sqlite3_bind_text(res, 6, context, -1, SQLITE_STATIC);
1415 + if (unlikely(rc != SQLITE_OK))
1416 + goto bind_fail;
1417 +
1418 + param++;
1419 + rc = sqlite3_bind_text(res, 7, title, -1, SQLITE_STATIC);
1420 + if (unlikely(rc != SQLITE_OK))
1421 + goto bind_fail;
1422 +
1423 + param++;
1424 + rc = sqlite3_bind_text(res, 8, units, -1, SQLITE_STATIC);
1425 + if (unlikely(rc != SQLITE_OK))
1426 + goto bind_fail;
1427 +
1428 + param++;
1429 + rc = sqlite3_bind_text(res, 9, plugin, -1, SQLITE_STATIC);
1430 + if (unlikely(rc != SQLITE_OK))
1431 + goto bind_fail;
1432 +
1433 + param++;
1434 + rc = sqlite3_bind_text(res, 10, module, -1, SQLITE_STATIC);
1435 + if (unlikely(rc != SQLITE_OK))
1436 + goto bind_fail;
1437 +
1438 + param++;
1439 + rc = sqlite3_bind_int(res, 11, (int) priority);
1440 + if (unlikely(rc != SQLITE_OK))
1441 + goto bind_fail;
1442 +
1443 + param++;
1444 + rc = sqlite3_bind_int(res, 12, chart_type);
1445 + if (unlikely(rc != SQLITE_OK))
1446 + goto bind_fail;
1447 +
1448 + param++;
1449 + rc = sqlite3_bind_blob(res, 13, chart_id, sizeof(*chart_id), SQLITE_STATIC);
1450 + if (unlikely(rc != SQLITE_OK))
1451 + goto bind_fail;
1452 +
1453 + rc = execute_insert(res);
1454 + if (unlikely(rc != SQLITE_DONE))
1455 + error_report("Failed to store chart hash_id, rc = %d", rc);
1456 +
1457 + rc = sqlite3_reset(res);
1458 + if (unlikely(rc != SQLITE_OK))
1459 + error_report("Failed to reset statement in chart hash_id store function, rc = %d", rc);
1460 +
1461 + return 0;
1462 +
1463 + bind_fail:
1464 + error_report("Failed to bind parameter %d to store chart hash_id, rc = %d", param, rc);
1465 + rc = sqlite3_reset(res);
1466 + if (unlikely(rc != SQLITE_OK))
1467 + error_report("Failed to reset statement in chart hash_id store function, rc = %d", rc);
1468 + return 1;
1469 +}
1470 +
1471 +void compute_chart_hash(RRDSET *st)
1472 +{
1473 + EVP_MD_CTX *evpctx;
1474 + unsigned char hash_value[EVP_MAX_MD_SIZE];
1475 + unsigned int hash_len;
1476 + char priority_str[32];
1477 +
1478 + sprintf(priority_str, "%ld", st->priority);
1479 +
1480 + evpctx = EVP_MD_CTX_create();
1481 + EVP_DigestInit_ex(evpctx, EVP_sha256(), NULL);
1482 + //EVP_DigestUpdate(evpctx, st->type, strlen(st->type));
1483 + EVP_DigestUpdate(evpctx, st->id, strlen(st->id));
1484 + EVP_DigestUpdate(evpctx, st->name, strlen(st->name));
1485 + EVP_DigestUpdate(evpctx, st->family, strlen(st->family));
1486 + EVP_DigestUpdate(evpctx, st->context, strlen(st->context));
1487 + EVP_DigestUpdate(evpctx, st->title, strlen(st->title));
1488 + EVP_DigestUpdate(evpctx, st->units, strlen(st->units));
1489 + EVP_DigestUpdate(evpctx, st->plugin_name, strlen(st->plugin_name));
1490 + if (st->module_name)
1491 + EVP_DigestUpdate(evpctx, st->module_name, strlen(st->module_name));
1492 +// EVP_DigestUpdate(evpctx, priority_str, strlen(priority_str));
1493 + EVP_DigestUpdate(evpctx, &st->priority, sizeof(st->priority));
1494 + EVP_DigestUpdate(evpctx, &st->chart_type, sizeof(st->chart_type));
1495 + EVP_DigestFinal_ex(evpctx, hash_value, &hash_len);
1496 + EVP_MD_CTX_destroy(evpctx);
1497 + fatal_assert(hash_len > sizeof(uuid_t));
1498 +
1499 + char uuid_str[GUID_LEN + 1];
1500 + uuid_unparse_lower(*((uuid_t *) &hash_value), uuid_str);
1501 + //info("Calculating HASH %s for chart %s", uuid_str, st->name);
1502 + uuid_copy(st->state->hash_id, *((uuid_t *) &hash_value));
1503 +
1504 + (void)sql_store_chart_hash(
1505 + (uuid_t *)&hash_value,
1506 + st->chart_uuid,
1507 + st->type,
1508 + st->id,
1509 + st->name,
1510 + st->family,
1511 + st->context,
1512 + st->title,
1513 + st->units,
1514 + st->plugin_name,
1515 + st->module_name,
1516 + st->priority,
1517 + st->chart_type);
1518 + return;
1519 +}
1520 +
1521 #define SQL_STORE_CLAIM_ID "insert into node_instance " \
1522 "(host_id, claim_id, date_created) values (@host_id, @claim_id, strftime('%s')) " \
1523 "on conflict(host_id) do update set claim_id = excluded.claim_id;"
@@ -1389,9 +1576,16 @@ static inline void set_host_node_id(RRDHOST *host, uuid_t *node_id)
1576 return;
1577 }
1578
1579 + struct aclk_database_worker_config *wc = host->dbsync_worker;
1580 +
1581 if (unlikely(!host->node_id))
1582 host->node_id = mallocz(sizeof(*host->node_id));
1583 uuid_copy(*(host->node_id), *node_id);
1584 +
1585 + if (unlikely(!wc))
1586 + sql_create_aclk_table(host, &host->host_uuid, node_id);
1587 + else
1588 + uuid_unparse_lower(*node_id, wc->node_id);
1589 return;
1590 }
1591
@@ -1581,7 +1775,7 @@ struct node_instance_list *get_node_list(void)
1775
1776 rc = sqlite3_prepare_v2(db_meta, SQL_GET_NODE_INSTANCE_LIST, -1, &res, 0);
1777 if (unlikely(rc != SQLITE_OK)) {
1584 - error_report("Failed to prepare statement store chart labels");
1778 + error_report("Failed to prepare statement to get node instance information");
1779 return NULL;
1780 };
1781
@@ -1591,7 +1785,7 @@ struct node_instance_list *get_node_list(void)
1785 row++;
1786
1787 if (sqlite3_reset(res) != SQLITE_OK) {
1594 - error_report("Failed to reset the prepared statement fetching storing node instance information");
1788 + error_report("Failed to reset the prepared statement while fetching node instance information");
1789 goto failed;
1790 }
1791 node_list = callocz(row + 1, sizeof(*node_list));
@@ -1603,7 +1797,7 @@ struct node_instance_list *get_node_list(void)
1797 if (sqlite3_column_bytes(res, 1) == sizeof(uuid_t)) {
1798 uuid_t *host_id = (uuid_t *)sqlite3_column_blob(res, 1);
1799 uuid_copy(node_list[row].host_id, *host_id);
1606 - node_list[row].querable = 1;
1800 + node_list[row].queryable = 1;
1801 uuid_unparse_lower(*host_id, host_guid);
1802 node_list[row].live = rrdhost_find_by_guid(host_guid, 0) ? 1 : 0;
1803 node_list[row].hops = uuid_compare(*host_id, localhost->host_uuid) ? 1 : 0;
@@ -1617,7 +1811,7 @@ struct node_instance_list *get_node_list(void)
1811
1812 failed:
1813 if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
1620 - error_report("Failed to finalize the prepared statement when storing node instance information");
1814 + error_report("Failed to finalize the prepared statement when fetching node instance information");
1815
1816 return node_list;
1817 };
@@ -1637,13 +1831,13 @@ void sql_load_node_id(RRDHOST *host)
1831
1832 rc = sqlite3_prepare_v2(db_meta, SQL_GET_HOST_NODE_ID, -1, &res, 0);
1833 if (unlikely(rc != SQLITE_OK)) {
1640 - error_report("Failed to prepare statement store chart labels");
1834 + error_report("Failed to prepare statement to fetch node id");
1835 return;
1836 };
1837
1838 rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
1839 if (unlikely(rc != SQLITE_OK)) {
1646 - error_report("Failed to bind host_id parameter to store node instance information");
1840 + error_report("Failed to bind host_id parameter to load node instance information");
1841 goto failed;
1842 }
1843
@@ -1657,7 +1851,7 @@ void sql_load_node_id(RRDHOST *host)
1851
1852 failed:
1853 if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
1660 - error_report("Failed to finalize the prepared statement when storing node instance information");
1854 + error_report("Failed to finalize the prepared statement when loading node instance information");
1855
1856 return;
1857 };
database/sqlite/sqlite_functions.h
+2 -3
@@ -12,7 +12,7 @@ struct node_instance_list {
12 uuid_t host_id;
13 char *hostname;
14 int live;
15 - int querable;
15 + int queryable;
16 int hops;
17 };
18
@@ -90,6 +90,5 @@ extern int get_host_id(uuid_t *node_id, uuid_t *host_id);
90 extern void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id);
91 extern struct node_instance_list *get_node_list(void);
92 extern void sql_load_node_id(RRDHOST *host);
93 -extern int execute_insert(sqlite3_stmt *res);
94 -extern int prepare_statement(sqlite3 *database, char *query, sqlite3_stmt **statement);
93 +extern void compute_chart_hash(RRDSET *st);
94 #endif //NETDATA_SQLITE_FUNCTIONS_H
database/sqlite/sqlite_health.c
+13 -2
@@ -109,7 +109,12 @@ void sql_health_alarm_log_update(RRDHOST *host, ALARM_ENTRY *ae) {
109 /* Health related SQL queries
110 Inserts an entry in the table
111 */
112 -#define SQL_INSERT_HEALTH_LOG(guid) "INSERT INTO health_log_%s(hostname, unique_id, alarm_id, alarm_event_id, config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, name, chart, family, exec, recipient, source, units, info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, class, component, type) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);", guid
112 +#define SQL_INSERT_HEALTH_LOG(guid) "INSERT INTO health_log_%s(hostname, unique_id, alarm_id, alarm_event_id, " \
113 + "config_hash_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, " \
114 + "exec_run_timestamp, delay_up_to_timestamp, name, chart, family, exec, recipient, source, " \
115 + "units, info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, " \
116 + "class, component, type) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);", guid
117 +
118 void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
119 sqlite3_stmt *res = NULL;
120 int rc;
@@ -639,7 +644,13 @@ void sql_health_alarm_log_load(RRDHOST *host) {
644 /*
645 * Store an alert config hash in the database
646 */
642 -#define SQL_STORE_ALERT_CONFIG_HASH "insert or replace into alert_hash (hash_id, date_updated, alarm, template, on_key, class, component, type, os, hosts, lookup, every, units, calc, families, plugin, module, charts, green, red, warn, crit, exec, to_key, info, delay, options, repeat, host_labels, p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after, p_db_lookup_before, p_update_every) values (?1,strftime('%s'),?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16,?17,?18,?19,?20,?21,?22,?23,?24,?25,?26,?27,?28,?29,?30,?31,?32,?33,?34);"
647 +#define SQL_STORE_ALERT_CONFIG_HASH "insert or replace into alert_hash (hash_id, date_updated, alarm, template, " \
648 + "on_key, class, component, type, os, hosts, lookup, every, units, calc, families, plugin, module, " \
649 + "charts, green, red, warn, crit, exec, to_key, info, delay, options, repeat, host_labels, " \
650 + "p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after, " \
651 + "p_db_lookup_before, p_update_every) values (?1,strftime('%s'),?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12," \
652 + "?13,?14,?15,?16,?17,?18,?19,?20,?21,?22,?23,?24,?25,?26,?27,?28,?29,?30,?31,?32,?33,?34);"
653 +
654 int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg)
655 {
656 static __thread sqlite3_stmt *res = NULL;
health/health_log.c
+1 -4
@@ -162,10 +162,7 @@ inline void health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae) {
162
163 #ifdef ENABLE_ACLK
164 if (netdata_cloud_setting) {
165 - if ((ae->new_status == RRDCALC_STATUS_WARNING || ae->new_status == RRDCALC_STATUS_CRITICAL) ||
166 - ((ae->old_status == RRDCALC_STATUS_WARNING || ae->old_status == RRDCALC_STATUS_CRITICAL))) {
167 - aclk_update_alarm(host, ae);
168 - }
165 + sql_queue_alarm_to_aclk(host, ae);
166 }
167 #endif
168 }