@cryptotaxi247 / netdata-1 / commits / 4c6a13e5b

Use one thread for ACLK synchonization (#14281)

* Remove aclk sync threads * Disable functions if compiled with --disable-cloud * Allocate and reuse buffer when scanning hosts Tune transactions when writing metadata Error checking when executing db_execute (it is already within a loop with retries) * Schedule host context load in parallel Child connection will be delayed if context load is not complete Event loop cleanup * Delay retention check if context is not loaded Remove context load check from regular metadata host scan * Improve checks to check finished threads * Cleanup warnings when compiling with --disable-cloud * Clean chart labels that were created before our current maximum retention * Fix sql statement * Remove structures members that of no use Remove buffer allocations when not needed * Fix compilation error * Don't check for service running when not from a worker * Code cleanup if agent is compiled with --disable-cloud Setup ACLK tables in the database if needed Submit node status update messages to the cloud * Fix compilation warning when --disable-cloud is specified * Address codacy issues * Remove empty file -- has already been moved under contexts * Use enum instead of numbers * Use UUID_STR_LEN * Add newline at the end of file * Release node_id to prevent memory leak under certain cases * Add queries in defines * Ignore rc from transaction start -- if there is an active transaction, we will use it (same with commit) should further improve in a future PR * Remove commented out code * If host is null (it should not be) do not allocate config (coverity reports Resource leak) * Do garbage collection when contexts is initialized * Handle the case when config is not yet available for a host

Stelios Fragkakis committed Mar 16, 2023 at 17:27 UTC 4c6a13e5bd09afd7405d0c309a76b88961e08630
21 files changed +1188 -1318
aclk/aclk.c
+1 -1
@@ -1222,7 +1222,7 @@ void add_aclk_host_labels(void) {
1222 }
1223
1224 void aclk_queue_node_info(RRDHOST *host) {
1225 - struct aclk_database_worker_config *wc = (struct aclk_database_worker_config *) host->dbsync_worker;
1225 + struct aclk_sync_host_config *wc = (struct aclk_sync_host_config *) host->aclk_sync_host_config;
1226 if (likely(wc)) {
1227 wc->node_info_send = 1;
1228 }
daemon/event_loop.c
+1
@@ -49,6 +49,7 @@ void register_libuv_worker_jobs() {
49 worker_register_job_name(UV_EVENT_DBENGINE_SHUTDOWN, "dbengine shutdown");
50
51 // metadata
52 + worker_register_job_name(UV_EVENT_HOST_CONTEXT_LOAD, "metadata load host context");
53 worker_register_job_name(UV_EVENT_METADATA_STORE, "metadata store host");
54 worker_register_job_name(UV_EVENT_METADATA_CLEANUP, "metadata cleanup");
55
daemon/event_loop.h
+1
@@ -41,6 +41,7 @@ enum event_loop_job {
41 UV_EVENT_DBENGINE_SHUTDOWN,
42
43 // metadata
44 + UV_EVENT_HOST_CONTEXT_LOAD,
45 UV_EVENT_METADATA_STORE,
46 UV_EVENT_METADATA_CLEANUP,
47
daemon/main.c
+1 -5
@@ -348,6 +348,7 @@ void netdata_cleanup_and_exit(int ret) {
348 | ABILITY_WEB_REQUESTS
349 | ABILITY_STREAMING_CONNECTIONS
350 | SERVICE_ACLK
351 + | SERVICE_ACLKSYNC
352 );
353
354 delta_shutdown_time("stop replication, exporters, ML training, health and web servers threads");
@@ -388,11 +389,6 @@ void netdata_cleanup_and_exit(int ret) {
389
390 metadata_sync_shutdown_prepare();
391
391 -#ifdef ENABLE_ACLK
392 - delta_shutdown_time("signal aclk sync to stop");
393 - aclk_sync_exit_all();
394 -#endif
395 -
392 delta_shutdown_time("stop aclk threads");
393
394 timeout = !service_wait_exit(
daemon/main.h
+1
@@ -43,6 +43,7 @@ typedef enum {
43 SERVICE_CONTEXT = (1 << 12),
44 SERVICE_ANALYTICS = (1 << 13),
45 SERVICE_EXPORTERS = (1 << 14),
46 + SERVICE_ACLKSYNC = (1 << 15)
47 } SERVICE_TYPE;
48
49 typedef enum {
database/contexts/worker.c
+4 -3
@@ -327,7 +327,7 @@ static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker_jo
327
328 RRDCONTEXT *rc;
329 dfe_start_reentrant(host->rrdctx.contexts, rc) {
330 - if(unlikely(!service_running(SERVICE_CONTEXT))) break;
330 + if(unlikely(worker_jobs && !service_running(SERVICE_CONTEXT))) break;
331
332 if(worker_jobs) worker_is_busy(WORKER_JOB_CLEANUP);
333
@@ -335,7 +335,7 @@ static void rrdcontext_garbage_collect_single_host(RRDHOST *host, bool worker_jo
335
336 RRDINSTANCE *ri;
337 dfe_start_reentrant(rc->rrdinstances, ri) {
338 - if(unlikely(!service_running(SERVICE_CONTEXT))) break;
338 + if(unlikely(worker_jobs && !service_running(SERVICE_CONTEXT))) break;
339
340 RRDMETRIC *rm;
341 dfe_start_write(ri->rrdmetrics, rm) {
@@ -1080,7 +1080,8 @@ void *rrdcontext_main(void *ptr) {
1080 dictionary_garbage_collect(host->rrdctx.hub_queue);
1081 }
1082
1083 - dictionary_garbage_collect(host->rrdctx.contexts);
1083 + if (host->rrdctx.contexts)
1084 + dictionary_garbage_collect(host->rrdctx.contexts);
1085 }
1086 dfe_done(host);
1087
database/rrd.h
+25 -19
@@ -773,35 +773,41 @@ bool rrdset_memory_load_or_create_map_save(RRDSET *st_on_file, RRD_MEMORY_MODE m
773 // and may lead to missing information.
774
775 typedef enum __attribute__ ((__packed__)) rrdhost_flags {
776 +
777 + // Careful not to overlap with rrdhost_options to avoid bugs if
778 + // rrdhost_flags_xxx is used instead of rrdhost_option_xxx or vice-versa
779 // Orphan, Archived and Obsolete flags
777 - RRDHOST_FLAG_ORPHAN = (1 << 10), // this host is orphan (not receiving data)
778 - RRDHOST_FLAG_ARCHIVED = (1 << 11), // The host is archived, no collected charts yet
779 - RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS = (1 << 12), // the host has pending chart obsoletions
780 - RRDHOST_FLAG_PENDING_OBSOLETE_DIMENSIONS = (1 << 13), // the host has pending dimension obsoletions
780 + RRDHOST_FLAG_ORPHAN = (1 << 8), // this host is orphan (not receiving data)
781 + RRDHOST_FLAG_ARCHIVED = (1 << 9), // The host is archived, no collected charts yet
782 + RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS = (1 << 10), // the host has pending chart obsoletions
783 + RRDHOST_FLAG_PENDING_OBSOLETE_DIMENSIONS = (1 << 11), // the host has pending dimension obsoletions
784
785 // Streaming sender
783 - RRDHOST_FLAG_RRDPUSH_SENDER_INITIALIZED = (1 << 14), // the host has initialized rrdpush structures
784 - RRDHOST_FLAG_RRDPUSH_SENDER_SPAWN = (1 << 15), // When set, the sender thread is running
785 - RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED = (1 << 16), // When set, the host is connected to a parent
786 - RRDHOST_FLAG_RRDPUSH_SENDER_READY_4_METRICS = (1 << 17), // when set, rrdset_done() should push metrics to parent
787 - RRDHOST_FLAG_RRDPUSH_SENDER_LOGGED_STATUS = (1 << 18), // when set, we have logged the status of metrics streaming
786 + RRDHOST_FLAG_RRDPUSH_SENDER_INITIALIZED = (1 << 12), // the host has initialized rrdpush structures
787 + RRDHOST_FLAG_RRDPUSH_SENDER_SPAWN = (1 << 13), // When set, the sender thread is running
788 + RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED = (1 << 14), // When set, the host is connected to a parent
789 + RRDHOST_FLAG_RRDPUSH_SENDER_READY_4_METRICS = (1 << 15), // when set, rrdset_done() should push metrics to parent
790 + RRDHOST_FLAG_RRDPUSH_SENDER_LOGGED_STATUS = (1 << 16), // when set, we have logged the status of metrics streaming
791
792 // Health
790 - RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION = (1 << 20), // contains charts and dims with uninitialized variables
791 - RRDHOST_FLAG_INITIALIZED_HEALTH = (1 << 21), // the host has initialized health structures
793 + RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION = (1 << 17), // contains charts and dims with uninitialized variables
794 + RRDHOST_FLAG_INITIALIZED_HEALTH = (1 << 18), // the host has initialized health structures
795
796 // Exporting
794 - RRDHOST_FLAG_EXPORTING_SEND = (1 << 22), // send it to external databases
795 - RRDHOST_FLAG_EXPORTING_DONT_SEND = (1 << 23), // don't send it to external databases
797 + RRDHOST_FLAG_EXPORTING_SEND = (1 << 19), // send it to external databases
798 + RRDHOST_FLAG_EXPORTING_DONT_SEND = (1 << 20), // don't send it to external databases
799
800 // ACLK
798 - RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 24), // when set, we should send ACLK stream context updates
801 + RRDHOST_FLAG_ACLK_STREAM_CONTEXTS = (1 << 21), // when set, we should send ACLK stream context updates
802 + RRDHOST_FLAG_ACLK_STREAM_ALERTS = (1 << 22), // set when the receiver part is disconnected
803 // Metadata
800 - RRDHOST_FLAG_METADATA_UPDATE = (1 << 25), // metadata needs to be stored in the database
801 - RRDHOST_FLAG_METADATA_LABELS = (1 << 26), // metadata needs to be stored in the database
802 - RRDHOST_FLAG_METADATA_INFO = (1 << 27), // metadata needs to be stored in the database
803 - RRDHOST_FLAG_METADATA_CLAIMID = (1 << 28), // metadata needs to be stored in the database
804 + RRDHOST_FLAG_METADATA_UPDATE = (1 << 23), // metadata needs to be stored in the database
805 + RRDHOST_FLAG_METADATA_LABELS = (1 << 24), // metadata needs to be stored in the database
806 + RRDHOST_FLAG_METADATA_INFO = (1 << 25), // metadata needs to be stored in the database
807 + RRDHOST_FLAG_PENDING_CONTEXT_LOAD = (1 << 26), // metadata needs to be stored in the database
808 + RRDHOST_FLAG_CONTEXT_LOAD_IN_PROGRESS = (1 << 27), // metadata needs to be stored in the database
809
810 + RRDHOST_FLAG_METADATA_CLAIMID = (1 << 28), // metadata needs to be stored in the database
811 RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED = (1 << 29), // set when the receiver part is disconnected
812 } RRDHOST_FLAGS;
813
@@ -1028,7 +1034,7 @@ struct rrdhost {
1034 struct sender_state *sender;
1035 netdata_thread_t rrdpush_sender_thread; // the sender thread
1036 size_t rrdpush_sender_replicating_charts; // the number of charts currently being replicated to a parent
1031 - void *dbsync_worker;
1037 + void *aclk_sync_host_config;
1038
1039 // ------------------------------------------------------------------------
1040 // streaming of data from remote hosts - rrdpush receiver
database/rrdhost.c
+28 -49
@@ -519,15 +519,14 @@ int is_legacy = 1;
519 , string2str(host->health.health_default_recipient)
520 );
521
522 - if(!archived)
522 + if(!archived) {
523 metaqueue_host_update_info(host);
524 -
525 - rrdhost_load_rrdcontext_data(host);
526 - if (!archived) {
524 + rrdhost_load_rrdcontext_data(host);
525 +// rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_INFO | RRDHOST_FLAG_METADATA_UPDATE);
526 ml_host_new(host);
527 ml_host_start_training_thread(host);
528 } else
530 - rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED | RRDHOST_FLAG_ORPHAN);
529 + rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD | RRDHOST_FLAG_ARCHIVED | RRDHOST_FLAG_ORPHAN);
530
531 return host;
532 }
@@ -719,31 +718,30 @@ RRDHOST *rrdhost_find_or_create(
718 );
719 }
720 else {
722 -
723 - rrdhost_update(host
724 - , hostname
725 - , registry_hostname
726 - , guid
727 - , os
728 - , timezone
729 - , abbrev_timezone
730 - , utc_offset
731 - , tags
732 - , program_name
733 - , program_version
734 - , update_every
735 - , history
736 - , mode
737 - , health_enabled
738 - , rrdpush_enabled
739 - , rrdpush_destination
740 - , rrdpush_api_key
741 - , rrdpush_send_charts_matching
742 - , rrdpush_enable_replication
743 - , rrdpush_seconds_to_replicate
744 - , rrdpush_replication_step
745 - , system_info);
746 -
721 + if (likely(!rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)))
722 + rrdhost_update(host
723 + , hostname
724 + , registry_hostname
725 + , guid
726 + , os
727 + , timezone
728 + , abbrev_timezone
729 + , utc_offset
730 + , tags
731 + , program_name
732 + , program_version
733 + , update_every
734 + , history
735 + , mode
736 + , health_enabled
737 + , rrdpush_enabled
738 + , rrdpush_destination
739 + , rrdpush_api_key
740 + , rrdpush_send_charts_matching
741 + , rrdpush_enable_replication
742 + , rrdpush_seconds_to_replicate
743 + , rrdpush_replication_step
744 + , system_info);
745 }
746
747 return host;
@@ -1167,21 +1165,6 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
1165 return;
1166 }
1167
1170 -#ifdef ENABLE_ACLK
1171 - struct aclk_database_worker_config *wc = host->dbsync_worker;
1172 - if (wc && !netdata_exit) {
1173 - struct aclk_database_cmd cmd;
1174 - memset(&cmd, 0, sizeof(cmd));
1175 - cmd.opcode = ACLK_DATABASE_ORPHAN_HOST;
1176 - struct aclk_completion compl ;
1177 - init_aclk_completion(&compl );
1178 - cmd.completion = &compl ;
1179 - aclk_database_enq_cmd(wc, &cmd);
1180 - wait_for_aclk_completion(&compl );
1181 - destroy_aclk_completion(&compl );
1182 - }
1183 -#endif
1184 -
1168 // ------------------------------------------------------------------------
1169 // free it
1170
@@ -1218,10 +1201,6 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
1201 string_freez(host->hostname);
1202 __atomic_sub_fetch(&netdata_buffers_statistics.rrdhost_allocations_size, sizeof(RRDHOST), __ATOMIC_RELAXED);
1203 freez(host);
1221 -#ifdef ENABLE_ACLK
1222 - if (wc)
1223 - wc->is_orphan = 0;
1224 -#endif
1204 }
1205
1206 void rrdhost_free_all(void) {
database/sqlite/sqlite_aclk.c
+373 -544
@@ -5,76 +5,174 @@
5
6 #include "sqlite_aclk_node.h"
7
8 +struct aclk_sync_config_s {
9 + uv_thread_t thread;
10 + uv_loop_t loop;
11 + uv_timer_t timer_req;
12 + time_t cleanup_after; // Start a cleanup after this timestamp
13 + uv_async_t async;
14 + /* FIFO command queue */
15 + uv_mutex_t cmd_mutex;
16 + uv_cond_t cmd_cond;
17 + bool initialized;
18 + volatile unsigned queue_size;
19 + struct aclk_database_cmdqueue cmd_queue;
20 +} aclk_sync_config = { 0 };
21 +
22 +
23 void sanity_check(void) {
24 // make sure the compiler will stop on misconfigurations
25 BUILD_BUG_ON(WORKER_UTILIZATION_MAX_JOB_TYPES < ACLK_MAX_ENUMERATIONS_DEFINED);
26 }
27
13 -void schedule_node_info_update(RRDHOST *host)
28 +
29 +int aclk_database_enq_cmd_noblock(struct aclk_database_cmd *cmd)
30 {
15 - if (unlikely(!host))
16 - return;
31 + unsigned queue_size;
32
18 - struct aclk_database_worker_config *wc = host->dbsync_worker;
33 + /* wait for free space in queue */
34 + uv_mutex_lock(&aclk_sync_config.cmd_mutex);
35 + if ((queue_size = aclk_sync_config.queue_size) == ACLK_DATABASE_CMD_Q_MAX_SIZE) {
36 + uv_mutex_unlock(&aclk_sync_config.cmd_mutex);
37 + return 1;
38 + }
39
20 - if (unlikely(!wc))
21 - return;
40 + fatal_assert(queue_size < ACLK_DATABASE_CMD_Q_MAX_SIZE);
41 + /* enqueue command */
42 + aclk_sync_config.cmd_queue.cmd_array[aclk_sync_config.cmd_queue.tail] = *cmd;
43 + aclk_sync_config.cmd_queue.tail = aclk_sync_config.cmd_queue.tail != ACLK_DATABASE_CMD_Q_MAX_SIZE - 1 ?
44 + aclk_sync_config.cmd_queue.tail + 1 : 0;
45 + aclk_sync_config.queue_size = queue_size + 1;
46 + uv_mutex_unlock(&aclk_sync_config.cmd_mutex);
47 + return 0;
48 +}
49
23 - struct aclk_database_cmd cmd;
24 - memset(&cmd, 0, sizeof(cmd));
25 - cmd.opcode = ACLK_DATABASE_NODE_STATE;
26 - cmd.completion = NULL;
27 - aclk_database_enq_cmd(wc, &cmd);
50 +static void aclk_database_enq_cmd(struct aclk_database_cmd *cmd)
51 +{
52 + unsigned queue_size;
53 +
54 + /* wait for free space in queue */
55 + uv_mutex_lock(&aclk_sync_config.cmd_mutex);
56 + while ((queue_size = aclk_sync_config.queue_size) == ACLK_DATABASE_CMD_Q_MAX_SIZE) {
57 + uv_cond_wait(&aclk_sync_config.cmd_cond, &aclk_sync_config.cmd_mutex);
58 + }
59 + fatal_assert(queue_size < ACLK_DATABASE_CMD_Q_MAX_SIZE);
60 + /* enqueue command */
61 + aclk_sync_config.cmd_queue.cmd_array[aclk_sync_config.cmd_queue.tail] = *cmd;
62 + aclk_sync_config.cmd_queue.tail = aclk_sync_config.cmd_queue.tail != ACLK_DATABASE_CMD_Q_MAX_SIZE - 1 ?
63 + aclk_sync_config.cmd_queue.tail + 1 : 0;
64 + aclk_sync_config.queue_size = queue_size + 1;
65 + uv_mutex_unlock(&aclk_sync_config.cmd_mutex);
66 +
67 + /* wake up event loop */
68 + int rc = uv_async_send(&aclk_sync_config.async);
69 + if (unlikely(rc))
70 + debug(D_ACLK_SYNC, "Failed to wake up event loop");
71 }
72
30 -#ifdef ENABLE_ACLK
31 -static int sql_check_aclk_table(void *data, int argc, char **argv, char **column)
73 +enum {
74 + IDX_HOST_ID,
75 + IDX_HOSTNAME,
76 + IDX_REGISTRY,
77 + IDX_UPDATE_EVERY,
78 + IDX_OS,
79 + IDX_TIMEZONE,
80 + IDX_TAGS,
81 + IDX_HOPS,
82 + IDX_MEMORY_MODE,
83 + IDX_ABBREV_TIMEZONE,
84 + IDX_UTC_OFFSET,
85 + IDX_PROGRAM_NAME,
86 + IDX_PROGRAM_VERSION,
87 + IDX_ENTRIES,
88 + IDX_HEALTH_ENABLED,
89 +};
90 +
91 +static int create_host_callback(void *data, int argc, char **argv, char **column)
92 {
33 - struct aclk_database_worker_config *wc = data;
93 + UNUSED(data);
94 UNUSED(argc);
95 UNUSED(column);
96
37 - debug(D_ACLK_SYNC,"Scheduling aclk sync table check for node %s", (char *) argv[0]);
38 - struct aclk_database_cmd cmd;
39 - memset(&cmd, 0, sizeof(cmd));
40 - cmd.opcode = ACLK_DATABASE_DELETE_HOST;
41 - cmd.data = strdupz((char *) argv[0]);
42 - aclk_database_enq_cmd_noblock(wc, &cmd);
43 - return 0;
44 -}
97 + char guid[UUID_STR_LEN];
98 + uuid_unparse_lower(*(uuid_t *)argv[IDX_HOST_ID], guid);
99
46 -#define SQL_SELECT_ACLK_ACTIVE_LIST "SELECT REPLACE(SUBSTR(name,19),'_','-') FROM sqlite_schema " \
47 - "WHERE name LIKE 'aclk_chart_latest_%' AND type IN ('table');"
100 + struct rrdhost_system_info *system_info = callocz(1, sizeof(struct rrdhost_system_info));
101 + __atomic_sub_fetch(&netdata_buffers_statistics.rrdhost_allocations_size, sizeof(struct rrdhost_system_info), __ATOMIC_RELAXED);
102
49 -static void sql_check_aclk_table_list(struct aclk_database_worker_config *wc)
50 -{
51 - char *err_msg = NULL;
52 - debug(D_ACLK_SYNC,"Cleaning tables for nodes that do not exist");
53 - int rc = sqlite3_exec_monitored(db_meta, SQL_SELECT_ACLK_ACTIVE_LIST, sql_check_aclk_table, (void *) wc, &err_msg);
54 - if (rc != SQLITE_OK) {
55 - error_report("Query failed when trying to check for obsolete ACLK sync tables, %s", err_msg);
56 - sqlite3_free(err_msg);
57 - }
103 + system_info->hops = str2i((const char *) argv[IDX_HOPS]);
104 +
105 + sql_build_host_system_info((uuid_t *)argv[IDX_HOST_ID], system_info);
106 +
107 + RRDHOST *host = rrdhost_find_or_create(
108 + (const char *) argv[IDX_HOSTNAME]
109 + , (const char *) argv[IDX_REGISTRY]
110 + , guid
111 + , (const char *) argv[IDX_OS]
112 + , (const char *) argv[IDX_TIMEZONE]
113 + , (const char *) argv[IDX_ABBREV_TIMEZONE]
114 + , (int32_t) (argv[IDX_UTC_OFFSET] ? str2uint32_t(argv[IDX_UTC_OFFSET], NULL) : 0)
115 + , (const char *) argv[IDX_TAGS]
116 + , (const char *) (argv[IDX_PROGRAM_NAME] ? argv[IDX_PROGRAM_NAME] : "unknown")
117 + , (const char *) (argv[IDX_PROGRAM_VERSION] ? argv[IDX_PROGRAM_VERSION] : "unknown")
118 + , argv[IDX_UPDATE_EVERY] ? str2i(argv[IDX_UPDATE_EVERY]) : 1
119 + , argv[IDX_ENTRIES] ? str2i(argv[IDX_ENTRIES]) : 0
120 + , default_rrd_memory_mode
121 + , 0 // health
122 + , 0 // rrdpush enabled
123 + , NULL //destination
124 + , NULL // api key
125 + , NULL // send charts matching
126 + , false // rrdpush_enable_replication
127 + , 0 // rrdpush_seconds_to_replicate
128 + , 0 // rrdpush_replication_step
129 + , system_info
130 + , 1
131 + );
132 + if (likely(host))
133 + host->rrdlabels = sql_load_host_labels((uuid_t *)argv[IDX_HOST_ID]);
134 +
135 +#ifdef NETDATA_INTERNAL_CHECKS
136 + char node_str[UUID_STR_LEN] = "<none>";
137 + if (likely(host->node_id))
138 + uuid_unparse_lower(*host->node_id, node_str);
139 + internal_error(true, "Adding archived host \"%s\" with GUID \"%s\" node id = \"%s\"", rrdhost_hostname(host), host->machine_guid, node_str);
140 +#endif
141 + return 0;
142 }
143
60 -static void sql_maint_aclk_sync_database(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
144 +#ifdef ENABLE_ACLK
145 +static struct aclk_database_cmd aclk_database_deq_cmd(void)
146 {
62 - UNUSED(cmd);
63 -
64 - debug(D_ACLK, "Checking database for %s", wc->host_guid);
147 + struct aclk_database_cmd ret;
148 + unsigned queue_size;
149
66 - BUFFER *sql = buffer_create(ACLK_SYNC_QUERY_SIZE, &netdata_buffers_statistics.buffers_sqlite);
150 + uv_mutex_lock(&aclk_sync_config.cmd_mutex);
151 + queue_size = aclk_sync_config.queue_size;
152 + if (queue_size == 0) {
153 + memset(&ret, 0, sizeof(ret));
154 + ret.opcode = ACLK_DATABASE_NOOP;
155 + ret.completion = NULL;
156
68 - buffer_sprintf(sql,"DELETE FROM aclk_alert_%s WHERE date_submitted IS NOT NULL AND "
69 - "CAST(date_cloud_ack AS INT) < unixepoch()-%d;", wc->uuid_str, ACLK_DELETE_ACK_ALERTS_INTERNAL);
70 - db_execute(buffer_tostring(sql));
157 + } else {
158 + /* dequeue command */
159 + ret = aclk_sync_config.cmd_queue.cmd_array[aclk_sync_config.cmd_queue.head];
160 + if (queue_size == 1) {
161 + aclk_sync_config.cmd_queue.head = aclk_sync_config.cmd_queue.tail = 0;
162 + } else {
163 + aclk_sync_config.cmd_queue.head = aclk_sync_config.cmd_queue.head != ACLK_DATABASE_CMD_Q_MAX_SIZE - 1 ?
164 + aclk_sync_config.cmd_queue.head + 1 : 0;
165 + }
166 + aclk_sync_config.queue_size = queue_size - 1;
167 + /* wake up producers */
168 + uv_cond_signal(&aclk_sync_config.cmd_cond);
169 + }
170 + uv_mutex_unlock(&aclk_sync_config.cmd_mutex);
171
72 - buffer_free(sql);
172 + return ret;
173 }
174
75 -
175 #define SQL_SELECT_HOST_BY_UUID "SELECT host_id FROM host WHERE host_id = @host_id;"
77 -
176 static int is_host_available(uuid_t *host_id)
177 {
178 sqlite3_stmt *res = NULL;
@@ -94,7 +192,7 @@ static int is_host_available(uuid_t *host_id)
192
193 rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);
194 if (unlikely(rc != SQLITE_OK)) {
97 - error_report("Failed to bind host_id parameter to select node instance information");
195 + error_report("Failed to bind host_id parameter to check host existence");
196 goto failed;
197 }
198 rc = sqlite3_step_monitored(res);
@@ -107,15 +205,13 @@ failed:
205 }
206
207 // OPCODE: ACLK_DATABASE_DELETE_HOST
110 -void sql_delete_aclk_table_list(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
208 +static void sql_delete_aclk_table_list(char *host_guid)
209 {
112 - UNUSED(wc);
113 - char uuid_str[GUID_LEN + 1];
114 - char host_str[GUID_LEN + 1];
210 + char uuid_str[UUID_STR_LEN];
211 + char host_str[UUID_STR_LEN];
212
213 int rc;
214 uuid_t host_uuid;
118 - char *host_guid = (char *)cmd.data;
215
216 if (unlikely(!host_guid))
217 return;
@@ -157,274 +253,67 @@ void sql_delete_aclk_table_list(struct aclk_database_worker_config *wc, struct a
253 if (unlikely(rc != SQLITE_OK))
254 error_report("Failed to finalize statement to clean up aclk tables, rc = %d", rc);
255
160 - db_execute(buffer_tostring(sql));
256 + rc = db_execute(buffer_tostring(sql));
257 + if (unlikely(rc))
258 + error("Failed to drop unused ACLK tables");
259
260 fail:
261 buffer_free(sql);
262 }
165 -#endif
166 -
167 -uv_mutex_t aclk_async_lock;
168 -struct aclk_database_worker_config *aclk_thread_head = NULL;
169 -
170 -int claimed()
171 -{
172 - int rc;
173 - rrdhost_aclk_state_lock(localhost);
174 - rc = (localhost->aclk_state.claimed_id != NULL);
175 - rrdhost_aclk_state_unlock(localhost);
176 - return rc;
177 -}
263
179 -void aclk_add_worker_thread(struct aclk_database_worker_config *wc)
264 +static int sql_check_aclk_table(void *data __maybe_unused, int argc __maybe_unused, char **argv __maybe_unused, char **column __maybe_unused)
265 {
181 - if (unlikely(!wc))
182 - return;
183 -
184 - uv_mutex_lock(&aclk_async_lock);
185 - if (unlikely(!wc->host)) {
186 - wc->next = aclk_thread_head;
187 - aclk_thread_head = wc;
188 - }
189 - uv_mutex_unlock(&aclk_async_lock);
266 + debug(D_ACLK_SYNC,"Scheduling aclk sync table check for node %s", (char *) argv[0]);
267 + struct aclk_database_cmd cmd;
268 + memset(&cmd, 0, sizeof(cmd));
269 + cmd.opcode = ACLK_DATABASE_DELETE_HOST;
270 + cmd.param[0] = strdupz((char *) argv[0]);
271 + aclk_database_enq_cmd_noblock(&cmd);
272 + return 0;
273 }
274
192 -void aclk_del_worker_thread(struct aclk_database_worker_config *wc)
193 -{
194 - if (unlikely(!wc))
195 - return;
196 -
197 - uv_mutex_lock(&aclk_async_lock);
198 - struct aclk_database_worker_config **tmp = &aclk_thread_head;
199 - while (*tmp && (*tmp) != wc)
200 - tmp = &(*tmp)->next;
201 - if (*tmp)
202 - *tmp = wc->next;
203 - uv_mutex_unlock(&aclk_async_lock);
204 -}
275 +#define SQL_SELECT_ACLK_ACTIVE_LIST "SELECT REPLACE(SUBSTR(name,19),'_','-') FROM sqlite_schema " \
276 + "WHERE name LIKE 'aclk_chart_latest_%' AND type IN ('table');"
277
206 -int aclk_worker_thread_exists(char *guid)
278 +static void sql_check_aclk_table_list(void)
279 {
208 - int rc = 0;
209 - uv_mutex_lock(&aclk_async_lock);
210 -
211 - struct aclk_database_worker_config *tmp = aclk_thread_head;
212 -
213 - while (tmp && !rc) {
214 - rc = strcmp(tmp->uuid_str, guid) == 0;
215 - tmp = tmp->next;
280 + char *err_msg = NULL;
281 + debug(D_ACLK_SYNC,"Cleaning tables for nodes that do not exist");
282 + int rc = sqlite3_exec_monitored(db_meta, SQL_SELECT_ACLK_ACTIVE_LIST, sql_check_aclk_table, NULL, &err_msg);
283 + if (rc != SQLITE_OK) {
284 + error_report("Query failed when trying to check for obsolete ACLK sync tables, %s", err_msg);
285 + sqlite3_free(err_msg);
286 }
217 - uv_mutex_unlock(&aclk_async_lock);
218 - return rc;
287 }
288
221 -void aclk_database_init_cmd_queue(struct aclk_database_worker_config *wc)
222 -{
223 - wc->cmd_queue.head = wc->cmd_queue.tail = 0;
224 - wc->queue_size = 0;
225 - fatal_assert(0 == uv_cond_init(&wc->cmd_cond));
226 - fatal_assert(0 == uv_mutex_init(&wc->cmd_mutex));
227 -}
289 +#define SQL_ALERT_CLEANUP "DELETE FROM aclk_alert_%s WHERE date_submitted IS NOT NULL AND CAST(date_cloud_ack AS INT) < unixepoch()-%d;"
290
229 -int aclk_database_enq_cmd_noblock(struct aclk_database_worker_config *wc, struct aclk_database_cmd *cmd)
291 +static int sql_maint_aclk_sync_database(void *data __maybe_unused, int argc __maybe_unused, char **argv, char **column __maybe_unused)
292 {
231 - unsigned queue_size;
232 -
233 - /* wait for free space in queue */
234 - uv_mutex_lock(&wc->cmd_mutex);
235 - if ((queue_size = wc->queue_size) == ACLK_DATABASE_CMD_Q_MAX_SIZE || wc->is_shutting_down) {
236 - uv_mutex_unlock(&wc->cmd_mutex);
237 - return 1;
238 - }
239 -
240 - fatal_assert(queue_size < ACLK_DATABASE_CMD_Q_MAX_SIZE);
241 - /* enqueue command */
242 - wc->cmd_queue.cmd_array[wc->cmd_queue.tail] = *cmd;
243 - wc->cmd_queue.tail = wc->cmd_queue.tail != ACLK_DATABASE_CMD_Q_MAX_SIZE - 1 ?
244 - wc->cmd_queue.tail + 1 : 0;
245 - wc->queue_size = queue_size + 1;
246 - uv_mutex_unlock(&wc->cmd_mutex);
293 + char sql[512];
294 + snprintfz(sql,511, SQL_ALERT_CLEANUP, (char *) argv[0], ACLK_DELETE_ACK_ALERTS_INTERNAL);
295 + if (unlikely(db_execute(sql)))
296 + error_report("Failed to clean stale ACLK alert entries");
297 return 0;
298 }
299
250 -void aclk_database_enq_cmd(struct aclk_database_worker_config *wc, struct aclk_database_cmd *cmd)
251 -{
252 - unsigned queue_size;
253 -
254 - /* wait for free space in queue */
255 - uv_mutex_lock(&wc->cmd_mutex);
256 - if (wc->is_shutting_down) {
257 - uv_mutex_unlock(&wc->cmd_mutex);
258 - return;
259 - }
260 - while ((queue_size = wc->queue_size) == ACLK_DATABASE_CMD_Q_MAX_SIZE) {
261 - uv_cond_wait(&wc->cmd_cond, &wc->cmd_mutex);
262 - }
263 - fatal_assert(queue_size < ACLK_DATABASE_CMD_Q_MAX_SIZE);
264 - /* enqueue command */
265 - wc->cmd_queue.cmd_array[wc->cmd_queue.tail] = *cmd;
266 - wc->cmd_queue.tail = wc->cmd_queue.tail != ACLK_DATABASE_CMD_Q_MAX_SIZE - 1 ?
267 - wc->cmd_queue.tail + 1 : 0;
268 - wc->queue_size = queue_size + 1;
269 - uv_mutex_unlock(&wc->cmd_mutex);
270 -
271 - /* wake up event loop */
272 - int rc = uv_async_send(&wc->async);
273 - if (unlikely(rc))
274 - debug(D_ACLK_SYNC, "Failed to wake up event loop");
275 -}
276 -
277 -struct aclk_database_cmd aclk_database_deq_cmd(struct aclk_database_worker_config* wc)
278 -{
279 - struct aclk_database_cmd ret;
280 - unsigned queue_size;
281 -
282 - uv_mutex_lock(&wc->cmd_mutex);
283 - queue_size = wc->queue_size;
284 - if (queue_size == 0 || wc->is_shutting_down) {
285 - memset(&ret, 0, sizeof(ret));
286 - ret.opcode = ACLK_DATABASE_NOOP;
287 - ret.completion = NULL;
288 - if (wc->is_shutting_down)
289 - uv_cond_signal(&wc->cmd_cond);
290 - } else {
291 - /* dequeue command */
292 - ret = wc->cmd_queue.cmd_array[wc->cmd_queue.head];
293 - if (queue_size == 1) {
294 - wc->cmd_queue.head = wc->cmd_queue.tail = 0;
295 - } else {
296 - wc->cmd_queue.head = wc->cmd_queue.head != ACLK_DATABASE_CMD_Q_MAX_SIZE - 1 ?
297 - wc->cmd_queue.head + 1 : 0;
298 - }
299 - wc->queue_size = queue_size - 1;
300 - /* wake up producers */
301 - uv_cond_signal(&wc->cmd_cond);
302 - }
303 - uv_mutex_unlock(&wc->cmd_mutex);
300
305 - return ret;
306 -}
301 +#define SQL_SELECT_ACLK_ALERT_LIST "SELECT SUBSTR(name,12) FROM sqlite_schema WHERE name LIKE 'aclk_alert_%' AND type IN ('table');"
302
308 -struct aclk_database_worker_config *find_inactive_wc_by_node_id(char *node_id)
303 +static void sql_maint_aclk_sync_database_all(void)
304 {
310 - if (unlikely(!node_id))
311 - return NULL;
312 -
313 - uv_mutex_lock(&aclk_async_lock);
314 - struct aclk_database_worker_config *wc = aclk_thread_head;
315 -
316 - while (wc) {
317 - if (!strcmp(wc->node_id, node_id))
318 - break;
319 - wc = wc->next;
305 + char *err_msg = NULL;
306 + debug(D_ACLK_SYNC,"Cleaning tables for nodes that do not exist");
307 + int rc = sqlite3_exec_monitored(db_meta, SQL_SELECT_ACLK_ALERT_LIST, sql_maint_aclk_sync_database, NULL, &err_msg);
308 + if (rc != SQLITE_OK) {
309 + error_report("Query failed when trying to check for obsolete ACLK sync tables, %s", err_msg);
310 + sqlite3_free(err_msg);
311 }
321 - uv_mutex_unlock(&aclk_async_lock);
322 -
323 - return (wc);
312 }
313
326 -void aclk_sync_exit_all()
327 -{
328 - rrd_rdlock();
329 - RRDHOST *host;
330 - rrdhost_foreach_read(host) {
331 - struct aclk_database_worker_config *wc = host->dbsync_worker;
332 - if (wc) {
333 - wc->is_shutting_down = 1;
334 - (void) aclk_database_deq_cmd(wc);
335 - uv_cond_signal(&wc->cmd_cond);
336 - }
337 - }
338 - rrd_unlock();
339 -
340 - uv_mutex_lock(&aclk_async_lock);
341 - struct aclk_database_worker_config *wc = aclk_thread_head;
342 - while (wc) {
343 - wc->is_shutting_down = 1;
344 - wc = wc->next;
345 - }
346 - uv_mutex_unlock(&aclk_async_lock);
347 -}
348 -
349 -enum {
350 - IDX_HOST_ID,
351 - IDX_HOSTNAME,
352 - IDX_REGISTRY,
353 - IDX_UPDATE_EVERY,
354 - IDX_OS,
355 - IDX_TIMEZONE,
356 - IDX_TAGS,
357 - IDX_HOPS,
358 - IDX_MEMORY_MODE,
359 - IDX_ABBREV_TIMEZONE,
360 - IDX_UTC_OFFSET,
361 - IDX_PROGRAM_NAME,
362 - IDX_PROGRAM_VERSION,
363 - IDX_ENTRIES,
364 - IDX_HEALTH_ENABLED,
365 -};
366 -
367 -static int create_host_callback(void *data, int argc, char **argv, char **column)
368 -{
369 - UNUSED(data);
370 - UNUSED(argc);
371 - UNUSED(column);
372 -
373 - char guid[UUID_STR_LEN];
374 - uuid_unparse_lower(*(uuid_t *)argv[IDX_HOST_ID], guid);
375 -
376 - struct rrdhost_system_info *system_info = callocz(1, sizeof(struct rrdhost_system_info));
377 - __atomic_sub_fetch(&netdata_buffers_statistics.rrdhost_allocations_size, sizeof(struct rrdhost_system_info), __ATOMIC_RELAXED);
378 -
379 - system_info->hops = str2i((const char *) argv[IDX_HOPS]);
380 -
381 - sql_build_host_system_info((uuid_t *)argv[IDX_HOST_ID], system_info);
382 -
383 - RRDHOST *host = rrdhost_find_or_create(
384 - (const char *) argv[IDX_HOSTNAME]
385 - , (const char *) argv[IDX_REGISTRY]
386 - , guid
387 - , (const char *) argv[IDX_OS]
388 - , (const char *) argv[IDX_TIMEZONE]
389 - , (const char *) argv[IDX_ABBREV_TIMEZONE]
390 - , argv[IDX_UTC_OFFSET] ? str2uint32_t(argv[IDX_UTC_OFFSET], NULL) : 0
391 - , (const char *) argv[IDX_TAGS]
392 - , (const char *) (argv[IDX_PROGRAM_NAME] ? argv[IDX_PROGRAM_NAME] : "unknown")
393 - , (const char *) (argv[IDX_PROGRAM_VERSION] ? argv[IDX_PROGRAM_VERSION] : "unknown")
394 - , argv[3] ? str2i(argv[IDX_UPDATE_EVERY]) : 1
395 - , argv[13] ? str2i(argv[IDX_ENTRIES]) : 0
396 - , default_rrd_memory_mode
397 - , 0 // health
398 - , 0 // rrdpush enabled
399 - , NULL //destination
400 - , NULL // api key
401 - , NULL // send charts matching
402 - , false // rrdpush_enable_replication
403 - , 0 // rrdpush_seconds_to_replicate
404 - , 0 // rrdpush_replication_step
405 - , system_info
406 - , 1
407 - );
408 - if (likely(host))
409 - host->rrdlabels = sql_load_host_labels((uuid_t *)argv[IDX_HOST_ID]);
410 -
411 -#ifdef NETDATA_INTERNAL_CHECKS
412 - char node_str[UUID_STR_LEN] = "<none>";
413 - if (likely(host->node_id))
414 - uuid_unparse_lower(*host->node_id, node_str);
415 - internal_error(true, "Adding archived host \"%s\" with GUID \"%s\" node id = \"%s\"", rrdhost_hostname(host), host->machine_guid, node_str);
416 -#endif
417 - return 0;
418 -}
419 -
420 -#ifdef ENABLE_ACLK
421 -static int aclk_start_sync_thread(void *data, int argc, char **argv, char **column)
314 +static int aclk_config_parameters(void *data __maybe_unused, int argc __maybe_unused, char **argv, char **column __maybe_unused)
315 {
316 char uuid_str[GUID_LEN + 1];
424 - UNUSED(data);
425 - UNUSED(argc);
426 - UNUSED(column);
427 -
317 uuid_unparse_lower(*((uuid_t *) argv[0]), uuid_str);
318
319 RRDHOST *host = rrdhost_find_by_guid(uuid_str);
@@ -434,155 +323,81 @@ static int aclk_start_sync_thread(void *data, int argc, char **argv, char **colu
323 sql_create_aclk_table(host, (uuid_t *) argv[0], (uuid_t *) argv[1]);
324 return 0;
325 }
437 -#endif
438 -void sql_aclk_sync_init(void)
439 -{
440 - char *err_msg = NULL;
441 - int rc;
442 -
443 - if (unlikely(!db_meta)) {
444 - if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) {
445 - return;
446 - }
447 - error_report("Database has not been initialized");
448 - return;
449 - }
450 -
451 - info("Creating archived hosts");
452 - rc = sqlite3_exec_monitored(db_meta, "SELECT host_id, hostname, registry_hostname, update_every, os, "
453 - "timezone, tags, hops, memory_mode, abbrev_timezone, utc_offset, program_name, "
454 - "program_version, entries, health_enabled FROM host WHERE hops >0;",
455 - create_host_callback, NULL, &err_msg);
456 - if (rc != SQLITE_OK) {
457 - error_report("SQLite error when loading archived hosts, rc = %d (%s)", rc, err_msg);
458 - sqlite3_free(err_msg);
459 - }
326
461 -#ifdef ENABLE_ACLK
462 - fatal_assert(0 == uv_mutex_init(&aclk_async_lock));
463 - rc = sqlite3_exec_monitored(db_meta, "SELECT ni.host_id, ni.node_id FROM host h, node_instance ni WHERE "
464 - "h.host_id = ni.host_id AND ni.node_id IS NOT NULL;", aclk_start_sync_thread, NULL, &err_msg);
465 - if (rc != SQLITE_OK) {
466 - error_report("SQLite error when starting ACLK sync threads, rc = %d (%s)", rc, err_msg);
467 - sqlite3_free(err_msg);
468 - }
469 - info("ACLK sync initialization completed");
470 -#endif
471 -}
472 -#ifdef ENABLE_ACLK
327 static void async_cb(uv_async_t *handle)
328 {
329 uv_stop(handle->loop);
330 uv_update_time(handle->loop);
477 - debug(D_ACLK_SYNC, "%s called, active=%d.", __func__, uv_is_active((uv_handle_t *)handle));
331 }
332
333 #define TIMER_PERIOD_MS (1000)
334
482 -static void timer_cb(uv_timer_t* handle)
335 +static void timer_cb(uv_timer_t *handle)
336 {
337 uv_stop(handle->loop);
338 uv_update_time(handle->loop);
339
487 - struct aclk_database_worker_config *wc = handle->data;
340 + struct aclk_sync_config_s *config = handle->data;
341 struct aclk_database_cmd cmd;
342 memset(&cmd, 0, sizeof(cmd));
490 - cmd.opcode = ACLK_DATABASE_TIMER;
491 - aclk_database_enq_cmd_noblock(wc, &cmd);
343
344 time_t now = now_realtime_sec();
345
495 - if (wc->cleanup_after && wc->cleanup_after < now) {
346 + if (config->cleanup_after && config->cleanup_after < now) {
347 cmd.opcode = ACLK_DATABASE_CLEANUP;
497 - if (!aclk_database_enq_cmd_noblock(wc, &cmd))
498 - wc->cleanup_after += ACLK_DATABASE_CLEANUP_INTERVAL;
348 + if (!aclk_database_enq_cmd_noblock(&cmd))
349 + config->cleanup_after += ACLK_DATABASE_CLEANUP_INTERVAL;
350 }
351
352 if (aclk_connected) {
502 - if (wc->alert_updates && !wc->pause_alert_updates) {
503 - cmd.opcode = ACLK_DATABASE_PUSH_ALERT;
504 - cmd.count = ACLK_MAX_ALERT_UPDATES;
505 - aclk_database_enq_cmd_noblock(wc, &cmd);
506 - }
353 + cmd.opcode = ACLK_DATABASE_PUSH_ALERT;
354 + aclk_database_enq_cmd_noblock(&cmd);
355 +
356 + aclk_check_node_info_and_collectors();
357 }
358 }
359
510 -static void aclk_database_worker(void *arg)
360 +static void aclk_synchronization(void *arg __maybe_unused)
361 {
512 - service_register(SERVICE_THREAD_TYPE_EVENT_LOOP, NULL, NULL, NULL, true);
362 + struct aclk_sync_config_s *config = arg;
363 + uv_thread_set_name_np(config->thread, "ACLKSYNC");
364 worker_register("ACLKSYNC");
365 + service_register(SERVICE_THREAD_TYPE_EVENT_LOOP, NULL, NULL, NULL, true);
366 +
367 worker_register_job_name(ACLK_DATABASE_NOOP, "noop");
515 - worker_register_job_name(ACLK_DATABASE_ORPHAN_HOST, "node orphan");
368 worker_register_job_name(ACLK_DATABASE_ALARM_HEALTH_LOG, "alert log");
369 worker_register_job_name(ACLK_DATABASE_CLEANUP, "cleanup");
370 worker_register_job_name(ACLK_DATABASE_DELETE_HOST, "node delete");
519 - worker_register_job_name(ACLK_DATABASE_NODE_INFO, "node info");
371 worker_register_job_name(ACLK_DATABASE_NODE_STATE, "node state");
521 - worker_register_job_name(ACLK_DATABASE_NODE_COLLECTORS, "node collectors");
372 worker_register_job_name(ACLK_DATABASE_PUSH_ALERT, "alert push");
373 worker_register_job_name(ACLK_DATABASE_PUSH_ALERT_CONFIG, "alert conf push");
374 worker_register_job_name(ACLK_DATABASE_PUSH_ALERT_SNAPSHOT, "alert snapshot");
375 worker_register_job_name(ACLK_DATABASE_QUEUE_REMOVED_ALERTS, "alerts check");
376 worker_register_job_name(ACLK_DATABASE_TIMER, "timer");
377
528 - struct aclk_database_worker_config *wc = arg;
529 - uv_loop_t *loop;
530 - int ret;
531 - enum aclk_database_opcode opcode;
532 - uv_timer_t timer_req;
533 - struct aclk_database_cmd cmd;
534 -
535 - char threadname[NETDATA_THREAD_NAME_MAX+1];
536 - if (wc->host)
537 - snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "ACLK[%s]", rrdhost_hostname(wc->host));
538 - else {
539 - snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "ACLK[%s]", wc->uuid_str);
540 - threadname[11] = '\0';
541 - }
542 - uv_thread_set_name_np(wc->thread, threadname);
543 -
544 - loop = wc->loop = mallocz(sizeof(uv_loop_t));
545 - ret = uv_loop_init(loop);
546 - if (ret) {
547 - error("uv_loop_init(): %s", uv_strerror(ret));
548 - goto error_after_loop_init;
549 - }
550 - loop->data = wc;
551 -
552 - ret = uv_async_init(wc->loop, &wc->async, async_cb);
553 - if (ret) {
554 - error("uv_async_init(): %s", uv_strerror(ret));
555 - goto error_after_async_init;
556 - }
557 - wc->async.data = wc;
558 -
559 - ret = uv_timer_init(loop, &timer_req);
560 - if (ret) {
561 - error("uv_timer_init(): %s", uv_strerror(ret));
562 - goto error_after_timer_init;
563 - }
564 - timer_req.data = wc;
565 - fatal_assert(0 == uv_timer_start(&timer_req, timer_cb, TIMER_PERIOD_MS, TIMER_PERIOD_MS));
566 -
567 - wc->node_info_send = 1;
568 - info("Starting ACLK sync thread for host %s -- scratch area %lu bytes", wc->host_guid, (unsigned long int) sizeof(*wc));
378 + uv_loop_t *loop = &config->loop;
379 + fatal_assert(0 == uv_loop_init(loop));
380 + fatal_assert(0 == uv_async_init(loop, &config->async, async_cb));
381
570 - memset(&cmd, 0, sizeof(cmd));
382 + fatal_assert(0 == uv_timer_init(loop, &config->timer_req));
383 + config->timer_req.data = config;
384 + fatal_assert(0 == uv_timer_start(&config->timer_req, timer_cb, TIMER_PERIOD_MS, TIMER_PERIOD_MS));
385
572 - wc->startup_time = now_realtime_sec();
573 - wc->cleanup_after = wc->startup_time + ACLK_DATABASE_CLEANUP_FIRST;
386 + info("Starting ACLK synchronization thread");
387
575 - debug(D_ACLK_SYNC,"Node %s reports pending message count = %u", wc->node_id, wc->chart_payload_count);
388 + config->cleanup_after = now_realtime_sec() + ACLK_DATABASE_CLEANUP_FIRST;
389 + config->initialized = true;
390
577 - while (likely(!netdata_exit)) {
391 + while (likely(service_running(SERVICE_ACLKSYNC))) {
392 + enum aclk_database_opcode opcode;
393 worker_is_idle();
394 uv_run(loop, UV_RUN_DEFAULT);
395
396 /* wait for commands */
397 do {
583 - cmd = aclk_database_deq_cmd(wc);
398 + struct aclk_database_cmd cmd = aclk_database_deq_cmd();
399
585 - if (netdata_exit)
400 + if (unlikely(!service_running(SERVICE_ACLKSYNC)))
401 break;
402
403 opcode = cmd.opcode;
@@ -594,206 +409,220 @@ static void aclk_database_worker(void *arg)
409 case ACLK_DATABASE_NOOP:
410 /* the command queue was empty, do nothing */
411 break;
597 -
412 // MAINTENANCE
413 case ACLK_DATABASE_CLEANUP:
600 - debug(D_ACLK_SYNC, "Database cleanup for %s", wc->host_guid);
601 -
602 - if (wc->startup_time + ACLK_DATABASE_CLEANUP_FIRST + 2 < now_realtime_sec() && claimed() && aclk_connected) {
603 - cmd.opcode = ACLK_DATABASE_NODE_INFO;
604 - cmd.completion = NULL;
605 - (void) aclk_database_enq_cmd_noblock(wc, &cmd);
606 - }
607 -
608 - sql_maint_aclk_sync_database(wc, cmd);
609 - if (wc->host == localhost)
610 - sql_check_aclk_table_list(wc);
414 + // Scan all aclk_alert_ tables and cleanup as needed
415 + sql_maint_aclk_sync_database_all();
416 + sql_check_aclk_table_list();
417 break;
418
419 case ACLK_DATABASE_DELETE_HOST:
614 - debug(D_ACLK_SYNC,"Cleaning ACLK tables for %s", (char *) cmd.data);
615 - sql_delete_aclk_table_list(wc, cmd);
420 + sql_delete_aclk_table_list(cmd.param[0]);
421 + break;
422 +// NODE STATE
423 + case ACLK_DATABASE_NODE_STATE:;
424 + RRDHOST *host = cmd.param[0];
425 + int live = (host == localhost || host->receiver || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) ? 1 : 0;
426 + struct aclk_sync_host_config *ahc = host->aclk_sync_host_config;
427 + if (unlikely(!ahc))
428 + sql_create_aclk_table(host, &host->host_uuid, host->node_id);
429 + aclk_host_state_update(host, live);
430 break;
617 -
431 // ALERTS
432 case ACLK_DATABASE_PUSH_ALERT_CONFIG:
620 - debug(D_ACLK_SYNC,"Pushing chart config info to the cloud for %s", wc->host_guid);
621 - aclk_push_alert_config_event(wc, cmd);
433 + aclk_push_alert_config_event(cmd.param[0], cmd.param[1]);
434 break;
435 case ACLK_DATABASE_PUSH_ALERT:
624 - debug(D_ACLK_SYNC, "Pushing alert info to the cloud for %s", wc->host_guid);
625 - aclk_push_alert_event(wc, cmd);
436 + aclk_push_alert_events_for_all_hosts();
437 break;
438 case ACLK_DATABASE_ALARM_HEALTH_LOG:
628 - debug(D_ACLK_SYNC, "Pushing alarm health log to the cloud for %s", wc->host_guid);
629 - aclk_push_alarm_health_log(wc, cmd);
439 + aclk_push_alarm_health_log(cmd.param[0]);
440 break;
631 - case ACLK_DATABASE_PUSH_ALERT_SNAPSHOT:
632 - debug(D_ACLK_SYNC, "Pushing alert snapshot to the cloud for node %s", wc->host_guid);
633 - aclk_push_alert_snapshot_event(wc, cmd);
441 + case ACLK_DATABASE_PUSH_ALERT_SNAPSHOT:;
442 + aclk_push_alert_snapshot_event(cmd.param[0]);
443 break;
444 case ACLK_DATABASE_QUEUE_REMOVED_ALERTS:
636 - debug(D_ACLK_SYNC, "Queueing removed alerts for node %s", wc->host_guid);
637 - sql_process_queue_removed_alerts_to_aclk(wc, cmd);
638 - break;
639 -// NODE OPERATIONS
640 - case ACLK_DATABASE_NODE_STATE:
641 - debug(D_ACLK_SYNC,"Sending state update for %s", wc->uuid_str);
642 - if (wc->host) {
643 - RRDHOST *host = wc->host;
644 - int live = (host == localhost || host->receiver || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) ? 1 : 0;
645 - aclk_host_state_update(wc->host, live);
646 - }
647 - break;
648 - case ACLK_DATABASE_NODE_INFO:
649 - debug(D_ACLK_SYNC,"Sending node info for %s", wc->uuid_str);
650 - sql_build_node_info(wc, cmd);
651 - break;
652 - case ACLK_DATABASE_NODE_COLLECTORS:
653 - debug(D_ACLK_SYNC,"Sending node collectors info for %s", wc->uuid_str);
654 - sql_build_node_collectors(wc);
655 - break;
656 -// NODE_INSTANCE DETECTION
657 - case ACLK_DATABASE_ORPHAN_HOST:
658 - wc->host = NULL;
659 - wc->is_orphan = 1;
660 - aclk_add_worker_thread(wc);
661 - break;
662 - case ACLK_DATABASE_TIMER:
663 - if (unlikely(localhost && !wc->host && !wc->is_orphan)) {
664 - if (claimed()) {
665 - wc->host = rrdhost_find_by_guid(wc->host_guid);
666 - if (wc->host) {
667 - info("HOST %s (%s) detected as active", rrdhost_hostname(wc->host), wc->host_guid);
668 - snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "ACLK[%s]", rrdhost_hostname(wc->host));
669 - uv_thread_set_name_np(wc->thread, threadname);
670 - wc->host->dbsync_worker = wc;
671 - if (unlikely(!wc->hostname))
672 - wc->hostname = strdupz(rrdhost_hostname(wc->host));
673 - aclk_del_worker_thread(wc);
674 - wc->node_info_send = 1;
675 - }
676 - }
677 - }
678 - if (wc->node_info_send && localhost && claimed() && aclk_connected) {
679 - cmd.opcode = ACLK_DATABASE_NODE_INFO;
680 - cmd.completion = NULL;
681 - wc->node_info_send = aclk_database_enq_cmd_noblock(wc, &cmd);
682 - }
683 - if (wc->node_collectors_send && wc->node_collectors_send + 30 < now_realtime_sec()) {
684 - cmd.opcode = ACLK_DATABASE_NODE_COLLECTORS;
685 - cmd.completion = NULL;
686 - wc->node_collectors_send = aclk_database_enq_cmd_noblock(wc, &cmd);
687 - }
688 - if (localhost == wc->host)
689 - (void) sqlite3_wal_checkpoint(db_meta, NULL);
445 + sql_process_queue_removed_alerts_to_aclk(cmd.param[0]);
446 break;
447 default:
448 debug(D_ACLK_SYNC, "%s: default.", __func__);
449 break;
450 }
451 if (cmd.completion)
696 - aclk_complete(cmd.completion);
452 + completion_mark_complete(cmd.completion);
453 } while (opcode != ACLK_DATABASE_NOOP);
454 }
455
700 - if (!uv_timer_stop(&timer_req))
701 - uv_close((uv_handle_t *)&timer_req, NULL);
702 -
703 - /* cleanup operations of the event loop */
704 - //info("Shutting down ACLK sync event loop for %s", wc->host_guid);
705 -
706 - /*
707 - * uv_async_send after uv_close does not seem to crash in linux at the moment,
708 - * it is however undocumented behaviour we need to be aware if this becomes
709 - * an issue in the future.
710 - */
711 - uv_close((uv_handle_t *)&wc->async, NULL);
712 - uv_run(loop, UV_RUN_DEFAULT);
713 -
714 - info("Shutting down ACLK sync event loop complete for host %s", wc->host_guid);
715 - /* TODO: don't let the API block by waiting to enqueue commands */
716 - uv_cond_destroy(&wc->cmd_cond);
456 + if (!uv_timer_stop(&config->timer_req))
457 + uv_close((uv_handle_t *)&config->timer_req, NULL);
458
718 - int rc;
719 - do {
720 - rc = uv_loop_close(loop);
721 - } while (rc != UV_EBUSY);
459 + uv_close((uv_handle_t *)&config->async, NULL);
460 +// uv_close((uv_handle_t *)&config->async_exit, NULL);
461 + uv_cond_destroy(&config->cmd_cond);
462 + (void) uv_loop_close(loop);
463
723 - freez(loop);
464 + worker_unregister();
465 + service_exits();
466 + info("ACLK SYNC: Shutting down ACLK synchronization event loop");
467 +}
468
725 - rrd_rdlock();
726 - if (likely(wc->host))
727 - wc->host->dbsync_worker = NULL;
728 - freez(wc->hostname);
729 - freez(wc);
730 - rrd_unlock();
469 +static void aclk_synchronization_init(void)
470 +{
471 + aclk_sync_config.cmd_queue.head = aclk_sync_config.cmd_queue.tail = 0;
472 + aclk_sync_config.queue_size = 0;
473 + fatal_assert(0 == uv_cond_init(&aclk_sync_config.cmd_cond));
474 + fatal_assert(0 == uv_mutex_init(&aclk_sync_config.cmd_mutex));
475
732 - worker_unregister();
733 - return;
734 -
735 -error_after_timer_init:
736 - uv_close((uv_handle_t *)&wc->async, NULL);
737 -error_after_async_init:
738 - fatal_assert(0 == uv_loop_close(loop));
739 -error_after_loop_init:
740 - freez(loop);
741 - worker_unregister();
476 + fatal_assert(0 == uv_thread_create(&aclk_sync_config.thread, aclk_synchronization, &aclk_sync_config));
477 }
478 #endif
479
480 // -------------------------------------------------------------
481
747 -void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id)
482 +void sql_create_aclk_table(RRDHOST *host __maybe_unused, uuid_t *host_uuid __maybe_unused, uuid_t *node_id __maybe_unused)
483 {
484 #ifdef ENABLE_ACLK
485 char uuid_str[GUID_LEN + 1];
486 char host_guid[GUID_LEN + 1];
487 + int rc;
488
489 uuid_unparse_lower_fix(host_uuid, uuid_str);
754 -
755 - if (aclk_worker_thread_exists(uuid_str))
756 - return;
757 -
490 uuid_unparse_lower(*host_uuid, host_guid);
491
760 - BUFFER *sql = buffer_create(ACLK_SYNC_QUERY_SIZE, &netdata_buffers_statistics.buffers_sqlite);
761 -
762 - buffer_sprintf(sql, TABLE_ACLK_ALERT, uuid_str);
763 - db_execute(buffer_tostring(sql));
764 - buffer_flush(sql);
765 -
766 - buffer_sprintf(sql, INDEX_ACLK_ALERT, uuid_str, uuid_str);
767 - db_execute(buffer_tostring(sql));
492 + char sql[ACLK_SYNC_QUERY_SIZE];
493
769 - buffer_free(sql);
494 + snprintfz(sql, ACLK_SYNC_QUERY_SIZE-1, TABLE_ACLK_ALERT, uuid_str);
495 + rc = db_execute(sql);
496 + if (unlikely(rc))
497 + error_report("Failed to create ACLK alert table for host %s", host ? rrdhost_hostname(host) : host_guid);
498 + else {
499 + snprintfz(sql, ACLK_SYNC_QUERY_SIZE -1, INDEX_ACLK_ALERT, uuid_str, uuid_str);
500 + rc = db_execute(sql);
501 + if (unlikely(rc))
502 + error_report("Failed to create ACLK alert table index for host %s", host ? string2str(host->hostname) : host_guid);
503 + }
504 + if (likely(host) && unlikely(host->aclk_sync_host_config))
505 + return;
506
771 - if (likely(host) && unlikely(host->dbsync_worker))
507 + if (unlikely(!host))
508 return;
509
774 - struct aclk_database_worker_config *wc = callocz(1, sizeof(struct aclk_database_worker_config));
510 + struct aclk_sync_host_config *wc = callocz(1, sizeof(struct aclk_sync_host_config));
511 if (node_id && !uuid_is_null(*node_id))
512 uuid_unparse_lower(*node_id, wc->node_id);
777 - if (likely(host)) {
778 - host->dbsync_worker = (void *)wc;
779 - wc->hostname = strdupz(rrdhost_hostname(host));
780 - if (node_id && !host->node_id) {
781 - host->node_id = mallocz(sizeof(*host->node_id));
782 - uuid_copy(*host->node_id, *node_id);
783 - }
513 +
514 + host->aclk_sync_host_config = (void *)wc;
515 + if (node_id && !host->node_id) {
516 + host->node_id = mallocz(sizeof(*host->node_id));
517 + uuid_copy(*host->node_id, *node_id);
518 }
785 - else
786 - wc->hostname = get_hostname_by_node_id(wc->node_id);
519 +
520 wc->host = host;
521 strcpy(wc->uuid_str, uuid_str);
789 - strcpy(wc->host_guid, host_guid);
522 wc->alert_updates = 0;
791 - aclk_database_init_cmd_queue(wc);
792 - aclk_add_worker_thread(wc);
793 - fatal_assert(0 == uv_thread_create(&(wc->thread), aclk_database_worker, wc));
794 -#else
795 - UNUSED(host);
796 - UNUSED(host_uuid);
797 - UNUSED(node_id);
523 + wc->node_info_send = 1;
524 #endif
799 -}
\ No newline at end of file
525 +}
526 +
527 +#define SQL_FETCH_ALL_HOSTS "SELECT host_id, hostname, registry_hostname, update_every, os, " \
528 + "timezone, tags, hops, memory_mode, abbrev_timezone, utc_offset, program_name, " \
529 + "program_version, entries, health_enabled FROM host WHERE hops >0;"
530 +
531 +#define SQL_FETCH_ALL_INSTANCES "SELECT ni.host_id, ni.node_id FROM host h, node_instance ni " \
532 + "WHERE h.host_id = ni.host_id AND ni.node_id IS NOT NULL; "
533 +void sql_aclk_sync_init(void)
534 +{
535 + char *err_msg = NULL;
536 + int rc;
537 +
538 + if (unlikely(!db_meta)) {
539 + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) {
540 + return;
541 + }
542 + error_report("Database has not been initialized");
543 + return;
544 + }
545 +
546 + info("Creating archived hosts");
547 + rc = sqlite3_exec_monitored(db_meta, SQL_FETCH_ALL_HOSTS, create_host_callback, NULL, &err_msg);
548 +
549 + if (rc != SQLITE_OK) {
550 + error_report("SQLite error when loading archived hosts, rc = %d (%s)", rc, err_msg);
551 + sqlite3_free(err_msg);
552 + }
553 + // Trigger host context load for hosts that have been created
554 + metadata_queue_load_host_context(NULL);
555 +
556 +#ifdef ENABLE_ACLK
557 + rc = sqlite3_exec_monitored(db_meta, SQL_FETCH_ALL_INSTANCES,aclk_config_parameters, NULL,&err_msg);
558 +
559 + if (rc != SQLITE_OK) {
560 + error_report("SQLite error when configuring host ACLK synchonization parameters, rc = %d (%s)", rc, err_msg);
561 + sqlite3_free(err_msg);
562 + }
563 + aclk_synchronization_init();
564 +
565 + info("ACLK sync initialization completed");
566 +#endif
567 +}
568 +
569 +// Public
570 +
571 +static inline void queue_aclk_sync_cmd(enum aclk_database_opcode opcode, const void *param0, const void *param1)
572 +{
573 + struct aclk_database_cmd cmd;
574 + cmd.opcode = opcode;
575 + cmd.param[0] = (void *) param0;
576 + cmd.param[1] = (void *) param1;
577 + cmd.completion = NULL;
578 + aclk_database_enq_cmd(&cmd);
579 +}
580 +
581 +// Public
582 +void aclk_push_alert_config(const char *node_id, const char *config_hash)
583 +{
584 + if (unlikely(!aclk_sync_config.initialized))
585 + return;
586 +
587 + queue_aclk_sync_cmd(ACLK_DATABASE_PUSH_ALERT_CONFIG, node_id, config_hash);
588 +}
589 +
590 +void aclk_push_node_alert_snapshot(const char *node_id)
591 +{
592 + if (unlikely(!aclk_sync_config.initialized))
593 + return;
594 +
595 + queue_aclk_sync_cmd(ACLK_DATABASE_PUSH_ALERT_SNAPSHOT, strdupz(node_id), NULL);
596 +}
597 +
598 +
599 +void aclk_push_node_health_log(const char *node_id)
600 +{
601 + if (unlikely(!aclk_sync_config.initialized))
602 + return;
603 +
604 + queue_aclk_sync_cmd(ACLK_DATABASE_ALARM_HEALTH_LOG, strdupz(node_id), NULL);
605 +}
606 +
607 +void aclk_push_node_removed_alerts(const char *node_id)
608 +{
609 + if (unlikely(!aclk_sync_config.initialized))
610 + return;
611 +
612 + queue_aclk_sync_cmd(ACLK_DATABASE_QUEUE_REMOVED_ALERTS, strdupz(node_id), NULL);
613 +}
614 +
615 +void schedule_node_info_update(RRDHOST *host __maybe_unused)
616 +{
617 +#ifdef ENABLE_ACLK
618 + if (unlikely(!host))
619 + return;
620 +
621 + struct aclk_database_cmd cmd;
622 + memset(&cmd, 0, sizeof(cmd));
623 + cmd.opcode = ACLK_DATABASE_NODE_STATE;
624 + cmd.param[0] = host;
625 + cmd.completion = NULL;
626 + aclk_database_enq_cmd(&cmd);
627 +#endif
628 +}
database/sqlite/sqlite_aclk.h
+22 -80
@@ -18,45 +18,6 @@
18 #define ACLK_DELETE_ACK_ALERTS_INTERNAL (86400)
19 #define ACLK_SYNC_QUERY_SIZE 512
20
21 -struct aclk_completion {
22 - uv_mutex_t mutex;
23 - uv_cond_t cond;
24 - volatile unsigned completed;
25 -};
26 -
27 -static inline void init_aclk_completion(struct aclk_completion *p)
28 -{
29 - p->completed = 0;
30 - fatal_assert(0 == uv_cond_init(&p->cond));
31 - fatal_assert(0 == uv_mutex_init(&p->mutex));
32 -}
33 -
34 -static inline void destroy_aclk_completion(struct aclk_completion *p)
35 -{
36 - uv_cond_destroy(&p->cond);
37 - uv_mutex_destroy(&p->mutex);
38 -}
39 -
40 -static inline void wait_for_aclk_completion(struct aclk_completion *p)
41 -{
42 - uv_mutex_lock(&p->mutex);
43 - while (0 == p->completed) {
44 - uv_cond_wait(&p->cond, &p->mutex);
45 - }
46 - fatal_assert(1 == p->completed);
47 - uv_mutex_unlock(&p->mutex);
48 -}
49 -
50 -static inline void aclk_complete(struct aclk_completion *p)
51 -{
52 - uv_mutex_lock(&p->mutex);
53 - p->completed = 1;
54 - uv_mutex_unlock(&p->mutex);
55 - uv_cond_broadcast(&p->cond);
56 -}
57 -
58 -extern uv_mutex_t aclk_async_lock;
59 -
21 static inline void uuid_unparse_lower_fix(uuid_t *uuid, char *out)
22 {
23 uuid_unparse_lower(*uuid, out);
@@ -66,6 +27,12 @@ static inline void uuid_unparse_lower_fix(uuid_t *uuid, char *out)
27 out[23] = '_';
28 }
29
30 +static inline int claimed()
31 +{
32 + return localhost->aclk_state.claimed_id != NULL;
33 +}
34 +
35 +
36 #define TABLE_ACLK_ALERT "CREATE TABLE IF NOT EXISTS aclk_alert_%s (sequence_id INTEGER PRIMARY KEY, " \
37 "alert_unique_id, date_created, date_submitted, date_cloud_ack, filtered_alert_unique_id NOT NULL, " \
38 "unique(alert_unique_id));"
@@ -74,17 +41,14 @@ static inline void uuid_unparse_lower_fix(uuid_t *uuid, char *out)
41 enum aclk_database_opcode {
42 ACLK_DATABASE_NOOP = 0,
43
77 - ACLK_DATABASE_ORPHAN_HOST,
44 ACLK_DATABASE_ALARM_HEALTH_LOG,
45 ACLK_DATABASE_CLEANUP,
46 ACLK_DATABASE_DELETE_HOST,
81 - ACLK_DATABASE_NODE_INFO,
47 ACLK_DATABASE_NODE_STATE,
48 ACLK_DATABASE_PUSH_ALERT,
49 ACLK_DATABASE_PUSH_ALERT_CONFIG,
50 ACLK_DATABASE_PUSH_ALERT_SNAPSHOT,
51 ACLK_DATABASE_QUEUE_REMOVED_ALERTS,
87 - ACLK_DATABASE_NODE_COLLECTORS,
52 ACLK_DATABASE_TIMER,
53
54 // leave this last
@@ -94,10 +58,8 @@ enum aclk_database_opcode {
58
59 struct aclk_database_cmd {
60 enum aclk_database_opcode opcode;
97 - void *data;
98 - void *data_param;
99 - int count;
100 - struct aclk_completion *completion;
61 + void *param[2];
62 + struct completion *completion;
63 };
64
65 #define ACLK_DATABASE_CMD_Q_MAX_SIZE (1024)
@@ -107,44 +69,23 @@ struct aclk_database_cmdqueue {
69 struct aclk_database_cmd cmd_array[ACLK_DATABASE_CMD_Q_MAX_SIZE];
70 };
71
110 -struct aclk_database_worker_config {
111 - uv_thread_t thread;
112 - char uuid_str[GUID_LEN + 1];
113 - char node_id[GUID_LEN + 1];
114 - char host_guid[GUID_LEN + 1];
115 - char *hostname; // hostname to avoid constant lookups
116 - time_t cleanup_after; // Start a cleanup after this timestamp
117 - time_t startup_time; // When the sync thread started
118 - uint64_t alerts_batch_id; // batch id for alerts to use
119 - uint64_t alerts_start_seq_id; // cloud has asked to start streaming from
120 - uint64_t alert_sequence_id; // last alert sequence_id
121 - int pause_alert_updates;
122 - uint32_t chart_payload_count;
123 - uint64_t alerts_snapshot_id; //will contain the snapshot_id value if snapshot was requested
124 - uint64_t alerts_ack_sequence_id; //last sequence_id ack'ed from cloud via sendsnapshot message
125 - uv_loop_t *loop;
72 +struct aclk_sync_host_config {
73 RRDHOST *host;
127 - uv_async_t async;
128 - /* FIFO command queue */
129 - uv_mutex_t cmd_mutex;
130 - uv_cond_t cmd_cond;
131 - volatile unsigned queue_size;
132 - struct aclk_database_cmdqueue cmd_queue;
74 int alert_updates;
75 int node_info_send;
76 time_t node_collectors_send;
136 - volatile unsigned is_shutting_down;
137 - volatile unsigned is_orphan;
138 - struct aclk_database_worker_config *next;
77 + char uuid_str[UUID_STR_LEN];
78 + char node_id[UUID_STR_LEN];
79 + uint64_t alerts_batch_id; // batch id for alerts to use
80 + uint64_t alerts_start_seq_id; // cloud has asked to start streaming from
81 + uint64_t alerts_snapshot_id; // will contain the snapshot_id value if snapshot was requested
82 + uint64_t alerts_ack_sequence_id; // last sequence_id ack'ed from cloud via sendsnapshot message
83 };
84
85 static inline RRDHOST *find_host_by_node_id(char *node_id)
86 {
87 uuid_t node_uuid;
144 - if (unlikely(!node_id))
145 - return NULL;
146 -
147 - if (uuid_parse(node_id, node_uuid))
88 + if (unlikely(!node_id || uuid_parse(node_id, node_uuid)))
89 return NULL;
90
91 rrd_rdlock();
@@ -163,12 +104,13 @@ static inline RRDHOST *find_host_by_node_id(char *node_id)
104
105 extern sqlite3 *db_meta;
106
166 -int aclk_database_enq_cmd_noblock(struct aclk_database_worker_config *wc, struct aclk_database_cmd *cmd);
167 -void aclk_database_enq_cmd(struct aclk_database_worker_config *wc, struct aclk_database_cmd *cmd);
107 +int aclk_database_enq_cmd_noblock(struct aclk_database_cmd *cmd);
108 void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id);
109 void sql_aclk_sync_init(void);
170 -int claimed();
171 -void aclk_sync_exit_all();
110 +void aclk_push_alert_config(const char *node_id, const char *config_hash);
111 +void aclk_push_node_alert_snapshot(const char *node_id);
112 +void aclk_push_node_health_log(const char *node_id);
113 +void aclk_push_node_removed_alerts(const char *node_id);
114 void schedule_node_info_update(RRDHOST *host);
173 -struct aclk_database_worker_config *find_inactive_wc_by_node_id(char *node_id);
115 +
116 #endif //NETDATA_SQLITE_ACLK_H
database/sqlite/sqlite_aclk_alert.c
+227 -237
@@ -5,20 +5,20 @@
5
6 #ifdef ENABLE_ACLK
7 #include "../../aclk/aclk_alarm_api.h"
8 -#include "../../aclk/aclk.h"
8 #endif
9
10 +#define SQL_GET_ALERT_REMOVE_TIME "SELECT when_key FROM health_log_%s WHERE alarm_id = %u " \
11 + "AND unique_id > %u AND unique_id < %u " \
12 + "AND new_status = -2;"
13 +
14 time_t removed_when(uint32_t alarm_id, uint32_t before_unique_id, uint32_t after_unique_id, char *uuid_str) {
15 sqlite3_stmt *res = NULL;
13 - int rc = 0;
16 time_t when = 0;
17 char sql[ACLK_SYNC_QUERY_SIZE];
18
17 - snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, "select when_key from health_log_%s where alarm_id = %u " \
18 - "and unique_id > %u and unique_id < %u " \
19 - "and new_status = -2;", uuid_str, alarm_id, after_unique_id, before_unique_id);
19 + snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, SQL_GET_ALERT_REMOVE_TIME, uuid_str, alarm_id, after_unique_id, before_unique_id);
20
21 - rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
21 + int rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
22 if (rc != SQLITE_OK) {
23 error_report("Failed to prepare statement when trying to find removed gap.");
24 return 0;
@@ -36,22 +36,26 @@ time_t removed_when(uint32_t alarm_id, uint32_t before_unique_id, uint32_t after
36 return when;
37 }
38
39 +#define SQL_UPDATE_FILTERED_ALERT "UPDATE aclk_alert_%s SET filtered_alert_unique_id = %u where filtered_alert_unique_id = %u"
40 +
41 void update_filtered(ALARM_ENTRY *ae, uint32_t unique_id, char *uuid_str) {
42 char sql[ACLK_SYNC_QUERY_SIZE];
41 - snprintfz(sql, ACLK_SYNC_QUERY_SIZE-1, "UPDATE aclk_alert_%s SET filtered_alert_unique_id = %u where filtered_alert_unique_id = %u", uuid_str, ae->unique_id, unique_id);
43 + snprintfz(sql, ACLK_SYNC_QUERY_SIZE-1, SQL_UPDATE_FILTERED_ALERT, uuid_str, ae->unique_id, unique_id);
44 sqlite3_exec_monitored(db_meta, sql, 0, 0, NULL);
45 ae->flags |= HEALTH_ENTRY_FLAG_ACLK_QUEUED;
46 }
47
48 +#define SQL_SELECT_ALERT_BY_UNIQUE_ID "SELECT hl.unique_id FROM health_log_%s hl, alert_hash ah WHERE hl.unique_id = %u " \
49 + "AND hl.config_hash_id = ah.hash_id " \
50 + "AND ah.warn IS NULL AND ah.crit IS NULL;"
51 +
52 static inline bool is_event_from_alert_variable_config(uint32_t unique_id, char *uuid_str) {
53 sqlite3_stmt *res = NULL;
54 int rc = 0;
55 bool ret = false;
56
57 char sql[ACLK_SYNC_QUERY_SIZE];
52 - snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, "select hl.unique_id from health_log_%s hl, alert_hash ah where hl.unique_id = %u " \
53 - "and hl.config_hash_id = ah.hash_id " \
54 - "and ah.warn is null and ah.crit is null;", uuid_str, unique_id);
58 + snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, SQL_SELECT_ALERT_BY_UNIQUE_ID, uuid_str, unique_id);
59
60 rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
61 if (rc != SQLITE_OK) {
@@ -73,12 +77,18 @@ static inline bool is_event_from_alert_variable_config(uint32_t unique_id, char
77
78 #define MAX_REMOVED_PERIOD 86400
79 //decide if some events should be sent or not
80 +
81 +#define SQL_SELECT_ALERT_BY_ID "SELECT hl.new_status, hl.config_hash_id, hl.unique_id FROM health_log_%s hl, aclk_alert_%s aa " \
82 + "WHERE hl.unique_id = aa.filtered_alert_unique_id " \
83 + "AND hl.alarm_id = %u " \
84 + "ORDER BY alarm_event_id DESC LIMIT 1;"
85 +
86 int should_send_to_cloud(RRDHOST *host, ALARM_ENTRY *ae)
87 {
88 sqlite3_stmt *res = NULL;
79 - char uuid_str[GUID_LEN + 1];
89 + char uuid_str[UUID_STR_LEN];
90 uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
81 - int send = 1, rc = 0;
91 + int send = 1;
92
93 if (ae->new_status == RRDCALC_STATUS_REMOVED || ae->new_status == RRDCALC_STATUS_UNINITIALIZED) {
94 return 0;
@@ -97,12 +107,9 @@ int should_send_to_cloud(RRDHOST *host, ALARM_ENTRY *ae)
107
108 //get the previous sent event of this alarm_id
109 //base the search on the last filtered event
100 - snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, "select hl.new_status, hl.config_hash_id, hl.unique_id from health_log_%s hl, aclk_alert_%s aa \
101 - where hl.unique_id = aa.filtered_alert_unique_id \
102 - and hl.alarm_id = %u \
103 - order by alarm_event_id desc LIMIT 1;", uuid_str, uuid_str, ae->alarm_id);
110 + snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, SQL_SELECT_ALERT_BY_ID, uuid_str, uuid_str, ae->alarm_id);
111
105 - rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
112 + int rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
113 if (rc != SQLITE_OK) {
114 error_report("Failed to prepare statement when trying to filter alert events.");
115 send = 1;
@@ -162,6 +169,10 @@ done:
169
170 // will replace call to aclk_update_alarm in health/health_log.c
171 // and handle both cases
172 +
173 +#define SQL_QUEUE_ALERT_TO_CLOUD "INSERT INTO aclk_alert_%s (alert_unique_id, date_created, filtered_alert_unique_id) " \
174 + "VALUES (@alert_unique_id, unixepoch(), @alert_unique_id) ON CONFLICT (alert_unique_id) do nothing;"
175 +
176 int sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, int skip_filter)
177 {
178 if(!service_running(SERVICE_ACLK))
@@ -182,27 +193,21 @@ int sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, int skip_filter)
193 }
194 }
195
185 - int rc = 0;
196 sqlite3_stmt *res_alert = NULL;
197 char uuid_str[GUID_LEN + 1];
198 uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
199
190 - BUFFER *sql = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
200 + char sql[ACLK_SYNC_QUERY_SIZE];
201
192 - buffer_sprintf(
193 - sql,
194 - "INSERT INTO aclk_alert_%s (alert_unique_id, date_created, filtered_alert_unique_id) "
195 - "VALUES (@alert_unique_id, unixepoch(), @alert_unique_id) on conflict (alert_unique_id) do nothing; ",
196 - uuid_str);
202 + snprintfz(sql, ACLK_SYNC_QUERY_SIZE - 1, SQL_QUEUE_ALERT_TO_CLOUD, uuid_str);
203
198 - rc = sqlite3_prepare_v2(db_meta, buffer_tostring(sql), -1, &res_alert, 0);
204 + int rc = sqlite3_prepare_v2(db_meta, sql, -1, &res_alert, 0);
205 if (unlikely(rc != SQLITE_OK)) {
206 error_report("Failed to prepare statement to store alert event");
201 - buffer_free(sql);
207 return 1;
208 }
209
205 - rc = sqlite3_bind_int(res_alert, 1, ae->unique_id);
210 + rc = sqlite3_bind_int(res_alert, 1, (int) ae->unique_id);
211 if (unlikely(rc != SQLITE_OK))
212 goto bind_fail;
213
@@ -213,16 +218,12 @@ int sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, int skip_filter)
218 }
219
220 ae->flags |= HEALTH_ENTRY_FLAG_ACLK_QUEUED;
216 - struct aclk_database_worker_config *wc = (struct aclk_database_worker_config *)host->dbsync_worker;
217 - if (wc) {
218 - wc->pause_alert_updates = 0;
219 - }
221 + rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
222
223 bind_fail:
224 if (unlikely(sqlite3_finalize(res_alert) != SQLITE_OK))
225 error_report("Failed to reset statement in store alert event, rc = %d", rc);
226
225 - buffer_free(sql);
227 return 0;
228 }
229
@@ -254,11 +255,10 @@ int rrdcalc_status_to_proto_enum(RRDCALC_STATUS status)
255 #endif
256 }
257
257 -void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
258 +void aclk_push_alert_event(struct aclk_sync_host_config *wc)
259 {
260 #ifndef ENABLE_ACLK
261 UNUSED(wc);
261 - UNUSED(cmd);
262 #else
263 int rc;
264
@@ -292,12 +292,13 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
292 wc->alerts_start_seq_id,
293 wc->uuid_str,
294 wc->alerts_start_seq_id);
295 - db_execute(buffer_tostring(sql));
295 + if (unlikely(db_execute(buffer_tostring(sql))))
296 + error_report("Failed to reset ACLK alert entries");
297 buffer_reset(sql);
298 wc->alerts_start_seq_id = 0;
299 }
300
300 - int limit = cmd.count > 0 ? cmd.count : 1;
301 + int limit = ACLK_MAX_ALERT_UPDATES;
302
303 sqlite3_stmt *res = NULL;
304
@@ -318,10 +319,15 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
319
320 BUFFER *sql_fix = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
321 buffer_sprintf(sql_fix, TABLE_ACLK_ALERT, wc->uuid_str);
321 - db_execute(buffer_tostring(sql_fix));
322 - buffer_flush(sql_fix);
323 - buffer_sprintf(sql_fix, INDEX_ACLK_ALERT, wc->uuid_str, wc->uuid_str);
324 - db_execute(buffer_tostring(sql_fix));
322 + rc = db_execute(buffer_tostring(sql_fix));
323 + if (unlikely(rc))
324 + error_report("Failed to create ACLK alert table for host %s", rrdhost_hostname(wc->host));
325 + else {
326 + buffer_flush(sql_fix);
327 + buffer_sprintf(sql_fix, INDEX_ACLK_ALERT, wc->uuid_str, wc->uuid_str);
328 + if (unlikely(db_execute(buffer_tostring(sql_fix))))
329 + error_report("Failed to create ACLK alert table for host %s", rrdhost_hostname(wc->host));
330 + }
331 buffer_free(sql_fix);
332
333 // Try again
@@ -429,7 +435,13 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
435 buffer_sprintf(sql, "UPDATE aclk_alert_%s SET date_submitted=unixepoch() "
436 "WHERE date_submitted IS NULL AND sequence_id BETWEEN %" PRIu64 " AND %" PRIu64 ";",
437 wc->uuid_str, first_sequence_id, last_sequence_id);
432 - db_execute(buffer_tostring(sql));
438 +
439 + if (unlikely(db_execute(buffer_tostring(sql))))
440 + error_report("Failed to mark ACLK alert entries as submitted for host %s", rrdhost_hostname(wc->host));
441 +
442 + // Mark to do one more check
443 + rrdhost_flag_set(wc->host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
444 +
445 } else {
446 if (log_first_sequence_id)
447 log_access(
@@ -441,7 +453,6 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
453 wc->alerts_batch_id);
454 log_first_sequence_id = 0;
455 log_last_sequence_id = 0;
444 - wc->pause_alert_updates = 1;
456 }
457
458 rc = sqlite3_finalize(res);
@@ -451,8 +462,24 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d
462 freez(claim_id);
463 buffer_free(sql);
464 #endif
465 +}
466 +
467 +void aclk_push_alert_events_for_all_hosts(void)
468 +{
469 + RRDHOST *host;
470 +
471 + dfe_start_reentrant(rrdhost_root_index, host) {
472 + if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED) || !rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS))
473 + continue;
474 +
475 + internal_error(true, "ACLK SYNC: Scanning host %s", rrdhost_hostname(host));
476 + rrdhost_flag_clear(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
477
455 - return;
478 + struct aclk_sync_host_config *wc = host->aclk_sync_host_config;
479 + if (likely(wc))
480 + aclk_push_alert_event(wc);
481 + }
482 + dfe_done(host);
483 }
484
485 void sql_queue_existing_alerts_to_aclk(RRDHOST *host)
@@ -467,14 +494,11 @@ void sql_queue_existing_alerts_to_aclk(RRDHOST *host)
494 "where new_status <> 0 and new_status <> -2 and config_hash_id is not null and updated_by_id = 0 " \
495 "order by unique_id asc on conflict (alert_unique_id) do nothing;", uuid_str, uuid_str, uuid_str);
496
470 - db_execute(buffer_tostring(sql));
497 + if (unlikely(db_execute(buffer_tostring(sql))))
498 + error_report("Failed to queue existing ACLK alert events for host %s", rrdhost_hostname(host));
499
500 buffer_free(sql);
473 -
474 - struct aclk_database_worker_config *wc = (struct aclk_database_worker_config *)host->dbsync_worker;
475 - if (wc) {
476 - wc->pause_alert_updates = 0;
477 - }
501 + rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
502 }
503
504 void aclk_send_alarm_health_log(char *node_id)
@@ -482,72 +506,61 @@ void aclk_send_alarm_health_log(char *node_id)
506 if (unlikely(!node_id))
507 return;
508
485 - struct aclk_database_worker_config *wc = find_inactive_wc_by_node_id(node_id);
509 + struct aclk_sync_host_config *wc = NULL;
510 + RRDHOST *host = find_host_by_node_id(node_id);
511
487 - if (likely(!wc)) {
488 - RRDHOST *host = find_host_by_node_id(node_id);
489 - if (likely(host))
490 - wc = (struct aclk_database_worker_config *)host->dbsync_worker;
491 - }
512 + if (unlikely(!host))
513 + return;
514
493 - if (!wc) {
515 + wc = (struct aclk_sync_host_config *)host->aclk_sync_host_config;
516 + if (unlikely(!wc)) {
517 log_access("ACLK REQ [%s (N/A)]: HEALTH LOG REQUEST RECEIVED FOR INVALID NODE", node_id);
518 return;
519 }
520
498 - log_access("ACLK REQ [%s (%s)]: HEALTH LOG REQUEST RECEIVED", node_id, wc->hostname ? wc->hostname : "N/A");
499 -
500 - struct aclk_database_cmd cmd;
501 - memset(&cmd, 0, sizeof(cmd));
502 - cmd.opcode = ACLK_DATABASE_ALARM_HEALTH_LOG;
521 + log_access("ACLK REQ [%s (%s)]: HEALTH LOG REQUEST RECEIVED", node_id, rrdhost_hostname(host));
522
504 - aclk_database_enq_cmd(wc, &cmd);
505 - return;
523 + aclk_push_node_health_log(node_id);
524 }
525
508 -void aclk_push_alarm_health_log(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
526 +void aclk_push_alarm_health_log(char *node_id __maybe_unused)
527 {
510 - UNUSED(cmd);
511 -#ifndef ENABLE_ACLK
512 - UNUSED(wc);
513 -#else
528 +#ifdef ENABLE_ACLK
529 int rc;
530
531 char *claim_id = get_agent_claimid();
517 - if (unlikely(!claim_id))
532 + if (unlikely(!claim_id)) {
533 + freez(node_id);
534 return;
535 + }
536
520 - RRDHOST *host = wc->host;
521 - if (unlikely(!host)) {
522 - host = find_host_by_node_id(wc->node_id);
537 + RRDHOST *host = find_host_by_node_id(node_id);
538
524 - if (unlikely(!host)) {
525 - log_access(
526 - "AC [%s (N/A)]: ACLK synchronization thread for %s is not yet linked to HOST.",
527 - wc->node_id,
528 - wc->host_guid);
529 - freez(claim_id);
530 - return;
531 - }
539 + if (unlikely(!host)) {
540 + log_access("AC [%s (N/A)]: Node id not found", node_id);
541 + freez(claim_id);
542 + freez(node_id);
543 + return;
544 }
545
534 - uint64_t first_sequence = 0;
535 - uint64_t last_sequence = 0;
546 + struct aclk_sync_host_config *wc = host->aclk_sync_host_config;
547 +
548 + int64_t first_sequence = 0;
549 + int64_t last_sequence = 0;
550 struct timeval first_timestamp;
551 struct timeval last_timestamp;
552
539 - BUFFER *sql = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
553 + char sql[ACLK_SYNC_QUERY_SIZE];
554
555 sqlite3_stmt *res = NULL;
556
557 //TODO: make this better: include info from health log too
544 - buffer_sprintf(sql, "SELECT MIN(sequence_id), MIN(date_created), MAX(sequence_id), MAX(date_created) " \
545 - "FROM aclk_alert_%s;", wc->uuid_str);
558 + snprintfz(sql, ACLK_SYNC_QUERY_SIZE - 1, "SELECT MIN(sequence_id), MIN(date_created), " \
559 + "MAX(sequence_id), MAX(date_created) FROM aclk_alert_%s;", wc->uuid_str);
560
547 - rc = sqlite3_prepare_v2(db_meta, buffer_tostring(sql), -1, &res, 0);
561 + rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
562 if (rc != SQLITE_OK) {
563 error_report("Failed to prepare statement to get health log statistics from the database");
550 - buffer_free(sql);
564 freez(claim_id);
565 return;
566 }
@@ -558,12 +571,12 @@ void aclk_push_alarm_health_log(struct aclk_database_worker_config *wc, struct a
571 last_timestamp.tv_usec = 0;
572
573 while (sqlite3_step_monitored(res) == SQLITE_ROW) {
561 - first_sequence = sqlite3_column_bytes(res, 0) > 0 ? (uint64_t) sqlite3_column_int64(res, 0) : 0;
574 + first_sequence = sqlite3_column_bytes(res, 0) > 0 ? (int64_t) sqlite3_column_int64(res, 0) : 0;
575 if (sqlite3_column_bytes(res, 1) > 0) {
576 first_timestamp.tv_sec = sqlite3_column_int64(res, 1);
577 }
578
566 - last_sequence = sqlite3_column_bytes(res, 2) > 0 ? (uint64_t) sqlite3_column_int64(res, 2) : 0;
579 + last_sequence = sqlite3_column_bytes(res, 2) > 0 ? (int64_t) sqlite3_column_int64(res, 2) : 0;
580 if (sqlite3_column_bytes(res, 3) > 0) {
581 last_timestamp.tv_sec = sqlite3_column_int64(res, 3);
582 }
@@ -577,27 +590,23 @@ void aclk_push_alarm_health_log(struct aclk_database_worker_config *wc, struct a
590
591 struct alarm_log_health alarm_log;
592 alarm_log.claim_id = claim_id;
580 - alarm_log.node_id = wc->node_id;
593 + alarm_log.node_id = wc->node_id;
594 alarm_log.log_entries = log_entries;
595 alarm_log.status = wc->alert_updates == 0 ? 2 : 1;
596 alarm_log.enabled = (int)host->health.health_enabled;
597
585 - wc->alert_sequence_id = last_sequence;
586 -
598 aclk_send_alarm_log_health(&alarm_log);
588 - log_access("ACLK RES [%s (%s)]: HEALTH LOG SENT from %"PRIu64" to %"PRIu64, wc->node_id, wc->hostname ? wc->hostname : "N/A", first_sequence, last_sequence);
599 + log_access("ACLK RES [%s (%s)]: HEALTH LOG SENT from %ld to %ld", wc->node_id, rrdhost_hostname(host), first_sequence, last_sequence);
600
601 rc = sqlite3_finalize(res);
602 if (unlikely(rc != SQLITE_OK))
603 error_report("Failed to reset statement to get health log statistics from the database, rc = %d", rc);
604
605 freez(claim_id);
595 - buffer_free(sql);
606 + freez(node_id);
607
608 aclk_alert_reloaded = 1;
609 #endif
599 -
600 - return;
610 }
611
612 void aclk_send_alarm_configuration(char *config_hash)
@@ -605,22 +614,14 @@ void aclk_send_alarm_configuration(char *config_hash)
614 if (unlikely(!config_hash))
615 return;
616
608 - struct aclk_database_worker_config *wc = (struct aclk_database_worker_config *) localhost->dbsync_worker;
617 + struct aclk_sync_host_config *wc = (struct aclk_sync_host_config *) localhost->aclk_sync_host_config;
618
610 - if (unlikely(!wc)) {
619 + if (unlikely(!wc))
620 return;
612 - }
621
622 log_access("ACLK REQ [%s (%s)]: Request to send alert config %s.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", config_hash);
623
616 - struct aclk_database_cmd cmd;
617 - memset(&cmd, 0, sizeof(cmd));
618 - cmd.opcode = ACLK_DATABASE_PUSH_ALERT_CONFIG;
619 - cmd.data_param = (void *) strdupz(config_hash);
620 - cmd.completion = NULL;
621 - aclk_database_enq_cmd(wc, &cmd);
622 -
623 - return;
624 + aclk_push_alert_config(wc->node_id, config_hash);
625 }
626
627 #define SQL_SELECT_ALERT_CONFIG "SELECT alarm, template, on_key, class, type, component, os, hosts, plugin," \
@@ -628,19 +629,29 @@ void aclk_send_alarm_configuration(char *config_hash)
629 "options, host_labels, p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after," \
630 "p_db_lookup_before, p_update_every FROM alert_hash WHERE hash_id = @hash_id;"
631
631 -int aclk_push_alert_config_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
632 +int aclk_push_alert_config_event(char *node_id __maybe_unused, char *config_hash __maybe_unused)
633 {
633 - UNUSED(wc);
634 -#ifndef ENABLE_ACLK
635 - UNUSED(cmd);
636 -#else
634 int rc = 0;
635
636 +#ifdef ENABLE_ACLK
637 +
638 CHECK_SQLITE_CONNECTION(db_meta);
639
640 sqlite3_stmt *res = NULL;
641
643 - char *config_hash = (char *) cmd.data_param;
642 + struct aclk_sync_host_config *wc = NULL;
643 + RRDHOST *host = find_host_by_node_id(node_id);
644 +
645 + if (unlikely(!host)) {
646 + freez(config_hash);
647 + return 1;
648 + }
649 +
650 + wc = (struct aclk_sync_host_config *)host->aclk_sync_host_config;
651 + if (unlikely(!wc)) {
652 + freez(config_hash);
653 + return 1;
654 + }
655
656 rc = sqlite3_prepare_v2(db_meta, SQL_SELECT_ALERT_CONFIG, -1, &res, 0);
657 if (rc != SQLITE_OK) {
@@ -723,7 +734,7 @@ int aclk_push_alert_config_event(struct aclk_database_worker_config *wc, struct
734 if (likely(p_alarm_config.cfg_hash)) {
735 log_access("ACLK RES [%s (%s)]: Sent alert config %s.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", config_hash);
736 aclk_send_provide_alarm_cfg(&p_alarm_config);
726 - freez((char *) cmd.data_param);
737 + freez((char *) config_hash);
738 freez(p_alarm_config.cfg_hash);
739 destroy_aclk_alarm_configuration(&alarm_config);
740 }
@@ -735,9 +746,8 @@ bind_fail:
746 if (unlikely(rc != SQLITE_OK))
747 error_report("Failed to reset statement when pushing alarm config hash, rc = %d", rc);
748
738 - return rc;
749 #endif
740 - return 0;
750 + return rc;
751 }
752
753
@@ -747,100 +757,97 @@ void aclk_start_alert_streaming(char *node_id, uint64_t batch_id, uint64_t start
757 if (unlikely(!node_id))
758 return;
759
750 - //log_access("ACLK REQ [%s (N/A)]: ALERTS STREAM from %"PRIu64" batch=%"PRIu64".", node_id, start_seq_id, batch_id);
751 -
760 uuid_t node_uuid;
761 if (uuid_parse(node_id, node_uuid))
762 return;
763
756 - struct aclk_database_worker_config *wc = NULL;
764 RRDHOST *host = find_host_by_node_id(node_id);
758 - if (likely(host)) {
759 - wc = (struct aclk_database_worker_config *)host->dbsync_worker ?
760 - (struct aclk_database_worker_config *)host->dbsync_worker :
761 - (struct aclk_database_worker_config *)find_inactive_wc_by_node_id(node_id);
765
763 - if (unlikely(!host->health.health_enabled)) {
764 - log_access("ACLK STA [%s (N/A)]: Ignoring request to stream alert state changes, health is disabled.", node_id);
765 - return;
766 - }
766 + if (unlikely(!host))
767 + return;
768
768 - if (unlikely(batch_id == 1) && unlikely(start_seq_id == 1))
769 - sql_queue_existing_alerts_to_aclk(host);
770 - } else
771 - wc = (struct aclk_database_worker_config *)find_inactive_wc_by_node_id(node_id);
769 + struct aclk_sync_host_config *wc = host->aclk_sync_host_config;
770
773 - if (likely(wc)) {
774 - log_access("ACLK REQ [%s (%s)]: ALERTS STREAM from %"PRIu64" batch=%"PRIu64, node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", start_seq_id, batch_id);
775 - __sync_synchronize();
776 - wc->alerts_batch_id = batch_id;
777 - wc->alerts_start_seq_id = start_seq_id;
778 - wc->alert_updates = 1;
779 - wc->pause_alert_updates = 0;
780 - __sync_synchronize();
771 + if (unlikely(!wc))
772 + return;
773 +
774 + if (unlikely(!host->health.health_enabled)) {
775 + log_access("ACLK STA [%s (N/A)]: Ignoring request to stream alert state changes, health is disabled.", node_id);
776 + return;
777 }
782 - else
783 - log_access("ACLK STA [%s (N/A)]: ACLK synchronization thread is not active.", node_id);
778
785 - return;
786 -}
779 + // TODO: CHECK
780 + if (unlikely(batch_id == 1) && unlikely(start_seq_id == 1))
781 + sql_queue_existing_alerts_to_aclk(host);
782
788 -void sql_process_queue_removed_alerts_to_aclk(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
789 -{
790 - UNUSED(cmd);
783 + log_access("ACLK REQ [%s (%s)]: ALERTS STREAM from %"PRIu64" batch=%"PRIu64, node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", start_seq_id, batch_id);
784 + wc->alerts_batch_id = batch_id;
785 + wc->alerts_start_seq_id = start_seq_id;
786 + wc->alert_updates = 1;
787 +}
788
792 - BUFFER *sql = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
789 +#define SQL_QUEUE_REMOVE_ALERTS "INSERT INTO aclk_alert_%s (alert_unique_id, date_created, filtered_alert_unique_id) " \
790 + "SELECT unique_id alert_unique_id, UNIXEPOCH(), unique_id alert_unique_id FROM health_log_%s " \
791 + "WHERE new_status = -2 AND updated_by_id = 0 AND unique_id NOT IN " \
792 + "(SELECT alert_unique_id FROM aclk_alert_%s) ORDER BY unique_id ASC " \
793 + "ON CONFLICT (alert_unique_id) DO NOTHING;"
794
794 - buffer_sprintf(sql,"insert into aclk_alert_%s (alert_unique_id, date_created, filtered_alert_unique_id) " \
795 - "select unique_id alert_unique_id, unixepoch(), unique_id alert_unique_id from health_log_%s " \
796 - "where new_status = -2 and updated_by_id = 0 and unique_id not in " \
797 - "(select alert_unique_id from aclk_alert_%s) order by unique_id asc " \
798 - "on conflict (alert_unique_id) do nothing;", wc->uuid_str, wc->uuid_str, wc->uuid_str);
795 +void sql_process_queue_removed_alerts_to_aclk(char *node_id)
796 +{
797 + struct aclk_sync_host_config *wc;
798 + RRDHOST *host = find_host_by_node_id(node_id);
799 + freez(node_id);
800
800 - db_execute(buffer_tostring(sql));
801 + if (unlikely(!host || !(wc = host->aclk_sync_host_config)))
802 + return;
803
802 - log_access("ACLK STA [%s (%s)]: QUEUED REMOVED ALERTS", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A");
804 + char sql[ACLK_SYNC_QUERY_SIZE * 2];
805
804 - buffer_free(sql);
806 + snprintfz(sql,ACLK_SYNC_QUERY_SIZE * 2 - 1, SQL_QUEUE_REMOVE_ALERTS, wc->uuid_str, wc->uuid_str, wc->uuid_str);
807
806 - wc->pause_alert_updates = 0;
807 - return;
808 + if (unlikely(db_execute(sql))) {
809 + log_access("ACLK STA [%s (%s)]: QUEUED REMOVED ALERTS FAILED", wc->node_id, rrdhost_hostname(wc->host));
810 + error_report("Failed to queue ACLK alert removed entries for host %s", rrdhost_hostname(wc->host));
811 + }
812 + else
813 + log_access("ACLK STA [%s (%s)]: QUEUED REMOVED ALERTS", wc->node_id, rrdhost_hostname(wc->host));
814 + rrdhost_flag_set(wc->host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
815 }
816
817 void sql_queue_removed_alerts_to_aclk(RRDHOST *host)
818 {
812 - if (unlikely(!host->dbsync_worker))
819 + if (unlikely(!host->aclk_sync_host_config))
820 return;
821
815 - if (!claimed())
822 + if (!claimed() || !host->node_id)
823 return;
824
818 - struct aclk_database_cmd cmd;
819 - memset(&cmd, 0, sizeof(cmd));
820 - cmd.opcode = ACLK_DATABASE_QUEUE_REMOVED_ALERTS;
821 - cmd.data = NULL;
822 - cmd.data_param = NULL;
823 - cmd.completion = NULL;
824 - aclk_database_enq_cmd((struct aclk_database_worker_config *) host->dbsync_worker, &cmd);
825 + char node_id[UUID_STR_LEN];
826 + uuid_unparse_lower(*host->node_id, node_id);
827 +
828 + aclk_push_node_removed_alerts(node_id);
829 }
830
827 -void aclk_process_send_alarm_snapshot(char *node_id, char *claim_id, uint64_t snapshot_id, uint64_t sequence_id)
831 +void aclk_process_send_alarm_snapshot(char *node_id, char *claim_id __maybe_unused, uint64_t snapshot_id, uint64_t sequence_id)
832 {
829 - UNUSED(claim_id);
830 - if (unlikely(!node_id))
831 - return;
832 -
833 uuid_t node_uuid;
834 - if (uuid_parse(node_id, node_uuid))
834 + if (unlikely(!node_id || uuid_parse(node_id, node_uuid)))
835 return;
836
837 - struct aclk_database_worker_config *wc = NULL;
837 RRDHOST *host = find_host_by_node_id(node_id);
839 - if (likely(host))
840 - wc = (struct aclk_database_worker_config *)host->dbsync_worker;
838 + if (unlikely(!host)) {
839 + log_access("ACLK STA [%s (N/A)]: ACLK node id does not exist", node_id);
840 + return;
841 + }
842
842 - if (likely(wc)) {
843 - log_access(
843 + struct aclk_sync_host_config *wc = (struct aclk_sync_host_config *)host->aclk_sync_host_config;
844 +
845 + if (unlikely(!wc)) {
846 + log_access("ACLK STA [%s (N/A)]: ACLK node id does not exist", node_id);
847 + return;
848 + }
849 +
850 + log_access(
851 "IN [%s (%s)]: Request to send alerts snapshot, snapshot_id %" PRIu64 " and ack_sequence_id %" PRIu64,
852 wc->node_id,
853 wc->host ? rrdhost_hostname(wc->host) : "N/A",
@@ -853,32 +860,7 @@ void aclk_process_send_alarm_snapshot(char *node_id, char *claim_id, uint64_t sn
860 wc->alerts_ack_sequence_id = sequence_id;
861 __sync_synchronize();
862
856 - struct aclk_database_cmd cmd;
857 - memset(&cmd, 0, sizeof(cmd));
858 - cmd.opcode = ACLK_DATABASE_PUSH_ALERT_SNAPSHOT;
859 - cmd.data_param = NULL;
860 - cmd.completion = NULL;
861 - aclk_database_enq_cmd(wc, &cmd);
862 - } else
863 - log_access("ACLK STA [%s (N/A)]: ACLK synchronization thread is not active.", node_id);
864 -
865 - return;
866 -}
867 -
868 -void aclk_mark_alert_cloud_ack(char *uuid_str, uint64_t alerts_ack_sequence_id)
869 -{
870 - BUFFER *sql = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
871 -
872 - if (alerts_ack_sequence_id != 0) {
873 - buffer_sprintf(
874 - sql,
875 - "UPDATE aclk_alert_%s SET date_cloud_ack = unixepoch() WHERE sequence_id <= %" PRIu64 "",
876 - uuid_str,
877 - alerts_ack_sequence_id);
878 - db_execute(buffer_tostring(sql));
879 - }
880 -
881 - buffer_free(sql);
863 + aclk_push_node_alert_snapshot(node_id);
864 }
865
866 #ifdef ENABLE_ACLK
@@ -949,21 +931,28 @@ static int have_recent_alarm(RRDHOST *host, uint32_t alarm_id, uint32_t mark)
931 #endif
932
933 #define ALARM_EVENTS_PER_CHUNK 10
952 -void aclk_push_alert_snapshot_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
934 +#define SQL_ALERT_CLOUD_ACK "UPDATE aclk_alert_%s SET date_cloud_ack=unixepoch() WHERE sequence_id <= %"PRIu64
935 +
936 +void aclk_push_alert_snapshot_event(char *node_id __maybe_unused)
937 {
954 -#ifndef ENABLE_ACLK
955 - UNUSED(wc);
956 - UNUSED(cmd);
957 -#else
958 - UNUSED(cmd);
959 - // we perhaps we don't need this for snapshots
960 - if (unlikely(!wc->alert_updates)) {
961 - log_access("ACLK STA [%s (%s)]: Ignoring alert snapshot event, updates have been turned off for this node.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A");
938 +#ifdef ENABLE_ACLK
939 + RRDHOST *host = find_host_by_node_id(node_id);
940 +
941 + if (unlikely(!host)) {
942 + log_access("AC [%s (N/A)]: Node id not found", node_id);
943 + freez(node_id);
944 return;
945 }
946 + freez(node_id);
947
965 - if (unlikely(!wc->host)) {
966 - error_report("ACLK synchronization thread for %s is not linked to HOST", wc->host_guid);
948 + struct aclk_sync_host_config *wc = host->aclk_sync_host_config;
949 +
950 + // we perhaps we don't need this for snapshots
951 + if (unlikely(!wc->alert_updates)) {
952 + log_access(
953 + "ACLK STA [%s (%s)]: Ignoring alert snapshot event, updates have been turned off for this node.",
954 + wc->node_id,
955 + wc->host ? rrdhost_hostname(wc->host) : "N/A");
956 return;
957 }
958
@@ -974,11 +963,15 @@ void aclk_push_alert_snapshot_event(struct aclk_database_worker_config *wc, stru
963 if (unlikely(!claim_id))
964 return;
965
977 - log_access("ACLK REQ [%s (%s)]: Sending alerts snapshot, snapshot_id %" PRIu64, wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", wc->alerts_snapshot_id);
966 + log_access("ACLK REQ [%s (%s)]: Sending alerts snapshot, snapshot_id %" PRIu64, wc->node_id, rrdhost_hostname(wc->host), wc->alerts_snapshot_id);
967
979 - aclk_mark_alert_cloud_ack(wc->uuid_str, wc->alerts_ack_sequence_id);
968 + if (wc->alerts_ack_sequence_id) {
969 + char sql[512];
970 + snprintfz(sql, 511, SQL_ALERT_CLOUD_ACK, wc->uuid_str, wc->alerts_ack_sequence_id);
971 + if (unlikely(db_execute(sql)))
972 + error_report("Failed to set ACLK alert entries cloud ACK status for host %s", rrdhost_hostname(host));
973 + }
974
981 - RRDHOST *host = wc->host;
975 uint32_t cnt = 0;
976
977 netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
@@ -1065,55 +1058,53 @@ void aclk_push_alert_snapshot_event(struct aclk_database_worker_config *wc, stru
1058
1059 freez(claim_id);
1060 #endif
1068 - return;
1061 }
1062
1063 +#define SQL_DELETE_ALERT_ENTRIES "DELETE FROM aclk_alert_%s WHERE filtered_alert_unique_id NOT IN (SELECT unique_id FROM health_log_%s);"
1064 +
1065 void sql_aclk_alert_clean_dead_entries(RRDHOST *host)
1066 {
1067 if (!claimed())
1068 return;
1069
1076 - char uuid_str[GUID_LEN + 1];
1070 + char uuid_str[UUID_STR_LEN];
1071 uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
1072
1079 - BUFFER *sql = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
1073 + char sql[512];
1074 + snprintfz(sql,511,SQL_DELETE_ALERT_ENTRIES, uuid_str, uuid_str);
1075
1081 - buffer_sprintf(sql,"delete from aclk_alert_%s where filtered_alert_unique_id not in "
1082 - " (select unique_id from health_log_%s); ", uuid_str, uuid_str);
1083 -
1076 char *err_msg = NULL;
1085 - int rc = sqlite3_exec_monitored(db_meta, buffer_tostring(sql), NULL, NULL, &err_msg);
1077 + int rc = sqlite3_exec_monitored(db_meta, sql, NULL, NULL, &err_msg);
1078 if (rc != SQLITE_OK) {
1087 - error_report("Failed when trying to clean stale ACLK alert entries from aclk_alert_%s, error message \"%s""",
1088 - uuid_str, err_msg);
1079 + error_report("Failed when trying to clean stale ACLK alert entries from aclk_alert_%s, error message \"%s\"", uuid_str, err_msg);
1080 sqlite3_free(err_msg);
1081 }
1091 - buffer_free(sql);
1082 }
1083
1084 +#define SQL_GET_MIN_MAX_ALERT_SEQ "SELECT MIN(sequence_id), MAX(sequence_id), " \
1085 + "(SELECT MAX(sequence_id) FROM aclk_alert_%s WHERE date_cloud_ack IS NOT NULL), " \
1086 + "(SELECT MAX(sequence_id) FROM aclk_alert_%s WHERE date_submitted IS NOT NULL) " \
1087 + "FROM aclk_alert_%s WHERE date_submitted IS NULL;"
1088 +
1089 int get_proto_alert_status(RRDHOST *host, struct proto_alert_status *proto_alert_status)
1090 {
1091 int rc;
1097 - struct aclk_database_worker_config *wc = NULL;
1098 - wc = (struct aclk_database_worker_config *)host->dbsync_worker;
1092 + struct aclk_sync_host_config *wc = NULL;
1093 + wc = (struct aclk_sync_host_config *)host->aclk_sync_host_config;
1094 if (!wc)
1095 return 1;
1096
1097 proto_alert_status->alert_updates = wc->alert_updates;
1098 proto_alert_status->alerts_batch_id = wc->alerts_batch_id;
1099
1105 - BUFFER *sql = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
1100 + char sql[ACLK_SYNC_QUERY_SIZE];
1101 sqlite3_stmt *res = NULL;
1102
1108 - buffer_sprintf(sql, "SELECT MIN(sequence_id), MAX(sequence_id), " \
1109 - "(select MAX(sequence_id) from aclk_alert_%s where date_cloud_ack is not NULL), " \
1110 - "(select MAX(sequence_id) from aclk_alert_%s where date_submitted is not NULL) " \
1111 - "FROM aclk_alert_%s where date_submitted is null;", wc->uuid_str, wc->uuid_str, wc->uuid_str);
1103 + snprintfz(sql, ACLK_SYNC_QUERY_SIZE - 1, SQL_GET_MIN_MAX_ALERT_SEQ, wc->uuid_str, wc->uuid_str, wc->uuid_str);
1104
1113 - rc = sqlite3_prepare_v2(db_meta, buffer_tostring(sql), -1, &res, 0);
1105 + rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
1106 if (rc != SQLITE_OK) {
1107 error_report("Failed to prepare statement to get alert log status from the database.");
1116 - buffer_free(sql);
1108 return 1;
1109 }
1110
@@ -1128,6 +1119,5 @@ int get_proto_alert_status(RRDHOST *host, struct proto_alert_status *proto_alert
1119 if (unlikely(rc != SQLITE_OK))
1120 error_report("Failed to finalize statement to get alert log status from the database, rc = %d", rc);
1121
1131 - buffer_free(sql);
1122 return 0;
1123 }
database/sqlite/sqlite_aclk_alert.h
+9 -6
@@ -14,18 +14,21 @@ struct proto_alert_status {
14 uint64_t last_submitted_sequence_id;
15 };
16
17 -int aclk_add_alert_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
18 -void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
17 +int aclk_add_alert_event(struct aclk_sync_host_config *wc, struct aclk_database_cmd cmd);
18 +void aclk_push_alert_event(struct aclk_sync_host_config *wc);
19 void aclk_send_alarm_health_log(char *node_id);
20 -void aclk_push_alarm_health_log(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
20 +void aclk_push_alarm_health_log(char *node_id);
21 void aclk_send_alarm_configuration (char *config_hash);
22 -int aclk_push_alert_config_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
22 +int aclk_push_alert_config_event(char *node_id, char *config_hash);
23 void aclk_start_alert_streaming(char *node_id, uint64_t batch_id, uint64_t start_seq_id);
24 void sql_queue_removed_alerts_to_aclk(RRDHOST *host);
25 -void sql_process_queue_removed_alerts_to_aclk(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
26 -void aclk_push_alert_snapshot_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
25 +void sql_process_queue_removed_alerts_to_aclk(char *node_id);
26 +
27 +void aclk_push_alert_snapshot_event(char *node_id);
28 void aclk_process_send_alarm_snapshot(char *node_id, char *claim_id, uint64_t snapshot_id, uint64_t sequence_id);
29 int get_proto_alert_status(RRDHOST *host, struct proto_alert_status *proto_alert_status);
30 int sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, int skip_filter);
31 +void aclk_push_alert_events_for_all_hosts(void);
32 +
33
34 #endif //NETDATA_SQLITE_ACLK_ALERT_H
database/sqlite/sqlite_aclk_node.c
+71 -27
@@ -25,12 +25,17 @@ DICTIONARY *collectors_from_charts(RRDHOST *host, DICTIONARY *dict) {
25
26 return dict;
27 }
28 -#endif
28
30 -void sql_build_node_collectors(struct aclk_database_worker_config *wc)
29 +static void build_node_collectors(char *node_id __maybe_unused)
30 {
32 -#ifdef ENABLE_ACLK
33 - if (!wc->host)
31 +
32 + RRDHOST *host = find_host_by_node_id(node_id);
33 +
34 + if (unlikely(!host))
35 + return;
36 +
37 + struct aclk_sync_host_config *wc = (struct aclk_sync_host_config *) host->aclk_sync_host_config;
38 + if (unlikely(!wc))
39 return;
40
41 struct update_node_collectors upd_node_collectors;
@@ -39,46 +44,49 @@ void sql_build_node_collectors(struct aclk_database_worker_config *wc)
44 upd_node_collectors.node_id = wc->node_id;
45 upd_node_collectors.claim_id = get_agent_claimid();
46
42 - upd_node_collectors.node_collectors = collectors_from_charts(wc->host, dict);
47 + upd_node_collectors.node_collectors = collectors_from_charts(host, dict);
48 aclk_update_node_collectors(&upd_node_collectors);
49
50 dictionary_destroy(dict);
51 freez(upd_node_collectors.claim_id);
52
48 - log_access("ACLK RES [%s (%s)]: NODE COLLECTORS SENT", wc->node_id, rrdhost_hostname(wc->host));
49 -#else
50 - UNUSED(wc);
51 -#endif
53 + log_access("ACLK RES [%s (%s)]: NODE COLLECTORS SENT", node_id, rrdhost_hostname(host));
54 +
55 + freez(node_id);
56 }
57
54 -void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
58 +static void build_node_info(char *node_id __maybe_unused)
59 {
56 - UNUSED(cmd);
57 -
58 -#ifdef ENABLE_ACLK
60 struct update_node_info node_info;
61
61 - char *claim_id = get_agent_claimid();
62 - if (!wc->host || !claim_id) {
63 - wc->node_info_send = 1;
64 - freez(claim_id);
62 + RRDHOST *host = find_host_by_node_id(node_id);
63 +
64 + if (unlikely((!host))) {
65 + freez(node_id);
66 return;
67 }
68
68 - wc->node_info_send = 0;
69 + struct aclk_sync_host_config *wc = (struct aclk_sync_host_config *) host->aclk_sync_host_config;
70 +
71 + if (unlikely(!wc)) {
72 + freez(node_id);
73 + return;
74 + }
75 +
76 + wc->node_info_send = 1;
77 +
78 rrd_rdlock();
79 node_info.node_id = wc->node_id;
71 - node_info.claim_id = claim_id;
72 - node_info.machine_guid = wc->host_guid;
80 + node_info.claim_id = get_agent_claimid();
81 + node_info.machine_guid = host->machine_guid;
82 node_info.child = (wc->host != localhost);
74 - node_info.ml_info.ml_capable = ml_capable(localhost);
83 + node_info.ml_info.ml_capable = ml_capable();
84 node_info.ml_info.ml_enabled = ml_enabled(wc->host);
85
86 node_info.node_instance_capabilities = aclk_get_node_instance_capas(wc->host);
87
88 now_realtime_timeval(&node_info.updated_at);
89
81 - RRDHOST *host = wc->host;
90 char *host_version = NULL;
91 if (host != localhost) {
92 netdata_mutex_lock(&host->receiver_lock);
@@ -103,7 +111,7 @@ void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_dat
111 node_info.data.virtualization_type = host->system_info->virtualization ? host->system_info->virtualization : "unknown";
112 node_info.data.container_type = host->system_info->container ? host->system_info->container : "unknown";
113 node_info.data.custom_info = config_get(CONFIG_SECTION_WEB, "custom dashboard_info.js", "");
106 - node_info.data.machine_guid = wc->host_guid;
114 + node_info.data.machine_guid = host->machine_guid;
115
116 struct capability node_caps[] = {
117 { .name = "ml", .version = host->system_info->ml_capable, .enabled = host->system_info->ml_enabled },
@@ -118,7 +126,7 @@ void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_dat
126 node_info.data.host_labels_ptr = host->rrdlabels;
127
128 aclk_update_node_info(&node_info);
121 - log_access("ACLK RES [%s (%s)]: NODE INFO SENT for guid [%s] (%s)", wc->node_id, rrdhost_hostname(wc->host), wc->host_guid, wc->host == localhost ? "parent" : "child");
129 + log_access("ACLK RES [%s (%s)]: NODE INFO SENT for guid [%s] (%s)", wc->node_id, rrdhost_hostname(wc->host), host->machine_guid, wc->host == localhost ? "parent" : "child");
130
131 rrd_unlock();
132 freez(node_info.claim_id);
@@ -126,7 +134,43 @@ void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_dat
134 freez(host_version);
135
136 wc->node_collectors_send = now_realtime_sec();
129 -#else
130 - UNUSED(wc);
131 -#endif
137 + freez(node_id);
138 +
139 }
140 +
141 +
142 +void aclk_check_node_info_and_collectors(void)
143 +{
144 + RRDHOST *host;
145 +
146 + if (unlikely(!aclk_connected))
147 + return;
148 +
149 + dfe_start_reentrant(rrdhost_root_index, host) {
150 +
151 + if (unlikely(rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD))) {
152 + info("ACLK: 'host:%s' not sending node info, context load is pending", rrdhost_hostname(host));
153 + continue;
154 + }
155 +
156 + struct aclk_sync_host_config *wc = host->aclk_sync_host_config;
157 +
158 + if (unlikely(!wc))
159 + continue;
160 +
161 + if (wc->node_info_send) {
162 + build_node_info(strdupz(wc->node_id));
163 + internal_error(true, "ACLK SYNC: Sending node info for %s", rrdhost_hostname(host));
164 + wc->node_info_send = 0;
165 + }
166 +
167 + if (wc->node_collectors_send && wc->node_collectors_send + 30 < now_realtime_sec()) {
168 + build_node_collectors(strdupz(wc->node_id));
169 + internal_error(true, "ACLK SYNC: Sending collectors for %s", rrdhost_hostname(host));
170 + wc->node_collectors_send = 0;
171 + }
172 + }
173 + dfe_done(host);
174 +}
175 +
176 +#endif
database/sqlite/sqlite_aclk_node.h
+1 -2
@@ -3,6 +3,5 @@
3 #ifndef NETDATA_SQLITE_ACLK_NODE_H
4 #define NETDATA_SQLITE_ACLK_NODE_H
5
6 -void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);
7 -void sql_build_node_collectors(struct aclk_database_worker_config *wc);
6 +void aclk_check_node_info_and_collectors(void);
7 #endif //NETDATA_SQLITE_ACLK_NODE_H
database/sqlite/sqlite_context.c
+2 -2
@@ -369,9 +369,9 @@ int ctx_store_context(uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data)
369 goto skip_store;
370 }
371
372 - rc = sqlite3_bind_int(res, 10, (time_t) context_data->deleted);
372 + rc = sqlite3_bind_int(res, 10, context_data->deleted);
373 if (unlikely(rc != SQLITE_OK)) {
374 - error_report("Failed to bind last_time_t to store context details");
374 + error_report("Failed to bind deleted flag to store context details");
375 goto skip_store;
376 }
377
database/sqlite/sqlite_functions.c
+18 -55
@@ -117,7 +117,6 @@ int execute_insert(sqlite3_stmt *res)
117 break;
118 }
119 }
120 -
120 return rc;
121 }
122
@@ -510,8 +509,9 @@ skip:
509 error_report("Failed to finalize statement %s, rc = %d", sql, rc);
510 return result;
511 }
513 -
514 -void db_execute(const char *cmd)
512 +// Return 0 OK
513 +// Return 1 Failed
514 +int db_execute(const char *cmd)
515 {
516 int rc;
517 int cnt = 0;
@@ -532,6 +532,7 @@ void db_execute(const char *cmd)
532
533 ++cnt;
534 }
535 + return (rc != SQLITE_OK);
536 }
537
538 static inline void set_host_node_id(RRDHOST *host, uuid_t *node_id)
@@ -545,7 +546,7 @@ static inline void set_host_node_id(RRDHOST *host, uuid_t *node_id)
546 return;
547 }
548
548 - struct aclk_database_worker_config *wc = host->dbsync_worker;
549 + struct aclk_sync_host_config *wc = host->aclk_sync_host_config;
550
551 if (unlikely(!host->node_id))
552 host->node_id = mallocz(sizeof(*host->node_id));
@@ -599,7 +600,7 @@ int update_node_id(uuid_t *host_id, uuid_t *node_id)
600 rrd_wrlock();
601 host = rrdhost_find_by_guid(host_guid);
602 if (likely(host))
602 - set_host_node_id(host, node_id);
603 + set_host_node_id(host, node_id);
604 rrd_unlock();
605
606 failed:
@@ -609,48 +610,6 @@ failed:
610 return rc - 1;
611 }
612
612 -#define SQL_SELECT_HOSTNAME_BY_NODE_ID "SELECT h.hostname FROM node_instance ni, " \
613 -"host h WHERE ni.host_id = h.host_id AND ni.node_id = @node_id;"
614 -
615 -char *get_hostname_by_node_id(char *node)
616 -{
617 - sqlite3_stmt *res = NULL;
618 - char *hostname = NULL;
619 - int rc;
620 -
621 - if (unlikely(!db_meta)) {
622 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
623 - error_report("Database has not been initialized");
624 - return NULL;
625 - }
626 -
627 - uuid_t node_id;
628 - if (uuid_parse(node, node_id))
629 - return NULL;
630 -
631 - rc = sqlite3_prepare_v2(db_meta, SQL_SELECT_HOSTNAME_BY_NODE_ID, -1, &res, 0);
632 - if (unlikely(rc != SQLITE_OK)) {
633 - error_report("Failed to prepare statement to fetch hostname by node id");
634 - return NULL;
635 - }
636 -
637 - rc = sqlite3_bind_blob(res, 1, &node_id, sizeof(node_id), SQLITE_STATIC);
638 - if (unlikely(rc != SQLITE_OK)) {
639 - error_report("Failed to bind host_id parameter to select node instance information");
640 - goto failed;
641 - }
642 -
643 - rc = sqlite3_step_monitored(res);
644 - if (likely(rc == SQLITE_ROW))
645 - hostname = strdupz((char *)sqlite3_column_text(res, 0));
646 -
647 -failed:
648 - if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
649 - error_report("Failed to finalize the prepared statement when search for hostname by node id");
650 -
651 - return hostname;
652 -}
653 -
613 #define SQL_SELECT_HOST_BY_NODE_ID "select host_id from node_instance where node_id = @node_id;"
614
615 int get_host_id(uuid_t *node_id, uuid_t *host_id)
@@ -689,7 +648,7 @@ failed:
648 return (rc == SQLITE_ROW) ? 0 : -1;
649 }
650
692 -#define SQL_SELECT_NODE_ID "select node_id from node_instance where host_id = @host_id and node_id not null;"
651 +#define SQL_SELECT_NODE_ID "SELECT node_id FROM node_instance WHERE host_id = @host_id AND node_id IS NOT NULL;"
652
653 int get_node_id(uuid_t *host_id, uuid_t *node_id)
654 {
@@ -725,8 +684,8 @@ failed:
684 return (rc == SQLITE_ROW) ? 0 : -1;
685 }
686
728 -#define SQL_INVALIDATE_NODE_INSTANCES "update node_instance set node_id = NULL where exists " \
729 - "(select host_id from node_instance where host_id = @host_id and (@claim_id is null or claim_id <> @claim_id));"
687 +#define SQL_INVALIDATE_NODE_INSTANCES "UPDATE node_instance SET node_id = NULL WHERE EXISTS " \
688 + "(SELECT host_id FROM node_instance WHERE host_id = @host_id AND (@claim_id IS NULL OR claim_id <> @claim_id));"
689
690 void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id)
691 {
@@ -770,8 +729,8 @@ failed:
729 error_report("Failed to finalize the prepared statement when invalidating node instance information");
730 }
731
773 -#define SQL_GET_NODE_INSTANCE_LIST "select ni.node_id, ni.host_id, h.hostname " \
774 - "from node_instance ni, host h where ni.host_id = h.host_id AND h.hops >=0;"
732 +#define SQL_GET_NODE_INSTANCE_LIST "SELECT ni.node_id, ni.host_id, h.hostname " \
733 + "FROM node_instance ni, host h WHERE ni.host_id = h.host_id AND h.hops >=0;"
734
735 struct node_instance_list *get_node_list(void)
736 {
@@ -810,12 +769,16 @@ struct node_instance_list *get_node_list(void)
769 uuid_copy(node_list[row].node_id, *((uuid_t *)sqlite3_column_blob(res, 0)));
770 if (sqlite3_column_bytes(res, 1) == sizeof(uuid_t)) {
771 uuid_t *host_id = (uuid_t *)sqlite3_column_blob(res, 1);
813 - uuid_copy(node_list[row].host_id, *host_id);
814 - node_list[row].queryable = 1;
772 uuid_unparse_lower(*host_id, host_guid);
773 RRDHOST *host = rrdhost_find_by_guid(host_guid);
774 + if (rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)) {
775 + info("ACLK: 'host:%s' skipping get node list because context is initializing", rrdhost_hostname(host));
776 + continue;
777 + }
778 + uuid_copy(node_list[row].host_id, *host_id);
779 + node_list[row].queryable = 1;
780 node_list[row].live = (host && (host == localhost || host->receiver
818 - || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)))) ? 1 : 0;
781 + || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)))) ? 1 : 0;
782 node_list[row].hops = (host && host->system_info) ? host->system_info->hops :
783 uuid_compare(*host_id, localhost->host_uuid) ? 1 : 0;
784 node_list[row].hostname =
database/sqlite/sqlite_functions.h
+1 -1
@@ -55,7 +55,7 @@ int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_b
55 int prepare_statement(sqlite3 *database, const char *query, sqlite3_stmt **statement);
56 int execute_insert(sqlite3_stmt *res);
57 int exec_statement_with_uuid(const char *sql, uuid_t *uuid);
58 -void db_execute(const char *cmd);
58 +int db_execute(const char *cmd);
59 void initialize_thread_key_pool(void);
60
61 // Look up functions
database/sqlite/sqlite_health.c
+64 -99
@@ -12,7 +12,7 @@
12 #define SQL_CREATE_HEALTH_LOG_TABLE(guid) "CREATE TABLE IF NOT EXISTS health_log_%s(hostname text, unique_id int, alarm_id int, alarm_event_id int, config_hash_id blob, updated_by_id int, updates_id int, when_key int, duration int, non_clear_duration int, flags int, exec_run_timestamp int, delay_up_to_timestamp int, name text, chart text, family text, exec text, recipient text, source text, units text, info text, exec_code int, new_status real, old_status real, delay int, new_value double, old_value double, last_repeat int, class text, component text, type text, chart_context text);", guid
13 int sql_create_health_log_table(RRDHOST *host) {
14 int rc;
15 - char *err_msg = NULL, command[MAX_HEALTH_SQL_SIZE + 1];
15 + char command[MAX_HEALTH_SQL_SIZE + 1];
16
17 if (unlikely(!db_meta)) {
18 if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
@@ -25,18 +25,17 @@ int sql_create_health_log_table(RRDHOST *host) {
25
26 snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_CREATE_HEALTH_LOG_TABLE(uuid_str));
27
28 - rc = sqlite3_exec_monitored(db_meta, command, 0, 0, &err_msg);
29 - if (rc != SQLITE_OK) {
30 - error_report("HEALTH [%s]: SQLite error during creation of health log table, rc = %d (%s)", rrdhost_hostname(host), rc, err_msg);
31 - sqlite3_free(err_msg);
32 - return 1;
28 + rc = db_execute(command);
29 + if (unlikely(rc))
30 + error_report("HEALTH [%s]: SQLite error during creation of health log table", rrdhost_hostname(host));
31 + else {
32 + snprintfz(command, MAX_HEALTH_SQL_SIZE, "CREATE INDEX IF NOT EXISTS health_log_index_%s ON health_log_%s (unique_id); ", uuid_str, uuid_str);
33 + rc = db_execute(command);
34 + if (unlikely(unlikely(rc)))
35 + error_report("HEALTH [%s]: SQLite error during creation of health log table index", rrdhost_hostname(host));
36 }
37
35 - snprintfz(command, MAX_HEALTH_SQL_SIZE, "CREATE INDEX IF NOT EXISTS "
36 - "health_log_index_%s ON health_log_%s (unique_id); ", uuid_str, uuid_str);
37 - db_execute(command);
38 -
39 - return 0;
38 + return rc;
39 }
40
41 /* Health related SQL queries
@@ -104,7 +103,7 @@ void sql_health_alarm_log_update(RRDHOST *host, ALARM_ENTRY *ae) {
103 error_report("HEALTH [%s]: Failed to update health log, rc = %d", rrdhost_hostname(host), rc);
104 }
105
107 - failed:
106 +failed:
107 if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
108 error_report("HEALTH [%s]: Failed to finalize the prepared statement for updating health log.", rrdhost_hostname(host));
109 }
@@ -345,7 +344,7 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
344 ae->flags |= HEALTH_ENTRY_FLAG_SAVED;
345 host->health.health_log_entries_written++;
346
348 - failed:
347 +failed:
348 if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
349 error_report("HEALTH [%s]: Failed to finalize the prepared statement for inserting to health log.", rrdhost_hostname(host));
350 }
@@ -452,7 +451,7 @@ void sql_health_alarm_log_count(RRDHOST *host) {
451 #define SQL_INJECT_REMOVED_UPDATE(guid) "update health_log_%s set flags = flags | ?1, updated_by_id = ?2 where unique_id = ?3; ", guid
452 void sql_inject_removed_status(char *uuid_str, uint32_t alarm_id, uint32_t alarm_event_id, uint32_t unique_id, uint32_t max_unique_id)
453 {
455 - int rc = 0;
454 + int rc;
455 char command[MAX_HEALTH_SQL_SIZE + 1];
456
457 if (!alarm_id || !alarm_event_id || !unique_id || !max_unique_id)
@@ -546,7 +545,7 @@ failed:
545 #define SQL_SELECT_MAX_UNIQUE_ID(guid) "SELECT MAX(unique_id) from health_log_%s", guid
546 uint32_t sql_get_max_unique_id (char *uuid_str)
547 {
549 - int rc = 0;
548 + int rc;
549 char command[MAX_HEALTH_SQL_SIZE + 1];
550 uint32_t max_unique_id = 0;
551
@@ -573,10 +572,9 @@ uint32_t sql_get_max_unique_id (char *uuid_str)
572 #define SQL_SELECT_LAST_STATUSES(guid) "SELECT new_status, unique_id, alarm_id, alarm_event_id from health_log_%s group by alarm_id having max(alarm_event_id)", guid
573 void sql_check_removed_alerts_state(char *uuid_str)
574 {
576 - int rc = 0;
575 + int rc;
576 char command[MAX_HEALTH_SQL_SIZE + 1];
578 - RRDCALC_STATUS status;
579 - uint32_t alarm_id = 0, alarm_event_id = 0, unique_id = 0, max_unique_id = 0;
577 + uint32_t max_unique_id = 0;
578
579 sqlite3_stmt *res = NULL;
580
@@ -588,6 +586,9 @@ void sql_check_removed_alerts_state(char *uuid_str)
586 }
587
588 while (sqlite3_step_monitored(res) == SQLITE_ROW) {
589 + uint32_t alarm_id, alarm_event_id, unique_id;
590 + RRDCALC_STATUS status;
591 +
592 status = (RRDCALC_STATUS) sqlite3_column_int(res, 0);
593 unique_id = (uint32_t) sqlite3_column_int64(res, 1);
594 alarm_id = (uint32_t) sqlite3_column_int64(res, 2);
@@ -683,8 +684,7 @@ void sql_health_alarm_log_load(RRDHOST *host) {
684 }
685
686 // Check if we got last_repeat field
686 - time_t last_repeat = 0;
687 - last_repeat = (time_t)sqlite3_column_int64(res, 27);
687 + time_t last_repeat = (time_t)sqlite3_column_int64(res, 27);
688
689 rc = dictionary_get(all_rrdcalcs, (char *) sqlite3_column_text(res, 14));
690 if(unlikely(rc)) {
@@ -847,196 +847,161 @@ int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg)
847 }
848 }
849
850 - param++;
851 - rc = sqlite3_bind_blob(res, param, hash_id, sizeof(*hash_id), SQLITE_STATIC);
850 + rc = sqlite3_bind_blob(res, ++param, hash_id, sizeof(*hash_id), SQLITE_STATIC);
851 if (unlikely(rc != SQLITE_OK))
852 goto bind_fail;
853
855 - param++;
856 - rc = sqlite3_bind_string_or_null(res, cfg->alarm, param);
854 + rc = sqlite3_bind_string_or_null(res, cfg->alarm, ++param);
855 if (unlikely(rc != SQLITE_OK))
856 goto bind_fail;
857
860 - param++;
861 - rc = sqlite3_bind_string_or_null(res, cfg->template_key, param);
858 + rc = sqlite3_bind_string_or_null(res, cfg->template_key, ++param);
859 if (unlikely(rc != SQLITE_OK))
860 goto bind_fail;
861
865 - param++;
866 - rc = sqlite3_bind_string_or_null(res, cfg->on, param);
862 + rc = sqlite3_bind_string_or_null(res, cfg->on, ++param);
863 if (unlikely(rc != SQLITE_OK))
864 goto bind_fail;
865
870 - param++;
871 - rc = sqlite3_bind_string_or_null(res, cfg->classification, param);
866 + rc = sqlite3_bind_string_or_null(res, cfg->classification, ++param);
867 if (unlikely(rc != SQLITE_OK))
868 goto bind_fail;
869
875 - param++;
876 - rc = sqlite3_bind_string_or_null(res, cfg->component, param);
870 + rc = sqlite3_bind_string_or_null(res, cfg->component, ++param);
871 if (unlikely(rc != SQLITE_OK))
872 goto bind_fail;
873
880 - param++;
881 - rc = sqlite3_bind_string_or_null(res, cfg->type, param);
874 + rc = sqlite3_bind_string_or_null(res, cfg->type, ++param);
875 if (unlikely(rc != SQLITE_OK))
876 goto bind_fail;
877
885 - param++;
886 - rc = sqlite3_bind_string_or_null(res, cfg->os, param);
878 + rc = sqlite3_bind_string_or_null(res, cfg->os, ++param);
879 if (unlikely(rc != SQLITE_OK))
880 goto bind_fail;
881
890 - param++;
891 - rc = sqlite3_bind_string_or_null(res, cfg->host, param);
882 + rc = sqlite3_bind_string_or_null(res, cfg->host, ++param);
883 if (unlikely(rc != SQLITE_OK))
884 goto bind_fail;
885
895 - param++;
896 - rc = sqlite3_bind_string_or_null(res, cfg->lookup, param);
886 + rc = sqlite3_bind_string_or_null(res, cfg->lookup, ++param);
887 if (unlikely(rc != SQLITE_OK))
888 goto bind_fail;
889
900 - param++;
901 - rc = sqlite3_bind_string_or_null(res, cfg->every, param);
890 + rc = sqlite3_bind_string_or_null(res, cfg->every, ++param);
891 if (unlikely(rc != SQLITE_OK))
892 goto bind_fail;
893
905 - param++;
906 - rc = sqlite3_bind_string_or_null(res, cfg->units, param);
894 + rc = sqlite3_bind_string_or_null(res, cfg->units, ++param);
895 if (unlikely(rc != SQLITE_OK))
896 goto bind_fail;
897
910 - param++;
911 - rc = sqlite3_bind_string_or_null(res, cfg->calc, param);
898 + rc = sqlite3_bind_string_or_null(res, cfg->calc, ++param);
899 if (unlikely(rc != SQLITE_OK))
900 goto bind_fail;
901
915 - param++;
916 - rc = sqlite3_bind_string_or_null(res, cfg->families, param);
902 + rc = sqlite3_bind_string_or_null(res, cfg->families, ++param);
903 if (unlikely(rc != SQLITE_OK))
904 goto bind_fail;
905
920 - param++;
921 - rc = sqlite3_bind_string_or_null(res, cfg->plugin, param);
906 + rc = sqlite3_bind_string_or_null(res, cfg->plugin, ++param);
907 if (unlikely(rc != SQLITE_OK))
908 goto bind_fail;
909
925 - param++;
926 - rc = sqlite3_bind_string_or_null(res, cfg->module, param);
910 + rc = sqlite3_bind_string_or_null(res, cfg->module, ++param);
911 if (unlikely(rc != SQLITE_OK))
912 goto bind_fail;
913
930 - param++;
931 - rc = sqlite3_bind_string_or_null(res, cfg->charts, param);
914 + rc = sqlite3_bind_string_or_null(res, cfg->charts, ++param);
915 if (unlikely(rc != SQLITE_OK))
916 goto bind_fail;
917
935 - param++;
936 - rc = sqlite3_bind_string_or_null(res, cfg->green, param);
918 + rc = sqlite3_bind_string_or_null(res, cfg->green, ++param);
919 if (unlikely(rc != SQLITE_OK))
920 goto bind_fail;
921
940 - param++;
941 - rc = sqlite3_bind_string_or_null(res, cfg->red, param);
922 + rc = sqlite3_bind_string_or_null(res, cfg->red, ++param);
923 if (unlikely(rc != SQLITE_OK))
924 goto bind_fail;
925
945 - param++;
946 - rc = sqlite3_bind_string_or_null(res, cfg->warn, param);
926 + rc = sqlite3_bind_string_or_null(res, cfg->warn, ++param);
927 if (unlikely(rc != SQLITE_OK))
928 goto bind_fail;
929
950 - param++;
951 - rc = sqlite3_bind_string_or_null(res, cfg->crit, param);
930 + rc = sqlite3_bind_string_or_null(res, cfg->crit, ++param);
931 if (unlikely(rc != SQLITE_OK))
932 goto bind_fail;
933
955 - param++;
956 - rc = sqlite3_bind_string_or_null(res, cfg->exec, param);
934 + rc = sqlite3_bind_string_or_null(res, cfg->exec, ++param);
935 if (unlikely(rc != SQLITE_OK))
936 goto bind_fail;
937
960 - param++;
961 - rc = sqlite3_bind_string_or_null(res, cfg->to, param);
938 + rc = sqlite3_bind_string_or_null(res, cfg->to, ++param);
939 if (unlikely(rc != SQLITE_OK))
940 goto bind_fail;
941
965 - param++;
966 - rc = sqlite3_bind_string_or_null(res, cfg->info, param);
942 + rc = sqlite3_bind_string_or_null(res, cfg->info, ++param);
943 if (unlikely(rc != SQLITE_OK))
944 goto bind_fail;
945
970 - param++;
971 - rc = sqlite3_bind_string_or_null(res, cfg->delay, param);
946 + rc = sqlite3_bind_string_or_null(res, cfg->delay, ++param);
947 if (unlikely(rc != SQLITE_OK))
948 goto bind_fail;
949
975 - param++;
976 - rc = sqlite3_bind_string_or_null(res, cfg->options, param);
950 + rc = sqlite3_bind_string_or_null(res, cfg->options, ++param);
951 if (unlikely(rc != SQLITE_OK))
952 goto bind_fail;
953
980 - param++;
981 - rc = sqlite3_bind_string_or_null(res, cfg->repeat, param);
954 + rc = sqlite3_bind_string_or_null(res, cfg->repeat, ++param);
955 if (unlikely(rc != SQLITE_OK))
956 goto bind_fail;
957
985 - param++;
986 - rc = sqlite3_bind_string_or_null(res, cfg->host_labels, param);
958 + rc = sqlite3_bind_string_or_null(res, cfg->host_labels, ++param);
959 if (unlikely(rc != SQLITE_OK))
960 goto bind_fail;
961
962 if (cfg->p_db_lookup_after) {
991 - param++;
992 - rc = sqlite3_bind_string_or_null(res, cfg->p_db_lookup_dimensions, param);
963 + rc = sqlite3_bind_string_or_null(res, cfg->p_db_lookup_dimensions, ++param);
964 if (unlikely(rc != SQLITE_OK))
965 goto bind_fail;
966
996 - param++;
997 - rc = sqlite3_bind_string_or_null(res, cfg->p_db_lookup_method, param);
967 + rc = sqlite3_bind_string_or_null(res, cfg->p_db_lookup_method, ++param);
968 if (unlikely(rc != SQLITE_OK))
969 goto bind_fail;
970
1001 - param++;
1002 - rc = sqlite3_bind_int(res, 31, cfg->p_db_lookup_options);
971 + rc = sqlite3_bind_int(res, ++param, (int) cfg->p_db_lookup_options);
972 if (unlikely(rc != SQLITE_OK))
973 goto bind_fail;
974
1006 - param++;
1007 - rc = sqlite3_bind_int(res, 32, cfg->p_db_lookup_after);
975 + rc = sqlite3_bind_int(res, ++param, (int) cfg->p_db_lookup_after);
976 if (unlikely(rc != SQLITE_OK))
977 goto bind_fail;
978
1011 - param++;
1012 - rc = sqlite3_bind_int(res, 33, cfg->p_db_lookup_before);
979 + rc = sqlite3_bind_int(res, ++param, (int) cfg->p_db_lookup_before);
980 if (unlikely(rc != SQLITE_OK))
981 goto bind_fail;
982 } else {
1016 - param++;
1017 - rc = sqlite3_bind_null(res, 29);
983 + rc = sqlite3_bind_null(res, ++param);
984 if (unlikely(rc != SQLITE_OK))
985 goto bind_fail;
1020 - param++;
1021 - rc = sqlite3_bind_null(res, 30);
986 +
987 + rc = sqlite3_bind_null(res, ++param);
988 if (unlikely(rc != SQLITE_OK))
989 goto bind_fail;
1024 - param++;
1025 - rc = sqlite3_bind_null(res, 31);
990 +
991 + rc = sqlite3_bind_null(res, ++param);
992 if (unlikely(rc != SQLITE_OK))
993 goto bind_fail;
1028 - param++;
1029 - rc = sqlite3_bind_null(res, 32);
994 +
995 + rc = sqlite3_bind_null(res, ++param);
996 if (unlikely(rc != SQLITE_OK))
997 goto bind_fail;
1032 - param++;
1033 - rc = sqlite3_bind_null(res, 33);
998 +
999 + rc = sqlite3_bind_null(res, ++param);
1000 if (unlikely(rc != SQLITE_OK))
1001 goto bind_fail;
1002 }
1003
1038 - param++;
1039 - rc = sqlite3_bind_int(res, 34, cfg->p_update_every);
1004 + rc = sqlite3_bind_int(res, ++param, cfg->p_update_every);
1005 if (unlikely(rc != SQLITE_OK))
1006 goto bind_fail;
1007
@@ -1050,7 +1015,7 @@ int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg)
1015
1016 return 0;
1017
1053 - bind_fail:
1018 +bind_fail:
1019 error_report("Failed to bind parameter %d to store alert hash_id, rc = %d", param, rc);
1020 rc = sqlite3_reset(res);
1021 if (unlikely(rc != SQLITE_OK))
database/sqlite/sqlite_metadata.c
+331 -188
@@ -64,6 +64,7 @@ enum metadata_opcode {
64 METADATA_STORE_CLAIM_ID,
65 METADATA_ADD_HOST_INFO,
66 METADATA_SCAN_HOSTS,
67 + METADATA_LOAD_HOST_CONTEXT,
68 METADATA_MAINTENANCE,
69 METADATA_SYNC_SHUTDOWN,
70 METADATA_UNITTEST,
@@ -154,19 +155,43 @@ static int chart_label_store_to_sql_callback(const char *name, const char *value
155 return 1;
156 }
157
157 -static void check_and_update_chart_labels(RRDSET *st, BUFFER *work_buffer)
158 +#define SQL_DELETE_CHART_LABEL "DELETE FROM chart_label WHERE chart_id = @chart_id;"
159 +#define SQL_DELETE_CHART_LABEL_HISTORY "DELETE FROM chart_label WHERE date_created < %ld AND chart_id = @chart_id;"
160 +
161 +static void clean_old_chart_labels(RRDSET *st)
162 +{
163 + char sql[512];
164 + time_t first_time_s = rrdset_first_entry_s(st);
165 +
166 + if (unlikely(!first_time_s))
167 + snprintfz(sql, 511,SQL_DELETE_CHART_LABEL);
168 + else
169 + snprintfz(sql, 511,SQL_DELETE_CHART_LABEL_HISTORY, first_time_s);
170 +
171 + int rc = exec_statement_with_uuid(sql, &st->chart_uuid);
172 + if (unlikely(rc))
173 + error_report("METADATA: 'host:%s' Failed to clean old labels for chart %s", rrdhost_hostname(st->rrdhost), rrdset_name(st));
174 +}
175 +
176 +static int check_and_update_chart_labels(RRDSET *st, BUFFER *work_buffer, size_t *query_counter)
177 {
178 size_t old_version = st->rrdlabels_last_saved_version;
179 size_t new_version = dictionary_version(st->rrdlabels);
180
162 - if(new_version != old_version) {
163 - buffer_flush(work_buffer);
164 - struct query_build tmp = {.sql = work_buffer, .count = 0};
165 - uuid_unparse_lower(st->chart_uuid, tmp.uuid_str);
166 - rrdlabels_walkthrough_read(st->rrdlabels, chart_label_store_to_sql_callback, &tmp);
181 + if (new_version == old_version)
182 + return 0;
183 +
184 + struct query_build tmp = {.sql = work_buffer, .count = 0};
185 + uuid_unparse_lower(st->chart_uuid, tmp.uuid_str);
186 + rrdlabels_walkthrough_read(st->rrdlabels, chart_label_store_to_sql_callback, &tmp);
187 + int rc = db_execute(buffer_tostring(work_buffer));
188 + if (likely(!rc)) {
189 st->rrdlabels_last_saved_version = new_version;
168 - db_execute(buffer_tostring(work_buffer));
190 + (*query_counter)++;
191 }
192 +
193 + clean_old_chart_labels(st);
194 + return rc;
195 }
196
197 // Migrate all hosts with hops zero to this host_uuid
@@ -177,12 +202,13 @@ void migrate_localhost(uuid_t *host_uuid)
202 rc = exec_statement_with_uuid(MIGRATE_LOCALHOST_TO_NEW_MACHINE_GUID, host_uuid);
203 if (!rc)
204 rc = exec_statement_with_uuid(DELETE_NON_EXISTING_LOCALHOST, host_uuid);
180 - if (!rc)
181 - db_execute(DELETE_MISSING_NODE_INSTANCES);
182 -
205 + if (!rc) {
206 + if (unlikely(db_execute(DELETE_MISSING_NODE_INSTANCES)))
207 + error_report("Failed to remove deleted hosts from node instances");
208 + }
209 }
210
185 -static void store_claim_id(uuid_t *host_id, uuid_t *claim_id)
211 +static int store_claim_id(uuid_t *host_id, uuid_t *claim_id)
212 {
213 sqlite3_stmt *res = NULL;
214 int rc;
@@ -190,18 +216,18 @@ static void store_claim_id(uuid_t *host_id, uuid_t *claim_id)
216 if (unlikely(!db_meta)) {
217 if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
218 error_report("Database has not been initialized");
193 - return;
219 + return 1;
220 }
221
222 rc = sqlite3_prepare_v2(db_meta, SQL_STORE_CLAIM_ID, -1, &res, 0);
223 if (unlikely(rc != SQLITE_OK)) {
198 - error_report("Failed to prepare statement store chart labels");
199 - return;
224 + error_report("Failed to prepare statement to store host claim id");
225 + return 1;
226 }
227
228 rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);
229 if (unlikely(rc != SQLITE_OK)) {
204 - error_report("Failed to bind host_id parameter to store node instance information");
230 + error_report("Failed to bind host_id parameter to store claim id");
231 goto failed;
232 }
233
@@ -210,17 +236,19 @@ static void store_claim_id(uuid_t *host_id, uuid_t *claim_id)
236 else
237 rc = sqlite3_bind_null(res, 2);
238 if (unlikely(rc != SQLITE_OK)) {
213 - error_report("Failed to bind claim_id parameter to store node instance information");
239 + error_report("Failed to bind claim_id parameter to host claim id");
240 goto failed;
241 }
242
243 rc = execute_insert(res);
244 if (unlikely(rc != SQLITE_DONE))
219 - error_report("Failed to store node instance information, rc = %d", rc);
245 + error_report("Failed to store host claim id rc = %d", rc);
246
247 failed:
248 if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
223 - error_report("Failed to finalize the prepared statement when storing node instance information");
249 + error_report("Failed to finalize the prepared statement when storing a host claim id");
250 +
251 + return rc != SQLITE_DONE;
252 }
253
254 static void delete_dimension_uuid(uuid_t *dimension_uuid)
@@ -252,7 +280,7 @@ skip_execution:
280
281 //
282 // Store host and host system info information in the database
255 -static int sql_store_host_info(RRDHOST *host)
283 +static int store_host_metadata(RRDHOST *host)
284 {
285 static __thread sqlite3_stmt *res = NULL;
286 int rc, param = 0;
@@ -340,7 +368,7 @@ static int sql_store_host_info(RRDHOST *host)
368 if (unlikely(rc != SQLITE_OK))
369 error_report("Failed to reset statement to store host %s, rc = %d", rrdhost_hostname(host), rc);
370
343 - return !(store_rc == SQLITE_DONE);
371 + return store_rc != SQLITE_DONE;
372 bind_fail:
373 error_report("Failed to bind %d parameter to store host %s, rc = %d", param, rrdhost_hostname(host), rc);
374 rc = sqlite3_reset(res);
@@ -349,7 +377,7 @@ bind_fail:
377 return 1;
378 }
379
352 -static void sql_store_host_system_info_key_value(const char *name, const char *value, void *data)
380 +static void add_host_sysinfo_key_value(const char *name, const char *value, void *data)
381 {
382 struct query_build *lb = data;
383
@@ -365,44 +393,43 @@ static void sql_store_host_system_info_key_value(const char *name, const char *v
393 lb->count++;
394 }
395
368 -static BUFFER *sql_store_host_system_info(RRDHOST *host)
396 +static bool build_host_system_info_statements(RRDHOST *host, BUFFER *work_buffer)
397 {
398 struct rrdhost_system_info *system_info = host->system_info;
399
400 if (unlikely(!system_info))
373 - return NULL;
374 -
375 - BUFFER *work_buffer = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
401 + return false;
402
403 + buffer_flush(work_buffer);
404 struct query_build key_data = {.sql = work_buffer, .count = 0};
405 uuid_unparse_lower(host->host_uuid, key_data.uuid_str);
406
380 - sql_store_host_system_info_key_value("NETDATA_CONTAINER_OS_NAME", system_info->container_os_name, &key_data);
381 - sql_store_host_system_info_key_value("NETDATA_CONTAINER_OS_ID", system_info->container_os_id, &key_data);
382 - sql_store_host_system_info_key_value("NETDATA_CONTAINER_OS_ID_LIKE", system_info->container_os_id_like, &key_data);
383 - sql_store_host_system_info_key_value("NETDATA_CONTAINER_OS_VERSION", system_info->container_os_version, &key_data);
384 - sql_store_host_system_info_key_value("NETDATA_CONTAINER_OS_VERSION_ID", system_info->container_os_version_id, &key_data);
385 - sql_store_host_system_info_key_value("NETDATA_CONTAINER_OS_DETECTION", system_info->host_os_detection, &key_data);
386 - sql_store_host_system_info_key_value("NETDATA_HOST_OS_NAME", system_info->host_os_name, &key_data);
387 - sql_store_host_system_info_key_value("NETDATA_HOST_OS_ID", system_info->host_os_id, &key_data);
388 - sql_store_host_system_info_key_value("NETDATA_HOST_OS_ID_LIKE", system_info->host_os_id_like, &key_data);
389 - sql_store_host_system_info_key_value("NETDATA_HOST_OS_VERSION", system_info->host_os_version, &key_data);
390 - sql_store_host_system_info_key_value("NETDATA_HOST_OS_VERSION_ID", system_info->host_os_version_id, &key_data);
391 - sql_store_host_system_info_key_value("NETDATA_HOST_OS_DETECTION", system_info->host_os_detection, &key_data);
392 - sql_store_host_system_info_key_value("NETDATA_SYSTEM_KERNEL_NAME", system_info->kernel_name, &key_data);
393 - sql_store_host_system_info_key_value("NETDATA_SYSTEM_CPU_LOGICAL_CPU_COUNT", system_info->host_cores, &key_data);
394 - sql_store_host_system_info_key_value("NETDATA_SYSTEM_CPU_FREQ", system_info->host_cpu_freq, &key_data);
395 - sql_store_host_system_info_key_value("NETDATA_SYSTEM_TOTAL_RAM", system_info->host_ram_total, &key_data);
396 - sql_store_host_system_info_key_value("NETDATA_SYSTEM_TOTAL_DISK_SIZE", system_info->host_disk_space, &key_data);
397 - sql_store_host_system_info_key_value("NETDATA_SYSTEM_KERNEL_VERSION", system_info->kernel_version, &key_data);
398 - sql_store_host_system_info_key_value("NETDATA_SYSTEM_ARCHITECTURE", system_info->architecture, &key_data);
399 - sql_store_host_system_info_key_value("NETDATA_SYSTEM_VIRTUALIZATION", system_info->virtualization, &key_data);
400 - sql_store_host_system_info_key_value("NETDATA_SYSTEM_VIRT_DETECTION", system_info->virt_detection, &key_data);
401 - sql_store_host_system_info_key_value("NETDATA_SYSTEM_CONTAINER", system_info->container, &key_data);
402 - sql_store_host_system_info_key_value("NETDATA_SYSTEM_CONTAINER_DETECTION", system_info->container_detection, &key_data);
403 - sql_store_host_system_info_key_value("NETDATA_HOST_IS_K8S_NODE", system_info->is_k8s_node, &key_data);
404 -
405 - return work_buffer;
407 + add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_NAME", system_info->container_os_name, &key_data);
408 + add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_ID", system_info->container_os_id, &key_data);
409 + add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_ID_LIKE", system_info->container_os_id_like, &key_data);
410 + add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_VERSION", system_info->container_os_version, &key_data);
411 + add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_VERSION_ID", system_info->container_os_version_id, &key_data);
412 + add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_DETECTION", system_info->host_os_detection, &key_data);
413 + add_host_sysinfo_key_value("NETDATA_HOST_OS_NAME", system_info->host_os_name, &key_data);
414 + add_host_sysinfo_key_value("NETDATA_HOST_OS_ID", system_info->host_os_id, &key_data);
415 + add_host_sysinfo_key_value("NETDATA_HOST_OS_ID_LIKE", system_info->host_os_id_like, &key_data);
416 + add_host_sysinfo_key_value("NETDATA_HOST_OS_VERSION", system_info->host_os_version, &key_data);
417 + add_host_sysinfo_key_value("NETDATA_HOST_OS_VERSION_ID", system_info->host_os_version_id, &key_data);
418 + add_host_sysinfo_key_value("NETDATA_HOST_OS_DETECTION", system_info->host_os_detection, &key_data);
419 + add_host_sysinfo_key_value("NETDATA_SYSTEM_KERNEL_NAME", system_info->kernel_name, &key_data);
420 + add_host_sysinfo_key_value("NETDATA_SYSTEM_CPU_LOGICAL_CPU_COUNT", system_info->host_cores, &key_data);
421 + add_host_sysinfo_key_value("NETDATA_SYSTEM_CPU_FREQ", system_info->host_cpu_freq, &key_data);
422 + add_host_sysinfo_key_value("NETDATA_SYSTEM_TOTAL_RAM", system_info->host_ram_total, &key_data);
423 + add_host_sysinfo_key_value("NETDATA_SYSTEM_TOTAL_DISK_SIZE", system_info->host_disk_space, &key_data);
424 + add_host_sysinfo_key_value("NETDATA_SYSTEM_KERNEL_VERSION", system_info->kernel_version, &key_data);
425 + add_host_sysinfo_key_value("NETDATA_SYSTEM_ARCHITECTURE", system_info->architecture, &key_data);
426 + add_host_sysinfo_key_value("NETDATA_SYSTEM_VIRTUALIZATION", system_info->virtualization, &key_data);
427 + add_host_sysinfo_key_value("NETDATA_SYSTEM_VIRT_DETECTION", system_info->virt_detection, &key_data);
428 + add_host_sysinfo_key_value("NETDATA_SYSTEM_CONTAINER", system_info->container, &key_data);
429 + add_host_sysinfo_key_value("NETDATA_SYSTEM_CONTAINER_DETECTION", system_info->container_detection, &key_data);
430 + add_host_sysinfo_key_value("NETDATA_HOST_IS_K8S_NODE", system_info->is_k8s_node, &key_data);
431 +
432 + return true;
433 }
434
435
@@ -410,13 +437,10 @@ static BUFFER *sql_store_host_system_info(RRDHOST *host)
437 * Store a chart in the database
438 */
439
413 -static int sql_store_chart(
414 - uuid_t *chart_uuid, uuid_t *host_uuid, const char *type, const char *id, const char *name, const char *family,
415 - const char *context, const char *title, const char *units, const char *plugin, const char *module, long priority,
416 - int update_every, int chart_type, int memory_mode, long history_entries)
440 +static int store_chart_metadata(RRDSET *st)
441 {
442 static __thread sqlite3_stmt *res = NULL;
419 - int rc, param = 0;
443 + int rc, param = 0, store_rc = 0;
444
445 if (unlikely(!db_meta)) {
446 if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
@@ -433,98 +457,83 @@ static int sql_store_chart(
457 }
458 }
459
436 - param++;
437 - rc = sqlite3_bind_blob(res, 1, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC);
460 + rc = sqlite3_bind_blob(res, ++param, &st->chart_uuid, sizeof(st->chart_uuid), SQLITE_STATIC);
461 if (unlikely(rc != SQLITE_OK))
462 goto bind_fail;
463
441 - param++;
442 - rc = sqlite3_bind_blob(res, 2, host_uuid, sizeof(*host_uuid), SQLITE_STATIC);
464 + rc = sqlite3_bind_blob(res, ++param, &st->rrdhost->host_uuid, sizeof(st->rrdhost->host_uuid), SQLITE_STATIC);
465 if (unlikely(rc != SQLITE_OK))
466 goto bind_fail;
467
446 - param++;
447 - rc = sqlite3_bind_text(res, 3, type, -1, SQLITE_STATIC);
468 + rc = sqlite3_bind_text(res, ++param, string2str(st->parts.type), -1, SQLITE_STATIC);
469 if (unlikely(rc != SQLITE_OK))
470 goto bind_fail;
471
451 - param++;
452 - rc = sqlite3_bind_text(res, 4, id, -1, SQLITE_STATIC);
472 + rc = sqlite3_bind_text(res, ++param, string2str(st->parts.id), -1, SQLITE_STATIC);
473 if (unlikely(rc != SQLITE_OK))
474 goto bind_fail;
475
456 - param++;
476 + const char *name = string2str(st->parts.name);
477 if (name && *name)
458 - rc = sqlite3_bind_text(res, 5, name, -1, SQLITE_STATIC);
478 + rc = sqlite3_bind_text(res, ++param, name, -1, SQLITE_STATIC);
479 else
460 - rc = sqlite3_bind_null(res, 5);
480 + rc = sqlite3_bind_null(res, ++param);
481 if (unlikely(rc != SQLITE_OK))
482 goto bind_fail;
483
464 - param++;
465 - rc = sqlite3_bind_text(res, 6, family, -1, SQLITE_STATIC);
484 + rc = sqlite3_bind_text(res, ++param, rrdset_family(st), -1, SQLITE_STATIC);
485 if (unlikely(rc != SQLITE_OK))
486 goto bind_fail;
487
469 - param++;
470 - rc = sqlite3_bind_text(res, 7, context, -1, SQLITE_STATIC);
488 + rc = sqlite3_bind_text(res, ++param, rrdset_context(st), -1, SQLITE_STATIC);
489 if (unlikely(rc != SQLITE_OK))
490 goto bind_fail;
491
474 - param++;
475 - rc = sqlite3_bind_text(res, 8, title, -1, SQLITE_STATIC);
492 + rc = sqlite3_bind_text(res, ++param, rrdset_title(st), -1, SQLITE_STATIC);
493 if (unlikely(rc != SQLITE_OK))
494 goto bind_fail;
495
479 - param++;
480 - rc = sqlite3_bind_text(res, 9, units, -1, SQLITE_STATIC);
496 + rc = sqlite3_bind_text(res, ++param, rrdset_units(st), -1, SQLITE_STATIC);
497 if (unlikely(rc != SQLITE_OK))
498 goto bind_fail;
499
484 - param++;
485 - rc = sqlite3_bind_text(res, 10, plugin, -1, SQLITE_STATIC);
500 + rc = sqlite3_bind_text(res, ++param, rrdset_plugin_name(st), -1, SQLITE_STATIC);
501 if (unlikely(rc != SQLITE_OK))
502 goto bind_fail;
503
489 - param++;
490 - rc = sqlite3_bind_text(res, 11, module, -1, SQLITE_STATIC);
504 + rc = sqlite3_bind_text(res, ++param, rrdset_module_name(st), -1, SQLITE_STATIC);
505 if (unlikely(rc != SQLITE_OK))
506 goto bind_fail;
507
494 - param++;
495 - rc = sqlite3_bind_int(res, 12, (int) priority);
508 + rc = sqlite3_bind_int(res, ++param, (int) st->priority);
509 if (unlikely(rc != SQLITE_OK))
510 goto bind_fail;
511
499 - param++;
500 - rc = sqlite3_bind_int(res, 13, update_every);
512 + rc = sqlite3_bind_int(res, ++param, st->update_every);
513 if (unlikely(rc != SQLITE_OK))
514 goto bind_fail;
515
504 - param++;
505 - rc = sqlite3_bind_int(res, 14, chart_type);
516 + rc = sqlite3_bind_int(res, ++param, st->chart_type);
517 if (unlikely(rc != SQLITE_OK))
518 goto bind_fail;
519
509 - param++;
510 - rc = sqlite3_bind_int(res, 15, memory_mode);
520 + rc = sqlite3_bind_int(res, ++param, st->rrd_memory_mode);
521 if (unlikely(rc != SQLITE_OK))
522 goto bind_fail;
523
514 - param++;
515 - rc = sqlite3_bind_int(res, 16, (int) history_entries);
524 + rc = sqlite3_bind_int(res, ++param, (int) st->entries);
525 if (unlikely(rc != SQLITE_OK))
526 goto bind_fail;
527
519 - rc = execute_insert(res);
520 - if (unlikely(rc != SQLITE_DONE))
521 - error_report("Failed to store chart, rc = %d", rc);
528 + store_rc = execute_insert(res);
529 + if (unlikely(store_rc != SQLITE_DONE))
530 + error_report("Failed to store chart, rc = %d", store_rc);
531
532 rc = sqlite3_reset(res);
533 if (unlikely(rc != SQLITE_OK))
534 error_report("Failed to reset statement in chart store function, rc = %d", rc);
535
527 - return 0;
536 + return store_rc != SQLITE_DONE;
537
538 bind_fail:
539 error_report("Failed to bind parameter %d to store chart, rc = %d", param, rc);
@@ -537,9 +546,7 @@ bind_fail:
546 /*
547 * Store a dimension
548 */
540 -static int sql_store_dimension(
541 - uuid_t *dim_uuid, uuid_t *chart_uuid, const char *id, const char *name, collected_number multiplier,
542 - collected_number divisor, int algorithm, bool hidden)
549 +static int store_dimension_metadata(RRDDIM *rd)
550 {
551 static __thread sqlite3_stmt *res = NULL;
552 int rc, param = 0;
@@ -559,35 +566,35 @@ static int sql_store_dimension(
566 }
567 }
568
562 - rc = sqlite3_bind_blob(res, ++param, dim_uuid, sizeof(*dim_uuid), SQLITE_STATIC);
569 + rc = sqlite3_bind_blob(res, ++param, &rd->metric_uuid, sizeof(rd->metric_uuid), SQLITE_STATIC);
570 if (unlikely(rc != SQLITE_OK))
571 goto bind_fail;
572
566 - rc = sqlite3_bind_blob(res, ++param, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC);
573 + rc = sqlite3_bind_blob(res, ++param, &rd->rrdset->chart_uuid, sizeof(rd->rrdset->chart_uuid), SQLITE_STATIC);
574 if (unlikely(rc != SQLITE_OK))
575 goto bind_fail;
576
570 - rc = sqlite3_bind_text(res, ++param, id, -1, SQLITE_STATIC);
577 + rc = sqlite3_bind_text(res, ++param, string2str(rd->id), -1, SQLITE_STATIC);
578 if (unlikely(rc != SQLITE_OK))
579 goto bind_fail;
580
574 - rc = sqlite3_bind_text(res, ++param, name, -1, SQLITE_STATIC);
581 + rc = sqlite3_bind_text(res, ++param, string2str(rd->name), -1, SQLITE_STATIC);
582 if (unlikely(rc != SQLITE_OK))
583 goto bind_fail;
584
578 - rc = sqlite3_bind_int(res, ++param, (int) multiplier);
585 + rc = sqlite3_bind_int(res, ++param, (int) rd->multiplier);
586 if (unlikely(rc != SQLITE_OK))
587 goto bind_fail;
588
582 - rc = sqlite3_bind_int(res, ++param, (int ) divisor);
589 + rc = sqlite3_bind_int(res, ++param, (int ) rd->divisor);
590 if (unlikely(rc != SQLITE_OK))
591 goto bind_fail;
592
586 - rc = sqlite3_bind_int(res, ++param, algorithm);
593 + rc = sqlite3_bind_int(res, ++param, rd->algorithm);
594 if (unlikely(rc != SQLITE_OK))
595 goto bind_fail;
596
590 - if (hidden)
597 + if (rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN))
598 rc = sqlite3_bind_text(res, ++param, "hidden", -1, SQLITE_STATIC);
599 else
600 rc = sqlite3_bind_null(res, ++param);
@@ -700,7 +707,6 @@ static void cleanup_health_log(void)
707 //
708 // EVENT LOOP STARTS HERE
709 //
703 -static uv_mutex_t metadata_async_lock;
710
711 static void metadata_init_cmd_queue(struct metadata_wc *wc)
712 {
@@ -856,6 +862,7 @@ static void start_metadata_cleanup(uv_work_t *req)
862 struct metadata_wc *wc = req->data;
863 check_dimension_metadata(wc);
864 cleanup_health_log();
865 + (void) sqlite3_wal_checkpoint(db_meta, NULL);
866 worker_is_idle();
867 }
868
@@ -863,9 +870,121 @@ struct scan_metadata_payload {
870 uv_work_t request;
871 struct metadata_wc *wc;
872 struct completion *completion;
873 + BUFFER *work_buffer;
874 uint32_t max_count;
875 };
876
877 +struct host_context_load_thread {
878 + uv_thread_t thread;
879 + RRDHOST *host;
880 + bool busy;
881 + bool finished;
882 +};
883 +
884 +static void restore_host_context(void *arg)
885 +{
886 + struct host_context_load_thread *hclt = arg;
887 + RRDHOST *host = hclt->host;
888 +
889 + usec_t started_ut = now_monotonic_usec(); (void)started_ut;
890 + rrdhost_load_rrdcontext_data(host);
891 + usec_t ended_ut = now_monotonic_usec(); (void)ended_ut;
892 +
893 + rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD | RRDHOST_FLAG_CONTEXT_LOAD_IN_PROGRESS);
894 +
895 + internal_error(true, "METADATA: 'host:%s' context load in %0.2f ms", rrdhost_hostname(host),
896 + (double)(ended_ut - started_ut) / USEC_PER_MS);
897 +
898 + __atomic_store_n(&hclt->finished, true, __ATOMIC_RELEASE);
899 +}
900 +
901 +// Callback after scan of hosts is done
902 +static void after_start_host_load_context(uv_work_t *req, int status __maybe_unused)
903 +{
904 + struct scan_metadata_payload *data = req->data;
905 + freez(data);
906 +}
907 +
908 +#define MAX_FIND_THREAD_RETRIES (10)
909 +
910 +static void cleanup_finished_threads(struct host_context_load_thread *hclt, size_t max_thread_slots, bool wait)
911 +{
912 + for (size_t index = 0; index < max_thread_slots; index++) {
913 + if (__atomic_load_n(&(hclt[index].finished), __ATOMIC_RELAXED)
914 + || (wait && __atomic_load_n(&(hclt[index].busy), __ATOMIC_ACQUIRE))) {
915 + int rc = uv_thread_join(&(hclt[index].thread));
916 + if (rc)
917 + error("Failed to join thread, rc = %d",rc);
918 + __atomic_store_n(&(hclt[index].busy), false, __ATOMIC_RELEASE);
919 + __atomic_store_n(&(hclt[index].finished), false, __ATOMIC_RELEASE);
920 + }
921 + }
922 +}
923 +
924 +static size_t find_available_thread_slot(struct host_context_load_thread *hclt, size_t max_thread_slots, size_t *found_index)
925 +{
926 + size_t retries = MAX_FIND_THREAD_RETRIES;
927 + while (retries--) {
928 + size_t index = 0;
929 + while (index < max_thread_slots) {
930 + if (false == __atomic_load_n(&(hclt[index].busy), __ATOMIC_ACQUIRE)) {
931 + *found_index = index;
932 + return true;
933 + }
934 + index++;
935 + }
936 + sleep_usec(10 * USEC_PER_MS);
937 + }
938 + return false;
939 +}
940 +
941 +static void start_all_host_load_context(uv_work_t *req __maybe_unused)
942 +{
943 + register_libuv_worker_jobs();
944 +
945 + struct scan_metadata_payload *data = req->data;
946 + UNUSED(data);
947 +
948 + worker_is_busy(UV_EVENT_HOST_CONTEXT_LOAD);
949 + usec_t started_ut = now_monotonic_usec(); (void)started_ut;
950 +
951 + RRDHOST *host;
952 +
953 + size_t max_threads = MIN(get_netdata_cpus() / 2, 6);
954 + struct host_context_load_thread *hclt = callocz(max_threads, sizeof(*hclt));
955 +
956 + size_t thread_index;
957 + dfe_start_reentrant(rrdhost_root_index, host) {
958 + if (rrdhost_flag_check(host, RRDHOST_FLAG_CONTEXT_LOAD_IN_PROGRESS) ||
959 + !rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD))
960 + continue;
961 +
962 + rrdhost_flag_set(host, RRDHOST_FLAG_CONTEXT_LOAD_IN_PROGRESS);
963 + internal_error(true, "METADATA: 'host:%s' loading context", rrdhost_hostname(host));
964 +
965 + cleanup_finished_threads(hclt, max_threads, false);
966 + bool found_slot = find_available_thread_slot(hclt, max_threads, &thread_index);
967 +
968 + if (unlikely(!found_slot)) {
969 + struct host_context_load_thread hclt_sync = {.host = host};
970 + restore_host_context(&hclt_sync);
971 + }
972 + else {
973 + __atomic_store_n(&hclt[thread_index].busy, true, __ATOMIC_RELAXED);
974 + hclt[thread_index].host = host;
975 + assert(0 == uv_thread_create(&hclt[thread_index].thread, restore_host_context, &hclt[thread_index]));
976 + }
977 + }
978 + dfe_done(host);
979 +
980 + cleanup_finished_threads(hclt, max_threads, true);
981 + freez(hclt);
982 + usec_t ended_ut = now_monotonic_usec(); (void)ended_ut;
983 + internal_error(true, "METADATA: 'host:ALL' contexts loaded in %0.2f ms", (double)(ended_ut - started_ut) / USEC_PER_MS);
984 +
985 + worker_is_idle();
986 +}
987 +
988 // Callback after scan of hosts is done
989 static void after_metadata_hosts(uv_work_t *req, int status __maybe_unused)
990 {
@@ -881,13 +1000,15 @@ static void after_metadata_hosts(uv_work_t *req, int status __maybe_unused)
1000 freez(data);
1001 }
1002
884 -static bool metadata_scan_host(RRDHOST *host, uint32_t max_count, size_t *query_counter) {
1003 +static bool metadata_scan_host(RRDHOST *host, uint32_t max_count, bool use_transaction, BUFFER *work_buffer, size_t *query_counter) {
1004 RRDSET *st;
1005 int rc;
1006
1007 bool more_to_do = false;
1008 uint32_t scan_count = 1;
890 - BUFFER *work_buffer = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
1009 +
1010 + if (use_transaction)
1011 + (void)db_execute("BEGIN TRANSACTION;");
1012
1013 rrdset_foreach_reentrant(st, host) {
1014 if (scan_count == max_count) {
@@ -900,27 +1021,16 @@ static bool metadata_scan_host(RRDHOST *host, uint32_t max_count, size_t *query_
1021 rrdset_flag_clear(st, RRDSET_FLAG_METADATA_UPDATE);
1022 scan_count++;
1023
903 - check_and_update_chart_labels(st, work_buffer);
904 -
905 - rc = sql_store_chart(
906 - &st->chart_uuid,
907 - &st->rrdhost->host_uuid,
908 - string2str(st->parts.type),
909 - string2str(st->parts.id),
910 - string2str(st->parts.name),
911 - rrdset_family(st),
912 - rrdset_context(st),
913 - rrdset_title(st),
914 - rrdset_units(st),
915 - rrdset_plugin_name(st),
916 - rrdset_module_name(st),
917 - st->priority,
918 - st->update_every,
919 - st->chart_type,
920 - st->rrd_memory_mode,
921 - st->entries);
1024 + buffer_flush(work_buffer);
1025 + rc = check_and_update_chart_labels(st, work_buffer, query_counter);
1026 if (unlikely(rc))
923 - internal_error(true, "METADATA: Failed to store chart metadata %s", string2str(st->id));
1027 + error_report("METADATA: 'host:%s': Failed to update labels for chart %s", rrdhost_hostname(host), rrdset_name(st));
1028 + else
1029 + (*query_counter)++;
1030 +
1031 + rc = store_chart_metadata(st);
1032 + if (unlikely(rc))
1033 + error_report("METADATA: 'host:%s': Failed to store metadata for chart %s", rrdhost_hostname(host), rrdset_name(st));
1034 }
1035
1036 RRDDIM *rd;
@@ -935,44 +1045,53 @@ static bool metadata_scan_host(RRDHOST *host, uint32_t max_count, size_t *query_
1045 else
1046 rrddim_flag_clear(rd, RRDDIM_FLAG_META_HIDDEN);
1047
938 - rc = sql_store_dimension(
939 - &rd->metric_uuid,
940 - &rd->rrdset->chart_uuid,
941 - string2str(rd->id),
942 - string2str(rd->name),
943 - rd->multiplier,
944 - rd->divisor,
945 - rd->algorithm,
946 - rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN));
947 -
1048 + rc = store_dimension_metadata(rd);
1049 if (unlikely(rc))
949 - error_report("METADATA: Failed to store dimension %s", string2str(rd->id));
1050 + error_report("METADATA: 'host:%s': Failed to dimension metadata for chart %s. dimension %s",
1051 + rrdhost_hostname(host), rrdset_name(st),
1052 + rrddim_name(rd));
1053 }
1054 }
1055 rrddim_foreach_done(rd);
1056 }
1057 rrdset_foreach_done(st);
1058
956 - buffer_free(work_buffer);
1059 + if (use_transaction)
1060 + (void)db_execute("COMMIT TRANSACTION;");
1061 +
1062 return more_to_do;
1063 }
1064
960 -static void store_host_and_system_info(RRDHOST *host, size_t *query_counter)
1065 +static void store_host_and_system_info(RRDHOST *host, BUFFER *work_buffer, size_t *query_counter)
1066 {
962 - BUFFER *work_buffer = sql_store_host_system_info(host);
963 - if (work_buffer) {
964 - db_execute(buffer_tostring(work_buffer));
965 - buffer_free(work_buffer);
966 - if (likely(query_counter))
967 - (*query_counter)++;
1067 + bool free_work_buffer = (NULL == work_buffer);
1068 +
1069 + if (unlikely(free_work_buffer))
1070 + work_buffer = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
1071 +
1072 + if (build_host_system_info_statements(host, work_buffer)) {
1073 + int rc = db_execute(buffer_tostring(work_buffer));
1074 + if (unlikely(rc)) {
1075 + error_report("METADATA: 'host:%s': Failed to store host updated information in the database", rrdhost_hostname(host));
1076 + rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_INFO | RRDHOST_FLAG_METADATA_UPDATE);
1077 + }
1078 + else {
1079 + if (likely(query_counter))
1080 + (*query_counter)++;
1081 + }
1082 }
1083
970 - int rc = sql_store_host_info(host);
971 - if (unlikely(rc))
972 - error_report("METADATA: 'host:%s': failed to store host info", string2str(host->hostname));
973 - else
1084 + if (unlikely(store_host_metadata(host))) {
1085 + error_report("METADATA: 'host:%s': Failed to store host info in the database", rrdhost_hostname(host));
1086 + rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_INFO | RRDHOST_FLAG_METADATA_UPDATE);
1087 + }
1088 + else {
1089 if (likely(query_counter))
1090 (*query_counter)++;
1091 + }
1092 +
1093 + if (unlikely(free_work_buffer))
1094 + buffer_free(work_buffer);
1095 }
1096
1097 // Worker thread to scan hosts for pending metadata to store
@@ -981,76 +1100,90 @@ static void start_metadata_hosts(uv_work_t *req __maybe_unused)
1100 register_libuv_worker_jobs();
1101
1102 RRDHOST *host;
1103 + int transaction_started = 0;
1104
1105 struct scan_metadata_payload *data = req->data;
1106 struct metadata_wc *wc = data->wc;
1107
1108 + BUFFER *work_buffer = data->work_buffer;
1109 usec_t all_started_ut = now_monotonic_usec(); (void)all_started_ut;
1110 internal_error(true, "METADATA: checking all hosts...");
1111 + usec_t started_ut = now_monotonic_usec(); (void)started_ut;
1112
1113 bool run_again = false;
1114 worker_is_busy(UV_EVENT_METADATA_STORE);
1115
1116 if (!data->max_count)
995 - db_execute("BEGIN TRANSACTION;");
1117 + transaction_started = !db_execute("BEGIN TRANSACTION;");
1118 +
1119 dfe_start_reentrant(rrdhost_root_index, host) {
1120 if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED) || !rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_UPDATE))
1121 continue;
1122
1123 size_t query_counter = 0; (void)query_counter;
1001 - usec_t started_ut = now_monotonic_usec(); (void)started_ut;
1124
1125 rrdhost_flag_clear(host,RRDHOST_FLAG_METADATA_UPDATE);
1126
1127 if (unlikely(rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_LABELS))) {
1128 rrdhost_flag_clear(host, RRDHOST_FLAG_METADATA_LABELS);
1129 +
1130 int rc = exec_statement_with_uuid(SQL_DELETE_HOST_LABELS, &host->host_uuid);
1008 - if (likely(rc == SQLITE_OK)) {
1009 - BUFFER *work_buffer = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
1131 + if (likely(!rc)) {
1132 + query_counter++;
1133 +
1134 + buffer_flush(work_buffer);
1135 struct query_build tmp = {.sql = work_buffer, .count = 0};
1136 uuid_unparse_lower(host->host_uuid, tmp.uuid_str);
1137 rrdlabels_walkthrough_read(host->rrdlabels, host_label_store_to_sql_callback, &tmp);
1013 - db_execute(buffer_tostring(work_buffer));
1014 - buffer_free(work_buffer);
1015 - query_counter++;
1138 + rc = db_execute(buffer_tostring(work_buffer));
1139 +
1140 + if (unlikely(rc)) {
1141 + error_report("METADATA: 'host:%s': failed to update metadata host labels", rrdhost_hostname(host));
1142 + rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE);
1143 + }
1144 + else
1145 + query_counter++;
1146 + } else {
1147 + error_report("METADATA: 'host:%s': failed to delete old host labels", rrdhost_hostname(host));
1148 + rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE);
1149 }
1150 }
1151
1152 if (unlikely(rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_CLAIMID))) {
1153 rrdhost_flag_clear(host, RRDHOST_FLAG_METADATA_CLAIMID);
1154 uuid_t uuid;
1022 -
1155 + int rc;
1156 if (likely(host->aclk_state.claimed_id && !uuid_parse(host->aclk_state.claimed_id, uuid)))
1024 - store_claim_id(&host->host_uuid, &uuid);
1157 + rc = store_claim_id(&host->host_uuid, &uuid);
1158 else
1026 - store_claim_id(&host->host_uuid, NULL);
1159 + rc = store_claim_id(&host->host_uuid, NULL);
1160
1028 - query_counter++;
1161 + if (unlikely(rc))
1162 + rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_CLAIMID | RRDHOST_FLAG_METADATA_UPDATE);
1163 + else
1164 + query_counter++;
1165 }
1030 -
1166 if (unlikely(rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_INFO))) {
1167 rrdhost_flag_clear(host, RRDHOST_FLAG_METADATA_INFO);
1033 - store_host_and_system_info(host, &query_counter);
1168 + store_host_and_system_info(host, work_buffer, &query_counter);
1169 }
1170
1036 - if (data->max_count)
1037 - db_execute("BEGIN TRANSACTION;");
1038 - if (unlikely(metadata_scan_host(host, data->max_count, &query_counter))) {
1171 + // For clarity
1172 + bool use_transaction = data->max_count;
1173 + if (unlikely(metadata_scan_host(host, data->max_count, use_transaction, work_buffer, &query_counter))) {
1174 run_again = true;
1175 rrdhost_flag_set(host,RRDHOST_FLAG_METADATA_UPDATE);
1176 internal_error(true,"METADATA: 'host:%s': scheduling another run, more charts to store", rrdhost_hostname(host));
1177 }
1043 - if (data->max_count)
1044 - db_execute("COMMIT TRANSACTION;");
1045 -
1178 usec_t ended_ut = now_monotonic_usec(); (void)ended_ut;
1179 internal_error(true, "METADATA: 'host:%s': saved metadata with %zu SQL statements, in %0.2f ms",
1180 rrdhost_hostname(host), query_counter,
1181 (double)(ended_ut - started_ut) / USEC_PER_MS);
1182 }
1183 dfe_done(host);
1052 - if (!data->max_count)
1053 - db_execute("COMMIT TRANSACTION;");
1184 +
1185 + if (!data->max_count && transaction_started)
1186 + transaction_started = db_execute("COMMIT TRANSACTION;");
1187
1188 usec_t all_ended_ut = now_monotonic_usec(); (void)all_ended_ut;
1189 internal_error(true, "METADATA: checking all hosts completed in %0.2f ms",
@@ -1065,7 +1198,6 @@ static void start_metadata_hosts(uv_work_t *req __maybe_unused)
1198
1199 static void metadata_event_loop(void *arg)
1200 {
1068 - service_register(SERVICE_THREAD_TYPE_EVENT_LOOP, NULL, NULL, NULL, true);
1201 worker_register("METASYNC");
1202 worker_register_job_name(METADATA_DATABASE_NOOP, "noop");
1203 worker_register_job_name(METADATA_DATABASE_TIMER, "timer");
@@ -1082,6 +1214,7 @@ static void metadata_event_loop(void *arg)
1214 uv_work_t metadata_cleanup_worker;
1215
1216 uv_thread_set_name_np(wc->thread, "METASYNC");
1217 +// service_register(SERVICE_THREAD_TYPE_EVENT_LOOP, NULL, NULL, NULL, true);
1218 loop = wc->loop = mallocz(sizeof(uv_loop_t));
1219 ret = uv_loop_init(loop);
1220 if (ret) {
@@ -1118,6 +1251,8 @@ static void metadata_event_loop(void *arg)
1251 int shutdown = 0;
1252 wc->row_id = 0;
1253 completion_mark_complete(&wc->init_complete);
1254 + BUFFER *work_buffer = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
1255 + struct scan_metadata_payload *data;
1256
1257 while (shutdown == 0 || (wc->flags & METADATA_WORKER_BUSY)) {
1258 uuid_t *uuid;
@@ -1163,11 +1298,7 @@ static void metadata_event_loop(void *arg)
1298 break;
1299 case METADATA_ADD_HOST_INFO:
1300 host = (RRDHOST *) cmd.param[0];
1166 - store_host_and_system_info(host, NULL);
1167 - if (unlikely(!host->dbsync_worker))
1168 - sql_create_aclk_table(host, &host->host_uuid, host->node_id);
1169 - else
1170 - schedule_node_info_update(host);
1301 + store_host_and_system_info(host, NULL, NULL);
1302 break;
1303 case METADATA_SCAN_HOSTS:
1304 if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SCANNING_HOSTS)))
@@ -1176,10 +1307,11 @@ static void metadata_event_loop(void *arg)
1307 if (unittest_running)
1308 break;
1309
1179 - struct scan_metadata_payload *data = mallocz(sizeof(*data));
1310 + data = mallocz(sizeof(*data));
1311 data->request.data = data;
1312 data->wc = wc;
1313 data->completion = cmd.completion; // Completion by the worker
1314 + data->work_buffer = work_buffer;
1315
1316 if (unlikely(cmd.completion)) {
1317 data->max_count = 0; // 0 will process all pending updates
@@ -1199,6 +1331,19 @@ static void metadata_event_loop(void *arg)
1331 metadata_flag_clear(wc, METADATA_FLAG_SCANNING_HOSTS);
1332 }
1333 break;
1334 + case METADATA_LOAD_HOST_CONTEXT:;
1335 + if (unittest_running)
1336 + break;
1337 +
1338 + data = callocz(1,sizeof(*data));
1339 + data->request.data = data;
1340 + data->wc = wc;
1341 + if (unlikely(
1342 + uv_queue_work(loop,&data->request, start_all_host_load_context,
1343 + after_start_host_load_context))) {
1344 + freez(data);
1345 + }
1346 + break;
1347 case METADATA_MAINTENANCE:
1348 if (unlikely(metadata_flag_check(wc, METADATA_FLAG_CLEANUP)))
1349 break;
@@ -1227,21 +1372,14 @@ static void metadata_event_loop(void *arg)
1372 if (!uv_timer_stop(&wc->timer_req))
1373 uv_close((uv_handle_t *)&wc->timer_req, NULL);
1374
1230 - /*
1231 - * uv_async_send after uv_close does not seem to crash in linux at the moment,
1232 - * it is however undocumented behaviour we need to be aware if this becomes
1233 - * an issue in the future.
1234 - */
1375 uv_close((uv_handle_t *)&wc->async, NULL);
1236 - uv_run(loop, UV_RUN_DEFAULT);
1237 -
1376 uv_cond_destroy(&wc->cmd_cond);
1377 int rc;
1240 -
1378 do {
1379 rc = uv_loop_close(loop);
1380 } while (rc != UV_EBUSY);
1381
1382 + buffer_free(work_buffer);
1383 freez(loop);
1384 worker_unregister();
1385
@@ -1313,8 +1451,6 @@ void metadata_sync_init(void)
1451 {
1452 struct metadata_wc *wc = &metasync_worker;
1453
1316 - fatal_assert(0 == uv_mutex_init(&metadata_async_lock));
1317 -
1454 memset(wc, 0, sizeof(*wc));
1455 metadata_init_cmd_queue(wc);
1456 completion_init(&wc->init_complete);
@@ -1374,6 +1510,13 @@ void metaqueue_host_update_info(RRDHOST *host)
1510 queue_metadata_cmd(METADATA_ADD_HOST_INFO, host, NULL);
1511 }
1512
1513 +void metadata_queue_load_host_context(RRDHOST *host)
1514 +{
1515 + if (unlikely(!metasync_worker.loop))
1516 + return;
1517 + queue_metadata_cmd(METADATA_LOAD_HOST_CONTEXT, host, NULL);
1518 +}
1519 +
1520 //
1521 // unitests
1522 //
database/sqlite/sqlite_metadata.h
+1
@@ -15,6 +15,7 @@ void metaqueue_delete_dimension_uuid(uuid_t *uuid);
15 void metaqueue_store_claim_id(uuid_t *host_uuid, uuid_t *claim_uuid);
16 void metaqueue_host_update_info(RRDHOST *host);
17 void migrate_localhost(uuid_t *host_uuid);
18 +void metadata_queue_load_host_context(RRDHOST *host);
19
20 // UNIT TEST
21 int metadata_unittest(void);
streaming/receiver.c
+6
@@ -692,6 +692,12 @@ static int rrdpush_receive(struct receiver_state *rpt)
692 return 1;
693 }
694
695 + if (unlikely(rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD))) {
696 + rrdpush_receive_log_status(rpt, "host is initializing", "INITIALIZATION IN PROGRESS RETRY LATER");
697 + close(rpt->fd);
698 + return 1;
699 + }
700 +
701 // system_info has been consumed by the host structure
702 rpt->system_info = NULL;
703