Fix cloud node stale status when a virtual host is created (#14660)
* Schedule direct metadata update on host creation Virtual hosts do not have a receiver but they are not orphan Schedule node info update on host activation New function to store host info and host_system_info If the host is just created, create tables and sync thread If the host exists during startup it is not live but reschedule node update if it is reactivated * New opcode to send current node state * Remove debug messages * Fix system host info
Stelios Fragkakis committed
Mar 8, 2023 at 17:19 UTC
34737e3fda17312a4c73ff4d5c3154e733edac8b
7 files changed
+80
-44
collectors/plugins.d/pluginsd_parser.c
+1
@@ -409,6 +409,7 @@ static PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, size_t nu
409
410
rrdhost_flag_clear(host, RRDHOST_FLAG_ORPHAN);
411
rrdcontext_host_child_connected(host);
412
+ schedule_node_info_update(host);
413
414
return PARSER_RC_OK;
415
}
database/rrdhost.c
+20
-21
@@ -366,7 +366,6 @@ int is_legacy = 1;
366
rrdfamily_index_init(host);
367
rrdcalctemplate_index_init(host);
368
rrdcalc_rrdhost_index_init(host);
369
- metaqueue_host_update_info(host);
369
370
if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
371
#ifdef ENABLE_DBENGINE
@@ -521,7 +520,7 @@ int is_legacy = 1;
520
);
521
522
if(!archived)
524
- rrdhost_flag_set(host,RRDHOST_FLAG_METADATA_INFO | RRDHOST_FLAG_METADATA_UPDATE);
523
+ metaqueue_host_update_info(host);
524
525
rrdhost_load_rrdcontext_data(host);
526
if (!archived) {
@@ -1268,25 +1267,25 @@ struct rrdhost_system_info *rrdhost_labels_to_system_info(DICTIONARY *labels) {
1267
struct rrdhost_system_info *info = callocz(1, sizeof(struct rrdhost_system_info));
1268
info->hops = 1;
1269
1271
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->cloud_provider_type, "_cloud_provider_type");
1272
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->cloud_instance_type, "_cloud_instance_type");
1273
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->cloud_instance_region, "_cloud_instance_region");
1274
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->host_os_name, "_os_name");
1275
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->host_os_version, "_os_version");
1276
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->kernel_version, "_kernel_version");
1277
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->host_cores, "_system_cores");
1278
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->host_cpu_freq, "_system_cpu_freq");
1279
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->host_ram_total, "_system_ram_total");
1280
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->host_disk_space, "_system_disk_space");
1281
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->architecture, "_architecture");
1282
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->virtualization, "_virtualization");
1283
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->container, "_container");
1284
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->container_detection, "_container_detection");
1285
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->virt_detection, "_virt_detection");
1286
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->is_k8s_node, "_is_k8s_node");
1287
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->install_type, "_install_type");
1288
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->prebuilt_arch, "_prebuilt_arch");
1289
- rrdlabels_get_value_strdup_or_null(labels, &localhost->system_info->prebuilt_dist, "_prebuilt_dist");
1270
+ rrdlabels_get_value_strdup_or_null(labels, &info->cloud_provider_type, "_cloud_provider_type");
1271
+ rrdlabels_get_value_strdup_or_null(labels, &info->cloud_instance_type, "_cloud_instance_type");
1272
+ rrdlabels_get_value_strdup_or_null(labels, &info->cloud_instance_region, "_cloud_instance_region");
1273
+ rrdlabels_get_value_strdup_or_null(labels, &info->host_os_name, "_os_name");
1274
+ rrdlabels_get_value_strdup_or_null(labels, &info->host_os_version, "_os_version");
1275
+ rrdlabels_get_value_strdup_or_null(labels, &info->kernel_version, "_kernel_version");
1276
+ rrdlabels_get_value_strdup_or_null(labels, &info->host_cores, "_system_cores");
1277
+ rrdlabels_get_value_strdup_or_null(labels, &info->host_cpu_freq, "_system_cpu_freq");
1278
+ rrdlabels_get_value_strdup_or_null(labels, &info->host_ram_total, "_system_ram_total");
1279
+ rrdlabels_get_value_strdup_or_null(labels, &info->host_disk_space, "_system_disk_space");
1280
+ rrdlabels_get_value_strdup_or_null(labels, &info->architecture, "_architecture");
1281
+ rrdlabels_get_value_strdup_or_null(labels, &info->virtualization, "_virtualization");
1282
+ rrdlabels_get_value_strdup_or_null(labels, &info->container, "_container");
1283
+ rrdlabels_get_value_strdup_or_null(labels, &info->container_detection, "_container_detection");
1284
+ rrdlabels_get_value_strdup_or_null(labels, &info->virt_detection, "_virt_detection");
1285
+ rrdlabels_get_value_strdup_or_null(labels, &info->is_k8s_node, "_is_k8s_node");
1286
+ rrdlabels_get_value_strdup_or_null(labels, &info->install_type, "_install_type");
1287
+ rrdlabels_get_value_strdup_or_null(labels, &info->prebuilt_arch, "_prebuilt_arch");
1288
+ rrdlabels_get_value_strdup_or_null(labels, &info->prebuilt_dist, "_prebuilt_dist");
1289
1290
return info;
1291
}
database/sqlite/sqlite_aclk.c
+26
@@ -10,6 +10,23 @@ void sanity_check(void) {
10
BUILD_BUG_ON(WORKER_UTILIZATION_MAX_JOB_TYPES < ACLK_MAX_ENUMERATIONS_DEFINED);
11
}
12
13
+void schedule_node_info_update(RRDHOST *host)
14
+{
15
+ if (unlikely(!host))
16
+ return;
17
+
18
+ struct aclk_database_worker_config *wc = host->dbsync_worker;
19
+
20
+ if (unlikely(!wc))
21
+ return;
22
+
23
+ struct aclk_database_cmd cmd;
24
+ memset(&cmd, 0, sizeof(cmd));
25
+ cmd.opcode = ACLK_DATABASE_NODE_STATE;
26
+ cmd.completion = NULL;
27
+ aclk_database_enq_cmd(wc, &cmd);
28
+}
29
+
30
static int sql_check_aclk_table(void *data, int argc, char **argv, char **column)
31
{
32
struct aclk_database_worker_config *wc = data;
@@ -500,6 +517,7 @@ static void aclk_database_worker(void *arg)
517
worker_register_job_name(ACLK_DATABASE_CLEANUP, "cleanup");
518
worker_register_job_name(ACLK_DATABASE_DELETE_HOST, "node delete");
519
worker_register_job_name(ACLK_DATABASE_NODE_INFO, "node info");
520
+ worker_register_job_name(ACLK_DATABASE_NODE_STATE, "node state");
521
worker_register_job_name(ACLK_DATABASE_NODE_COLLECTORS, "node collectors");
522
worker_register_job_name(ACLK_DATABASE_PUSH_ALERT, "alert push");
523
worker_register_job_name(ACLK_DATABASE_PUSH_ALERT_CONFIG, "alert conf push");
@@ -620,6 +638,14 @@ static void aclk_database_worker(void *arg)
638
break;
639
640
// NODE OPERATIONS
641
+ case ACLK_DATABASE_NODE_STATE:
642
+ debug(D_ACLK_SYNC,"Sending state update for %s", wc->uuid_str);
643
+ if (wc->host) {
644
+ RRDHOST *host = wc->host;
645
+ int live = (host == localhost || host->receiver || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) ? 1 : 0;
646
+ aclk_host_state_update(wc->host, live);
647
+ }
648
+ break;
649
case ACLK_DATABASE_NODE_INFO:
650
debug(D_ACLK_SYNC,"Sending node info for %s", wc->uuid_str);
651
sql_build_node_info(wc, cmd);
database/sqlite/sqlite_aclk.h
+2
@@ -79,6 +79,7 @@ enum aclk_database_opcode {
79
ACLK_DATABASE_CLEANUP,
80
ACLK_DATABASE_DELETE_HOST,
81
ACLK_DATABASE_NODE_INFO,
82
+ ACLK_DATABASE_NODE_STATE,
83
ACLK_DATABASE_PUSH_ALERT,
84
ACLK_DATABASE_PUSH_ALERT_CONFIG,
85
ACLK_DATABASE_PUSH_ALERT_SNAPSHOT,
@@ -168,5 +169,6 @@ void sql_create_aclk_table(RRDHOST *host, uuid_t *host_uuid, uuid_t *node_id);
169
void sql_aclk_sync_init(void);
170
int claimed();
171
void aclk_sync_exit_all();
172
+void schedule_node_info_update(RRDHOST *host);
173
struct aclk_database_worker_config *find_inactive_wc_by_node_id(char *node_id);
174
#endif //NETDATA_SQLITE_ACLK_H
database/sqlite/sqlite_aclk_node.c
+5
-5
@@ -49,7 +49,6 @@ void sql_build_node_collectors(struct aclk_database_worker_config *wc)
49
#else
50
UNUSED(wc);
51
#endif
52
- return;
52
}
53
54
void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)
@@ -59,14 +58,17 @@ void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_dat
58
#ifdef ENABLE_ACLK
59
struct update_node_info node_info;
60
62
- if (!wc->host) {
61
+ char *claim_id = get_agent_claimid();
62
+ if (!wc->host || !claim_id) {
63
wc->node_info_send = 1;
64
+ freez(claim_id);
65
return;
66
}
67
68
+ wc->node_info_send = 0;
69
rrd_rdlock();
70
node_info.node_id = wc->node_id;
69
- node_info.claim_id = get_agent_claimid();
71
+ node_info.claim_id = claim_id;
72
node_info.machine_guid = wc->host_guid;
73
node_info.child = (wc->host != localhost);
74
node_info.ml_info.ml_capable = ml_capable(localhost);
@@ -127,6 +129,4 @@ void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_dat
129
#else
130
UNUSED(wc);
131
#endif
130
-
131
- return;
132
}
database/sqlite/sqlite_functions.c
+2
-1
@@ -814,7 +814,8 @@ struct node_instance_list *get_node_list(void)
814
node_list[row].queryable = 1;
815
uuid_unparse_lower(*host_id, host_guid);
816
RRDHOST *host = rrdhost_find_by_guid(host_guid);
817
- node_list[row].live = host && (host == localhost || host->receiver) ? 1 : 0;
817
+ node_list[row].live = (host && (host == localhost || host->receiver
818
+ || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)))) ? 1 : 0;
819
node_list[row].hops = (host && host->system_info) ? host->system_info->hops :
820
uuid_compare(*host_id, localhost->host_uuid) ? 1 : 0;
821
node_list[row].hostname =
database/sqlite/sqlite_metadata.c
+24
-17
@@ -957,6 +957,24 @@ static bool metadata_scan_host(RRDHOST *host, uint32_t max_count, size_t *query_
957
return more_to_do;
958
}
959
960
+static void store_host_and_system_info(RRDHOST *host, size_t *query_counter)
961
+{
962
+ BUFFER *work_buffer = sql_store_host_system_info(host);
963
+ if (work_buffer) {
964
+ db_execute(buffer_tostring(work_buffer));
965
+ buffer_free(work_buffer);
966
+ if (likely(query_counter))
967
+ (*query_counter)++;
968
+ }
969
+
970
+ int rc = sql_store_host_info(host);
971
+ if (unlikely(rc))
972
+ error_report("METADATA: 'host:%s': failed to store host info", string2str(host->hostname));
973
+ else
974
+ if (likely(query_counter))
975
+ (*query_counter)++;
976
+}
977
+
978
// Worker thread to scan hosts for pending metadata to store
979
static void start_metadata_hosts(uv_work_t *req __maybe_unused)
980
{
@@ -1012,19 +1030,7 @@ static void start_metadata_hosts(uv_work_t *req __maybe_unused)
1030
1031
if (unlikely(rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_INFO))) {
1032
rrdhost_flag_clear(host, RRDHOST_FLAG_METADATA_INFO);
1015
-
1016
- BUFFER *work_buffer = sql_store_host_system_info(host);
1017
- if(work_buffer) {
1018
- db_execute(buffer_tostring(work_buffer));
1019
- buffer_free(work_buffer);
1020
- query_counter++;
1021
- }
1022
-
1023
- int rc = sql_store_host_info(host);
1024
- if (unlikely(rc))
1025
- error_report("METADATA: 'host:%s': failed to store host info", string2str(host->hostname));
1026
- else
1027
- query_counter++;
1033
+ store_host_and_system_info(host, &query_counter);
1034
}
1035
1036
if (data->max_count)
@@ -1116,7 +1122,6 @@ static void metadata_event_loop(void *arg)
1122
while (shutdown == 0 || (wc->flags & METADATA_WORKER_BUSY)) {
1123
uuid_t *uuid;
1124
RRDHOST *host = NULL;
1119
- int rc;
1125
1126
worker_is_idle();
1127
uv_run(loop, UV_RUN_DEFAULT);
@@ -1158,9 +1163,11 @@ static void metadata_event_loop(void *arg)
1163
break;
1164
case METADATA_ADD_HOST_INFO:
1165
host = (RRDHOST *) cmd.param[0];
1161
- rc = sql_store_host_info(host);
1162
- if (unlikely(rc))
1163
- error_report("Failed to store host info in the database for %s", string2str(host->hostname));
1166
+ store_host_and_system_info(host, NULL);
1167
+ if (unlikely(!host->dbsync_worker))
1168
+ sql_create_aclk_table(host, &host->host_uuid, host->node_id);
1169
+ else
1170
+ schedule_node_info_update(host);
1171
break;
1172
case METADATA_SCAN_HOSTS:
1173
if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SCANNING_HOSTS)))