@cryptotaxi247 / netdata-1 / commits / 85f359fc2

Handle ephemeral hosts (#16381)

* Handle ephemeral hosts * Node empheral removal timeout 86400 seconds (1 day) * Move config from health to global section * Set a node to queryable false when it is ephemeral and is removed * Log queryable. Send queryable=0 only when forcing host deletion (the node is ephemeral) * Switch to "is ephemeral node" Document stream.conf * Unregister node id

Stelios Fragkakis committed Nov 23, 2023 at 23:56 UTC 85f359fc2677752ef7501fbee92d9caecc924538

15 files changed +305 -162

aclk/aclk.c

+5 -6

@@ -917,7 +917,7 @@ exit:

917 return NULL;

918 }

919

920 -void aclk_host_state_update(RRDHOST *host, int cmd)

920 +void aclk_host_state_update(RRDHOST *host, int cmd, int queryable)

921 {

922 uuid_t node_id;

923 int ret = 0;

@@ -962,7 +962,7 @@ void aclk_host_state_update(RRDHOST *host, int cmd)

962 node_instance_connection_t node_state_update = {

963 .hops = host->system_info->hops,

964 .live = cmd,

965 - .queryable = 1,

965 + .queryable = queryable,

966 .session_id = aclk_session_newarch

967 };

968 node_state_update.node_id = mallocz(UUID_STR_LEN);

@@ -976,9 +976,8 @@ void aclk_host_state_update(RRDHOST *host, int cmd)

976 rrdhost_aclk_state_unlock(localhost);

977

978 nd_log(NDLS_DAEMON, NDLP_DEBUG,

979 - "Queuing status update for node=%s, live=%d, hops=%u",

980 - (char*)node_state_update.node_id, cmd, host->system_info->hops);

981 -

979 + "Queuing status update for node=%s, live=%d, hops=%u, queryable=%d",

980 + (char*)node_state_update.node_id, cmd, host->system_info->hops, queryable);

981 freez((void*)node_state_update.node_id);

982 query->data.bin_payload.msg_name = "UpdateNodeInstanceConnection";

983 query->data.bin_payload.topic = ACLK_TOPICID_NODE_CONN;

@@ -1022,7 +1021,7 @@ void aclk_send_node_instances()

1021 rrdhost_aclk_state_unlock(localhost);

1022

1023 nd_log(NDLS_DAEMON, NDLP_DEBUG,

1025 - "Queuing status update for node=%s, live=%d, hops=%d",

1024 + "Queuing status update for node=%s, live=%d, hops=%d, queryable=1",

1025 (char*)node_state_update.node_id, list->live, list->hops);

1026

1027 freez((void*)node_state_update.capabilities);

aclk/aclk.h

+1 -1

@@ -75,7 +75,7 @@ extern struct aclk_shared_state {

75 int mqtt_shutdown_msg_rcvd;

76 } aclk_shared_state;

78 -void aclk_host_state_update(RRDHOST *host, int cmd);

78 +void aclk_host_state_update(RRDHOST *host, int cmd, int queryable);

79 void aclk_send_node_instances(void);

81 void aclk_send_bin_msg(char *msg, size_t msg_len, enum aclk_topics subtopic, const char *msgname);

collectors/plugins.d/pluginsd_parser.c

+33 -24

@@ -566,30 +566,29 @@ static inline PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, si

566 return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_HOST_DEFINE_END, "missing initialization, send " PLUGINSD_KEYWORD_HOST_DEFINE " before this");

567

568 RRDHOST *host = rrdhost_find_or_create(

569 - string2str(parser->user.host_define.hostname),

570 - string2str(parser->user.host_define.hostname),

571 - parser->user.host_define.machine_guid_str,

572 - "Netdata Virtual Host 1.0",

573 - netdata_configured_timezone,

574 - netdata_configured_abbrev_timezone,

575 - netdata_configured_utc_offset,

576 - NULL,

577 - program_name,

578 - program_version,

579 - default_rrd_update_every,

580 - default_rrd_history_entries,

581 - default_rrd_memory_mode,

582 - default_health_enabled,

583 - default_rrdpush_enabled,

584 - default_rrdpush_destination,

585 - default_rrdpush_api_key,

586 - default_rrdpush_send_charts_matching,

587 - default_rrdpush_enable_replication,

588 - default_rrdpush_seconds_to_replicate,

589 - default_rrdpush_replication_step,

590 - rrdhost_labels_to_system_info(parser->user.host_define.rrdlabels),

591 - false

592 - );

569 + string2str(parser->user.host_define.hostname),

570 + string2str(parser->user.host_define.hostname),

571 + parser->user.host_define.machine_guid_str,

572 + "Netdata Virtual Host 1.0",

573 + netdata_configured_timezone,

574 + netdata_configured_abbrev_timezone,

575 + netdata_configured_utc_offset,

576 + NULL,

577 + program_name,

578 + program_version,

579 + default_rrd_update_every,

580 + default_rrd_history_entries,

581 + default_rrd_memory_mode,

582 + default_health_enabled,

583 + default_rrdpush_enabled,

584 + default_rrdpush_destination,

585 + default_rrdpush_api_key,

586 + default_rrdpush_send_charts_matching,

587 + default_rrdpush_enable_replication,

588 + default_rrdpush_seconds_to_replicate,

589 + default_rrdpush_replication_step,

590 + rrdhost_labels_to_system_info(parser->user.host_define.rrdlabels),

591 + false);

592

593 rrdhost_option_set(host, RRDHOST_OPTION_VIRTUAL_HOST);

594

@@ -1412,6 +1411,14 @@ static inline PARSER_RC pluginsd_label(char **words, size_t num_words, PARSER *p

1411 if(unlikely(!(parser->user.new_host_labels)))

1412 parser->user.new_host_labels = rrdlabels_create();

1413

1414 + if (strcmp(name,HOST_LABEL_IS_EPHEMERAL) == 0) {

1415 + int is_ephemeral = appconfig_test_boolean_value((char *) value);

1416 + if (is_ephemeral) {

1417 + RRDHOST *host = pluginsd_require_scope_host(parser, PLUGINSD_KEYWORD_LABEL);

1418 + rrdhost_option_set(host, RRDHOST_OPTION_EPHEMERAL_HOST);

1419 + }

1420 + }

1421 +

1422 rrdlabels_add(parser->user.new_host_labels, name, store, str2l(label_source));

1423

1424 if (allocated_store)

@@ -1430,6 +1437,8 @@ static inline PARSER_RC pluginsd_overwrite(char **words __maybe_unused, size_t n

1437 host->rrdlabels = rrdlabels_create();

1438

1439 rrdlabels_migrate_to_these(host->rrdlabels, parser->user.new_host_labels);

1440 + if (rrdhost_option_check(host, RRDHOST_OPTION_EPHEMERAL_HOST))

1441 + rrdlabels_add(host->rrdlabels, HOST_LABEL_IS_EPHEMERAL, "true", RRDLABEL_SRC_CONFIG);

1442 rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE);

1443

1444 rrdlabels_destroy(parser->user.new_host_labels);

collectors/plugins.d/pluginsd_parser.h

@@ -15,6 +15,7 @@

16 #define PLUGINSD_MIN_RRDSET_POINTERS_CACHE 1024

18 +#define HOST_LABEL_IS_EPHEMERAL "_is_ephemeral"

19 // PARSER return codes

20 typedef enum __attribute__ ((__packed__)) parser_rc {

21 PARSER_RC_OK, // Callback was successful, go on

daemon/main.c

@@ -1077,6 +1077,11 @@ static void backwards_compatible_config() {

1077

1078 config_move(CONFIG_SECTION_LOGS, "errors flood protection period",

1079 CONFIG_SECTION_LOGS, "logs flood protection period");

1080 + config_move(CONFIG_SECTION_HEALTH, "is ephemeral",

1081 + CONFIG_SECTION_GLOBAL, "is ephemeral node");

1082 +

1083 + config_move(CONFIG_SECTION_HEALTH, "has unstable connection",

1084 + CONFIG_SECTION_GLOBAL, "has unstable connection");

1085 }

1086

1087 static int get_hostname(char *buf, size_t buf_size) {

@@ -1223,6 +1228,7 @@ static void get_netdata_configured_variables() {

1228 // --------------------------------------------------------------------

1229

1230 rrdset_free_obsolete_time_s = config_get_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdset_free_obsolete_time_s);

1231 + rrdhost_free_ephemeral_time_s = config_get_number(CONFIG_SECTION_DB, "cleanup ephemeral hosts after secs", rrdhost_free_ephemeral_time_s);

1232 // Current chart locking and invalidation scheme doesn't prevent Netdata from segmentation faults if a short

1233 // cleanup delay is set. Extensive stress tests showed that 10 seconds is quite a safe delay. Look at

1234 // https://github.com/netdata/netdata/pull/11222#issuecomment-868367920 for more information.

daemon/service.c

+35 -12

@@ -272,22 +272,45 @@ restart_after_removal:

272 if(!rrdhost_should_be_removed(host, protected_host, now))

273 continue;

274

275 - netdata_log_info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", rrdhost_hostname(host), host->machine_guid);

276 -

277 - if (rrdhost_option_check(host, RRDHOST_OPTION_DELETE_ORPHAN_HOST)

278 - /* don't delete multi-host DB host files */

279 - && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->db[0].instance))

280 - ) {

281 - worker_is_busy(WORKER_JOB_DELETE_HOST_CHARTS);

282 - rrdhost_delete_charts(host);

275 + bool is_archived = rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED);

276 + if (!is_archived) {

277 + netdata_log_info("Host '%s' with machine guid '%s' is obsolete - cleaning up.", rrdhost_hostname(host), host->machine_guid);

278 +

279 + if (rrdhost_option_check(host, RRDHOST_OPTION_DELETE_ORPHAN_HOST)

280 + /* don't delete multi-host DB host files */

281 + && !(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && is_storage_engine_shared(host->db[0].instance))

282 + ) {

283 + worker_is_busy(WORKER_JOB_DELETE_HOST_CHARTS);

284 + rrdhost_delete_charts(host);

285 + }

286 + else {

287 + worker_is_busy(WORKER_JOB_SAVE_HOST_CHARTS);

288 + rrdhost_save_charts(host);

289 + }

290 }

284 - else {

285 - worker_is_busy(WORKER_JOB_SAVE_HOST_CHARTS);

286 - rrdhost_save_charts(host);

291 +

292 + bool force = false;

293 +

294 + if (rrdhost_option_check(host, RRDHOST_OPTION_EPHEMERAL_HOST) && now - host->last_connected > rrdhost_free_ephemeral_time_s)

295 + force = true;

296 +

297 + if (!force && is_archived)

298 + continue;

299 +

300 + if (force) {

301 + netdata_log_info("Host '%s' with machine guid '%s' is archived, ephemeral clean up.", rrdhost_hostname(host), host->machine_guid);

302 }

303

304 worker_is_busy(WORKER_JOB_FREE_HOST);

290 - rrdhost_free___while_having_rrd_wrlock(host, false);

305 +#ifdef ENABLE_ACLK

306 + // in case we have cloud connection we inform cloud

307 + // a child disconnected

308 + if (netdata_cloud_enabled && force) {

309 + aclk_host_state_update(host, 0, 0);

310 + unregister_node(host->machine_guid);

311 + }

312 +#endif

313 + rrdhost_free___while_having_rrd_wrlock(host, force);

314 goto restart_after_removal;

315 }

316

daemon/unit_test.c

+23 -24

@@ -1824,30 +1824,29 @@ static RRDHOST *dbengine_rrdhost_find_or_create(char *name)

1824 /* We don't want to drop metrics when generating load, we prefer to block data generation itself */

1825

1826 return rrdhost_find_or_create(

1827 - name

1828 - , name

1829 - , name

1830 - , os_type

1831 - , netdata_configured_timezone

1832 - , netdata_configured_abbrev_timezone

1833 - , netdata_configured_utc_offset

1834 - , ""

1835 - , program_name

1836 - , program_version

1837 - , default_rrd_update_every

1838 - , default_rrd_history_entries

1839 - , RRD_MEMORY_MODE_DBENGINE

1840 - , default_health_enabled

1841 - , default_rrdpush_enabled

1842 - , default_rrdpush_destination

1843 - , default_rrdpush_api_key

1844 - , default_rrdpush_send_charts_matching

1845 - , default_rrdpush_enable_replication

1846 - , default_rrdpush_seconds_to_replicate

1847 - , default_rrdpush_replication_step

1848 - , NULL

1849 - , 0

1850 - );

1827 + name,

1828 + name,

1829 + name,

1830 + os_type,

1831 + netdata_configured_timezone,

1832 + netdata_configured_abbrev_timezone,

1833 + netdata_configured_utc_offset,

1834 + "",

1835 + program_name,

1836 + program_version,

1837 + default_rrd_update_every,

1838 + default_rrd_history_entries,

1839 + RRD_MEMORY_MODE_DBENGINE,

1840 + default_health_enabled,

1841 + default_rrdpush_enabled,

1842 + default_rrdpush_destination,

1843 + default_rrdpush_api_key,

1844 + default_rrdpush_send_charts_matching,

1845 + default_rrdpush_enable_replication,

1846 + default_rrdpush_seconds_to_replicate,

1847 + default_rrdpush_replication_step,

1848 + NULL,

1849 + 0);

1850 }

1851

1852 // constants for test_dbengine

database/rrd.h

+25 -24

@@ -1043,6 +1043,7 @@ typedef enum __attribute__ ((__packed__)) {

1043 RRDHOST_OPTION_REPLICATION = (1 << 5), // when set, we support replication for this host

1044

1045 RRDHOST_OPTION_VIRTUAL_HOST = (1 << 6), // when set, this host is a virtual one

1046 + RRDHOST_OPTION_EPHEMERAL_HOST = (1 << 7), // when set, this host is an ephemeral one

1047 } RRDHOST_OPTIONS;

1048

1049 #define rrdhost_option_check(host, flag) ((host)->options & (flag))

@@ -1426,6 +1427,7 @@ void rrddim_index_destroy(RRDSET *st);

1427 // ----------------------------------------------------------------------------

1428

1429 extern time_t rrdhost_free_orphan_time_s;

1430 +extern time_t rrdhost_free_ephemeral_time_s;

1431

1432 int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unittest);

1433

@@ -1434,30 +1436,29 @@ RRDHOST *rrdhost_find_by_guid(const char *guid);

1436 RRDHOST *find_host_by_node_id(char *node_id);

1437

1438 RRDHOST *rrdhost_find_or_create(

1437 - const char *hostname

1438 - , const char *registry_hostname

1439 - , const char *guid

1440 - , const char *os

1441 - , const char *timezone

1442 - , const char *abbrev_timezone

1443 - , int32_t utc_offset

1444 - , const char *tags

1445 - , const char *program_name

1446 - , const char *program_version

1447 - , int update_every

1448 - , long history

1449 - , RRD_MEMORY_MODE mode

1450 - , unsigned int health_enabled

1451 - , unsigned int rrdpush_enabled

1452 - , char *rrdpush_destination

1453 - , char *rrdpush_api_key

1454 - , char *rrdpush_send_charts_matching

1455 - , bool rrdpush_enable_replication

1456 - , time_t rrdpush_seconds_to_replicate

1457 - , time_t rrdpush_replication_step

1458 - , struct rrdhost_system_info *system_info

1459 - , bool is_archived

1460 -);

1439 + const char *hostname,

1440 + const char *registry_hostname,

1441 + const char *guid,

1442 + const char *os,

1443 + const char *timezone,

1444 + const char *abbrev_timezone,

1445 + int32_t utc_offset,

1446 + const char *tags,

1447 + const char *program_name,

1448 + const char *program_version,

1449 + int update_every,

1450 + long history,

1451 + RRD_MEMORY_MODE mode,

1452 + unsigned int health_enabled,

1453 + unsigned int rrdpush_enabled,

1454 + char *rrdpush_destination,

1455 + char *rrdpush_api_key,

1456 + char *rrdpush_send_charts_matching,

1457 + bool rrdpush_enable_replication,

1458 + time_t rrdpush_seconds_to_replicate,

1459 + time_t rrdpush_replication_step,

1460 + struct rrdhost_system_info *system_info,

1461 + bool is_archived);

1462

1463 int rrdhost_set_system_info_variable(struct rrdhost_system_info *system_info, char *name, char *value);

1464

database/rrdhost.c

+10 -2

@@ -31,6 +31,7 @@ netdata_rwlock_t rrd_rwlock = NETDATA_RWLOCK_INITIALIZER;

32 time_t rrdset_free_obsolete_time_s = 3600;

33 time_t rrdhost_free_orphan_time_s = 3600;

34 +time_t rrdhost_free_ephemeral_time_s = 86400;

36 bool is_storage_engine_shared(STORAGE_INSTANCE *engine __maybe_unused) {

37 #ifdef ENABLE_DBENGINE

@@ -838,7 +839,7 @@ inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, tim

839 && rrdhost_receiver_replicating_charts(host) == 0

840 && rrdhost_sender_replicating_charts(host) == 0

841 && rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)

841 - && !rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)

842 + && !rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)

843 && !host->receiver

844 && host->child_disconnected_time

845 && host->child_disconnected_time + rrdhost_free_orphan_time_s < now_s)

@@ -1464,7 +1465,14 @@ static void rrdhost_load_auto_labels(void) {

1465

1466 add_aclk_host_labels();

1467

1467 - health_add_host_labels();

1468 + // The source should be CONF, but when it is set, these labels are exported by default ('send configured labels' in exporting.conf).

1469 + // Their export seems to break exporting to Graphite, see https://github.com/netdata/netdata/issues/14084.

1470 +

1471 + int is_ephemeral = appconfig_get_boolean(&netdata_config, CONFIG_SECTION_GLOBAL, "is ephemeral node", CONFIG_BOOLEAN_NO);

1472 + rrdlabels_add(labels, "_is_ephemeral", is_ephemeral ? "true" : "false", RRDLABEL_SRC_AUTO);

1473 +

1474 + int has_unstable_connection = appconfig_get_boolean(&netdata_config, CONFIG_SECTION_GLOBAL, "has unstable connection", CONFIG_BOOLEAN_NO);

1475 + rrdlabels_add(labels, "_has_unstable_connection", has_unstable_connection ? "true" : "false", RRDLABEL_SRC_AUTO);

1476

1477 rrdlabels_add(labels, "_is_parent", (localhost->connected_children_count > 0) ? "true" : "false", RRDLABEL_SRC_AUTO);

1478

database/sqlite/sqlite_aclk.c

+118 -28

@@ -71,6 +71,7 @@ enum {

71 IDX_ENTRIES,

72 IDX_HEALTH_ENABLED,

73 IDX_LAST_CONNECTED,

74 + IDX_IS_EPHEMERAL,

75 };

77 static int create_host_callback(void *data, int argc, char **argv, char **column)

@@ -79,9 +80,26 @@ static int create_host_callback(void *data, int argc, char **argv, char **column

80 UNUSED(argc);

81 UNUSED(column);

83 + time_t last_connected = (time_t) (argv[IDX_LAST_CONNECTED] ? str2uint64_t(argv[IDX_LAST_CONNECTED], NULL) : 0);

84 + time_t age = now_realtime_sec() - last_connected;

85 + int is_ephemeral = 0;

86 +

87 + if (argv[IDX_IS_EPHEMERAL])

88 + is_ephemeral = str2i(argv[IDX_IS_EPHEMERAL]);

89 +

90 char guid[UUID_STR_LEN];

91 uuid_unparse_lower(*(uuid_t *)argv[IDX_HOST_ID], guid);

93 + if (is_ephemeral && age > rrdhost_free_ephemeral_time_s) {

94 + netdata_log_info(

95 + "Skipping ephemeral hostname \"%s\" with GUID \"%s\", age = %ld seconds (limit %ld seconds)",

96 + (const char *)argv[IDX_HOSTNAME],

97 + guid,

98 + age,

99 + rrdhost_free_ephemeral_time_s);

100 + return 0;

101 + }

102 +

103 struct rrdhost_system_info *system_info = callocz(1, sizeof(struct rrdhost_system_info));

104 __atomic_sub_fetch(&netdata_buffers_statistics.rrdhost_allocations_size, sizeof(struct rrdhost_system_info), __ATOMIC_RELAXED);

105

@@ -90,33 +108,47 @@ static int create_host_callback(void *data, int argc, char **argv, char **column

108 sql_build_host_system_info((uuid_t *)argv[IDX_HOST_ID], system_info);

109

110 RRDHOST *host = rrdhost_find_or_create(

93 - (const char *) argv[IDX_HOSTNAME]

94 - , (const char *) argv[IDX_REGISTRY]

95 - , guid

96 - , (const char *) argv[IDX_OS]

97 - , (const char *) argv[IDX_TIMEZONE]

98 - , (const char *) argv[IDX_ABBREV_TIMEZONE]

99 - , (int32_t) (argv[IDX_UTC_OFFSET] ? str2uint32_t(argv[IDX_UTC_OFFSET], NULL) : 0)

100 - , (const char *) argv[IDX_TAGS]

101 - , (const char *) (argv[IDX_PROGRAM_NAME] ? argv[IDX_PROGRAM_NAME] : "unknown")

102 - , (const char *) (argv[IDX_PROGRAM_VERSION] ? argv[IDX_PROGRAM_VERSION] : "unknown")

103 - , argv[IDX_UPDATE_EVERY] ? str2i(argv[IDX_UPDATE_EVERY]) : 1

104 - , argv[IDX_ENTRIES] ? str2i(argv[IDX_ENTRIES]) : 0

105 - , default_rrd_memory_mode

106 - , 0 // health

107 - , 0 // rrdpush enabled

108 - , NULL //destination

109 - , NULL // api key

110 - , NULL // send charts matching

111 - , false // rrdpush_enable_replication

112 - , 0 // rrdpush_seconds_to_replicate

113 - , 0 // rrdpush_replication_step

114 - , system_info

115 - , 1

116 - );

111 + (const char *)argv[IDX_HOSTNAME],

112 + (const char *)argv[IDX_REGISTRY],

113 + guid,

114 + (const char *)argv[IDX_OS],

115 + (const char *)argv[IDX_TIMEZONE],

116 + (const char *)argv[IDX_ABBREV_TIMEZONE],

117 + (int32_t)(argv[IDX_UTC_OFFSET] ? str2uint32_t(argv[IDX_UTC_OFFSET], NULL) : 0),

118 + (const char *)argv[IDX_TAGS],

119 + (const char *)(argv[IDX_PROGRAM_NAME] ? argv[IDX_PROGRAM_NAME] : "unknown"),

120 + (const char *)(argv[IDX_PROGRAM_VERSION] ? argv[IDX_PROGRAM_VERSION] : "unknown"),

121 + argv[IDX_UPDATE_EVERY] ? str2i(argv[IDX_UPDATE_EVERY]) : 1,

122 + argv[IDX_ENTRIES] ? str2i(argv[IDX_ENTRIES]) : 0,

123 + default_rrd_memory_mode,

124 + 0 // health

125 + ,

126 + 0 // rrdpush enabled

127 + ,

128 + NULL //destination

129 + ,

130 + NULL // api key

131 + ,

132 + NULL // send charts matching

133 + ,

134 + false // rrdpush_enable_replication

135 + ,

136 + 0 // rrdpush_seconds_to_replicate

137 + ,

138 + 0 // rrdpush_replication_step

139 + ,

140 + system_info,

141 + 1);

142 +

143 if (likely(host)) {

144 + if (is_ephemeral)

145 + rrdhost_option_set(host, RRDHOST_OPTION_EPHEMERAL_HOST);

146 +

147 + if (is_ephemeral)

148 + host->child_disconnected_time = now_realtime_sec();

149 +

150 host->rrdlabels = sql_load_host_labels((uuid_t *)argv[IDX_HOST_ID]);

119 - host->last_connected = (time_t) (argv[IDX_LAST_CONNECTED] ? str2uint64_t(argv[IDX_LAST_CONNECTED], NULL) : 0);

151 + host->last_connected = last_connected;

152 }

153

154 (*number_of_chidren)++;

@@ -125,7 +157,7 @@ static int create_host_callback(void *data, int argc, char **argv, char **column

157 char node_str[UUID_STR_LEN] = "<none>";

158 if (likely(host->node_id))

159 uuid_unparse_lower(*host->node_id, node_str);

128 - internal_error(true, "Adding archived host \"%s\" with GUID \"%s\" node id = \"%s\"", rrdhost_hostname(host), host->machine_guid, node_str);

160 + internal_error(true, "Adding archived host \"%s\" with GUID \"%s\" node id = \"%s\" ephemeral=%d", rrdhost_hostname(host), host->machine_guid, node_str, is_ephemeral);

161 #endif

162 return 0;

163 }

@@ -215,6 +247,43 @@ fail:

247 buffer_free(sql);

248 }

249

250 +// OPCODE: ACLK_DATABASE_NODE_UNREGISTER

251 +static void sql_unregister_node(char *machine_guid)

252 +{

253 + int rc;

254 + uuid_t host_uuid;

255 +

256 + if (unlikely(!machine_guid))

257 + return;

258 +

259 + rc = uuid_parse(machine_guid, host_uuid);

260 + freez(machine_guid);

261 + if (rc)

262 + return;

263 +

264 + sqlite3_stmt *res = NULL;

265 +

266 + rc = sqlite3_prepare_v2(db_meta, "UPDATE node_instance SET node_id = NULL WHERE host_id = @host_id", -1, &res, 0);

267 + if (unlikely(rc != SQLITE_OK)) {

268 + error_report("Failed to prepare statement remote node id for a host");

269 + return;

270 + }

271 +

272 + rc = sqlite3_bind_blob(res, 1, &host_uuid, sizeof(host_uuid), SQLITE_STATIC);

273 + if (unlikely(rc != SQLITE_OK)) {

274 + error_report("Failed to bind host_id parameter to remove node id");

275 + goto failed;

276 + }

277 + rc = sqlite3_step_monitored(res);

278 + if (unlikely(rc != SQLITE_DONE))

279 + error_report("Failed to execute command to remove node id");

280 +

281 +failed:

282 + if (unlikely(sqlite3_finalize(res) != SQLITE_OK))

283 + error_report("Failed to finalize statement to remove node id");

284 +}

285 +

286 +

287 static int sql_check_aclk_table(void *data __maybe_unused, int argc __maybe_unused, char **argv __maybe_unused, char **column __maybe_unused)

288 {

289 struct aclk_database_cmd cmd;

@@ -375,7 +444,10 @@ static void aclk_synchronization(void *arg __maybe_unused)

444 struct aclk_sync_cfg_t *ahc = host->aclk_config;

445 if (unlikely(!ahc))

446 sql_create_aclk_table(host, &host->host_uuid, host->node_id);

378 - aclk_host_state_update(host, live);

447 + aclk_host_state_update(host, live, 1);

448 + break;

449 + case ACLK_DATABASE_NODE_UNREGISTER:

450 + sql_unregister_node(cmd.param[0]);

451 break;

452 // ALERTS

453 case ACLK_DATABASE_PUSH_ALERT_CONFIG:

@@ -472,7 +544,10 @@ void sql_create_aclk_table(RRDHOST *host __maybe_unused, uuid_t *host_uuid __may

544 #define SQL_FETCH_ALL_HOSTS \

545 "SELECT host_id, hostname, registry_hostname, update_every, os, " \

546 "timezone, tags, hops, memory_mode, abbrev_timezone, utc_offset, program_name, " \

475 - "program_version, entries, health_enabled, last_connected FROM host WHERE hops >0;"

547 + "program_version, entries, health_enabled, last_connected, " \

548 + "(SELECT CASE WHEN hl.label_value = 'true' THEN 1 ELSE 0 END FROM " \

549 + "host_label hl WHERE hl.host_id = h.host_id AND hl.label_key = '_is_ephemeral') " \

550 + "FROM host h WHERE hops > 0"

551

552 #define SQL_FETCH_ALL_INSTANCES \

553 "SELECT ni.host_id, ni.node_id FROM host h, node_instance ni " \

@@ -572,3 +647,18 @@ void schedule_node_info_update(RRDHOST *host __maybe_unused)

647 aclk_database_enq_cmd(&cmd);

648 #endif

649 }

650 +

651 +#ifdef ENABLE_ACLK

652 +void unregister_node(const char *machine_guid)

653 +{

654 + if (unlikely(!machine_guid))

655 + return;

656 +

657 + struct aclk_database_cmd cmd;

658 + memset(&cmd, 0, sizeof(cmd));

659 + cmd.opcode = ACLK_DATABASE_NODE_UNREGISTER;

660 + cmd.param[0] = strdupz(machine_guid);

661 + cmd.completion = NULL;

662 + aclk_database_enq_cmd(&cmd);

663 +}

664 +#endif

\ No newline at end of file

database/sqlite/sqlite_aclk.h

@@ -59,6 +59,7 @@ enum aclk_database_opcode {

59 ACLK_DATABASE_PUSH_ALERT_SNAPSHOT,

60 ACLK_DATABASE_PUSH_ALERT_CHECKPOINT,

61 ACLK_DATABASE_QUEUE_REMOVED_ALERTS,

62 + ACLK_DATABASE_NODE_UNREGISTER,

63 ACLK_DATABASE_TIMER,

65 // leave this last

@@ -93,5 +94,8 @@ void aclk_push_alert_config(const char *node_id, const char *config_hash);

94 void aclk_push_node_alert_snapshot(const char *node_id);

95 void aclk_push_node_removed_alerts(const char *node_id);

96 void schedule_node_info_update(RRDHOST *host);

97 +#ifdef ENABLE_ACLK

98 +void unregister_node(const char *machine_guid);

99 +#endif

100

101 #endif //NETDATA_SQLITE_ACLK_H

health/health.c

-14

@@ -1651,17 +1651,3 @@ void *health_main(void *ptr) {

1651 netdata_thread_cleanup_pop(1);

1652 return NULL;

1653 }

1654 -

1655 -void health_add_host_labels(void) {

1656 - RRDLABELS *labels = localhost->rrdlabels;

1657 -

1658 - // The source should be CONF, but when it is set, these labels are exported by default ('send configured labels' in exporting.conf).

1659 - // Their export seems to break exporting to Graphite, see https://github.com/netdata/netdata/issues/14084.

1660 -

1661 - int is_ephemeral = appconfig_get_boolean(&netdata_config, CONFIG_SECTION_HEALTH, "is ephemeral", CONFIG_BOOLEAN_NO);

1662 - rrdlabels_add(labels, "_is_ephemeral", is_ephemeral ? "true" : "false", RRDLABEL_SRC_AUTO);

1663 -

1664 - int has_unstable_connection = appconfig_get_boolean(&netdata_config, CONFIG_SECTION_HEALTH, "has unstable connection", CONFIG_BOOLEAN_NO);

1665 - rrdlabels_add(labels, "_has_unstable_connection", has_unstable_connection ? "true" : "false", RRDLABEL_SRC_AUTO);

1666 -}

1667 -

health/health.h

-1

@@ -102,7 +102,6 @@ void *health_cmdapi_thread(void *ptr);

102 char *health_edit_command_from_source(const char *source);

103 void sql_refresh_hashes(void);

104

105 -void health_add_host_labels(void);

105 void health_string2json(BUFFER *wb, const char *prefix, const char *label, const char *value, const char *suffix);

106

107 void health_log_alert_transition_with_trace(RRDHOST *host, ALARM_ENTRY *ae, int line, const char *file, const char *function);

streaming/receiver.c

+34 -26

@@ -623,6 +623,10 @@ static void rrdpush_receive(struct receiver_state *rpt)

623 rpt->config.rrdpush_compression = appconfig_get_boolean(&stream_config, rpt->key, "enable compression", rpt->config.rrdpush_compression);

624 rpt->config.rrdpush_compression = appconfig_get_boolean(&stream_config, rpt->machine_guid, "enable compression", rpt->config.rrdpush_compression);

625

626 + bool is_ephemeral = false;

627 + is_ephemeral = appconfig_get_boolean(&stream_config, rpt->key, "is ephemeral node", CONFIG_BOOLEAN_NO);

628 + is_ephemeral = appconfig_get_boolean(&stream_config, rpt->machine_guid, "is ephemeral node", is_ephemeral);

629 +

630 if(rpt->config.rrdpush_compression) {

631 char *order = appconfig_get(&stream_config, rpt->key, "compression algorithms order", RRDPUSH_COMPRESSION_ALGORITHMS_ORDER);

632 order = appconfig_get(&stream_config, rpt->machine_guid, "compression algorithms order", order);

@@ -635,30 +639,31 @@ static void rrdpush_receive(struct receiver_state *rpt)

639 {

640 // this will also update the host with our system_info

641 RRDHOST *host = rrdhost_find_or_create(

638 - rpt->hostname

639 - , rpt->registry_hostname

640 - , rpt->machine_guid

641 - , rpt->os

642 - , rpt->timezone

643 - , rpt->abbrev_timezone

644 - , rpt->utc_offset

645 - , rpt->tags

646 - , rpt->program_name

647 - , rpt->program_version

648 - , rpt->config.update_every

649 - , rpt->config.history

650 - , rpt->config.mode

651 - , (unsigned int)(rpt->config.health_enabled != CONFIG_BOOLEAN_NO)

652 - , (unsigned int)(rpt->config.rrdpush_enabled && rpt->config.rrdpush_destination && *rpt->config.rrdpush_destination && rpt->config.rrdpush_api_key && *rpt->config.rrdpush_api_key)

653 - , rpt->config.rrdpush_destination

654 - , rpt->config.rrdpush_api_key

655 - , rpt->config.rrdpush_send_charts_matching

656 - , rpt->config.rrdpush_enable_replication

657 - , rpt->config.rrdpush_seconds_to_replicate

658 - , rpt->config.rrdpush_replication_step

659 - , rpt->system_info

660 - , 0

661 - );

642 + rpt->hostname,

643 + rpt->registry_hostname,

644 + rpt->machine_guid,

645 + rpt->os,

646 + rpt->timezone,

647 + rpt->abbrev_timezone,

648 + rpt->utc_offset,

649 + rpt->tags,

650 + rpt->program_name,

651 + rpt->program_version,

652 + rpt->config.update_every,

653 + rpt->config.history,

654 + rpt->config.mode,

655 + (unsigned int)(rpt->config.health_enabled != CONFIG_BOOLEAN_NO),

656 + (unsigned int)(rpt->config.rrdpush_enabled && rpt->config.rrdpush_destination &&

657 + *rpt->config.rrdpush_destination && rpt->config.rrdpush_api_key &&

658 + *rpt->config.rrdpush_api_key),

659 + rpt->config.rrdpush_destination,

660 + rpt->config.rrdpush_api_key,

661 + rpt->config.rrdpush_send_charts_matching,

662 + rpt->config.rrdpush_enable_replication,

663 + rpt->config.rrdpush_seconds_to_replicate,

664 + rpt->config.rrdpush_replication_step,

665 + rpt->system_info,

666 + 0);

667

668 if(!host) {

669 rrdpush_receive_log_status(

@@ -807,11 +812,14 @@ static void rrdpush_receive(struct receiver_state *rpt)

812 // in case we have cloud connection we inform cloud

813 // new child connected

814 if (netdata_cloud_enabled)

810 - aclk_host_state_update(rpt->host, 1);

815 + aclk_host_state_update(rpt->host, 1, 1);

816 #endif

817

818 rrdhost_set_is_parent_label();

819

820 + if (is_ephemeral)

821 + rrdhost_option_set(rpt->host, RRDHOST_OPTION_EPHEMERAL_HOST);

822 +

823 // let it reconnect to parent immediately

824 rrdpush_reset_destinations_postpone_time(rpt->host);

825

@@ -837,7 +845,7 @@ static void rrdpush_receive(struct receiver_state *rpt)

845 // in case we have cloud connection we inform cloud

846 // a child disconnected

847 if (netdata_cloud_enabled)

840 - aclk_host_state_update(rpt->host, 0);

848 + aclk_host_state_update(rpt->host, 0, 1);

849 #endif

850

851 cleanup:

streaming/stream.conf

+10

@@ -183,6 +183,11 @@

183 # The duration we want to replicate per each step.

184 #replication_step = 600

185

186 + # Indicate whether this child is an ephemeral node. An ephemeral node will become unavailable

187 + # after the specified duration of "cleanup ephemeral hosts after secs" (as defined in the db section of netdata.conf)

188 + # from the time of the node's last connection.

189 + #is ephemeral node = false

190 +

191 # -----------------------------------------------------------------------------

192 # 3. PER SENDING HOST SETTINGS, ON PARENT NETDATA

193 # THIS IS OPTIONAL - YOU DON'T HAVE TO CONFIGURE IT

@@ -253,3 +258,8 @@

258

259 # The duration we want to replicate per each step.

260 #replication_step = 600

261 +

262 + # Indicate whether this child is an ephemeral node. An ephemeral node will become unavailable

263 + # after the specified duration of "cleanup ephemeral hosts after secs" (as defined in the db section of netdata.conf)

264 + # from the time of the node's last connection.

265 + #is ephemeral node = false