@cryptotaxi247 / netdata-1 / commits / 9bf07d05d

Delay child disconnect update (#18712)

* Calculate child restart time (start time + shutdown) Assume the child will return at 125% of that time Add timers to handle scheduluing of the node update events to the cloud Handle faster reconnection and reschedule timer to correct state * Remove unnecessary check * No need to get the remaining timer value (function may not be supported) * Switch message to DEBUG

Stelios Fragkakis committed Oct 10, 2024 at 00:18 UTC 9bf07d05d0f7434f56cae7ab989ab1141c664baf

9 files changed +89 -17

src/aclk/aclk.c

+1 -1

@@ -804,7 +804,7 @@ void *aclk_main(void *ptr)

804 goto exit_full;

805

806 if (schedule_node_update) {

807 - schedule_node_info_update(localhost);

807 + schedule_node_state_update(localhost, 0);

808 schedule_node_update = false;

809 }

810

src/database/sqlite/sqlite_aclk.c

+56 -6

@@ -206,8 +206,6 @@ fail:

206 static void invalidate_host_last_connected(nd_uuid_t *host_uuid)

207 {

208 sqlite3_stmt *res = NULL;

209 - if (!host_uuid)

210 - return;

209

210 if (!PREPARE_STATEMENT(db_meta, SQL_INVALIDATE_HOST_LAST_CONNECTED, &res))

211 return;

@@ -358,6 +356,27 @@ static int read_query_thread_count()

356 return threads;

357 }

358

359 +static void node_update_timer_cb(uv_timer_t *handle)

360 +{

361 + struct aclk_sync_cfg_t *ahc = handle->data;

362 + RRDHOST *host = ahc->host;

363 +

364 + spinlock_lock(&host->receiver_lock);

365 + int live = (host == localhost || host->receiver || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) ? 1 : 0;

366 + spinlock_unlock(&host->receiver_lock);

367 + nd_log(NDLS_ACLK, NDLP_DEBUG,"Timer: Sending node update info for %s, LIVE = %d", rrdhost_hostname(host), live);

368 + aclk_host_state_update(host, live, 1);

369 +}

370 +

371 +static void close_callback(uv_handle_t *handle, void *data __maybe_unused)

372 +{

373 + if (handle->type == UV_TIMER) {

374 + uv_timer_stop((uv_timer_t *)handle);

375 + }

376 +

377 + uv_close(handle, NULL); // Automatically close and free the handle

378 +}

379 +

380 static void aclk_synchronization(void *arg)

381 {

382 struct aclk_sync_config_s *config = arg;

@@ -414,11 +433,36 @@ static void aclk_synchronization(void *arg)

433 // NODE STATE

434 case ACLK_DATABASE_NODE_STATE:;

435 RRDHOST *host = cmd.param[0];

417 - int live = (host == localhost || host->receiver || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) ? 1 : 0;

436 struct aclk_sync_cfg_t *ahc = host->aclk_config;

437 if (unlikely(!ahc))

438 create_aclk_config(host, &host->host_id.uuid, &host->node_id.uuid);

439 +

440 + uint64_t schedule_time = (uint64_t)(uintptr_t)cmd.param[1];

441 +

442 + if (!ahc->timer_initialized) {

443 + int rc = uv_timer_init(loop, &ahc->timer);

444 + if (!rc) {

445 + ahc->timer_initialized = true;

446 + ahc->timer.data = ahc;

447 + }

448 + }

449 +

450 + if (ahc->timer_initialized) {

451 + if (uv_is_active((uv_handle_t *)&ahc->timer))

452 + uv_timer_stop(&ahc->timer);

453 +

454 + ahc->timer.data = ahc;

455 + int rc = uv_timer_start(&ahc->timer, node_update_timer_cb, schedule_time, 0);

456 + if (!rc)

457 + break; // Timer started, exit

458 + }

459 +

460 + // This is fallback if timer fails

461 + spinlock_lock(&host->receiver_lock);

462 + int live = (host == localhost || host->receiver || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN))) ? 1 : 0;

463 + spinlock_unlock(&host->receiver_lock);

464 aclk_host_state_update(host, live, 1);

465 + nd_log(NDLS_ACLK, NDLP_DEBUG,"Sending node update info for %s, LIVE = %d", rrdhost_hostname(host), live);

466 break;

467 case ACLK_DATABASE_NODE_UNREGISTER:

468 sql_unregister_node(cmd.param[0]);

@@ -467,6 +511,11 @@ static void aclk_synchronization(void *arg)

511 uv_close((uv_handle_t *)&config->timer_req, NULL);

512

513 uv_close((uv_handle_t *)&config->async, NULL);

514 + uv_run(loop, UV_RUN_DEFAULT);

515 +

516 + uv_walk(loop, (uv_walk_cb) close_callback, NULL);

517 + uv_run(loop, UV_RUN_DEFAULT);

518 +

519 (void) uv_loop_close(loop);

520

521 worker_unregister();

@@ -581,11 +630,12 @@ void aclk_query_init(mqtt_wss_client client) {

630 queue_aclk_sync_cmd(ACLK_MQTT_WSS_CLIENT, client, NULL);

631 }

632

584 -void schedule_node_info_update(RRDHOST *host __maybe_unused)

633 +void schedule_node_state_update(RRDHOST *host, uint64_t delay)

634 {

586 - if (unlikely(!host))

635 + if (unlikely(!aclk_sync_config.initialized || !host))

636 return;

588 - queue_aclk_sync_cmd(ACLK_DATABASE_NODE_STATE, host, NULL);

637 +

638 + queue_aclk_sync_cmd(ACLK_DATABASE_NODE_STATE, host, (void *)(uintptr_t)delay);

639 }

640

641 void unregister_node(const char *machine_guid)

src/database/sqlite/sqlite_aclk.h

+3 -1

@@ -39,6 +39,8 @@ struct aclk_database_cmd {

40 typedef struct aclk_sync_cfg_t {

41 RRDHOST *host;

42 + uv_timer_t timer;

43 + bool timer_initialized;

44 int8_t send_snapshot;

45 bool stream_alerts;

46 int alert_count;

@@ -53,7 +55,7 @@ typedef struct aclk_sync_cfg_t {

55 void create_aclk_config(RRDHOST *host, nd_uuid_t *host_uuid, nd_uuid_t *node_id);

56 void sql_aclk_sync_init(void);

57 void aclk_push_alert_config(const char *node_id, const char *config_hash);

56 -void schedule_node_info_update(RRDHOST *host);

58 +void schedule_node_state_update(RRDHOST *host, uint64_t delay);

59 void unregister_node(const char *machine_guid);

61 #endif //NETDATA_SQLITE_ACLK_H

src/health/health_event_loop.c

+2 -2

@@ -75,7 +75,7 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run)

75 time_t needed = now + rc->config.before + rc->config.after;

77 if(needed + update_every < first || needed - update_every > last) {

78 - netdata_log_info(

78 + netdata_log_debug(D_HEALTH,

79 "Health not examining alarm '%s.%s' yet (not enough data yet - we need %lu but got %lu - %lu).",

80 rrdcalc_chart_name(rc),

81 rrdcalc_name(rc),

@@ -229,7 +229,7 @@ static void health_event_loop(void) {

229 "Postponing alarm checks for %"PRId32" seconds, "

230 "because it seems that the system was just resumed from suspension.",

231 (int32_t)health_globals.config.postpone_alarms_during_hibernation_for_seconds);

232 - schedule_node_info_update(localhost);

232 + schedule_node_state_update(localhost, 0);

233 }

234

235 if (unlikely(silencers->all_alarms && silencers->stype == STYPE_DISABLE_ALARMS)) {

src/plugins.d/pluginsd_parser.c

+1 -1

@@ -218,7 +218,7 @@ static inline PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, si

218

219 rrdhost_flag_clear(host, RRDHOST_FLAG_ORPHAN);

220 rrdcontext_host_child_connected(host);

221 - schedule_node_info_update(host);

221 + schedule_node_state_update(host, 100);

222

223 return PARSER_RC_OK;

224 }

src/streaming/receiver.c

+5 -6

@@ -829,8 +829,7 @@ static void rrdpush_receive(struct receiver_state *rpt)

829

830 // in case we have cloud connection we inform cloud

831 // new child connected

832 - aclk_host_state_update(rpt->host, 1, 1);

833 -

832 + schedule_node_state_update(rpt->host, 300);

833 rrdhost_set_is_parent_label();

834

835 if (is_ephemeral)

@@ -848,14 +847,14 @@ static void rrdpush_receive(struct receiver_state *rpt)

847 {

848 char msg[100 + 1];

849 snprintfz(msg, sizeof(msg) - 1, "disconnected (completed %zu updates)", count);

851 - rrdpush_receive_log_status(

852 - rpt, msg,

853 - RRDPUSH_STATUS_DISCONNECTED, NDLP_WARNING);

850 + rrdpush_receive_log_status(rpt, msg, RRDPUSH_STATUS_DISCONNECTED, NDLP_WARNING);

851 }

852

853 // in case we have cloud connection we inform cloud

854 // a child disconnected

858 - aclk_host_state_update(rpt->host, 0, 1);

855 + STREAM_PATH tmp = rrdhost_stream_path_fetch(rpt->host);

856 + uint64_t total_reboot = (tmp.start_time + tmp.shutdown_time);

857 + schedule_node_state_update(rpt->host, MIN((total_reboot * MAX_CHILD_DISC_TOLERANCE), MAX_CHILD_DISC_DELAY));

858

859 cleanup:

860 ;

src/streaming/rrdpush.h

@@ -10,6 +10,11 @@

10 #include "database/rrd.h"

11 #include "stream_capabilities.h"

13 +// When a child disconnects this is the maximum we will wait

14 +// before we update the cloud that the child is offline

15 +#define MAX_CHILD_DISC_DELAY (30000)

16 +#define MAX_CHILD_DISC_TOLERANCE (125 / 100)

17 +

18 #define CONNECTED_TO_SIZE 100

19 #define CBUFFER_INITIAL_SIZE (16 * 1024)

20 #define THREAD_BUFFER_INITIAL_SIZE (CBUFFER_INITIAL_SIZE / 2)

src/streaming/stream_path.c

+15

@@ -101,6 +101,21 @@ static STREAM_PATH rrdhost_stream_path_self(RRDHOST *host) {

101 return p;

102 }

103

104 +STREAM_PATH rrdhost_stream_path_fetch(RRDHOST *host) {

105 + STREAM_PATH p = { 0 };

106 +

107 + spinlock_lock(&host->rrdpush.path.spinlock);

108 + for (size_t i = 0; i < host->rrdpush.path.used; i++) {

109 + STREAM_PATH *tmp_path = &host->rrdpush.path.array[i];

110 + if(UUIDeq(host->host_id, tmp_path->host_id)) {

111 + p = *tmp_path;

112 + break;

113 + }

114 + }

115 + spinlock_unlock(&host->rrdpush.path.spinlock);

116 + return p;

117 +}

118 +

119 void rrdhost_stream_path_to_json(BUFFER *wb, struct rrdhost *host, const char *key, bool add_version) {

120 if(add_version)

121 buffer_json_member_add_uint64(wb, "version", 1);

src/streaming/stream_path.h

@@ -47,6 +47,7 @@ void stream_path_node_id_updated(struct rrdhost *host);

48 void stream_path_child_disconnected(struct rrdhost *host);

49 void stream_path_parent_disconnected(struct rrdhost *host);

50 +STREAM_PATH rrdhost_stream_path_fetch(struct rrdhost *host);

52 bool stream_path_set_from_json(struct rrdhost *host, const char *json, bool from_parent);