@cryptotaxi247 / netdata-1 / commits / ccbae3ab0

ACLK timeout (#19425)

* fix aclk timeout * added workers to aclk thread

Costa Tsaousis committed Jan 17, 2025 at 13:02 UTC ccbae3ab031dc34f7959c566612bfb0262c90667

7 files changed +114 -13

CMakeLists.txt

@@ -1900,6 +1900,7 @@ set(MQTT_WEBSOCKETS_FILES

1900 src/aclk/mqtt_websockets/common_public.c

1901 src/aclk/mqtt_websockets/common_public.h

1902 src/aclk/mqtt_websockets/common_internal.h

1903 + src/aclk/mqtt_websockets/aclk_mqtt_workers.h

1904 )

1905

1906 set(ACLK_PROTO_DEFS

src/aclk/aclk.c

+37

@@ -3,6 +3,7 @@

3 #include "aclk.h"

5 #include "mqtt_websockets/mqtt_wss_client.h"

6 +#include "mqtt_websockets/aclk_mqtt_workers.h"

7 #include "aclk_otp.h"

8 #include "aclk_tx_msgs.h"

9 #include "aclk_query.h"

@@ -307,6 +308,7 @@ static int handle_connection(mqtt_wss_client client)

308 // timeout 1000 to check at least once a second

309 // for netdata_exit

310 if (mqtt_wss_service(client, 1000) < 0){

311 + worker_is_busy(WORKER_ACLK_DISCONNECTED);

312 error_report("Connection Error or Dropped");

313 return 1;

314 }

@@ -315,16 +317,20 @@ static int handle_connection(mqtt_wss_client client)

317 const char *reason;

318 switch (disconnect_req) {

319 case ACLK_CLOUD_DISCONNECT:

320 + worker_is_busy(WORKER_ACLK_CMD_DISCONNECT);

321 reason = "cloud request";

322 break;

323 case ACLK_PING_TIMEOUT:

324 + worker_is_busy(WORKER_ACLK_CMD_TIMEOUT);

325 reason = "ping timeout";

326 schedule_node_update = true;

327 break;

328 case ACLK_RELOAD_CONF:

329 + worker_is_busy(WORKER_ACLK_CMD_RELOAD_CONF);

330 reason = "reclaim";

331 break;

332 default:

333 + worker_is_busy(WORKER_ACLK_CMD_UNKNOWN);

334 reason = "unknown";

335 break;

336 }

@@ -769,6 +775,31 @@ void *aclk_main(void *ptr)

775 {

776 struct netdata_static_thread *static_thread = ptr;

777

778 + worker_register("ACLK");

779 + worker_register_job_name(WORKER_ACLK_WAIT_CLAIMING, "wait claim");

780 + worker_register_job_name(WORKER_ACLK_CONNECT, "connect");

781 + worker_register_job_name(WORKER_ACLK_NODE_UPDATE, "node update");

782 + worker_register_job_name(WORKER_ACLK_HANDLE_CONNECTION, "handle connection");

783 + worker_register_job_name(WORKER_ACLK_DISCONNECTED, "disconnected");

784 + worker_register_job_name(WORKER_ACLK_CMD_DISCONNECT, "cmd disconnect");

785 + worker_register_job_name(WORKER_ACLK_CMD_TIMEOUT, "cmd timeout");

786 + worker_register_job_name(WORKER_ACLK_CMD_RELOAD_CONF, "cmd reload");

787 + worker_register_job_name(WORKER_ACLK_CMD_UNKNOWN, "cmd unknown");

788 + worker_register_job_name(WORKER_ACLK_SENT_PING, "sent ping");

789 + worker_register_job_name(WORKER_ACLK_POLL_ERROR, "poll error");

790 + worker_register_job_name(WORKER_ACLK_POLL_OK, "poll ok");

791 + worker_register_job_name(WORKER_ACLK_RX, "rx");

792 + worker_register_job_name(WORKER_ACLK_RX_ERROR, "rx error");

793 + worker_register_job_name(WORKER_ACLK_PROCESS_RAW, "p-raw");

794 + worker_register_job_name(WORKER_ACLK_PROCESS_HANDSHAKE, "p-handshake");

795 + worker_register_job_name(WORKER_ACLK_PROCESS_ESTABLISHED, "p-established");

796 + worker_register_job_name(WORKER_ACLK_PROCESS_ERROR, "p-error");

797 + worker_register_job_name(WORKER_ACLK_PROCESS_CLOSED_GRACEFULLY, "p-closed");

798 + worker_register_job_name(WORKER_ACLK_PROCESS_UNKNOWN, "p-unknown");

799 + worker_register_job_name(WORKER_ACLK_HANDLE_MQTT_INTERNAL, "mqtt internal");

800 + worker_register_job_name(WORKER_ACLK_TX, "tx");

801 + worker_register_job_name(WORKER_ACLK_TX_ERROR, "tx error");

802 +

803 ACLK_PROXY_TYPE proxy_type;

804 aclk_get_proxy(&proxy_type);

805 if (proxy_type == PROXY_TYPE_SOCKS5) {

@@ -779,6 +810,7 @@ void *aclk_main(void *ptr)

810

811 aclk_init_rx_msg_handlers();

812

813 + worker_is_busy(WORKER_ACLK_WAIT_CLAIMING);

814 if (wait_till_agent_claim_ready())

815 goto exit;

816

@@ -809,21 +841,26 @@ void *aclk_main(void *ptr)

841 netdata_log_info("Starting ACLK query event loop");

842 aclk_query_init(mqttwss_client);

843 do {

844 + worker_is_busy(WORKER_ACLK_CONNECT);

845 if (aclk_attempt_to_connect(mqttwss_client))

846 goto exit_full;

847

848 if (schedule_node_update) {

849 + worker_is_busy(WORKER_ACLK_NODE_UPDATE);

850 schedule_node_state_update(localhost, 0);

851 schedule_node_update = false;

852 }

853

854 + worker_is_busy(WORKER_ACLK_HANDLE_CONNECTION);

855 if (handle_connection(mqttwss_client)) {

856 + worker_is_busy(WORKER_ACLK_DISCONNECTED);

857 last_disconnect_time = now_realtime_sec();

858 aclk_set_disconnected();

859 nd_log(NDLS_ACCESS, NDLP_WARNING, "ACLK DISCONNECTED");

860 }

861 } while (service_running(SERVICE_ACLK));

862

863 + worker_is_busy(WORKER_ACLK_DISCONNECTED);

864 aclk_graceful_disconnect(mqttwss_client);

865

866 #ifdef MQTT_WSS_DEBUG

src/aclk/mqtt_websockets/aclk_mqtt_workers.h new

+30

@@ -0,0 +1,30 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_ACLK_MQTT_WORKERS_H

4 +#define NETDATA_ACLK_MQTT_WORKERS_H

5 +

6 +#define WORKER_ACLK_WAIT_CLAIMING 0

7 +#define WORKER_ACLK_CONNECT 1

8 +#define WORKER_ACLK_NODE_UPDATE 2

9 +#define WORKER_ACLK_HANDLE_CONNECTION 3

10 +#define WORKER_ACLK_DISCONNECTED 4

11 +#define WORKER_ACLK_CMD_DISCONNECT 5

12 +#define WORKER_ACLK_CMD_TIMEOUT 6

13 +#define WORKER_ACLK_CMD_RELOAD_CONF 7

14 +#define WORKER_ACLK_CMD_UNKNOWN 8

15 +#define WORKER_ACLK_SENT_PING 9

16 +#define WORKER_ACLK_POLL_ERROR 10

17 +#define WORKER_ACLK_POLL_OK 11

18 +#define WORKER_ACLK_RX 12

19 +#define WORKER_ACLK_RX_ERROR 13

20 +#define WORKER_ACLK_PROCESS_RAW 14

21 +#define WORKER_ACLK_PROCESS_HANDSHAKE 15

22 +#define WORKER_ACLK_PROCESS_ESTABLISHED 16

23 +#define WORKER_ACLK_PROCESS_ERROR 17

24 +#define WORKER_ACLK_PROCESS_CLOSED_GRACEFULLY 18

25 +#define WORKER_ACLK_PROCESS_UNKNOWN 19

26 +#define WORKER_ACLK_HANDLE_MQTT_INTERNAL 20

27 +#define WORKER_ACLK_TX 21

28 +#define WORKER_ACLK_TX_ERROR 22

29 +

30 +#endif //NETDATA_ACLK_MQTT_WORKERS_H

src/aclk/mqtt_websockets/mqtt_wss_client.c

+37 -11

@@ -5,6 +5,7 @@

5 #endif

7 #include "libnetdata/libnetdata.h"

8 +#include "aclk_mqtt_workers.h"

9 #include "mqtt_wss_client.h"

10 #include "mqtt_ng.h"

11 #include "ws_client.h"

@@ -616,7 +617,7 @@ int mqtt_wss_connect(

617 client->poll_fds[POLLFD_SOCKET].events = POLLIN;

618 // wait till MQTT connection is established

619 while (!client->mqtt_connected) {

619 - if(mqtt_wss_service(client, -1)) {

620 + if(mqtt_wss_service(client, 60 * MSEC_PER_SEC)) {

621 nd_log(NDLS_DAEMON, NDLP_ERR, "Error connecting to MQTT WSS server \"%s\", port %d.", host, port);

622 return 2;

623 }

@@ -661,13 +662,13 @@ static const char *mqtt_wss_error_tos(int ec)

662

663 static int mqtt_wss_service_all(mqtt_wss_client client, int timeout_ms)

664 {

664 - uint64_t exit_by = boottime_usec() + (timeout_ms * NSEC_PER_MSEC);

665 + uint64_t exit_by_us = boottime_usec() + (timeout_ms * NSEC_PER_MSEC);

666 client->poll_fds[POLLFD_SOCKET].events |= POLLOUT; // TODO when entering mwtt_wss_service use out buffer size to arm POLLOUT

667 while (rbuf_bytes_available(client->ws_client->buf_write)) {

667 - const uint64_t now = boottime_usec();

668 - if (now >= exit_by)

668 + const uint64_t now_us = boottime_usec();

669 + if (now_us >= exit_by_us)

670 return MWS_TIMED_OUT;

670 - if (mqtt_wss_service(client, exit_by - now))

671 + if (mqtt_wss_service(client, (exit_by_us - now_us) / USEC_PER_SEC))

672 return MWS_ERROR;

673 }

674 return MWS_OK;

@@ -753,9 +754,23 @@ static int handle_mqtt_internal(mqtt_wss_client client)

754

755 static int t_till_next_keepalive_ms(mqtt_wss_client client)

756 {

756 - time_t last_send = mqtt_ng_last_send_time(client->mqtt);

757 - time_t next_mqtt_keep_alive = last_send + client->mqtt_keepalive * 0.75;

758 - return ((next_mqtt_keep_alive - now_realtime_sec()) * MSEC_PER_SEC);

757 + time_t last_send_ts = mqtt_ng_last_send_time(client->mqtt);

758 + time_t next_mqtt_keep_alive_ts = last_send_ts + client->mqtt_keepalive * 0.75;

759 +

760 + time_t now_ts = now_realtime_sec();

761 +

762 + if(now_ts >= next_mqtt_keep_alive_ts)

763 + return 0;

764 +

765 + int timeout_ms = (int)((next_mqtt_keep_alive_ts - now_ts) * MSEC_PER_SEC);

766 +

767 + if(timeout_ms < 1)

768 + timeout_ms = 1;

769 +

770 + if(timeout_ms > (int)(45 * MSEC_PER_SEC))

771 + timeout_ms = (int)(45 * MSEC_PER_SEC);

772 +

773 + return timeout_ms;

774 }

775

776 #ifdef MQTT_WSS_CPUSTATS

@@ -784,8 +799,6 @@ int mqtt_wss_service(mqtt_wss_client client, int timeout_ms)

799 // Check user requested TO doesn't interfere with MQTT keep alives

800 if (!ping_timeout) {

801 int till_next_keep_alive = t_till_next_keepalive_ms(client);

787 - if (till_next_keep_alive < 0)

788 - till_next_keep_alive = 0;

802 if (client->mqtt_connected && (timeout_ms < 0 || timeout_ms >= till_next_keep_alive)) {

803 timeout_ms = till_next_keep_alive;

804 send_keepalive = 1;

@@ -797,7 +810,10 @@ int mqtt_wss_service(mqtt_wss_client client, int timeout_ms)

810 client->stats.time_keepalive += t2 - t1;

811 #endif

812

813 + worker_is_idle();

814 if ((ret = poll(client->poll_fds, 2, timeout_ms >= 0 ? timeout_ms : -1)) < 0) {

815 + worker_is_busy(WORKER_ACLK_POLL_ERROR);

816 +

817 if (errno == EINTR) {

818 nd_log(NDLS_DAEMON, NDLP_WARNING, "poll interrupted by EINTR");

819 return 0;

@@ -805,6 +821,7 @@ int mqtt_wss_service(mqtt_wss_client client, int timeout_ms)

821 nd_log(NDLS_DAEMON, NDLP_ERR, "poll error \"%s\"", strerror(errno));

822 return -2;

823 }

824 + worker_is_busy(WORKER_ACLK_POLL_OK);

825

826 #ifdef MQTT_WSS_CPUSTATS

827 t1 = mqtt_wss_now_usec();

@@ -817,6 +834,7 @@ int mqtt_wss_service(mqtt_wss_client client, int timeout_ms)

834 // MQTT keep alives

835 mqtt_ng_ping(client->mqtt);

836 ping_timeout = now + PING_TIMEOUT;

837 + worker_is_busy(WORKER_ACLK_SENT_PING);

838 } else {

839 if (ping_timeout && ping_timeout < now) {

840 disconnect_req = ACLK_PING_TIMEOUT;

@@ -836,6 +854,8 @@ int mqtt_wss_service(mqtt_wss_client client, int timeout_ms)

854 client->poll_fds[POLLFD_SOCKET].events = 0;

855

856 if ((ptr = rbuf_get_linear_insert_range(client->ws_client->buf_read, &size))) {

857 + worker_is_busy(WORKER_ACLK_RX);

858 +

859 if((ret = SSL_read(client->ssl, ptr, size)) > 0) {

860 spinlock_lock(&client->stat_lock);

861 client->stats.bytes_rx += ret;

@@ -847,6 +867,7 @@ int mqtt_wss_service(mqtt_wss_client client, int timeout_ms)

867 set_socket_pollfds(client, ret);

868 if (ret != SSL_ERROR_WANT_READ &&

869 ret != SSL_ERROR_WANT_WRITE) {

870 + worker_is_busy(WORKER_ACLK_RX_ERROR);

871 nd_log(NDLS_DAEMON, NDLP_ERR, "SSL_read error: %d %s", ret, util_openssl_ret_err(ret));

872 if (ret == SSL_ERROR_SYSCALL)

873 nd_log(NDLS_DAEMON, NDLP_ERR, "SSL_read SYSCALL errno: %d %s", errnobkp, strerror(errnobkp));

@@ -882,9 +903,11 @@ int mqtt_wss_service(mqtt_wss_client client, int timeout_ms)

903 #endif

904

905 // process MQTT stuff

885 - if(client->ws_client->state == WS_ESTABLISHED)

906 + if(client->ws_client->state == WS_ESTABLISHED) {

907 + worker_is_busy(WORKER_ACLK_HANDLE_MQTT_INTERNAL);

908 if (handle_mqtt_internal(client))

909 return MQTT_WSS_ERR_PROTO_MQTT;

910 + }

911

912 if (client->mqtt_didnt_finish_write) {

913 client->mqtt_didnt_finish_write = 0;

@@ -897,6 +920,8 @@ int mqtt_wss_service(mqtt_wss_client client, int timeout_ms)

920 #endif

921

922 if ((ptr = rbuf_get_linear_read_range(client->ws_client->buf_write, &size))) {

923 + worker_is_busy(WORKER_ACLK_TX);

924 +

925 if ((ret = SSL_write(client->ssl, ptr, size)) > 0) {

926 spinlock_lock(&client->stat_lock);

927 client->stats.bytes_tx += ret;

@@ -908,6 +933,7 @@ int mqtt_wss_service(mqtt_wss_client client, int timeout_ms)

933 set_socket_pollfds(client, ret);

934 if (ret != SSL_ERROR_WANT_READ &&

935 ret != SSL_ERROR_WANT_WRITE) {

936 + worker_is_busy(WORKER_ACLK_TX_ERROR);

937 nd_log(NDLS_DAEMON, NDLP_ERR, "SSL_write error: %d %s", ret, util_openssl_ret_err(ret));

938 if (ret == SSL_ERROR_SYSCALL)

939 nd_log(NDLS_DAEMON, NDLP_ERR, "SSL_write SYSCALL errno: %d %s", errnobkp, strerror(errnobkp));

src/aclk/mqtt_websockets/mqtt_wss_client.h

+1 -2

@@ -5,7 +5,6 @@

6 #include "common_public.h"

8 -

8 #define MQTT_WSS_OK 0 // All OK call me at your earliest convinience

9 #define MQTT_WSS_OK_TO 1 // All OK, poll timeout you requested when calling mqtt_wss_service expired

10 //you might want to know if timeout

@@ -58,7 +57,7 @@ int mqtt_wss_connect(

57 int ssl_flags,

58 const struct mqtt_wss_proxy *proxy,

59 bool *fallback_ipv4);

61 -int mqtt_wss_service(mqtt_wss_client client, int timeout_ms);

60 +int mqtt_wss_service(mqtt_wss_client client, int t_ms);

61 void mqtt_wss_disconnect(mqtt_wss_client client, int timeout_ms);

63 // we redefine this instead of using MQTT-C flags as in future

src/aclk/mqtt_websockets/ws_client.c

@@ -4,6 +4,7 @@

5 #include "ws_client.h"

6 #include "common_internal.h"

7 +#include "aclk_mqtt_workers.h"

9 const char *websocket_upgrage_hdr = "GET /mqtt HTTP/1.1\x0D\x0A"

10 "Host: %s\x0D\x0A"

@@ -623,10 +624,12 @@ int ws_client_process(ws_client *client)

624 int ret;

625 switch(client->state) {

626 case WS_RAW:

627 + worker_is_busy(WORKER_ACLK_PROCESS_RAW);

628 if (ws_client_start_handshake(client))

629 return WS_CLIENT_INTERNAL_ERROR;

630 return WS_CLIENT_NEED_MORE_BYTES;

631 case WS_HANDSHAKE:

632 + worker_is_busy(WORKER_ACLK_PROCESS_HANDSHAKE);

633 do {

634 ret = ws_client_parse_handshake_resp(client);

635 if (ret == WS_CLIENT_PROTOCOL_ERROR)

@@ -636,6 +639,7 @@ int ws_client_process(ws_client *client)

639 } while (!ret);

640 break;

641 case WS_ESTABLISHED:

642 + worker_is_busy(WORKER_ACLK_PROCESS_ESTABLISHED);

643 do {

644 ret = ws_client_process_rx_ws(client);

645 switch(ret) {

@@ -655,12 +659,15 @@ int ws_client_process(ws_client *client)

659 } while (!ret || ret == WS_CLIENT_PARSING_DONE);

660 break;

661 case WS_ERROR:

662 + worker_is_busy(WORKER_ACLK_PROCESS_ERROR);

663 nd_log(NDLS_DAEMON, NDLP_ERR, "ws_client is in error state. Restart the connection!");

664 return WS_CLIENT_PROTOCOL_ERROR;

665 case WS_CONN_CLOSED_GRACEFUL:

666 + worker_is_busy(WORKER_ACLK_PROCESS_CLOSED_GRACEFULLY);

667 nd_log(NDLS_DAEMON, NDLP_ERR, "Connection has been gracefully closed. Calling this is useless (and probably bug) until you reconnect again.");

668 return WS_CLIENT_CONNECTION_CLOSED;

669 default:

670 + worker_is_busy(WORKER_ACLK_PROCESS_UNKNOWN);

671 nd_log(NDLS_DAEMON, NDLP_CRIT, "Unknown connection state! Probably memory corruption.");

672 return WS_CLIENT_INTERNAL_ERROR;

673 }

src/daemon/pulse/pulse-workers.c

@@ -125,6 +125,7 @@ static struct worker_utilization all_workers_utilization[] = {

125 { .name = "DBENGINE", .family = "workers dbengine instances", .priority = 1000000 },

126 { .name = "LIBUV", .family = "workers libuv threadpool", .priority = 1000000 },

127 { .name = "WEB", .family = "workers web server", .priority = 1000000 },

128 + { .name = "ACLK", .family = "workers aclk", .priority = 1000000 },

129 { .name = "ACLKSYNC", .family = "workers aclk sync", .priority = 1000000 },

130 { .name = "METASYNC", .family = "workers metadata sync", .priority = 1000000 },

131 { .name = "PLUGINSD", .family = "workers plugins.d", .priority = 1000000 },