@cryptotaxi247 / netdata-1 / commits / 14dafed8f

Pulse stream-parents (#19445)

* cleanup stream handshake reasons; introduce pulse-parents * mark function params as not used * more work towards pulse parents * added 2 charts with status per node (inbound, outbound) * log the reason the receiver left, when the sender is disconnected * fix receiver exit reasons * reject a duplicate streaming request with a different hostname * log already connected on children * do not retry too frequently * fix log * insist on connecting when the parent says already connected, but it is not in the stream path * fix last commit * log already connected on parent * added streaming events charts * streaming events are available in extended pulse * archived to stale * unify aclk connection status and connection failures/disconnection reasons

Costa Tsaousis committed Jan 22, 2025 at 18:31 UTC 14dafed8f827aed171f252842e2adf5fbb5beb1d
41 files changed +1478 -685
CMakeLists.txt
+2
@@ -1174,6 +1174,8 @@ set(DAEMON_FILES
1174 src/daemon/pulse/pulse-network.h
1175 src/daemon/pulse/pulse-db-dbengine-retention.c
1176 src/daemon/pulse/pulse-db-dbengine-retention.h
1177 + src/daemon/pulse/pulse-parents.c
1178 + src/daemon/pulse/pulse-parents.h
1179 )
1180
1181 set(H2O_FILES
src/aclk/aclk.c
+144 -127
@@ -93,7 +93,7 @@ static void aclk_ssl_keylog_cb(const SSL *ssl, const char *line)
93 if (!ssl_log_file)
94 ssl_log_file = fopen(ssl_log_filename, "a");
95 if (!ssl_log_file) {
96 - netdata_log_error("Couldn't open ssl_log file (%s) for append.", ssl_log_filename);
96 + netdata_log_error("ACLK: Couldn't open ssl_log file (%s) for append.", ssl_log_filename);
97 return;
98 }
99 fputs(line, ssl_log_file);
@@ -128,14 +128,14 @@ static int load_private_key()
128 long bytes_read;
129 char *private_key = read_by_filename(filename, &bytes_read);
130 if (!private_key) {
131 - netdata_log_error("Claimed agent cannot establish ACLK - unable to load private key '%s' failed.", filename);
131 + netdata_log_error("ACLK: Claimed agent cannot establish ACLK - unable to load private key '%s' failed.", filename);
132 return 1;
133 }
134 netdata_log_debug(D_ACLK, "Claimed agent loaded private key len=%ld bytes", bytes_read);
135
136 BIO *key_bio = BIO_new_mem_buf(private_key, -1);
137 if (key_bio==NULL) {
138 - netdata_log_error("Claimed agent cannot establish ACLK - failed to create BIO for key");
138 + netdata_log_error("ACLK: Claimed agent cannot establish ACLK - failed to create BIO for key");
139 goto biofailed;
140 }
141
@@ -146,13 +146,13 @@ static int load_private_key()
146 NULL, NULL);
147
148 if (!aclk_dctx) {
149 - netdata_log_error("Loading private key (from claiming) failed - no OpenSSL Decoders found");
149 + netdata_log_error("ACLK: Loading private key (from claiming) failed - no OpenSSL Decoders found");
150 goto biofailed;
151 }
152
153 // this is necesseary to avoid RSA key with wrong size
154 if (!OSSL_DECODER_from_bio(aclk_dctx, key_bio)) {
155 - netdata_log_error("Decoding private key (from claiming) failed - invalid format.");
155 + netdata_log_error("ACLK: Decoding private key (from claiming) failed - invalid format.");
156 goto biofailed;
157 }
158 #else
@@ -166,7 +166,7 @@ static int load_private_key()
166 }
167 char err[512];
168 ERR_error_string_n(ERR_get_error(), err, sizeof(err));
169 - netdata_log_error("Claimed agent cannot establish ACLK - cannot create private key: %s", err);
169 + netdata_log_error("ACLK: Claimed agent cannot establish ACLK - cannot create private key: %s", err);
170
171 biofailed:
172 freez(private_key);
@@ -212,7 +212,7 @@ static int wait_till_agent_claim_ready()
212 // We trap the impossible NULL here to keep the linter happy without using a fatal() in the code.
213 const char *cloud_base_url = cloud_config_url_get();
214 if (cloud_base_url == NULL) {
215 - netdata_log_error("Do not move the \"url\" out of netdata_conf_section_global_run_as_user!!");
215 + netdata_log_error("ACLK: Do not move the \"url\" out of netdata_conf_section_global_run_as_user!!");
216 return 1;
217 }
218
@@ -220,7 +220,7 @@ static int wait_till_agent_claim_ready()
220 // TODO make it without malloc/free
221 memset(&url, 0, sizeof(url_t));
222 if (url_parse(cloud_base_url, &url)) {
223 - netdata_log_error("Agent is claimed but the URL in configuration key \"url\" is invalid, please fix");
223 + netdata_log_error("ACLK: Agent is claimed but the URL in configuration key \"url\" is invalid, please fix");
224 url_t_destroy(&url);
225 sleep(5);
226 continue;
@@ -244,7 +244,7 @@ static void msg_callback(const char *topic, const void *msg, size_t msglen, int
244 netdata_log_debug(D_ACLK, "Got Message From Broker Topic \"%s\" QOS %d", topic, qos);
245
246 if (aclk_shared_state.mqtt_shutdown_msg_id > 0) {
247 - netdata_log_error("Link is shutting down. Ignoring incoming message.");
247 + netdata_log_error("ACLK: Link is shutting down. Ignoring incoming message.");
248 return;
249 }
250
@@ -266,7 +266,7 @@ static void msg_callback(const char *topic, const void *msg, size_t msglen, int
266 snprintf(filename, FN_MAX_LEN, ACLK_LOG_CONVERSATION_DIR "/%010d-rx-%s.bin", ACLK_GET_CONV_LOG_NEXT(), msgtype);
267 logfd = open(filename, O_CREAT | O_TRUNC | O_WRONLY, S_IRUSR | S_IWUSR );
268 if(logfd < 0)
269 - netdata_log_error("Error opening ACLK Conversation logfile \"%s\" for RX message.", filename);
269 + netdata_log_error("ACLK: Error opening ACLK Conversation logfile \"%s\" for RX message.", filename);
270 write(logfd, msg, msglen);
271 close(logfd);
272 #endif
@@ -307,9 +307,24 @@ static int handle_connection(mqtt_wss_client client)
307 while (service_running(SERVICE_ACLK)) {
308 // timeout 1000 to check at least once a second
309 // for netdata_exit
310 - if (mqtt_wss_service(client, 1000) < 0){
310 + int rc = mqtt_wss_service(client, 1000);
311 + if (rc < 0){
312 worker_is_busy(WORKER_ACLK_DISCONNECTED);
313 error_report("Connection Error or Dropped");
314 +
315 + if(rc == MQTT_WSS_ERR_REMOTE_CLOSED)
316 + aclk_status_set(ACLK_STATUS_OFFLINE_CLOSED_BY_REMOTE);
317 + else if(rc == MQTT_WSS_ERR_PROTO_MQTT)
318 + aclk_status_set(ACLK_STATUS_OFFLINE_MQTT_PROTOCOL_ERROR);
319 + else if(rc == MQTT_WSS_ERR_PROTO_WS)
320 + aclk_status_set(ACLK_STATUS_OFFLINE_WS_PROTOCOL_ERROR);
321 + else if(rc == MQTT_WSS_ERR_MSG_TOO_BIG)
322 + aclk_status_set(ACLK_STATUS_OFFLINE_MESSAGE_TOO_BIG);
323 + else if(rc == MQTT_WSS_ERR_POLL_FAILED)
324 + aclk_status_set(ACLK_STATUS_OFFLINE_POLL_ERROR);
325 + else /* if(rc == MQTT_WSS_ERR_CONN_DROP) */
326 + aclk_status_set(ACLK_STATUS_OFFLINE_SOCKET_ERROR);
327 +
328 return 1;
329 }
330
@@ -319,19 +334,23 @@ static int handle_connection(mqtt_wss_client client)
334 case ACLK_CLOUD_DISCONNECT:
335 worker_is_busy(WORKER_ACLK_CMD_DISCONNECT);
336 reason = "cloud request";
337 + aclk_status_set(ACLK_STATUS_OFFLINE_CLOUD_REQUESTED_DISCONNECT);
338 break;
339 case ACLK_PING_TIMEOUT:
340 worker_is_busy(WORKER_ACLK_CMD_TIMEOUT);
341 reason = "ping timeout";
342 schedule_node_update = true;
343 + aclk_status_set(ACLK_STATUS_OFFLINE_PING_TIMEOUT);
344 break;
345 case ACLK_RELOAD_CONF:
346 worker_is_busy(WORKER_ACLK_CMD_RELOAD_CONF);
347 reason = "reclaim";
348 + aclk_status_set(ACLK_STATUS_OFFLINE_RELOADING_CONFIG);
349 break;
350 default:
351 worker_is_busy(WORKER_ACLK_CMD_UNKNOWN);
352 reason = "unknown";
353 + aclk_status_set(ACLK_STATUS_OFFLINE);
354 break;
355 }
356
@@ -352,13 +371,13 @@ static inline void mqtt_connected_actions(mqtt_wss_client client)
371 char *topic = (char*)aclk_get_topic(ACLK_TOPICID_COMMAND);
372
373 if (!topic)
355 - netdata_log_error("Unable to fetch topic for COMMAND (to subscribe)");
374 + netdata_log_error("ACLK: Unable to fetch topic for COMMAND (to subscribe)");
375 else
376 mqtt_wss_subscribe(client, topic, 1);
377
378 topic = (char*)aclk_get_topic(ACLK_TOPICID_CMD_NG_V1);
379 if (!topic)
361 - netdata_log_error("Unable to fetch topic for protobuf COMMAND (to subscribe)");
380 + netdata_log_error("ACLK: Unable to fetch topic for protobuf COMMAND (to subscribe)");
381 else
382 mqtt_wss_subscribe(client, topic, 1);
383
@@ -384,7 +403,7 @@ void aclk_graceful_disconnect(mqtt_wss_client client)
403 time_t t = now_monotonic_sec();
404 while (!mqtt_wss_service(client, 100)) {
405 if (now_monotonic_sec() - t >= 2) {
387 - netdata_log_error("Wasn't able to gracefully shutdown ACLK in time!");
406 + netdata_log_error("ACLK: Wasn't able to gracefully shutdown ACLK in time!");
407 break;
408 }
409 if (aclk_shared_state.mqtt_shutdown_msg_rcvd) {
@@ -448,6 +467,7 @@ static int aclk_block_till_recon_allowed() {
467 "Wait before attempting to reconnect in %.3f seconds", recon_delay / (float)MSEC_PER_SEC);
468
469 // we want to wake up from time to time to check netdata_exit
470 + worker_is_busy(WORKER_ACLK_WAITING_TO_CONNECT);
471 while (recon_delay)
472 {
473 if (!service_running(SERVICE_ACLK))
@@ -460,6 +480,8 @@ static int aclk_block_till_recon_allowed() {
480 sleep_usec(recon_delay * USEC_PER_MS);
481 recon_delay = 0;
482 }
483 +
484 + worker_is_busy(WORKER_ACLK_CONNECT);
485 return !service_running(SERVICE_ACLK);
486 }
487
@@ -480,72 +502,100 @@ static int aclk_get_transport_idx(aclk_env_t *env) {
502 }
503 #endif
504
483 -ACLK_STATUS aclk_status = ACLK_STATUS_NONE;
505 +ACLK_STATUS aclk_status = ACLK_STATUS_OFFLINE;
506
507 const char *aclk_status_to_string(void) {
508 + if(aclk_status == ACLK_STATUS_CONNECTED)
509 + return "connected";
510 +
511 + if((int)aclk_status < (int)ND_SOCK_ERR_MAX)
512 + return ND_SOCK_ERROR_2str((ND_SOCK_ERROR)aclk_status);
513 +
514 + if((int)aclk_status < (int)HTTPS_CLIENT_RESP_MAX)
515 + return https_client_resp_t_2str((https_client_resp_t)aclk_status);
516 +
517 switch(aclk_status) {
518 case ACLK_STATUS_CONNECTED:
519 return "connected";
520
490 - case ACLK_STATUS_NONE:
491 - return "none";
521 + case ACLK_STATUS_OFFLINE:
522 + return "offline";
523
524 case ACLK_STATUS_DISABLED:
525 return "disabled";
526
496 - case ACLK_STATUS_NO_CLOUD_URL:
497 - return "no_cloud_url";
498 -
499 - case ACLK_STATUS_INVALID_CLOUD_URL:
500 - return "invalid_cloud_url";
501 -
502 - case ACLK_STATUS_NOT_CLAIMED:
503 - return "not_claimed";
504 -
505 - case ACLK_STATUS_ENV_ENDPOINT_UNREACHABLE:
506 - return "env_endpoint_unreachable";
507 -
508 - case ACLK_STATUS_ENV_RESPONSE_NOT_200:
509 - return "env_response_not_200";
510 -
511 - case ACLK_STATUS_ENV_RESPONSE_EMPTY:
512 - return "env_response_empty";
527 + case ACLK_STATUS_CANT_CONNECT_NO_CLOUD_URL:
528 + return "configuration error, no cloud url";
529
514 - case ACLK_STATUS_ENV_RESPONSE_NOT_JSON:
515 - return "env_response_not_json";
516 -
517 - case ACLK_STATUS_ENV_FAILED:
518 - return "env_failed";
530 + case ACLK_STATUS_CANT_CONNECT_INVALID_CLOUD_URL:
531 + return "configuration error, invalid cloud url";
532
533 case ACLK_STATUS_BLOCKED:
521 - return "blocked";
534 + return "agent is blocked";
535
536 case ACLK_STATUS_NO_OLD_PROTOCOL:
524 - return "no_old_protocol";
537 + return "can't connect, old protocol not supported";
538
539 case ACLK_STATUS_NO_PROTOCOL_CAPABILITY:
527 - return "no_protocol_capability";
540 + return "can't connect, protocol capability not supported";
541
542 case ACLK_STATUS_INVALID_ENV_AUTH_URL:
530 - return "invalid_env_auth_url";
543 + return "can't connect, invalid /env auth url";
544
545 case ACLK_STATUS_INVALID_ENV_TRANSPORT_IDX:
533 - return "invalid_env_transport_idx";
546 + return "can't connect, invalid /env transport idx";
547
548 case ACLK_STATUS_INVALID_ENV_TRANSPORT_URL:
536 - return "invalid_env_transport_url";
537 -
538 - case ACLK_STATUS_INVALID_OTP:
539 - return "invalid_otp";
549 + return "can't connect, invalid /env transport URL";
550
551 case ACLK_STATUS_NO_LWT_TOPIC:
542 - return "no_lwt_topic";
552 + return "can't connect, no LWT topic";
553 +
554 + case ACLK_STATUS_OFFLINE_CLOUD_REQUESTED_DISCONNECT:
555 + return "disconnected, due to remote request";
556 +
557 + case ACLK_STATUS_OFFLINE_PING_TIMEOUT:
558 + return "disconnected, ping timed out";
559 +
560 + case ACLK_STATUS_OFFLINE_RELOADING_CONFIG:
561 + return "disconnected, to reload config";
562 +
563 + case ACLK_STATUS_OFFLINE_POLL_ERROR:
564 + return "disconnected, poll() failed";
565 +
566 + case ACLK_STATUS_OFFLINE_CLOSED_BY_REMOTE:
567 + return "disconnected, closed by remote end";
568 +
569 + case ACLK_STATUS_OFFLINE_SOCKET_ERROR:
570 + return "disconnected, socket error";
571 +
572 + case ACLK_STATUS_OFFLINE_MQTT_PROTOCOL_ERROR:
573 + return "disconnected, MQTT protocol error";
574 +
575 + case ACLK_STATUS_OFFLINE_WS_PROTOCOL_ERROR:
576 + return "disconnected, WebSockets protocol error";
577 +
578 + case ACLK_STATUS_OFFLINE_MESSAGE_TOO_BIG:
579 + return "disconnected, message too big";
580
581 default:
545 - return "unknown";
582 + return "unknown status";
583 }
584 }
585
586 +void aclk_status_set(ACLK_STATUS status) {
587 + aclk_status = status;
588 +
589 + ND_LOG_STACK lgs[] = {
590 + ND_LOG_FIELD_UUID(NDF_MESSAGE_ID, &aclk_connection_msgid),
591 + ND_LOG_FIELD_END(),
592 + };
593 + ND_LOG_STACK_PUSH(lgs);
594 +
595 + nd_log(NDLS_DAEMON, status == ACLK_STATUS_CONNECTED ? NDLP_INFO : NDLP_ERR,
596 + "Netdata Cloud, ACLK connection status: %s", aclk_status_to_string());
597 +}
598 +
599 const char *aclk_cloud_base_url = NULL;
600
601 /* Attempts to make a connection to MQTT broker over WSS
@@ -562,7 +612,7 @@ const char *aclk_cloud_base_url = NULL;
612 #endif
613 static int aclk_attempt_to_connect(mqtt_wss_client client)
614 {
565 - int ret;
615 + https_client_resp_t rc;
616
617 url_t base_url;
618
@@ -575,23 +625,22 @@ static int aclk_attempt_to_connect(mqtt_wss_client client)
625 while (service_running(SERVICE_ACLK)) {
626 aclk_cloud_base_url = cloud_config_url_get();
627 if (aclk_cloud_base_url == NULL) {
578 - error_report("Do not move the \"url\" out of netdata_conf_section_global_run_as_user!!");
579 - aclk_status = ACLK_STATUS_NO_CLOUD_URL;
628 + error_report("ACLK: cloud base URL is empty.");
629 + aclk_status_set(ACLK_STATUS_CANT_CONNECT_NO_CLOUD_URL);
630 return -1;
631 }
632
633 if (aclk_block_till_recon_allowed()) {
584 - aclk_status = ACLK_STATUS_BLOCKED;
634 + aclk_status_set(ACLK_STATUS_BLOCKED);
635 return 1;
636 }
637
588 - nd_log(NDLS_DAEMON, NDLP_DEBUG,
589 - "Attempting connection now");
638 + nd_log(NDLS_DAEMON, NDLP_DEBUG, "ACLK: attempting to connect now");
639
640 memset(&base_url, 0, sizeof(url_t));
641 if (url_parse(aclk_cloud_base_url, &base_url)) {
593 - aclk_status = ACLK_STATUS_INVALID_CLOUD_URL;
594 - error_report("ACLK base URL configuration key could not be parsed. Will retry in %d seconds.", CLOUD_BASE_URL_READ_RETRY);
642 + aclk_status_set(ACLK_STATUS_CANT_CONNECT_INVALID_CLOUD_URL);
643 + error_report("ACLK: base URL '%s' cannot be parsed.", aclk_cloud_base_url);
644 sleep(CLOUD_BASE_URL_READ_RETRY);
645 url_t_destroy(&base_url);
646 continue;
@@ -618,79 +667,46 @@ static int aclk_attempt_to_connect(mqtt_wss_client client)
667 }
668 aclk_env = callocz(1, sizeof(aclk_env_t));
669
621 - ret = aclk_get_env(aclk_env, base_url.host, base_url.port, &fallback_ipv4);
670 + rc = aclk_get_env(aclk_env, base_url.host, base_url.port, &fallback_ipv4);
671 url_t_destroy(&base_url);
623 - if(ret) switch(ret) {
624 - case 1:
625 - aclk_status = ACLK_STATUS_NOT_CLAIMED;
626 - error_report("Failed to Get ACLK environment (agent is not claimed)");
627 - // delay handled by aclk_block_till_recon_allowed
628 - continue;
629 -
630 - case 2:
631 - aclk_status = ACLK_STATUS_ENV_ENDPOINT_UNREACHABLE;
632 - error_report("Failed to Get ACLK environment (cannot contact ENV endpoint)");
633 - // delay handled by aclk_block_till_recon_allowed
634 - continue;
635 -
636 - case 3:
637 - aclk_status = ACLK_STATUS_ENV_RESPONSE_NOT_200;
638 - error_report("Failed to Get ACLK environment (ENV response code is not 200)");
639 - // delay handled by aclk_block_till_recon_allowed
640 - continue;
641 -
642 - case 4:
643 - aclk_status = ACLK_STATUS_ENV_RESPONSE_EMPTY;
644 - error_report("Failed to Get ACLK environment (ENV response is empty)");
645 - // delay handled by aclk_block_till_recon_allowed
646 - continue;
647 -
648 - case 5:
649 - aclk_status = ACLK_STATUS_ENV_RESPONSE_NOT_JSON;
650 - error_report("Failed to Get ACLK environment (ENV response is not JSON)");
651 - // delay handled by aclk_block_till_recon_allowed
652 - continue;
653 -
654 - default:
655 - aclk_status = ACLK_STATUS_ENV_FAILED;
656 - error_report("Failed to Get ACLK environment (unknown error)");
657 - // delay handled by aclk_block_till_recon_allowed
658 - continue;
672 + if (rc != HTTPS_CLIENT_RESP_OK) {
673 + aclk_status_set((ACLK_STATUS)rc);
674 + continue;
675 }
676
677 if (!service_running(SERVICE_ACLK)) {
662 - aclk_status = ACLK_STATUS_DISABLED;
678 + aclk_status_set(ACLK_STATUS_DISABLED);
679 return 1;
680 }
681
682 if (aclk_env->encoding != ACLK_ENC_PROTO) {
667 - aclk_status = ACLK_STATUS_NO_OLD_PROTOCOL;
668 - error_report("This agent can only use the new cloud protocol but cloud requested old one.");
683 + aclk_status_set(ACLK_STATUS_NO_OLD_PROTOCOL);
684 + error_report("ACLK: this agent can only use the new cloud protocol but cloud requested old one.");
685 continue;
686 }
687
688 if (!aclk_env_has_capa("proto")) {
673 - aclk_status = ACLK_STATUS_NO_PROTOCOL_CAPABILITY;
674 - error_report("Can't use encoding=proto without at least \"proto\" capability.");
689 + aclk_status_set(ACLK_STATUS_NO_PROTOCOL_CAPABILITY);
690 + error_report("ACLK: can't use encoding=proto without at least \"proto\" capability.");
691 continue;
692 }
693
694 nd_log(NDLS_DAEMON, NDLP_DEBUG,
679 - "New ACLK protobuf protocol negotiated successfully (/env response).");
695 + "ACLK: new ACLK protobuf protocol negotiated successfully (/env response).");
696
697 memset(&auth_url, 0, sizeof(url_t));
698 if (url_parse(aclk_env->auth_endpoint, &auth_url)) {
683 - aclk_status = ACLK_STATUS_INVALID_ENV_AUTH_URL;
684 - error_report("Parsing URL returned by env endpoint for authentication failed. \"%s\"", aclk_env->auth_endpoint);
699 + aclk_status_set(ACLK_STATUS_INVALID_ENV_AUTH_URL);
700 + error_report("ACLK: parsing URL returned by env endpoint for authentication failed. \"%s\"", aclk_env->auth_endpoint);
701 url_t_destroy(&auth_url);
702 continue;
703 }
704
689 - ret = aclk_get_mqtt_otp(aclk_private_key, (char **)&mqtt_conn_params.clientid, (char **)&mqtt_conn_params.username, (char **)&mqtt_conn_params.password, &auth_url, &fallback_ipv4);
705 + rc = aclk_get_mqtt_otp(aclk_private_key, (char **)&mqtt_conn_params.clientid, (char **)&mqtt_conn_params.username, (char **)&mqtt_conn_params.password, &auth_url, &fallback_ipv4);
706 url_t_destroy(&auth_url);
691 - if (ret) {
692 - aclk_status = ACLK_STATUS_INVALID_OTP;
693 - error_report("Error passing Challenge/Response to get OTP");
707 + if (rc != HTTPS_CLIENT_RESP_OK) {
708 + aclk_status_set((ACLK_STATUS)rc);
709 + error_report("ACLK: error passing Challenge/Response to get OTP");
710 continue;
711 }
712
@@ -699,23 +715,23 @@ static int aclk_attempt_to_connect(mqtt_wss_client client)
715 mqtt_conn_params.will_topic = aclk_get_topic(ACLK_TOPICID_AGENT_CONN);
716
717 if (!mqtt_conn_params.will_topic) {
702 - aclk_status = ACLK_STATUS_NO_LWT_TOPIC;
703 - error_report("Couldn't get LWT topic. Will not send LWT.");
718 + aclk_status_set(ACLK_STATUS_NO_LWT_TOPIC);
719 + error_report("ACLK: couldn't get LWT topic. Will not send LWT.");
720 continue;
721 }
722
723 // Do the MQTT connection
708 - ret = aclk_get_transport_idx(aclk_env);
709 - if (ret < 0) {
710 - aclk_status = ACLK_STATUS_INVALID_ENV_TRANSPORT_IDX;
711 - error_report("Cloud /env endpoint didn't return any transport usable by this Agent.");
724 + int trp = aclk_get_transport_idx(aclk_env);
725 + if (trp < 0) {
726 + aclk_status_set(ACLK_STATUS_INVALID_ENV_TRANSPORT_IDX);
727 + error_report("ACLK: cloud /env endpoint didn't return any transport usable by this agent.");
728 continue;
729 }
730
731 memset(&mqtt_url, 0, sizeof(url_t));
716 - if (url_parse(aclk_env->transports[ret]->endpoint, &mqtt_url)){
717 - aclk_status = ACLK_STATUS_INVALID_ENV_TRANSPORT_URL;
718 - error_report("Failed to parse target URL for /env trp idx %d \"%s\"", ret, aclk_env->transports[ret]->endpoint);
732 + if (url_parse(aclk_env->transports[rc]->endpoint, &mqtt_url)){
733 + aclk_status_set(ACLK_STATUS_INVALID_ENV_TRANSPORT_URL);
734 + error_report("ACLK: failed to parse target URL for /env trp idx %d \"%s\"", trp, aclk_env->transports[rc]->endpoint);
735 url_t_destroy(&mqtt_url);
736 continue;
737 }
@@ -728,10 +744,10 @@ static int aclk_attempt_to_connect(mqtt_wss_client client)
744 mqtt_conn_params.will_msg = aclk_generate_lwt(&mqtt_conn_params.will_msg_len);
745
746 #ifdef ACLK_DISABLE_CHALLENGE
731 - ret = mqtt_wss_connect(client, base_url.host, base_url.port, &mqtt_conn_params, ACLK_SSL_FLAGS, &proxy_conf);
747 + int mqtt_rc = mqtt_wss_connect(client, base_url.host, base_url.port, &mqtt_conn_params, ACLK_SSL_FLAGS, &proxy_conf);
748 url_t_destroy(&base_url);
749 #else
734 - ret = mqtt_wss_connect(client, mqtt_url.host, mqtt_url.port, &mqtt_conn_params, ACLK_SSL_FLAGS, &proxy_conf, &fallback_ipv4);
750 + int mqtt_rc = mqtt_wss_connect(client, mqtt_url.host, mqtt_url.port, &mqtt_conn_params, ACLK_SSL_FLAGS, &proxy_conf, &fallback_ipv4);
751 url_t_destroy(&mqtt_url);
752
753 freez((char*)mqtt_conn_params.clientid);
@@ -744,20 +760,20 @@ static int aclk_attempt_to_connect(mqtt_wss_client client)
760 freez((char*)proxy_conf.username);
761 freez((char*)proxy_conf.password);
762
747 - if (!ret) {
763 + if (!mqtt_rc) {
764 last_conn_time_mqtt = now_realtime_sec();
749 - nd_log(NDLS_DAEMON, NDLP_INFO, "ACLK connection successfully established");
750 - aclk_status = ACLK_STATUS_CONNECTED;
765 + nd_log(NDLS_DAEMON, NDLP_INFO, "ACLK: connection successfully established");
766 + aclk_status_set(ACLK_STATUS_CONNECTED);
767 nd_log(NDLS_ACCESS, NDLP_INFO, "ACLK CONNECTED");
768 mqtt_connected_actions(client);
769 fallback_ipv4 = false;
770 return 0;
771 }
772
757 - error_report("Connect failed");
773 + error_report("ACLK: connection failed");
774 }
775
760 - aclk_status = ACLK_STATUS_DISABLED;
776 + aclk_status_set(ACLK_STATUS_DISABLED);
777 return 1;
778 }
779
@@ -810,11 +826,12 @@ void *aclk_main(void *ptr)
826 worker_register_job_name(WORKER_ACLK_CPT_UNKNOWN, "cpt unknown");
827 worker_register_job_name(WORKER_ACLK_SEND_FRAGMENT, "send fragment");
828 worker_register_job_name(WORKER_ACLK_MSG_CALLBACK, "msg callback");
829 + worker_register_job_name(WORKER_ACLK_WAITING_TO_CONNECT, "conn wait");
830
831 ACLK_PROXY_TYPE proxy_type;
832 aclk_get_proxy(&proxy_type);
833 if (proxy_type == PROXY_TYPE_SOCKS5) {
817 - netdata_log_error("SOCKS5 proxy is not supported by ACLK-NG yet.");
834 + netdata_log_error("ACLK: SOCKS5 proxy is not supported by ACLK-NG yet.");
835 static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
836 return NULL;
837 }
@@ -826,7 +843,7 @@ void *aclk_main(void *ptr)
843 goto exit;
844
845 if (!((mqttwss_client = mqtt_wss_new(msg_callback, puback_callback)))) {
829 - netdata_log_error("Couldn't initialize MQTT_WSS network library");
846 + netdata_log_error("ACLK: Couldn't initialize MQTT_WSS network library");
847 goto exit;
848 }
849
@@ -849,7 +866,7 @@ void *aclk_main(void *ptr)
866
867 // Keep reconnecting and talking until our time has come
868 // and the Grim Reaper (netdata_exit) calls
852 - netdata_log_info("Starting ACLK query event loop");
869 + netdata_log_info("ACLK: Starting ACLK query event loop");
870 aclk_query_init(mqttwss_client);
871 do {
872 worker_is_busy(WORKER_ACLK_CONNECT);
@@ -929,7 +946,7 @@ void aclk_host_state_update(RRDHOST *host, int cmd, int queryable)
946 int ret = get_node_id(&host->host_id.uuid, &node_id.uuid);
947 if (ret > 0) {
948 // this means we were not able to check if node_id already present
932 - netdata_log_error("Unable to check for node_id. Ignoring the host state update.");
949 + netdata_log_error("ACLK: Unable to check for node_id. Ignoring the host state update.");
950 return;
951 }
952 if (ret < 0) {
src/aclk/aclk.h
+23 -11
@@ -7,6 +7,8 @@
7 #include "aclk_util.h"
8 //#include "aclk_rrdhost_state.h"
9
10 +#include "https_client.h"
11 +
12 // How many MQTT PUBACKs we need to get to consider connection
13 // stable for the purposes of TBEB (truncated binary exponential backoff)
14 #define ACLK_PUBACKS_CONN_STABLE 3
@@ -18,26 +20,35 @@ typedef enum {
20 ACLK_PING_TIMEOUT = 3
21 } ACLK_DISCONNECT_ACTION;
22
21 -typedef enum __attribute__((packed)) {
23 +typedef enum {
24 ACLK_STATUS_CONNECTED = 0,
23 - ACLK_STATUS_NONE,
25 +
26 + // ND_SOCK_ERR_XXX is included here
27 + // HTTPS_CLIENT_RESP_XXX is included here
28 +
29 + ACLK_STATUS_OFFLINE = HTTPS_CLIENT_RESP_MAX,
30 ACLK_STATUS_DISABLED,
25 - ACLK_STATUS_NO_CLOUD_URL,
26 - ACLK_STATUS_INVALID_CLOUD_URL,
27 - ACLK_STATUS_NOT_CLAIMED,
28 - ACLK_STATUS_ENV_ENDPOINT_UNREACHABLE,
29 - ACLK_STATUS_ENV_RESPONSE_NOT_200,
30 - ACLK_STATUS_ENV_RESPONSE_EMPTY,
31 - ACLK_STATUS_ENV_RESPONSE_NOT_JSON,
32 - ACLK_STATUS_ENV_FAILED,
31 + ACLK_STATUS_CANT_CONNECT_NO_CLOUD_URL,
32 + ACLK_STATUS_CANT_CONNECT_INVALID_CLOUD_URL,
33 ACLK_STATUS_BLOCKED,
34 ACLK_STATUS_NO_OLD_PROTOCOL,
35 ACLK_STATUS_NO_PROTOCOL_CAPABILITY,
36 ACLK_STATUS_INVALID_ENV_AUTH_URL,
37 ACLK_STATUS_INVALID_ENV_TRANSPORT_IDX,
38 ACLK_STATUS_INVALID_ENV_TRANSPORT_URL,
39 - ACLK_STATUS_INVALID_OTP,
39 ACLK_STATUS_NO_LWT_TOPIC,
40 +
41 + // disconnection reasons
42 + ACLK_STATUS_OFFLINE_CLOUD_REQUESTED_DISCONNECT,
43 + ACLK_STATUS_OFFLINE_PING_TIMEOUT,
44 + ACLK_STATUS_OFFLINE_RELOADING_CONFIG,
45 + ACLK_STATUS_OFFLINE_POLL_ERROR,
46 + ACLK_STATUS_OFFLINE_CLOSED_BY_REMOTE,
47 + ACLK_STATUS_OFFLINE_SOCKET_ERROR,
48 + ACLK_STATUS_OFFLINE_MQTT_PROTOCOL_ERROR,
49 + ACLK_STATUS_OFFLINE_WS_PROTOCOL_ERROR,
50 + ACLK_STATUS_OFFLINE_MESSAGE_TOO_BIG,
51 +
52 } ACLK_STATUS;
53
54 extern ACLK_STATUS aclk_status;
@@ -95,5 +106,6 @@ void add_aclk_host_labels(void);
106 void aclk_queue_node_info(RRDHOST *host, bool immediate);
107
108 struct mqtt_wss_stats aclk_statistics(void);
109 +void aclk_status_set(ACLK_STATUS status);
110
111 #endif /* ACLK_H */
src/aclk/aclk_otp.c
+110 -97
@@ -4,8 +4,8 @@
4 #include "aclk_util.h"
5 #include "aclk.h"
6
7 -static int aclk_https_request(https_req_t *request, https_req_response_t *response, bool *fallback_ipv4) {
8 - int rc;
7 +static https_client_resp_t aclk_https_request(https_req_t *request, https_req_response_t *response, bool *fallback_ipv4) {
8 + https_client_resp_t rc;
9 // wrapper for ACLK only which loads ACLK specific proxy settings
10 // then only calls https_request
11 struct mqtt_wss_proxy proxy_conf = { .host = NULL, .port = 0, .username = NULL, .password = NULL, .type = MQTT_WSS_DIRECT };
@@ -33,7 +33,7 @@ struct auth_data {
33
34 #define PARSE_ENV_JSON_CHK_TYPE(it, type, name) \
35 if (json_object_get_type(json_object_iter_peek_value(it)) != type) { \
36 - netdata_log_error("value of key \"%s\" should be %s", name, #type); \
36 + netdata_log_error("ACLK: value of key \"%s\" should be %s", name, #type); \
37 goto exit; \
38 }
39
@@ -50,7 +50,7 @@ static int parse_passwd_response(const char *json_str, struct auth_data *auth) {
50
51 json = json_tokener_parse(json_str);
52 if (!json) {
53 - netdata_log_error("JSON-C failed to parse the payload of http response of /env endpoint");
53 + netdata_log_error("ACLK: JSON-C failed to parse the payload of http response of /env endpoint");
54 return 1;
55 }
56
@@ -83,26 +83,26 @@ static int parse_passwd_response(const char *json_str, struct auth_data *auth) {
83 PARSE_ENV_JSON_CHK_TYPE(&it, json_type_array, JSON_KEY_TOPICS)
84
85 if (aclk_generate_topic_cache(json_object_iter_peek_value(&it))) {
86 - netdata_log_error("Failed to generate topic cache!");
86 + netdata_log_error("ACLK: Failed to generate topic cache!");
87 goto exit;
88 }
89 json_object_iter_next(&it);
90 continue;
91 }
92 - netdata_log_error("Unknown key \"%s\" in passwd response payload. Ignoring", json_object_iter_peek_name(&it));
92 + netdata_log_error("ACLK: Unknown key \"%s\" in passwd response payload. Ignoring", json_object_iter_peek_name(&it));
93 json_object_iter_next(&it);
94 }
95
96 if (!auth->client_id) {
97 - netdata_log_error(JSON_KEY_CLIENTID " is compulsory key in /password response");
97 + netdata_log_error("ACLK: " JSON_KEY_CLIENTID " is compulsory key in /password response");
98 goto exit;
99 }
100 if (!auth->passwd) {
101 - netdata_log_error(JSON_KEY_PASS " is compulsory in /password response");
101 + netdata_log_error("ACLK: " JSON_KEY_PASS " is compulsory in /password response");
102 goto exit;
103 }
104 if (!auth->username) {
105 - netdata_log_error(JSON_KEY_USER " is compulsory in /password response");
105 + netdata_log_error("ACLK: " JSON_KEY_USER " is compulsory in /password response");
106 goto exit;
107 }
108
@@ -121,11 +121,11 @@ exit:
121 static const char *get_json_str_by_path(json_object *json, const char *path) {
122 json_object *ptr;
123 if (json_pointer_get(json, path, &ptr)) {
124 - netdata_log_error("Missing compulsory key \"%s\" in error response", path);
124 + netdata_log_error("ACLK: Missing compulsory key \"%s\" in error response", path);
125 return NULL;
126 }
127 if (json_object_get_type(ptr) != json_type_string) {
128 - netdata_log_error("Value of Key \"%s\" in error response should be string", path);
128 + netdata_log_error("ACLK: Value of Key \"%s\" in error response should be string", path);
129 return NULL;
130 }
131 return json_object_get_string(ptr);
@@ -142,7 +142,7 @@ static int aclk_parse_otp_error(const char *json_str) {
142
143 json = json_tokener_parse(json_str);
144 if (!json) {
145 - netdata_log_error("JSON-C failed to parse the payload of http response of /env endpoint");
145 + netdata_log_error("ACLK: JSON-C failed to parse the payload of http response of /env endpoint");
146 return 1;
147 }
148
@@ -158,7 +158,7 @@ static int aclk_parse_otp_error(const char *json_str) {
158 // optional field
159 if (!json_pointer_get(json, "/" JSON_KEY_ERTRY, &ptr)) {
160 if (json_object_get_type(ptr) != json_type_boolean) {
161 - netdata_log_error("Error response Key " "/" JSON_KEY_ERTRY " should be of boolean type");
161 + netdata_log_error("ACLK: Error response Key " "/" JSON_KEY_ERTRY " should be of boolean type");
162 goto exit;
163 }
164 block_retry = json_object_get_boolean(ptr);
@@ -167,7 +167,7 @@ static int aclk_parse_otp_error(const char *json_str) {
167 // optional field
168 if (!json_pointer_get(json, "/" JSON_KEY_EDELAY, &ptr)) {
169 if (json_object_get_type(ptr) != json_type_int) {
170 - netdata_log_error("Error response Key " "/" JSON_KEY_EDELAY " should be of integer type");
170 + netdata_log_error("ACLK: Error response Key " "/" JSON_KEY_EDELAY " should be of integer type");
171 goto exit;
172 }
173 backoff = json_object_get_int(ptr);
@@ -179,7 +179,7 @@ static int aclk_parse_otp_error(const char *json_str) {
179 if (backoff > 0)
180 aclk_block_until = now_monotonic_sec() + backoff;
181
182 - netdata_log_error("Cloud returned EC=\"%s\", Msg-Key:\"%s\", Msg:\"%s\", BlockRetry:%s, Backoff:%ds (-1 unset by cloud)", ec, ek, emsg, block_retry > 0 ? "true" : "false", backoff);
182 + netdata_log_error("ACLK: Cloud returned EC=\"%s\", Msg-Key:\"%s\", Msg:\"%s\", BlockRetry:%s, Backoff:%ds (-1 unset by cloud)", ec, ek, emsg, block_retry > 0 ? "true" : "false", backoff);
183 rc = 0;
184 exit:
185 json_object_put(json);
@@ -200,7 +200,7 @@ static int aclk_parse_otp_error(const char *json_str) {
200
201 json = json_tokener_parse(json_str);
202 if (!json) {
203 - netdata_log_error("JSON-C failed to parse the payload of http response of /env endpoint");
203 + netdata_log_error("ACLK: JSON-C failed to parse the payload of http response of /env endpoint");
204 return 1;
205 }
206
@@ -231,7 +231,7 @@ static int aclk_parse_otp_error(const char *json_str) {
231 }
232 if (!strcmp(json_object_iter_peek_name(&it), JSON_KEY_EDELAY)) {
233 if (json_object_get_type(json_object_iter_peek_value(&it)) != json_type_int) {
234 - netdata_log_error("value of key " JSON_KEY_EDELAY " should be integer");
234 + netdata_log_error("ACLK: value of key " JSON_KEY_EDELAY " should be integer");
235 goto exit;
236 }
237
@@ -241,7 +241,7 @@ static int aclk_parse_otp_error(const char *json_str) {
241 }
242 if (!strcmp(json_object_iter_peek_name(&it), JSON_KEY_ERTRY)) {
243 if (json_object_get_type(json_object_iter_peek_value(&it)) != json_type_boolean) {
244 - netdata_log_error("value of key " JSON_KEY_ERTRY " should be integer");
244 + netdata_log_error("ACLK: value of key " JSON_KEY_ERTRY " should be integer");
245 goto exit;
246 }
247
@@ -249,7 +249,7 @@ static int aclk_parse_otp_error(const char *json_str) {
249 json_object_iter_next(&it);
250 continue;
251 }
252 - netdata_log_error("Unknown key \"%s\" in error response payload. Ignoring", json_object_iter_peek_name(&it));
252 + netdata_log_error("ACLK: Unknown key \"%s\" in error response payload. Ignoring", json_object_iter_peek_name(&it));
253 json_object_iter_next(&it);
254 }
255
@@ -259,7 +259,7 @@ static int aclk_parse_otp_error(const char *json_str) {
259 if (backoff > 0)
260 aclk_block_until = now_monotonic_sec() + backoff;
261
262 - netdata_log_error("Cloud returned EC=\"%s\", Msg-Key:\"%s\", Msg:\"%s\", BlockRetry:%s, Backoff:%ds (-1 unset by cloud)", ec, ek, emsg, block_retry > 0 ? "true" : "false", backoff);
262 + netdata_log_error("ACLK: Cloud returned EC=\"%s\", Msg-Key:\"%s\", Msg:\"%s\", BlockRetry:%s, Backoff:%ds (-1 unset by cloud)", ec, ek, emsg, block_retry > 0 ? "true" : "false", backoff);
263 rc = 0;
264 exit:
265 json_object_put(json);
@@ -271,9 +271,9 @@ exit:
271 #define CHALLENGE_LEN_BASE64 344
272
273 #define OTP_URL_PREFIX "/api/v1/auth/node/"
274 -int aclk_get_otp_challenge(url_t *target, const char *agent_id, unsigned char **challenge, int *challenge_bytes, bool *fallback_ipv4)
274 +static https_client_resp_t aclk_get_otp_challenge(url_t *target, const char *agent_id, unsigned char **challenge, int *challenge_bytes, bool *fallback_ipv4)
275 {
276 - int rc = 1;
276 + https_client_resp_t rc;
277 https_req_t req = HTTPS_REQ_T_INITIALIZER;
278 https_req_response_t resp = HTTPS_REQ_RESPONSE_T_INITIALIZER;
279
@@ -284,13 +284,15 @@ int aclk_get_otp_challenge(url_t *target, const char *agent_id, unsigned char **
284 buffer_sprintf(url, "%s/node/%s/challenge", target->path, agent_id);
285 req.url = (char *)buffer_tostring(url);
286
287 - if (aclk_https_request(&req, &resp, fallback_ipv4)) {
288 - netdata_log_error("ACLK_OTP Challenge failed");
287 + rc = aclk_https_request(&req, &resp, fallback_ipv4);
288 + if (rc != HTTPS_CLIENT_RESP_OK) {
289 + netdata_log_error("ACLK: OTP Challenge failed");
290 buffer_free(url);
290 - return 1;
291 + return rc;
292 }
293 if (resp.http_code != 200) {
293 - netdata_log_error("ACLK_OTP Challenge HTTP code not 200 OK (got %d)", resp.http_code);
294 + rc = HTTPS_CLIENT_RESP_OTP_CHALLENGE_NOT_200;
295 + netdata_log_error("ACLK: OTP Challenge HTTP code not 200 OK (got %d)", resp.http_code);
296 buffer_free(url);
297 if (resp.payload_size)
298 aclk_parse_otp_error(resp.payload);
@@ -298,29 +300,32 @@ int aclk_get_otp_challenge(url_t *target, const char *agent_id, unsigned char **
300 }
301 buffer_free(url);
302
301 - netdata_log_info("ACLK_OTP Got Challenge from Cloud");
302 -
303 json_object *json = json_tokener_parse(resp.payload);
304 if (!json) {
305 - netdata_log_error("Couldn't parse HTTP GET challenge payload");
305 + rc = HTTPS_CLIENT_RESP_OTP_CHALLENGE_INVALID;
306 + netdata_log_error("ACLK: couldn't parse HTTP GET challenge payload");
307 goto cleanup_resp;
308 }
309 json_object *challenge_json;
310 if (!json_object_object_get_ex(json, "challenge", &challenge_json)) {
310 - netdata_log_error("No key named \"challenge\" in the returned JSON");
311 + rc = HTTPS_CLIENT_RESP_OTP_CHALLENGE_INVALID;
312 + netdata_log_error("ACLK: No key named \"challenge\" in the returned JSON");
313 goto cleanup_json;
314 }
315 if (!json_object_is_type(challenge_json, json_type_string)) {
314 - netdata_log_error("\"challenge\" is not a string JSON type");
316 + rc = HTTPS_CLIENT_RESP_OTP_CHALLENGE_INVALID;
317 + netdata_log_error("ACLK: \"challenge\" is not a string JSON type");
318 goto cleanup_json;
319 }
320 const char *challenge_base64;
321 if (!((challenge_base64 = json_object_get_string(challenge_json)))) {
319 - netdata_log_error("Failed to extract challenge from JSON object");
322 + rc = HTTPS_CLIENT_RESP_OTP_CHALLENGE_INVALID;
323 + netdata_log_error("ACLK: Failed to extract challenge from JSON object");
324 goto cleanup_json;
325 }
326 if (strlen(challenge_base64) != CHALLENGE_LEN_BASE64) {
323 - netdata_log_error("Received Challenge has unexpected length of %zu (expected %d)", strlen(challenge_base64), CHALLENGE_LEN_BASE64);
327 + rc = HTTPS_CLIENT_RESP_OTP_CHALLENGE_INVALID;
328 + netdata_log_error("ACLK: Received Challenge has unexpected length of %zu (expected %d)", strlen(challenge_base64), CHALLENGE_LEN_BASE64);
329 goto cleanup_json;
330 }
331
@@ -328,12 +333,14 @@ int aclk_get_otp_challenge(url_t *target, const char *agent_id, unsigned char **
333 *challenge_bytes = netdata_base64_decode(*challenge, (const unsigned char *) challenge_base64, CHALLENGE_LEN_BASE64);
334
335 if (*challenge_bytes != CHALLENGE_LEN) {
331 - netdata_log_error("Unexpected challenge length of %d instead of %d", *challenge_bytes, CHALLENGE_LEN);
336 + rc = HTTPS_CLIENT_RESP_OTP_CHALLENGE_INVALID;
337 + netdata_log_error("ACLK: Unexpected challenge length of %d instead of %d", *challenge_bytes, CHALLENGE_LEN);
338 freez(*challenge);
339 *challenge = NULL;
340 goto cleanup_json;
341 }
336 - rc = 0;
342 +
343 + rc = HTTPS_CLIENT_RESP_OK;
344
345 cleanup_json:
346 json_object_put(json);
@@ -342,9 +349,9 @@ cleanup_resp:
349 return rc;
350 }
351
345 -int aclk_send_otp_response(const char *agent_id, const unsigned char *response, int response_bytes, url_t *target, struct auth_data *mqtt_auth, bool *fallback_ipv4)
352 +static https_client_resp_t aclk_send_otp_response(const char *agent_id, const unsigned char *response, int response_bytes, url_t *target, struct auth_data *mqtt_auth, bool *fallback_ipv4)
353 {
347 - int rc = 1;
354 + https_client_resp_t rc;
355 https_req_t req = HTTPS_REQ_T_INITIALIZER;
356 https_req_response_t resp = HTTPS_REQ_RESPONSE_T_INITIALIZER;
357
@@ -367,28 +374,31 @@ int aclk_send_otp_response(const char *agent_id, const unsigned char *response,
374 req.payload = (char *)buffer_tostring(resp_json);
375 req.payload_size = strlen(req.payload);
376
370 - if (aclk_https_request(&req, &resp, fallback_ipv4)) {
371 - netdata_log_error("ACLK_OTP Password error trying to post result to password");
377 + rc = aclk_https_request(&req, &resp, fallback_ipv4);
378 + if (rc != HTTPS_CLIENT_RESP_OK) {
379 + netdata_log_error("ACLK: OTP Password error trying to post result to password");
380 goto cleanup_buffers;
381 }
382 if (resp.http_code != 201) {
375 - netdata_log_error("ACLK_OTP Password HTTP code not 201 Created (got %d)", resp.http_code);
383 + rc = HTTPS_CLIENT_RESP_OTP_PASSWORD_NOT_201;
384 + netdata_log_error("ACLK: OTP Password HTTP code not 201 Created (got %d)", resp.http_code);
385 if (resp.payload_size)
386 aclk_parse_otp_error(resp.payload);
387 goto cleanup_response;
388 }
389 if (resp.payload_size == 0 || resp.payload == NULL) {
381 - netdata_log_error("ACLK_OTP Password response payload is empty despite returning 201 Created!");
390 + rc = HTTPS_CLIENT_RESP_OTP_PASSWORD_EMPTY;
391 + netdata_log_error("ACLK: OTP Password response payload is empty despite returning 201 Created!");
392 goto cleanup_response;
393 }
384 - netdata_log_info("ACLK_OTP Got Password from Cloud");
394
395 if (parse_passwd_response(resp.payload, mqtt_auth)){
387 - netdata_log_error("Error parsing response of password endpoint");
396 + rc = HTTPS_CLIENT_RESP_OTP_PASSWORD_NOT_JSON;
397 + netdata_log_error("ACLK: Error parsing response of password endpoint");
398 goto cleanup_response;
399 }
400
391 - rc = 0;
401 + rc = HTTPS_CLIENT_RESP_OK;
402
403 cleanup_response:
404 https_req_response_free(&resp);
@@ -437,49 +447,52 @@ static int private_decrypt(RSA *p_key, unsigned char * enc_data, int data_len, u
447 {
448 char err[512];
449 ERR_error_string_n(ERR_get_error(), err, sizeof(err));
440 - netdata_log_error("Decryption of the challenge failed: %s", err);
450 + netdata_log_error("ACLK: Decryption of the challenge failed: %s", err);
451 }
452 return result;
453 }
454
455 #if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_300
446 -int aclk_get_mqtt_otp(EVP_PKEY *p_key, char **mqtt_id, char **mqtt_usr, char **mqtt_pass, url_t *target, bool *fallback_ipv4)
456 +https_client_resp_t aclk_get_mqtt_otp(EVP_PKEY *p_key, char **mqtt_id, char **mqtt_usr, char **mqtt_pass, url_t *target, bool *fallback_ipv4)
457 #else
448 -int aclk_get_mqtt_otp(RSA *p_key, char **mqtt_id, char **mqtt_usr, char **mqtt_pass, url_t *target, bool *fallback_ipv4)
458 +https_client_resp_t aclk_get_mqtt_otp(RSA *p_key, char **mqtt_id, char **mqtt_usr, char **mqtt_pass, url_t *target, bool *fallback_ipv4)
459 #endif
460 {
461 unsigned char *challenge = NULL;
462 int challenge_bytes;
463 + https_client_resp_t rc;
464
465 CLAIM_ID claim_id = claim_id_get();
466 if (!claim_id_is_set(claim_id)) {
456 - netdata_log_error("Agent was not claimed - cannot perform challenge/response");
457 - return 1;
467 + netdata_log_error("ACLK: Agent was not claimed - cannot perform challenge/response");
468 + return HTTPS_CLIENT_RESP_OTP_AGENT_NOT_CLAIMED;
469 }
470
471 // Get Challenge
461 - if (aclk_get_otp_challenge(target, claim_id.str, &challenge, &challenge_bytes, fallback_ipv4)) {
462 - netdata_log_error("Error getting challenge");
463 - return 1;
472 + rc = aclk_get_otp_challenge(target, claim_id.str, &challenge, &challenge_bytes, fallback_ipv4);
473 + if (rc != HTTPS_CLIENT_RESP_OK) {
474 + netdata_log_error("ACLK: error getting challenge");
475 + return rc;
476 }
477
478 // Decrypt Challenge / Get response
479 unsigned char *response_plaintext = NULL;
480 int response_plaintext_bytes = private_decrypt(p_key, challenge, challenge_bytes, &response_plaintext);
481 if (response_plaintext_bytes < 0) {
470 - netdata_log_error("Couldn't decrypt the challenge received");
482 + netdata_log_error("ACLK: Couldn't decrypt the challenge received");
483 freez(response_plaintext);
484 freez(challenge);
473 - return 1;
485 + return HTTPS_CLIENT_RESP_OTP_CHALLENGE_DECRYPTION_FAILED;
486 }
487 freez(challenge);
488
489 // Encode and Send Challenge
490 struct auth_data data = { .client_id = NULL, .passwd = NULL, .username = NULL };
479 - if (aclk_send_otp_response(claim_id.str, response_plaintext, response_plaintext_bytes, target, &data, fallback_ipv4)) {
480 - netdata_log_error("Error getting response");
491 + rc = aclk_send_otp_response(claim_id.str, response_plaintext, response_plaintext_bytes, target, &data, fallback_ipv4);
492 + if (rc != HTTPS_CLIENT_RESP_OK) {
493 + netdata_log_error("ACLK: Error getting response");
494 freez(response_plaintext);
482 - return 1;
495 + return rc;
496 }
497
498 *mqtt_pass = data.passwd;
@@ -487,7 +500,7 @@ int aclk_get_mqtt_otp(RSA *p_key, char **mqtt_id, char **mqtt_usr, char **mqtt_p
500 *mqtt_id = data.client_id;
501
502 freez(response_plaintext);
490 - return 0;
503 + return HTTPS_CLIENT_RESP_OK;
504 }
505
506 #define JSON_KEY_ENC "encoding"
@@ -512,12 +525,12 @@ static int parse_json_env_transport(json_object *json, aclk_transport_desc_t *tr
525 if (!strcmp(json_object_iter_peek_name(&it), JSON_KEY_TRP_TYPE)) {
526 PARSE_ENV_JSON_CHK_TYPE(&it, json_type_string, JSON_KEY_TRP_TYPE)
527 if (trp->type != ACLK_TRP_UNKNOWN) {
515 - netdata_log_error(JSON_KEY_TRP_TYPE " set already");
528 + netdata_log_error("ACLK: " JSON_KEY_TRP_TYPE " set already");
529 goto exit;
530 }
531 trp->type = aclk_transport_type_t_from_str(json_object_get_string(json_object_iter_peek_value(&it)));
532 if (trp->type == ACLK_TRP_UNKNOWN) {
520 - netdata_log_error(JSON_KEY_TRP_TYPE " unknown type \"%s\"", json_object_get_string(json_object_iter_peek_value(&it)));
533 + netdata_log_error("ACLK: " JSON_KEY_TRP_TYPE " unknown type \"%s\"", json_object_get_string(json_object_iter_peek_value(&it)));
534 goto exit;
535 }
536 json_object_iter_next(&it);
@@ -527,7 +540,7 @@ static int parse_json_env_transport(json_object *json, aclk_transport_desc_t *tr
540 if (!strcmp(json_object_iter_peek_name(&it), JSON_KEY_TRP_ENDPOINT)) {
541 PARSE_ENV_JSON_CHK_TYPE(&it, json_type_string, JSON_KEY_TRP_ENDPOINT)
542 if (trp->endpoint) {
530 - netdata_log_error(JSON_KEY_TRP_ENDPOINT " set already");
543 + netdata_log_error("ACLK: " JSON_KEY_TRP_ENDPOINT " set already");
544 goto exit;
545 }
546 trp->endpoint = strdupz(json_object_get_string(json_object_iter_peek_value(&it)));
@@ -535,17 +548,17 @@ static int parse_json_env_transport(json_object *json, aclk_transport_desc_t *tr
548 continue;
549 }
550
538 - netdata_log_error("unknown JSON key in dictionary (\"%s\")", json_object_iter_peek_name(&it));
551 + netdata_log_error("ACLK: unknown JSON key in dictionary (\"%s\")", json_object_iter_peek_name(&it));
552 json_object_iter_next(&it);
553 }
554
555 if (!trp->endpoint) {
543 - netdata_log_error(JSON_KEY_TRP_ENDPOINT " is missing from JSON dictionary");
556 + netdata_log_error("ACLK: " JSON_KEY_TRP_ENDPOINT " is missing from JSON dictionary");
557 goto exit;
558 }
559
560 if (trp->type == ACLK_TRP_UNKNOWN) {
548 - netdata_log_error("transport type not set");
561 + netdata_log_error("ACLK: transport type not set");
562 goto exit;
563 }
564
@@ -561,7 +574,7 @@ static int parse_json_env_transports(json_object *json_array, aclk_env_t *env) {
574 json_object *obj;
575
576 if (env->transports) {
564 - netdata_log_error("transports have been set already");
577 + netdata_log_error("ACLK: transports have been set already");
578 return 1;
579 }
580
@@ -573,7 +586,7 @@ static int parse_json_env_transports(json_object *json_array, aclk_env_t *env) {
586 trp = callocz(1, sizeof(aclk_transport_desc_t));
587 obj = json_object_array_get_idx(json_array, i);
588 if (parse_json_env_transport(obj, trp)) {
576 - netdata_log_error("error parsing transport idx %d", (int)i);
589 + netdata_log_error("ACLK: error parsing transport idx %d", (int)i);
590 freez(trp);
591 return 1;
592 }
@@ -589,14 +602,14 @@ static int parse_json_env_transports(json_object *json_array, aclk_env_t *env) {
602 static int parse_json_backoff_int(struct json_object_iterator *it, int *out, const char* name, int min, int max) {
603 if (!strcmp(json_object_iter_peek_name(it), name)) {
604 if (json_object_get_type(json_object_iter_peek_value(it)) != json_type_int) {
592 - netdata_log_error("Could not parse \"%s\". Not an integer as expected.", name);
605 + netdata_log_error("ACLK: Could not parse \"%s\". Not an integer as expected.", name);
606 return MATCHED_ERROR;
607 }
608
609 *out = json_object_get_int(json_object_iter_peek_value(it));
610
611 if (*out < min || *out > max) {
599 - netdata_log_error("Value of \"%s\"=%d out of range (%d-%d).", name, *out, min, max);
612 + netdata_log_error("ACLK: Value of \"%s\"=%d out of range (%d-%d).", name, *out, min, max);
613 return MATCHED_ERROR;
614 }
615
@@ -638,7 +651,7 @@ static int parse_json_backoff(json_object *json, aclk_backoff_t *backoff) {
651 continue;
652 }
653
641 - netdata_log_error("unknown JSON key in dictionary (\"%s\")", json_object_iter_peek_name(&it));
654 + netdata_log_error("ACLK: unknown JSON key in dictionary (\"%s\")", json_object_iter_peek_name(&it));
655 json_object_iter_next(&it);
656 }
657
@@ -650,7 +663,7 @@ static int parse_json_env_caps(json_object *json, aclk_env_t *env) {
663 const char *str;
664
665 if (env->capabilities) {
653 - netdata_log_error("transports have been set already");
666 + netdata_log_error("ACLK: transports have been set already");
667 return 1;
668 }
669
@@ -665,12 +678,12 @@ static int parse_json_env_caps(json_object *json, aclk_env_t *env) {
678 for (size_t i = 0; i < env->capability_count; i++) {
679 obj = json_object_array_get_idx(json, i);
680 if (json_object_get_type(obj) != json_type_string) {
668 - netdata_log_error("Capability at index %d not a string!", (int)i);
681 + netdata_log_error("ACLK: Capability at index %d not a string!", (int)i);
682 return 1;
683 }
684 str = json_object_get_string(obj);
685 if (!str) {
673 - netdata_log_error("Error parsing capabilities");
686 + netdata_log_error("ACLK: Error parsing capabilities");
687 return 1;
688 }
689 env->capabilities[i] = strdupz(str);
@@ -686,7 +699,7 @@ static int parse_json_env(const char *json_str, aclk_env_t *env) {
699
700 json = json_tokener_parse(json_str);
701 if (!json) {
689 - netdata_log_error("JSON-C failed to parse the payload of http response of /env endpoint");
702 + netdata_log_error("ACLK: JSON-C failed to parse the payload of http response of /env endpoint");
703 return 1;
704 }
705
@@ -697,7 +710,7 @@ static int parse_json_env(const char *json_str, aclk_env_t *env) {
710 if (!strcmp(json_object_iter_peek_name(&it), JSON_KEY_AUTH_ENDPOINT)) {
711 PARSE_ENV_JSON_CHK_TYPE(&it, json_type_string, JSON_KEY_AUTH_ENDPOINT)
712 if (env->auth_endpoint) {
700 - netdata_log_error("authEndpoint set already");
713 + netdata_log_error("ACLK: authEndpoint set already");
714 goto exit;
715 }
716 env->auth_endpoint = strdupz(json_object_get_string(json_object_iter_peek_value(&it)));
@@ -708,7 +721,7 @@ static int parse_json_env(const char *json_str, aclk_env_t *env) {
721 if (!strcmp(json_object_iter_peek_name(&it), JSON_KEY_ENC)) {
722 PARSE_ENV_JSON_CHK_TYPE(&it, json_type_string, JSON_KEY_ENC)
723 if (env->encoding != ACLK_ENC_UNKNOWN) {
711 - netdata_log_error(JSON_KEY_ENC " set already");
724 + netdata_log_error("ACLK: " JSON_KEY_ENC " set already");
725 goto exit;
726 }
727 env->encoding = aclk_encoding_type_t_from_str(json_object_get_string(json_object_iter_peek_value(&it)));
@@ -731,7 +744,7 @@ static int parse_json_env(const char *json_str, aclk_env_t *env) {
744
745 if (parse_json_backoff(json_object_iter_peek_value(&it), &env->backoff)) {
746 env->backoff.base = 0;
734 - netdata_log_error("Error parsing Backoff parameters in env");
747 + netdata_log_error("ACLK: Error parsing Backoff parameters in env");
748 goto exit;
749 }
750
@@ -743,7 +756,7 @@ static int parse_json_env(const char *json_str, aclk_env_t *env) {
756 PARSE_ENV_JSON_CHK_TYPE(&it, json_type_array, JSON_KEY_CAPS)
757
758 if (parse_json_env_caps(json_object_iter_peek_value(&it), env)) {
746 - netdata_log_error("Error parsing capabilities list");
759 + netdata_log_error("ACLK: Error parsing capabilities list");
760 goto exit;
761 }
762
@@ -751,25 +764,25 @@ static int parse_json_env(const char *json_str, aclk_env_t *env) {
764 continue;
765 }
766
754 - netdata_log_error("unknown JSON key in dictionary (\"%s\")", json_object_iter_peek_name(&it));
767 + netdata_log_error("ACLK: unknown JSON key in dictionary (\"%s\")", json_object_iter_peek_name(&it));
768 json_object_iter_next(&it);
769 }
770
771 // Check all compulsory keys have been set
772 if (env->transport_count < 1) {
760 - netdata_log_error("env has to return at least one transport");
773 + netdata_log_error("ACLK: env has to return at least one transport");
774 goto exit;
775 }
776 if (!env->auth_endpoint) {
764 - netdata_log_error(JSON_KEY_AUTH_ENDPOINT " is compulsory");
777 + netdata_log_error("ACLK: " JSON_KEY_AUTH_ENDPOINT " is compulsory");
778 goto exit;
779 }
780 if (env->encoding == ACLK_ENC_UNKNOWN) {
768 - netdata_log_error(JSON_KEY_ENC " is compulsory");
781 + netdata_log_error("ACLK: " JSON_KEY_ENC " is compulsory");
782 goto exit;
783 }
784 if (!env->backoff.base) {
772 - netdata_log_error(JSON_KEY_BACKOFF " is compulsory");
785 + netdata_log_error("ACLK: " JSON_KEY_BACKOFF " is compulsory");
786 goto exit;
787 }
788
@@ -782,9 +795,10 @@ exit:
795 return 1;
796 }
797
785 -int aclk_get_env(aclk_env_t *env, const char* aclk_hostname, int aclk_port, bool *fallback_ipv4) {
798 +https_client_resp_t aclk_get_env(aclk_env_t *env, const char* aclk_hostname, int aclk_port, bool *fallback_ipv4) {
799 BUFFER *buf = buffer_create(1024, &netdata_buffers_statistics.buffers_aclk);
800
801 + https_client_resp_t rc;
802 https_req_t req = HTTPS_REQ_T_INITIALIZER;
803 https_req_response_t resp = HTTPS_REQ_RESPONSE_T_INITIALIZER;
804
@@ -792,9 +806,9 @@ int aclk_get_env(aclk_env_t *env, const char* aclk_hostname, int aclk_port, bool
806
807 CLAIM_ID claim_id = claim_id_get();
808 if (!claim_id_is_set(claim_id)) {
795 - netdata_log_error("Agent was not claimed - cannot perform challenge/response");
809 + netdata_log_error("ACLK: failed to get ACLK environment (agent is not claimed)");
810 buffer_free(buf);
797 - return 1;
811 + return HTTPS_CLIENT_RESP_ENV_AGENT_NOT_CLAIMED;
812 }
813
814 buffer_sprintf(buf, "/api/v1/env?v=%s&cap=proto,ctx&claim_id=%s", &(NETDATA_VERSION[1]) /* skip 'v' at beginning */, claim_id.str);
@@ -802,38 +816,37 @@ int aclk_get_env(aclk_env_t *env, const char* aclk_hostname, int aclk_port, bool
816 req.host = (char*)aclk_hostname;
817 req.port = aclk_port;
818 req.url = buf->buffer;
805 - if (aclk_https_request(&req, &resp, fallback_ipv4)) {
806 - netdata_log_error("Error trying to contact env endpoint");
819 + rc = aclk_https_request(&req, &resp, fallback_ipv4);
820 + if (rc != HTTPS_CLIENT_RESP_OK) {
821 + netdata_log_error("ACLK: failed to get ACLK environment (cannot contact ENV endpoint)");
822 https_req_response_free(&resp);
823 buffer_free(buf);
809 - return 2;
824 + return rc;
825 }
826 if (resp.http_code != 200) {
812 - netdata_log_error("The HTTP code not 200 OK (Got %d)", resp.http_code);
827 + netdata_log_error("ACLK: failed to get ACLK environment (ENV response code is not 200) (got %d)", resp.http_code);
828 if (resp.payload_size)
829 aclk_parse_otp_error(resp.payload);
830 https_req_response_free(&resp);
831 buffer_free(buf);
817 - return 3;
832 + return HTTPS_CLIENT_RESP_ENV_NOT_200;
833 }
834
835 if (!resp.payload || !resp.payload_size) {
821 - netdata_log_error("Unexpected empty payload as response to /env call");
836 + netdata_log_error("ACLK: failed to get ACLK environment (ENV response is empty)");
837 https_req_response_free(&resp);
838 buffer_free(buf);
824 - return 4;
839 + return HTTPS_CLIENT_RESP_ENV_EMPTY;
840 }
841
842 if (parse_json_env(resp.payload, env)) {
828 - netdata_log_error("error parsing /env message");
843 + netdata_log_error("ACLK: failed to get ACLK environment (ENV response is not JSON)");
844 https_req_response_free(&resp);
845 buffer_free(buf);
831 - return 5;
846 + return HTTPS_CLIENT_RESP_ENV_NOT_JSON;
847 }
848
834 - netdata_log_info("Getting Cloud /env successful");
835 -
849 https_req_response_free(&resp);
850 buffer_free(buf);
838 - return 0;
851 + return HTTPS_CLIENT_RESP_OK;
852 }
src/aclk/aclk_otp.h
+3 -3
@@ -9,10 +9,10 @@
9 #include "aclk_util.h"
10
11 #if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_300
12 -int aclk_get_mqtt_otp(EVP_PKEY *p_key, char **mqtt_id, char **mqtt_usr, char **mqtt_pass, url_t *target, bool *fallback_ipv4);
12 +https_client_resp_t aclk_get_mqtt_otp(EVP_PKEY *p_key, char **mqtt_id, char **mqtt_usr, char **mqtt_pass, url_t *target, bool *fallback_ipv4);
13 #else
14 -int aclk_get_mqtt_otp(RSA *p_key, char **mqtt_id, char **mqtt_usr, char **mqtt_pass, url_t *target, bool *fallback_ipv4);
14 +https_client_resp_t aclk_get_mqtt_otp(RSA *p_key, char **mqtt_id, char **mqtt_usr, char **mqtt_pass, url_t *target, bool *fallback_ipv4);
15 #endif
16 -int aclk_get_env(aclk_env_t *env, const char *aclk_hostname, int aclk_port, bool *fallback_ipv4);
16 +https_client_resp_t aclk_get_env(aclk_env_t *env, const char *aclk_hostname, int aclk_port, bool *fallback_ipv4);
17
18 #endif /* ACLK_OTP_H */
src/aclk/https_client.c
+208 -71
@@ -8,6 +8,130 @@
8
9 #include "daemon/pulse/pulse.h"
10
11 +ENUM_STR_MAP_DEFINE(https_client_resp_t) = {
12 + {
13 + .id = HTTPS_CLIENT_RESP_OK,
14 + .name = "ok",
15 + },
16 + {
17 + .id = HTTPS_CLIENT_RESP_UNKNOWN_ERROR,
18 + .name = "unknown error",
19 + },
20 + {
21 + .id = HTTPS_CLIENT_RESP_NO_MEM,
22 + .name = "not enough memory",
23 + },
24 + {
25 + .id = HTTPS_CLIENT_RESP_NONBLOCK_FAILED,
26 + .name = "cannot set socket to non-blocking mode",
27 + },
28 + {
29 + .id = HTTPS_CLIENT_RESP_PROXY_NOT_200,
30 + .name = "proxy did not return http/200",
31 + },
32 + {
33 + .id = HTTPS_CLIENT_RESP_NO_SSL_CTX,
34 + .name = "cannot create SSL ctx",
35 + },
36 + {
37 + .id = HTTPS_CLIENT_RESP_NO_SSL_VERIFY_PATHS,
38 + .name = "cannot set SSL verify paths",
39 + },
40 + {
41 + .id = HTTPS_CLIENT_RESP_NO_SSL_NEW,
42 + .name = "cannot create SSL",
43 + },
44 + {
45 + .id = HTTPS_CLIENT_RESP_NO_TLS_SNI,
46 + .name = "cannot set TLS SNI",
47 + },
48 + {
49 + .id = HTTPS_CLIENT_RESP_SSL_CONNECT_FAILED,
50 + .name = "SSL_connect() failed",
51 + },
52 + {
53 + .id = HTTPS_CLIENT_RESP_SSL_START_FAILED,
54 + .name = "cannot start SSL connection",
55 + },
56 + {
57 + .id = HTTPS_CLIENT_RESP_UNKNOWN_REQUEST_TYPE,
58 + .name = "unknown https client request type",
59 + },
60 + {
61 + .id = HTTPS_CLIENT_RESP_HEADER_WRITE_FAILED,
62 + .name = "https client failed to write http header",
63 + },
64 + {
65 + .id = HTTPS_CLIENT_RESP_PAYLOAD_WRITE_FAILED,
66 + .name = "https client failed to write http payload",
67 + },
68 + {
69 + .id = HTTPS_CLIENT_RESP_POLL_ERROR,
70 + .name = "https client poll() error",
71 + },
72 + {
73 + .id = HTTPS_CLIENT_RESP_TIMEOUT,
74 + .name = "https client timeout",
75 + },
76 + {
77 + .id = HTTPS_CLIENT_RESP_READ_ERROR,
78 + .name = "https client read error",
79 + },
80 + {
81 + .id = HTTPS_CLIENT_RESP_PARSE_ERROR,
82 + .name = "https client parsing of response failed",
83 + },
84 + {
85 + .id = HTTPS_CLIENT_RESP_ENV_AGENT_NOT_CLAIMED,
86 + .name = "agent is not claimed (during /env)",
87 + },
88 + {
89 + .id = HTTPS_CLIENT_RESP_ENV_NOT_200,
90 + .name = "/env response code is not 200",
91 + },
92 + {
93 + .id = HTTPS_CLIENT_RESP_ENV_EMPTY,
94 + .name = "/env response is empty",
95 + },
96 + {
97 + .id = HTTPS_CLIENT_RESP_ENV_NOT_JSON,
98 + .name = "/env response is not JSON",
99 + },
100 + {
101 + .id = HTTPS_CLIENT_RESP_OTP_CHALLENGE_NOT_200,
102 + .name = "otp challenge response is not http/200",
103 + },
104 + {
105 + .id = HTTPS_CLIENT_RESP_OTP_CHALLENGE_INVALID,
106 + .name = "otp challenge response is invalid",
107 + },
108 + {
109 + .id = HTTPS_CLIENT_RESP_OTP_PASSWORD_NOT_201,
110 + .name = "otp password response is not http/201",
111 + },
112 + {
113 + .id = HTTPS_CLIENT_RESP_OTP_PASSWORD_EMPTY,
114 + .name = "otp password response is empty",
115 + },
116 + {
117 + .id = HTTPS_CLIENT_RESP_OTP_PASSWORD_NOT_JSON,
118 + .name = "otp password response is not JSON",
119 + },
120 + {
121 + .id = HTTPS_CLIENT_RESP_OTP_AGENT_NOT_CLAIMED,
122 + .name = "agent is not claimed (during otp)",
123 + },
124 + {
125 + .id = HTTPS_CLIENT_RESP_OTP_CHALLENGE_DECRYPTION_FAILED,
126 + .name = "otp challenge decryption failed",
127 + },
128 +
129 + // terminator
130 + {.name = NULL, .id = 0}
131 +};
132 +
133 +ENUM_STR_DEFINE_FUNCTIONS(https_client_resp_t, HTTPS_CLIENT_RESP_UNKNOWN_ERROR, "unknown error");
134 +
135 static const char *http_req_type_to_str(http_req_type_t req) {
136 switch (req) {
137 case HTTP_REQ_GET:
@@ -62,16 +186,16 @@ static int process_http_hdr(http_parse_ctx *parse_ctx, const char *key, const ch
186 // we can skip the rest
187 if (parse_ctx->content_length < 0 && !strcmp("content-length", key)) {
188 if (parse_ctx->content_length == TRANSFER_ENCODING_CHUNKED) {
65 - netdata_log_error("Content-length and transfer-encoding: chunked headers are mutually exclusive");
189 + netdata_log_error("ACLK: Content-length and transfer-encoding: chunked headers are mutually exclusive");
190 return 1;
191 }
192 if (parse_ctx->content_length != -1) {
69 - netdata_log_error("Duplicate content-length header");
193 + netdata_log_error("ACLK: Duplicate content-length header");
194 return 1;
195 }
196 parse_ctx->content_length = str2u(val);
197 if (parse_ctx->content_length < 0) {
74 - netdata_log_error("Invalid content-length %d", parse_ctx->content_length);
198 + netdata_log_error("ACLK: Invalid content-length %d", parse_ctx->content_length);
199 return 1;
200 }
201 return 0;
@@ -79,7 +203,7 @@ static int process_http_hdr(http_parse_ctx *parse_ctx, const char *key, const ch
203 if (!strcmp("transfer-encoding", key)) {
204 if (!strcmp("chunked", val)) {
205 if (parse_ctx->content_length != -1) {
82 - netdata_log_error("Content-length and transfer-encoding: chunked headers are mutually exclusive");
206 + netdata_log_error("ACLK: Content-length and transfer-encoding: chunked headers are mutually exclusive");
207 return 1;
208 }
209 parse_ctx->content_length = TRANSFER_ENCODING_CHUNKED;
@@ -108,17 +232,17 @@ static int parse_http_hdr(rbuf_t buf, http_parse_ctx *parse_ctx)
232 char *ptr;
233
234 if (!rbuf_find_bytes(buf, HTTP_LINE_TERM, strlen(HTTP_LINE_TERM), &idx_end)) {
111 - netdata_log_error("CRLF expected");
235 + netdata_log_error("ACLK: CRLF expected");
236 return 1;
237 }
238
239 char *separator = rbuf_find_bytes(buf, HTTP_KEYVAL_SEPARATOR, strlen(HTTP_KEYVAL_SEPARATOR), &idx);
240 if (!separator) {
117 - netdata_log_error("Missing Key/Value separator");
241 + netdata_log_error("ACLK: Missing Key/Value separator");
242 return 1;
243 }
244 if (idx >= HTTP_HDR_BUFFER_SIZE) {
121 - netdata_log_error("Key name is too long");
245 + netdata_log_error("ACLK: Key name is too long");
246 return 1;
247 }
248
@@ -128,7 +252,7 @@ static int parse_http_hdr(rbuf_t buf, http_parse_ctx *parse_ctx)
252 rbuf_bump_tail(buf, strlen(HTTP_KEYVAL_SEPARATOR));
253 idx_end -= strlen(HTTP_KEYVAL_SEPARATOR) + idx;
254 if (idx_end >= HTTP_HDR_BUFFER_SIZE) {
131 - netdata_log_error("Value of key \"%s\" too long", buf_key);
255 + netdata_log_error("ACLK: Value of key \"%s\" too long", buf_key);
256 return 1;
257 }
258
@@ -173,7 +297,7 @@ static int process_chunked_content(rbuf_t buf, http_parse_ctx *parse_ctx)
297 continue;
298 }
299 if (idx >= HTTP_HDR_BUFFER_SIZE) {
176 - netdata_log_error("Chunk size is too long");
300 + netdata_log_error("ACLK: Chunk size is too long");
301 return HTTP_PARSE_ERROR;
302 }
303 char buf_size[HTTP_HDR_BUFFER_SIZE];
@@ -181,13 +305,13 @@ static int process_chunked_content(rbuf_t buf, http_parse_ctx *parse_ctx)
305 buf_size[idx] = 0;
306 long chunk_size = strtol(buf_size, NULL, 16);
307 if (chunk_size < 0 || chunk_size == LONG_MAX) {
184 - netdata_log_error("Chunk size out of range");
308 + netdata_log_error("ACLK: Chunk size out of range");
309 return HTTP_PARSE_ERROR;
310 }
311 parse_ctx->chunk_size = chunk_size;
312 if (parse_ctx->chunk_size == 0) {
313 if (errno == EINVAL) {
190 - netdata_log_error("Invalid chunk size");
314 + netdata_log_error("ACLK: Invalid chunk size");
315 return HTTP_PARSE_ERROR;
316 }
317 parse_ctx->chunked_content_state = CHUNKED_CONTENT_CHUNK_END_CRLF;
@@ -217,12 +341,12 @@ static int process_chunked_content(rbuf_t buf, http_parse_ctx *parse_ctx)
341 char buf_crlf[strlen(HTTP_LINE_TERM)];
342 rbuf_pop(buf, buf_crlf, strlen(HTTP_LINE_TERM));
343 if (memcmp(buf_crlf, HTTP_LINE_TERM, strlen(HTTP_LINE_TERM))) {
220 - netdata_log_error("CRLF expected");
344 + netdata_log_error("ACLK: CRLF expected");
345 return HTTP_PARSE_ERROR;
346 }
347 if (parse_ctx->chunked_content_state == CHUNKED_CONTENT_FINAL_CRLF) {
348 if (parse_ctx->chunked_response_size != parse_ctx->chunked_response_written)
225 - netdata_log_error("Chunked response size mismatch");
349 + netdata_log_error("ACLK: Chunked response size mismatch");
350 chunked_response_buffer_grow_by(parse_ctx, 1);
351 parse_ctx->chunked_response[parse_ctx->chunked_response_written] = 0;
352 return HTTP_PARSE_SUCCESS;
@@ -252,28 +376,28 @@ http_parse_rc parse_http_response(rbuf_t buf, http_parse_ctx *parse_ctx)
376 if (parse_ctx->state == HTTP_PARSE_PROXY_CONNECT) {
377 if (rbuf_memcmp_n(buf, RESP_PROTO10, strlen(RESP_PROTO10))) {
378 netdata_log_error(
255 - "Expected response to start with \"%s\" or \"%s\"", RESP_PROTO, RESP_PROTO10);
379 + "ACLK: Expected response to start with \"%s\" or \"%s\"", RESP_PROTO, RESP_PROTO10);
380 return HTTP_PARSE_ERROR;
381 }
382 }
383 else {
260 - netdata_log_error("Expected response to start with \"%s\"", RESP_PROTO);
384 + netdata_log_error("ACLK: Expected response to start with \"%s\"", RESP_PROTO);
385 return HTTP_PARSE_ERROR;
386 }
387 }
388 rbuf_bump_tail(buf, strlen(RESP_PROTO));
389 if (rbuf_pop(buf, rc, 4) != 4) {
266 - netdata_log_error("Expected HTTP status code");
390 + netdata_log_error("ACLK: Expected HTTP status code");
391 return HTTP_PARSE_ERROR;
392 }
393 if (rc[3] != ' ') {
270 - netdata_log_error("Expected space after HTTP return code");
394 + netdata_log_error("ACLK: Expected space after HTTP return code");
395 return HTTP_PARSE_ERROR;
396 }
397 rc[3] = 0;
398 parse_ctx->http_code = atoi(rc);
399 if (parse_ctx->http_code < 100 || parse_ctx->http_code >= 600) {
276 - netdata_log_error("HTTP code not in range 100 to 599");
400 + netdata_log_error("ACLK: HTTP code not in range 100 to 599");
401 return HTTP_PARSE_ERROR;
402 }
403
@@ -332,7 +456,7 @@ typedef struct https_req_ctx {
456
457 static int https_req_check_timedout(https_req_ctx_t *ctx) {
458 if (now_realtime_sec() > ctx->req_start_time + ctx->request->timeout_s) {
335 - netdata_log_error("request timed out");
459 + netdata_log_error("ACLK: request timed out");
460 return 1;
461 }
462 return 0;
@@ -366,12 +490,12 @@ static int socket_write_all(https_req_ctx_t *ctx, char *data, size_t data_len) {
490 do {
491 int ret = poll(&ctx->poll_fd, 1, POLL_TO_MS);
492 if (ret < 0) {
369 - netdata_log_error("poll error");
493 + netdata_log_error("ACLK: poll error");
494 return 1;
495 }
496 if (ret == 0) {
497 if (https_req_check_timedout(ctx)) {
374 - netdata_log_error("Poll timed out");
498 + netdata_log_error("ACLK: Poll timed out");
499 return 2;
500 }
501 continue;
@@ -381,7 +505,7 @@ static int socket_write_all(https_req_ctx_t *ctx, char *data, size_t data_len) {
505 if (ret > 0) {
506 ctx->written += ret;
507 } else if (errno != EAGAIN && errno != EWOULDBLOCK) {
384 - netdata_log_error("Error writing to socket");
508 + netdata_log_error("ACLK: Error writing to socket");
509 return 3;
510 }
511 } while (ctx->written < data_len);
@@ -396,12 +520,12 @@ static int ssl_write_all(https_req_ctx_t *ctx, char *data, size_t data_len) {
520 do {
521 int ret = poll(&ctx->poll_fd, 1, POLL_TO_MS);
522 if (ret < 0) {
399 - netdata_log_error("poll error");
523 + netdata_log_error("ACLK: poll error");
524 return 1;
525 }
526 if (ret == 0) {
527 if (https_req_check_timedout(ctx)) {
404 - netdata_log_error("Poll timed out");
528 + netdata_log_error("ACLK: Poll timed out");
529 return 2;
530 }
531 continue;
@@ -421,7 +545,7 @@ static int ssl_write_all(https_req_ctx_t *ctx, char *data, size_t data_len) {
545 ctx->poll_fd.events |= POLLOUT;
546 break;
547 default:
424 - netdata_log_error("SSL_write Err: %s", _ssl_err_tos(ret));
548 + netdata_log_error("ACLK: SSL_write Err: %s", _ssl_err_tos(ret));
549 return 3;
550 }
551 }
@@ -436,7 +560,7 @@ static inline int https_client_write_all(https_req_ctx_t *ctx, char *data, size_
560 return socket_write_all(ctx, data, data_len);
561 }
562
439 -static int read_parse_response(https_req_ctx_t *ctx) {
563 +static https_client_resp_t read_parse_response(https_req_ctx_t *ctx) {
564 int ret;
565 char *ptr;
566 size_t size;
@@ -445,13 +569,13 @@ static int read_parse_response(https_req_ctx_t *ctx) {
569 do {
570 ret = poll(&ctx->poll_fd, 1, POLL_TO_MS);
571 if (ret < 0) {
448 - netdata_log_error("poll error");
449 - return 1;
572 + netdata_log_error("ACLK: poll error");
573 + return HTTPS_CLIENT_RESP_POLL_ERROR;
574 }
575 if (ret == 0) {
576 if (https_req_check_timedout(ctx)) {
453 - netdata_log_error("Poll timed out");
454 - return 2;
577 + netdata_log_error("ACLK: poll() timed out");
578 + return HTTPS_CLIENT_RESP_TIMEOUT;
579 }
580 if (!ctx->ssl_ctx)
581 continue;
@@ -479,13 +603,14 @@ static int read_parse_response(https_req_ctx_t *ctx) {
603 ctx->poll_fd.events |= POLLOUT;
604 break;
605 default:
482 - netdata_log_error("SSL_read Err: %s", _ssl_err_tos(ret));
483 - return 3;
606 + netdata_log_error("ACLK: SSL_read() Err: %s", _ssl_err_tos(ret));
607 + return HTTPS_CLIENT_RESP_READ_ERROR;
608 }
485 - } else {
609 + }
610 + else {
611 if (errno != EAGAIN && errno != EWOULDBLOCK) {
487 - netdata_log_error("write error");
488 - return 3;
612 + netdata_log_error("ACLK: read error");
613 + return HTTPS_CLIENT_RESP_READ_ERROR;
614 }
615 ctx->poll_fd.events |= POLLIN;
616 }
@@ -494,11 +619,11 @@ static int read_parse_response(https_req_ctx_t *ctx) {
619 } while (!(ret = parse_http_response(ctx->buf_rx, &ctx->parse_ctx)));
620
621 if (ret != HTTP_PARSE_SUCCESS) {
497 - netdata_log_error("Error parsing HTTP response");
498 - return 1;
622 + netdata_log_error("ACLK: error parsing HTTP response");
623 + return HTTPS_CLIENT_RESP_PARSE_ERROR;
624 }
625
501 - return 0;
626 + return HTTPS_CLIENT_RESP_OK;
627 }
628
629 static const char *http_methods[] = {
@@ -510,15 +635,15 @@ static const char *http_methods[] = {
635
636 #define TX_BUFFER_SIZE 8192
637 #define RX_BUFFER_SIZE (TX_BUFFER_SIZE*2)
513 -static int handle_http_request(https_req_ctx_t *ctx) {
638 +static https_client_resp_t handle_http_request(https_req_ctx_t *ctx) {
639 BUFFER *hdr = buffer_create(TX_BUFFER_SIZE, &netdata_buffers_statistics.buffers_aclk);
515 - int rc = 0;
640 + https_client_resp_t rc = HTTPS_CLIENT_RESP_OK;
641
642 http_req_type_t req_type = ctx->request->request_type;
643
644 if (req_type >= HTTP_REQ_INVALID) {
520 - netdata_log_error("Unknown HTTPS request type!");
521 - rc = 1;
645 + netdata_log_error("ACLK: unknown HTTPS request type!");
646 + rc = HTTPS_CLIENT_RESP_UNKNOWN_REQUEST_TYPE;
647 goto err_exit;
648 }
649 buffer_strcat(hdr, http_methods[req_type]);
@@ -565,25 +690,25 @@ static int handle_http_request(https_req_ctx_t *ctx) {
690
691 // Send the request
692 if (https_client_write_all(ctx, hdr->buffer, hdr->len)) {
568 - netdata_log_error("Couldn't write HTTP request header into SSL connection");
569 - rc = 2;
693 + netdata_log_error("ACLK: couldn't write HTTP request header into SSL connection");
694 + rc = HTTPS_CLIENT_RESP_HEADER_WRITE_FAILED;
695 goto err_exit;
696 }
697
698 if (req_type == HTTP_REQ_POST && ctx->request->payload && ctx->request->payload_size) {
699 if (https_client_write_all(ctx, ctx->request->payload, ctx->request->payload_size)) {
575 - netdata_log_error("Couldn't write payload into SSL connection");
576 - rc = 3;
700 + netdata_log_error("ACLK: couldn't write payload into SSL connection");
701 + rc = HTTPS_CLIENT_RESP_PAYLOAD_WRITE_FAILED;
702 goto err_exit;
703 }
704 }
705
706 // Read The Response
582 - if (read_parse_response(ctx)) {
583 - netdata_log_error("Error reading or parsing response from server");
707 + rc = read_parse_response(ctx);
708 + if (rc != HTTPS_CLIENT_RESP_OK) {
709 + netdata_log_error("ACLK: error reading or parsing response from server");
710 if (ctx->parse_ctx.chunked_response)
711 freez(ctx->parse_ctx.chunked_response);
586 - rc = 4;
712 }
713
714 err_exit:
@@ -613,16 +738,17 @@ static int cert_verify_callback(int preverify_ok, X509_STORE_CTX *ctx)
738 if (!preverify_ok && err == X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT)
739 {
740 preverify_ok = 1;
616 - netdata_log_error("Self Signed Certificate Accepted as the agent was built with ACLK_SSL_ALLOW_SELF_SIGNED");
741 + netdata_log_error("ACLK: Self Signed Certificate Accepted as the agent was built with ACLK_SSL_ALLOW_SELF_SIGNED");
742 }
743 #endif
744
745 return preverify_ok;
746 }
747
623 -int https_request(https_req_t *request, https_req_response_t *response, bool *fallback_ipv4)
748 +https_client_resp_t https_request(https_req_t *request, https_req_response_t *response, bool *fallback_ipv4)
749 {
625 - int rc = 1, ret;
750 + https_client_resp_t rc;
751 + int ret;
752 char connect_port_str[PORT_STR_MAX_BYTES];
753
754 const char *connect_host = request->proxy_host ? request->proxy_host : request->host;
@@ -634,7 +760,8 @@ int https_request(https_req_t *request, https_req_response_t *response, bool *fa
760
761 ctx->buf_rx = rbuf_create(RX_BUFFER_SIZE);
762 if (!ctx->buf_rx) {
637 - netdata_log_error("Couldn't allocate buffer for RX data");
763 + rc = HTTPS_CLIENT_RESP_NO_MEM;
764 + netdata_log_error("ACLK: couldn't allocate buffer for RX data");
765 goto exit_req_ctx;
766 }
767
@@ -642,12 +769,14 @@ int https_request(https_req_t *request, https_req_response_t *response, bool *fa
769
770 ctx->sock = connect_to_this_ip46(IPPROTO_TCP, SOCK_STREAM, connect_host, 0, connect_port_str, &timeout, fallback_ipv4);
771 if (ctx->sock < 0) {
645 - netdata_log_error("Error connecting TCP socket to \"%s\"", connect_host);
772 + rc = -ctx->sock;
773 + netdata_log_error("ACLK: error connecting TCP socket to \"%s\"", connect_host);
774 goto exit_buf_rx;
775 }
776
777 if (fcntl(ctx->sock, F_SETFL, fcntl(ctx->sock, F_GETFL, 0) | O_NONBLOCK) == -1) {
650 - netdata_log_error("Error setting O_NONBLOCK to TCP socket.");
778 + rc = HTTPS_CLIENT_RESP_NONBLOCK_FAILED;
779 + netdata_log_error("ACLK: error setting O_NONBLOCK to TCP socket.");
780 goto exit_sock;
781 }
782
@@ -664,48 +793,54 @@ int https_request(https_req_t *request, https_req_response_t *response, bool *fa
793 req.proxy_username = request->proxy_username;
794 req.proxy_password = request->proxy_password;
795 ctx->request = &req;
667 - if (handle_http_request(ctx)) {
668 - netdata_log_error("Failed to CONNECT with proxy");
796 + rc = handle_http_request(ctx);
797 + if (rc != HTTPS_CLIENT_RESP_OK) {
798 + netdata_log_error("ACLK: failed to CONNECT with proxy");
799 http_parse_ctx_destroy(&ctx->parse_ctx);
800 goto exit_sock;
801 }
802 if (ctx->parse_ctx.http_code != 200) {
673 - netdata_log_error("Proxy didn't return 200 OK (got %d)", ctx->parse_ctx.http_code);
803 + rc = HTTPS_CLIENT_RESP_PROXY_NOT_200;
804 + netdata_log_error("ACLK: proxy didn't return 200 OK (got %d)", ctx->parse_ctx.http_code);
805 http_parse_ctx_destroy(&ctx->parse_ctx);
806 goto exit_sock;
807 }
808 http_parse_ctx_destroy(&ctx->parse_ctx);
678 - netdata_log_info("Proxy accepted CONNECT upgrade");
809 }
810 ctx->request = request;
811
812 ctx->ssl_ctx = netdata_ssl_create_client_ctx(0);
813 if (ctx->ssl_ctx==NULL) {
684 - netdata_log_error("Cannot allocate SSL context");
814 + rc = HTTPS_CLIENT_RESP_NO_SSL_CTX;
815 + netdata_log_error("ACLK: cannot allocate SSL context");
816 goto exit_sock;
817 }
818
819 if (!SSL_CTX_set_default_verify_paths(ctx->ssl_ctx)) {
689 - netdata_log_error("Error setting default verify paths");
820 + rc = HTTPS_CLIENT_RESP_NO_SSL_VERIFY_PATHS;
821 + netdata_log_error("ACLK: error setting default verify paths");
822 goto exit_CTX;
823 }
824 SSL_CTX_set_verify(ctx->ssl_ctx, SSL_VERIFY_PEER | SSL_VERIFY_CLIENT_ONCE, cert_verify_callback);
825
826 ctx->ssl = SSL_new(ctx->ssl_ctx);
827 if (ctx->ssl==NULL) {
696 - netdata_log_error("Cannot allocate SSL");
828 + rc = HTTPS_CLIENT_RESP_NO_SSL_NEW;
829 + netdata_log_error("ACLK: cannot allocate SSL");
830 goto exit_CTX;
831 }
832
833 if (!SSL_set_tlsext_host_name(ctx->ssl, request->host)) {
701 - netdata_log_error("Error setting TLS SNI host");
834 + rc = HTTPS_CLIENT_RESP_NO_TLS_SNI;
835 + netdata_log_error("ACLK: error setting TLS SNI host");
836 goto exit_CTX;
837 }
838
839 SSL_set_fd(ctx->ssl, ctx->sock);
840 ret = SSL_connect(ctx->ssl);
841 if (ret != -1 && ret != 1) {
708 - netdata_log_error("SSL could not connect");
842 + rc = HTTPS_CLIENT_RESP_SSL_CONNECT_FAILED;
843 + netdata_log_error("ACLK: SSL failed to connect");
844 goto exit_SSL;
845 }
846 if (ret == -1) {
@@ -713,14 +848,16 @@ int https_request(https_req_t *request, https_req_response_t *response, bool *fa
848 // consult SSL_connect documentation for details
849 int ec = SSL_get_error(ctx->ssl, ret);
850 if (ec != SSL_ERROR_WANT_READ && ec != SSL_ERROR_WANT_WRITE) {
716 - netdata_log_error("Failed to start SSL connection");
851 + rc = HTTPS_CLIENT_RESP_SSL_START_FAILED;
852 + netdata_log_error("ACLK: failed to start SSL connection");
853 goto exit_SSL;
854 }
855 }
856
857 // The actual request here
722 - if (handle_http_request(ctx)) {
723 - netdata_log_error("Couldn't process request");
858 + rc = handle_http_request(ctx);
859 + if (rc != HTTPS_CLIENT_RESP_OK) {
860 + netdata_log_error("ACLK: couldn't process request");
861 http_parse_ctx_destroy(&ctx->parse_ctx);
862 goto exit_SSL;
863 }
@@ -735,7 +872,7 @@ int https_request(https_req_t *request, https_req_response_t *response, bool *fa
872 response->payload = mallocz(response->payload_size + 1);
873 ret = rbuf_pop(ctx->buf_rx, response->payload, response->payload_size);
874 if (ret != (int)response->payload_size) {
738 - netdata_log_error("Payload size doesn't match remaining data on the buffer!");
875 + netdata_log_error("ACLK: payload size doesn't match remaining data on the buffer!");
876 response->payload_size = ret;
877 }
878 // normally we take payload as it is and copy it
@@ -746,9 +883,9 @@ int https_request(https_req_t *request, https_req_response_t *response, bool *fa
883 // only exact data without affixed 0x00
884 ((char*)response->payload)[response->payload_size] = 0; // mallocz(response->payload_size + 1);
885 }
749 - netdata_log_info("HTTPS \"%s\" request to \"%s\" finished with HTTP code: %d", http_req_type_to_str(ctx->request->request_type), ctx->request->host, response->http_code);
886 + netdata_log_info("ACLK: HTTPS \"%s\" request to \"%s\" finished with HTTP code: %d", http_req_type_to_str(ctx->request->request_type), ctx->request->host, response->http_code);
887
751 - rc = 0;
888 + rc = HTTPS_CLIENT_RESP_OK;
889
890 exit_SSL:
891 SSL_free(ctx->ssl);
@@ -776,7 +913,7 @@ static inline char *UNUSED_FUNCTION(min_non_null)(char *a, char *b) {
913 }
914
915 #define URI_PROTO_SEPARATOR "://"
779 -#define URL_PARSER_LOG_PREFIX "url_parser "
916 +#define URL_PARSER_LOG_PREFIX "ACLK: url_parser "
917
918 static int parse_host_port(url_t *url) {
919 char *ptr = strrchr(url->host, ':');
src/aclk/https_client.h
+41 -1
@@ -5,6 +5,46 @@
5
6 #include "libnetdata/libnetdata.h"
7
8 +typedef enum https_client_resp {
9 + HTTPS_CLIENT_RESP_OK = 0,
10 +
11 + // all the ND_SOCK_ERR_XXX are place here
12 +
13 + HTTPS_CLIENT_RESP_UNKNOWN_ERROR = ND_SOCK_ERR_MAX,
14 + HTTPS_CLIENT_RESP_NO_MEM,
15 + HTTPS_CLIENT_RESP_NONBLOCK_FAILED,
16 + HTTPS_CLIENT_RESP_PROXY_NOT_200,
17 + HTTPS_CLIENT_RESP_NO_SSL_CTX,
18 + HTTPS_CLIENT_RESP_NO_SSL_VERIFY_PATHS,
19 + HTTPS_CLIENT_RESP_NO_SSL_NEW,
20 + HTTPS_CLIENT_RESP_NO_TLS_SNI,
21 + HTTPS_CLIENT_RESP_SSL_CONNECT_FAILED,
22 + HTTPS_CLIENT_RESP_SSL_START_FAILED,
23 + HTTPS_CLIENT_RESP_UNKNOWN_REQUEST_TYPE,
24 + HTTPS_CLIENT_RESP_HEADER_WRITE_FAILED,
25 + HTTPS_CLIENT_RESP_PAYLOAD_WRITE_FAILED,
26 + HTTPS_CLIENT_RESP_POLL_ERROR,
27 + HTTPS_CLIENT_RESP_TIMEOUT,
28 + HTTPS_CLIENT_RESP_READ_ERROR,
29 + HTTPS_CLIENT_RESP_PARSE_ERROR,
30 + HTTPS_CLIENT_RESP_ENV_AGENT_NOT_CLAIMED,
31 + HTTPS_CLIENT_RESP_ENV_NOT_200,
32 + HTTPS_CLIENT_RESP_ENV_EMPTY,
33 + HTTPS_CLIENT_RESP_ENV_NOT_JSON,
34 + HTTPS_CLIENT_RESP_OTP_CHALLENGE_NOT_200,
35 + HTTPS_CLIENT_RESP_OTP_CHALLENGE_INVALID,
36 + HTTPS_CLIENT_RESP_OTP_PASSWORD_NOT_201,
37 + HTTPS_CLIENT_RESP_OTP_PASSWORD_EMPTY,
38 + HTTPS_CLIENT_RESP_OTP_PASSWORD_NOT_JSON,
39 + HTTPS_CLIENT_RESP_OTP_AGENT_NOT_CLAIMED,
40 + HTTPS_CLIENT_RESP_OTP_CHALLENGE_DECRYPTION_FAILED,
41 +
42 + // terminator
43 + HTTPS_CLIENT_RESP_MAX,
44 +} https_client_resp_t;
45 +
46 +ENUM_STR_DEFINE_FUNCTIONS_EXTERN(https_client_resp_t);
47 +
48 typedef enum http_req_type {
49 HTTP_REQ_GET = 0,
50 HTTP_REQ_POST,
@@ -75,7 +115,7 @@ void https_req_response_free(https_req_response_t *res);
115 .proxy_port = 8080 \
116 }
117
78 -int https_request(https_req_t *request, https_req_response_t *response, bool *fallback_ipv4);
118 +https_client_resp_t https_request(https_req_t *request, https_req_response_t *response, bool *fallback_ipv4);
119
120 // we expose previously internal parser as this is usefull also from
121 // other parts of the code
src/aclk/mqtt_websockets/aclk_mqtt_workers.h
+1
@@ -37,5 +37,6 @@
37 #define WORKER_ACLK_CPT_UNKNOWN 31
38 #define WORKER_ACLK_SEND_FRAGMENT 32
39 #define WORKER_ACLK_MSG_CALLBACK 33
40 +#define WORKER_ACLK_WAITING_TO_CONNECT 34
41
42 #endif //NETDATA_ACLK_MQTT_WORKERS_H
src/aclk/mqtt_websockets/mqtt_wss_client.c
+18 -3
@@ -816,10 +816,10 @@ int mqtt_wss_service(mqtt_wss_client client, int timeout_ms)
816
817 if (errno == EINTR) {
818 nd_log(NDLS_DAEMON, NDLP_WARNING, "poll interrupted by EINTR");
819 - return 0;
819 + return MQTT_WSS_OK;
820 }
821 nd_log(NDLS_DAEMON, NDLP_ERR, "poll error \"%s\"", strerror(errno));
822 - return -2;
822 + return MQTT_WSS_ERR_POLL_FAILED;
823 }
824 worker_is_busy(WORKER_ACLK_POLL_OK);
825
@@ -842,7 +842,7 @@ int mqtt_wss_service(mqtt_wss_client client, int timeout_ms)
842 }
843 // if poll timed out and user requested timeout was being used
844 // return here let user do his work and he will call us back soon
845 - return 0;
845 + return MQTT_WSS_OK;
846 }
847 }
848
@@ -865,12 +865,20 @@ int mqtt_wss_service(mqtt_wss_client client, int timeout_ms)
865 int errnobkp = errno;
866 ret = SSL_get_error(client->ssl, ret);
867 set_socket_pollfds(client, ret);
868 +
869 if (ret != SSL_ERROR_WANT_READ &&
870 ret != SSL_ERROR_WANT_WRITE) {
871 worker_is_busy(WORKER_ACLK_RX_ERROR);
872 nd_log(NDLS_DAEMON, NDLP_ERR, "SSL_read error: %d %s", ret, util_openssl_ret_err(ret));
873 +
874 + if (ret == SSL_ERROR_ZERO_RETURN) {
875 + nd_log(NDLS_DAEMON, NDLP_ERR, "SSL_read connection closed by remote end");
876 + return MQTT_WSS_ERR_REMOTE_CLOSED;
877 + }
878 +
879 if (ret == SSL_ERROR_SYSCALL)
880 nd_log(NDLS_DAEMON, NDLP_ERR, "SSL_read SYSCALL errno: %d %s", errnobkp, strerror(errnobkp));
881 +
882 return MQTT_WSS_ERR_CONN_DROP;
883 }
884 }
@@ -935,8 +943,15 @@ int mqtt_wss_service(mqtt_wss_client client, int timeout_ms)
943 ret != SSL_ERROR_WANT_WRITE) {
944 worker_is_busy(WORKER_ACLK_TX_ERROR);
945 nd_log(NDLS_DAEMON, NDLP_ERR, "SSL_write error: %d %s", ret, util_openssl_ret_err(ret));
946 +
947 + if (ret == SSL_ERROR_ZERO_RETURN) {
948 + nd_log(NDLS_DAEMON, NDLP_ERR, "SSL_write connection closed by remote end");
949 + return MQTT_WSS_ERR_REMOTE_CLOSED;
950 + }
951 +
952 if (ret == SSL_ERROR_SYSCALL)
953 nd_log(NDLS_DAEMON, NDLP_ERR, "SSL_write SYSCALL errno: %d %s", errnobkp, strerror(errnobkp));
954 +
955 return MQTT_WSS_ERR_CONN_DROP;
956 }
957 }
src/aclk/mqtt_websockets/mqtt_wss_client.h
+2
@@ -15,6 +15,8 @@
15 #define MQTT_WSS_ERR_MSG_TOO_BIG -6 // Message size too big for server
16 #define MQTT_WSS_ERR_CANT_DO -8 // if client was initialized with MQTT 3 but MQTT 5 feature
17 // was requested by user of library
18 +#define MQTT_WSS_ERR_POLL_FAILED -9
19 +#define MQTT_WSS_ERR_REMOTE_CLOSED -10
20
21 typedef struct mqtt_wss_client_struct *mqtt_wss_client;
22
src/claim/cloud-status.c
+1
@@ -124,6 +124,7 @@ CLOUD_STATUS buffer_json_cloud_status(BUFFER *wb, time_t now_s) {
124 CLAIM_ID claim_id = rrdhost_claim_id_get(localhost);
125 buffer_json_member_add_string(wb, "claim_id", claim_id.str);
126 buffer_json_member_add_string(wb, "url", cloud_config_url_get());
127 + buffer_json_member_add_string(wb, "reason", cloud_status_aclk_offline_reason());
128 break;
129 }
130 }
src/collectors/systemd-journal.plugin/systemd-journal-annotations.c
+1
@@ -617,6 +617,7 @@ static void netdata_systemd_journal_message_ids_init(void) {
617 msgid_into_dict("23e93dfccbf64e11aac858b9410d8a82", "Netdata fatal message");
618 msgid_into_dict("8ddaf5ba33a74078b609250db1e951f3", "Sensor state transition");
619 msgid_into_dict("ec87a56120d5431bace51e2fb8bba243", "Netdata log flood protection");
620 + msgid_into_dict("acb33cb95778476baac702eb7e4e151d", "Netdata Cloud connection");
621 }
622
623 void netdata_systemd_journal_transform_message_id(FACETS *facets __maybe_unused, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope __maybe_unused, void *data __maybe_unused) {
src/daemon/pulse/pulse-parents.c new
+422
@@ -0,0 +1,422 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#define PULSE_INTERNALS
4 +#include "pulse.h"
5 +
6 +DEFINE_JUDYL_TYPED(PHOST, PULSE_HOST_STATUS);
7 +
8 +// --------------------------------------------------------------------------------------------------------------------
9 +// parents
10 +
11 +struct by_reason {
12 + size_t counters[STREAM_HANDSHAKE_NEGATIVE_MAX + 3];
13 + RRDSET *st;
14 + RRDDIM *rd[STREAM_HANDSHAKE_NEGATIVE_MAX + 3];
15 +};
16 +
17 +#define STREAM_HANDSHAKE_STREAM_INFO (STREAM_HANDSHAKE_NEGATIVE_MAX)
18 +#define STREAM_HANDSHAKE_CONNECT (STREAM_HANDSHAKE_NEGATIVE_MAX + 1)
19 +#define STREAM_HANDSHAKE_OTHER (STREAM_HANDSHAKE_NEGATIVE_MAX + 2)
20 +
21 +struct {
22 + SPINLOCK spinlock;
23 + PHOST_JudyLSet index;
24 +
25 + struct {
26 + // counters
27 + struct by_reason events_by_reason;
28 + struct by_reason disconnects_by_reason;
29 +
30 + // gauges
31 + ssize_t nodes_local;
32 + ssize_t nodes_virtual;
33 + ssize_t nodes_loading;
34 + ssize_t nodes_archived;
35 + ssize_t nodes_offline;
36 + ssize_t nodes_waiting;
37 + ssize_t nodes_replicating;
38 + ssize_t nodes_running;
39 + } parent;
40 +
41 + struct {
42 + // counters
43 + struct by_reason stream_info_failed_by_reason;
44 + struct by_reason events_by_reason;
45 + struct by_reason disconnects_by_reason;
46 +
47 + // gauges
48 + ssize_t nodes_offline;
49 + ssize_t nodes_connecting;
50 + ssize_t nodes_pending;
51 + ssize_t nodes_waiting;
52 + ssize_t nodes_replicating;
53 + ssize_t nodes_running;
54 + ssize_t nodes_no_dst;
55 + } sender;
56 +
57 +} p = { 0 };
58 +
59 +static PULSE_HOST_STATUS pulse_host_detect_receiver_status(RRDHOST *host) {
60 + RRDHOST_STATUS status = { 0 };
61 + rrdhost_status(host, now_realtime_sec(), &status);
62 +
63 + PULSE_HOST_STATUS rc = 0;
64 +
65 + if(status.db.status == RRDHOST_DB_STATUS_INITIALIZING || status.ingest.status == RRDHOST_INGEST_STATUS_INITIALIZING)
66 + rc = PULSE_HOST_STATUS_LOADING;
67 +
68 + else if(status.ingest.type == RRDHOST_INGEST_TYPE_LOCALHOST)
69 + rc = PULSE_HOST_STATUS_LOCAL;
70 +
71 + else if(status.ingest.type == RRDHOST_INGEST_TYPE_VIRTUAL)
72 + rc = PULSE_HOST_STATUS_VIRTUAL;
73 +
74 + else if(status.ingest.status == RRDHOST_INGEST_STATUS_ARCHIVED)
75 + rc = PULSE_HOST_STATUS_ARCHIVED;
76 +
77 + else if(status.ingest.status == RRDHOST_INGEST_STATUS_REPLICATING)
78 + rc = PULSE_HOST_STATUS_RCV_REPLICATING;
79 +
80 + else if(status.ingest.status == RRDHOST_INGEST_STATUS_OFFLINE)
81 + rc = PULSE_HOST_STATUS_RCV_OFFLINE;
82 +
83 + else if(status.ingest.status == RRDHOST_INGEST_STATUS_ONLINE)
84 + rc = PULSE_HOST_STATUS_RCV_RUNNING;
85 +
86 + return rc;
87 +}
88 +
89 +static void update_reason(struct by_reason *b, STREAM_HANDSHAKE reason) {
90 + int r = reason;
91 +
92 + if(r >= 0)
93 + r = 0;
94 + else if(r > -STREAM_HANDSHAKE_NEGATIVE_MAX)
95 + r = -reason;
96 + else
97 + r = STREAM_HANDSHAKE_NEGATIVE_MAX;
98 +
99 + __atomic_add_fetch(&b->counters[r], 1, __ATOMIC_RELAXED);
100 +}
101 +
102 +static void pulse_host_add_sub_status(PULSE_HOST_STATUS status, ssize_t val, STREAM_HANDSHAKE reason) {
103 + while(status) {
104 + PULSE_HOST_STATUS s = 1 << (__builtin_ffs(status) - 1);
105 + status &= ~s;
106 +
107 + bool do_parent_reason = false, do_sender_reason = false;
108 +
109 + switch(s) {
110 + default:
111 + break;
112 +
113 + case PULSE_HOST_STATUS_LOCAL:
114 + __atomic_add_fetch(&p.parent.nodes_local, val, __ATOMIC_RELAXED);
115 + break;
116 +
117 + case PULSE_HOST_STATUS_VIRTUAL:
118 + __atomic_add_fetch(&p.parent.nodes_virtual, val, __ATOMIC_RELAXED);
119 + break;
120 +
121 + case PULSE_HOST_STATUS_LOADING:
122 + __atomic_add_fetch(&p.parent.nodes_loading, val, __ATOMIC_RELAXED);
123 + break;
124 +
125 + case PULSE_HOST_STATUS_ARCHIVED:
126 + __atomic_add_fetch(&p.parent.nodes_archived, val, __ATOMIC_RELAXED);
127 + break;
128 +
129 + case PULSE_HOST_STATUS_RCV_OFFLINE:
130 + __atomic_add_fetch(&p.parent.nodes_offline, val, __ATOMIC_RELAXED);
131 + do_parent_reason = true;
132 + break;
133 +
134 + case PULSE_HOST_STATUS_RCV_WAITING:
135 + __atomic_add_fetch(&p.parent.nodes_waiting, val, __ATOMIC_RELAXED);
136 + do_parent_reason = true;
137 + reason = 0;
138 + break;
139 +
140 + case PULSE_HOST_STATUS_RCV_REPLICATING:
141 + __atomic_add_fetch(&p.parent.nodes_replicating, val, __ATOMIC_RELAXED);
142 + break;
143 +
144 + case PULSE_HOST_STATUS_RCV_RUNNING:
145 + __atomic_add_fetch(&p.parent.nodes_running, val, __ATOMIC_RELAXED);
146 + break;
147 +
148 + case PULSE_HOST_STATUS_SND_OFFLINE:
149 + __atomic_add_fetch(&p.sender.nodes_offline, val, __ATOMIC_RELAXED);
150 + do_sender_reason = true;
151 + break;
152 +
153 + case PULSE_HOST_STATUS_SND_PENDING:
154 + __atomic_add_fetch(&p.sender.nodes_pending, val, __ATOMIC_RELAXED);
155 + break;
156 +
157 + case PULSE_HOST_STATUS_SND_CONNECTING:
158 + __atomic_add_fetch(&p.sender.nodes_connecting, val, __ATOMIC_RELAXED);
159 + __atomic_add_fetch(&p.sender.events_by_reason.counters[STREAM_HANDSHAKE_CONNECT], 1, __ATOMIC_RELAXED);
160 + break;
161 +
162 + case PULSE_HOST_STATUS_SND_WAITING:
163 + __atomic_add_fetch(&p.sender.nodes_waiting, val, __ATOMIC_RELAXED);
164 + break;
165 +
166 + case PULSE_HOST_STATUS_SND_REPLICATING:
167 + __atomic_add_fetch(&p.sender.nodes_replicating, val, __ATOMIC_RELAXED);
168 + break;
169 +
170 + case PULSE_HOST_STATUS_SND_RUNNING:
171 + __atomic_add_fetch(&p.sender.nodes_running, val, __ATOMIC_RELAXED);
172 + break;
173 +
174 + case PULSE_HOST_STATUS_SND_NO_DST:
175 + __atomic_add_fetch(&p.sender.nodes_no_dst, val, __ATOMIC_RELAXED);
176 + break;
177 + }
178 +
179 + if(do_parent_reason && val > 0)
180 + update_reason(&p.parent.disconnects_by_reason, reason);
181 +
182 + if(do_sender_reason && val > 0)
183 + update_reason(&p.sender.disconnects_by_reason, reason);
184 + }
185 +}
186 +
187 +void pulse_host_status(RRDHOST *host, PULSE_HOST_STATUS status, STREAM_HANDSHAKE reason) {
188 + PULSE_HOST_STATUS remove = 0;
189 +
190 + if(!status)
191 + status = pulse_host_detect_receiver_status(host);
192 +
193 + PULSE_HOST_STATUS basic = PULSE_HOST_STATUS_LOCAL|PULSE_HOST_STATUS_VIRTUAL| PULSE_HOST_STATUS_LOADING |PULSE_HOST_STATUS_ARCHIVED|PULSE_HOST_STATUS_DELETED;
194 + PULSE_HOST_STATUS rcv = PULSE_HOST_STATUS_RCV_OFFLINE|PULSE_HOST_STATUS_RCV_WAITING|PULSE_HOST_STATUS_RCV_REPLICATING|PULSE_HOST_STATUS_RCV_RUNNING;
195 + PULSE_HOST_STATUS snd = PULSE_HOST_STATUS_SND_OFFLINE|PULSE_HOST_STATUS_SND_PENDING|PULSE_HOST_STATUS_SND_CONNECTING|PULSE_HOST_STATUS_SND_WAITING|PULSE_HOST_STATUS_SND_REPLICATING|PULSE_HOST_STATUS_SND_RUNNING|PULSE_HOST_STATUS_SND_NO_DST;
196 +
197 + if(status & basic)
198 + remove = ~0;
199 + else if(status & rcv)
200 + remove = basic | rcv;
201 + else if(status & snd)
202 + remove = snd;
203 +
204 + spinlock_lock(&p.spinlock);
205 + PULSE_HOST_STATUS old = PHOST_GET(&p.index, (uintptr_t)host);
206 + if(status == PULSE_HOST_STATUS_DELETED)
207 + PHOST_DEL(&p.index, (uintptr_t)host);
208 + else
209 + PHOST_SET(&p.index, (uintptr_t)host, (old & ~remove) | status);
210 + spinlock_unlock(&p.spinlock);
211 +
212 + remove &= old;
213 + pulse_host_add_sub_status(remove, -1, 0);
214 +
215 + if(status != PULSE_HOST_STATUS_DELETED)
216 + pulse_host_add_sub_status(status, 1, reason);
217 +}
218 +
219 +void pulse_parent_stream_info_received_request(void) {
220 + __atomic_add_fetch(&p.parent.events_by_reason.counters[STREAM_HANDSHAKE_STREAM_INFO], 1, __ATOMIC_RELAXED);
221 +}
222 +
223 +void pulse_parent_receiver_request(void) {
224 + __atomic_add_fetch(&p.parent.events_by_reason.counters[STREAM_HANDSHAKE_CONNECT], 1, __ATOMIC_RELAXED);
225 +}
226 +
227 +void pulse_parent_receiver_rejected(STREAM_HANDSHAKE reason) {
228 + update_reason(&p.parent.events_by_reason, reason);
229 +}
230 +
231 +// --------------------------------------------------------------------------------------------------------------------
232 +// children / senders
233 +
234 +void pulse_stream_info_sent_request(void) {
235 + __atomic_add_fetch(&p.sender.events_by_reason.counters[STREAM_HANDSHAKE_STREAM_INFO], 1, __ATOMIC_RELAXED);
236 +}
237 +
238 +void pulse_sender_stream_info_failed(const char *destination __maybe_unused, STREAM_HANDSHAKE reason) {
239 + update_reason(&p.sender.stream_info_failed_by_reason, reason);
240 +}
241 +
242 +void pulse_sender_connection_failed(const char *destination __maybe_unused, STREAM_HANDSHAKE reason) {
243 + update_reason(&p.sender.events_by_reason, reason);
244 +}
245 +
246 +// --------------------------------------------------------------------------------------------------------------------
247 +
248 +static void chart_by_reason(struct by_reason *b, const char *id, const char *context, const char *title, int priority) {
249 + if(!b->st) {
250 + b->st = rrdset_create_localhost(
251 + "netdata"
252 + , id
253 + , NULL
254 + , "Streaming"
255 + , context
256 + , title
257 + , "events"
258 + , "netdata"
259 + , "pulse"
260 + , priority
261 + , localhost->rrd_update_every
262 + , RRDSET_TYPE_LINE
263 + );
264 +
265 + for(size_t i = 0; i < STREAM_HANDSHAKE_NEGATIVE_MAX ;i++) {
266 + char buf[1024];
267 + if(!i)
268 + strncpyz(buf, "connected", sizeof(buf) - 1);
269 + else
270 + strncpyz(buf, stream_handshake_error_to_string(-i), sizeof(buf) - 1);
271 + for(int c = 0; buf[c] ;c++)
272 + buf[c] = (char)tolower(buf[c]);
273 +
274 + b->rd[i] = rrddim_add(b->st, buf, NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
275 + }
276 + b->rd[STREAM_HANDSHAKE_STREAM_INFO] = rrddim_add(b->st, "info", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
277 + b->rd[STREAM_HANDSHAKE_CONNECT] = rrddim_add(b->st, "connect", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
278 + b->rd[STREAM_HANDSHAKE_OTHER] = rrddim_add(b->st, "other", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
279 + }
280 +
281 + for(size_t i = 0; i <= STREAM_HANDSHAKE_OTHER ;i++)
282 + rrddim_set_by_pointer(b->st, b->rd[i], (collected_number)__atomic_load_n(&b->counters[i], __ATOMIC_RELAXED));
283 +
284 + rrdset_done(b->st);
285 +}
286 +
287 +void pulse_parents_do(bool extended) {
288 + if(netdata_conf_is_parent()) {
289 + {
290 + static RRDSET *st_nodes = NULL;
291 + static RRDDIM *rd_loading = NULL;
292 + static RRDDIM *rd_local = NULL;
293 + static RRDDIM *rd_virtual = NULL;
294 + static RRDDIM *rd_archived = NULL;
295 + static RRDDIM *rd_offline = NULL;
296 + static RRDDIM *rd_waiting = NULL;
297 + static RRDDIM *rd_replicating = NULL;
298 + static RRDDIM *rd_running = NULL;
299 +
300 + if (unlikely(!st_nodes)) {
301 + st_nodes = rrdset_create_localhost(
302 + "netdata"
303 + , "streaming_inbound"
304 + , NULL
305 + , "Streaming"
306 + , "netdata.streaming_inbound"
307 + , "Inbound Nodes"
308 + , "nodes"
309 + , "netdata"
310 + , "pulse"
311 + , 130150
312 + , localhost->rrd_update_every
313 + , RRDSET_TYPE_LINE
314 + );
315 +
316 + rd_local = rrddim_add(st_nodes, "local", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
317 + rd_virtual = rrddim_add(st_nodes, "virtual", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
318 + rd_loading = rrddim_add(st_nodes, "loading", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
319 + rd_archived = rrddim_add(st_nodes, "stale", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
320 + rd_offline = rrddim_add(st_nodes, "offline", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
321 + rd_waiting = rrddim_add(st_nodes, "waiting", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
322 + rd_replicating = rrddim_add(st_nodes, "replicating", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
323 + rd_running = rrddim_add(st_nodes, "running", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
324 + }
325 +
326 + rrddim_set_by_pointer(st_nodes, rd_local, (collected_number)__atomic_load_n(&p.parent.nodes_local, __ATOMIC_RELAXED));
327 + rrddim_set_by_pointer(st_nodes, rd_virtual, (collected_number)__atomic_load_n(&p.parent.nodes_virtual, __ATOMIC_RELAXED));
328 + rrddim_set_by_pointer(st_nodes,rd_loading, (collected_number)__atomic_load_n(&p.parent.nodes_loading, __ATOMIC_RELAXED));
329 + rrddim_set_by_pointer(st_nodes, rd_archived, (collected_number)__atomic_load_n(&p.parent.nodes_archived, __ATOMIC_RELAXED));
330 + rrddim_set_by_pointer(st_nodes, rd_offline, (collected_number)__atomic_load_n(&p.parent.nodes_offline, __ATOMIC_RELAXED));
331 + rrddim_set_by_pointer(st_nodes, rd_waiting, (collected_number)__atomic_load_n(&p.parent.nodes_waiting, __ATOMIC_RELAXED));
332 + rrddim_set_by_pointer(st_nodes, rd_replicating, (collected_number)__atomic_load_n(&p.parent.nodes_replicating, __ATOMIC_RELAXED));
333 + rrddim_set_by_pointer(st_nodes, rd_running, (collected_number)__atomic_load_n(&p.parent.nodes_running, __ATOMIC_RELAXED));
334 +
335 + rrdset_done(st_nodes);
336 + }
337 +
338 + if(extended) {
339 + chart_by_reason(
340 + &p.parent.events_by_reason,
341 + "streaming_rejections_inbound",
342 + "netdata.streaming_events_inbound",
343 + "Inbound Streaming Events",
344 + 130151);
345 + chart_by_reason(
346 + &p.parent.disconnects_by_reason,
347 + "streaming_disconnects_inbound",
348 + "netdata.streaming_events_inbound",
349 + "Inbound Streaming Events",
350 + 130151);
351 + }
352 + }
353 +
354 + if(stream_conf_is_child()) {
355 + {
356 + static RRDSET *st_nodes = NULL;
357 + static RRDDIM *rd_pending = NULL;
358 + static RRDDIM *rd_connecting = NULL;
359 + static RRDDIM *rd_offline = NULL;
360 + static RRDDIM *rd_waiting = NULL;
361 + static RRDDIM *rd_replicating = NULL;
362 + static RRDDIM *rd_running = NULL;
363 + static RRDDIM *rd_no_dst = NULL;
364 +
365 + if (unlikely(!st_nodes)) {
366 + st_nodes = rrdset_create_localhost(
367 + "netdata"
368 + , "streaming_outbound"
369 + , NULL
370 + , "Streaming"
371 + , "netdata.streaming_outbound"
372 + , "Outbound Nodes"
373 + , "nodes"
374 + , "netdata"
375 + , "pulse"
376 + , 130151
377 + , localhost->rrd_update_every
378 + , RRDSET_TYPE_LINE
379 + );
380 +
381 + rd_connecting = rrddim_add(st_nodes, "connecting", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
382 + rd_pending = rrddim_add(st_nodes, "pending", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
383 + rd_offline = rrddim_add(st_nodes, "offline", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
384 + rd_waiting = rrddim_add(st_nodes, "waiting", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
385 + rd_replicating = rrddim_add(st_nodes, "replicating", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
386 + rd_running = rrddim_add(st_nodes, "running", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
387 + rd_no_dst = rrddim_add(st_nodes, "no dst", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
388 + }
389 +
390 + rrddim_set_by_pointer(st_nodes, rd_connecting, (collected_number)__atomic_load_n(&p.sender.nodes_connecting, __ATOMIC_RELAXED));
391 + rrddim_set_by_pointer(st_nodes, rd_pending, (collected_number)__atomic_load_n(&p.sender.nodes_pending, __ATOMIC_RELAXED));
392 + rrddim_set_by_pointer(st_nodes, rd_offline, (collected_number)__atomic_load_n(&p.sender.nodes_offline, __ATOMIC_RELAXED));
393 + rrddim_set_by_pointer(st_nodes, rd_waiting, (collected_number)__atomic_load_n(&p.sender.nodes_waiting, __ATOMIC_RELAXED));
394 + rrddim_set_by_pointer(st_nodes, rd_replicating, (collected_number)__atomic_load_n(&p.sender.nodes_replicating, __ATOMIC_RELAXED));
395 + rrddim_set_by_pointer(st_nodes, rd_running, (collected_number)__atomic_load_n(&p.sender.nodes_running, __ATOMIC_RELAXED));
396 + rrddim_set_by_pointer(st_nodes, rd_no_dst, (collected_number)__atomic_load_n(&p.sender.nodes_no_dst, __ATOMIC_RELAXED));
397 +
398 + rrdset_done(st_nodes);
399 + }
400 +
401 + if(extended) {
402 + chart_by_reason(
403 + &p.sender.stream_info_failed_by_reason,
404 + "streaming_info_failed_outbound",
405 + "netdata.streaming_events_outbound",
406 + "Outbound Streaming Events",
407 + 130152);
408 + chart_by_reason(
409 + &p.sender.events_by_reason,
410 + "streaming_rejections_outbound",
411 + "netdata.streaming_events_outbound",
412 + "Outbound Streaming Events",
413 + 130152);
414 + chart_by_reason(
415 + &p.sender.disconnects_by_reason,
416 + "streaming_disconnects_outbound",
417 + "netdata.streaming_events_outbound",
418 + "Outbound Streaming Events",
419 + 130152);
420 + }
421 + }
422 +}
src/daemon/pulse/pulse-parents.h new
+44
@@ -0,0 +1,44 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_PULSE_PARENTS_H
4 +#define NETDATA_PULSE_PARENTS_H
5 +
6 +#include "libnetdata/libnetdata.h"
7 +#include "streaming/stream-handshake.h"
8 +
9 +typedef enum {
10 + PULSE_HOST_STATUS_NONE = 0,
11 + PULSE_HOST_STATUS_LOCAL = (1 << 0),
12 + PULSE_HOST_STATUS_VIRTUAL = (1 << 1),
13 + PULSE_HOST_STATUS_LOADING = (1 << 2),
14 + PULSE_HOST_STATUS_ARCHIVED = (1 << 3),
15 + PULSE_HOST_STATUS_RCV_OFFLINE = (1 << 4),
16 + PULSE_HOST_STATUS_RCV_WAITING = (1 << 5),
17 + PULSE_HOST_STATUS_RCV_REPLICATING = (1 << 6),
18 + PULSE_HOST_STATUS_RCV_RUNNING = (1 << 7),
19 + PULSE_HOST_STATUS_SND_OFFLINE = (1 << 8),
20 + PULSE_HOST_STATUS_SND_PENDING = (1 << 9),
21 + PULSE_HOST_STATUS_SND_CONNECTING = (1 << 10),
22 + PULSE_HOST_STATUS_SND_NO_DST = (1 << 11),
23 + PULSE_HOST_STATUS_SND_WAITING = (1 << 12),
24 + PULSE_HOST_STATUS_SND_REPLICATING = (1 << 13),
25 + PULSE_HOST_STATUS_SND_RUNNING = (1 << 14),
26 + PULSE_HOST_STATUS_DELETED = (1 << 15),
27 +} PULSE_HOST_STATUS;
28 +
29 +void pulse_host_status(RRDHOST *host, PULSE_HOST_STATUS status, STREAM_HANDSHAKE reason);
30 +
31 +// receiver events
32 +
33 +void pulse_parent_stream_info_received_request(void);
34 +void pulse_parent_receiver_request(void);
35 +void pulse_parent_receiver_rejected(STREAM_HANDSHAKE reason);
36 +
37 +// sender
38 +void pulse_stream_info_sent_request(void);
39 +void pulse_sender_stream_info_failed(const char *destination __maybe_unused, STREAM_HANDSHAKE reason);
40 +void pulse_sender_connection_failed(const char *destination __maybe_unused, STREAM_HANDSHAKE reason);
41 +
42 +void pulse_parents_do(bool extended);
43 +
44 +#endif //NETDATA_PULSE_PARENTS_H
src/daemon/pulse/pulse.c
+6 -1
@@ -19,8 +19,9 @@
19 #define WORKER_JOB_REGISTRY 13
20 #define WORKER_JOB_ARAL 14
21 #define WORKER_JOB_NETWORK 15
22 +#define WORKER_JOB_PARENTS 16
23
23 -#if WORKER_UTILIZATION_MAX_JOB_TYPES < 16
24 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 17
25 #error "WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 14"
26 #endif
27
@@ -46,6 +47,7 @@ static void pulse_register_workers(void) {
47 worker_register_job_name(WORKER_JOB_REGISTRY, "registry");
48 worker_register_job_name(WORKER_JOB_ARAL, "aral");
49 worker_register_job_name(WORKER_JOB_NETWORK, "network");
50 + worker_register_job_name(WORKER_JOB_PARENTS, "parents");
51 }
52
53 static void pulse_cleanup(void *pptr)
@@ -144,6 +146,9 @@ void *pulse_thread_main(void *ptr) {
146 worker_is_busy(WORKER_JOB_ARAL);
147 pulse_aral_do(pulse_extended_enabled);
148
149 + worker_is_busy(WORKER_JOB_PARENTS);
150 + pulse_parents_do(pulse_extended_enabled);
151 +
152 // keep this last to have access to the memory counters
153 // exposed by everyone else
154 worker_is_busy(WORKER_JOB_DAEMON);
src/daemon/pulse/pulse.h
+1
@@ -26,6 +26,7 @@ extern bool pulse_extended_enabled;
26 #include "pulse-trace-allocations.h"
27 #include "pulse-aral.h"
28 #include "pulse-network.h"
29 +#include "pulse-parents.h"
30
31 void *pulse_thread_main(void *ptr);
32 void *pulse_thread_sqlite3_main(void *ptr);
src/database/rrd.c
+1
@@ -147,6 +147,7 @@ int rrd_init(const char *hostname, struct rrdhost_system_info *system_info, bool
147
148 rrdhost_flag_set(localhost, RRDHOST_FLAG_COLLECTOR_ONLINE);
149 object_state_activate(&localhost->state_id);
150 + pulse_host_status(localhost, 0, 0); // this will detect the receiver status
151
152 ml_host_start(localhost);
153 dyncfg_host_init(localhost);
src/database/rrdhost.c
+2 -1
@@ -810,7 +810,7 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
810 stream_sender_structures_free(host);
811
812 if (netdata_exit || force)
813 - stream_receiver_signal_to_stop_and_wait(host, STREAM_HANDSHAKE_DISCONNECT_HOST_CLEANUP);
813 + stream_receiver_signal_to_stop_and_wait(host, STREAM_HANDSHAKE_SND_DISCONNECT_HOST_CLEANUP);
814
815
816 // ------------------------------------------------------------------------
@@ -867,6 +867,7 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
867 string_freez(host->hostname);
868 __atomic_sub_fetch(&netdata_buffers_statistics.rrdhost_allocations_size, sizeof(RRDHOST), __ATOMIC_RELAXED);
869
870 + pulse_host_status(host, PULSE_HOST_STATUS_DELETED, 0);
871 freez(host);
872 }
873
src/database/sqlite/sqlite_aclk.c
+2
@@ -204,6 +204,8 @@ static int create_host_callback(void *data, int argc, char **argv, char **column
204
205 host->rrdlabels = sql_load_host_labels((nd_uuid_t *)argv[IDX_HOST_ID]);
206 host->stream.snd.status.last_connected = last_connected;
207 +
208 + pulse_host_status(host, 0, 0); // this will detect the receiver status
209 }
210
211 (*number_of_chidren)++;
src/database/sqlite/sqlite_metadata.c
+1
@@ -1918,6 +1918,7 @@ static void restore_host_context(void *arg)
1918 nd_log_daemon(NDLP_DEBUG, "Contexts for host %s loaded in %s", rrdhost_hostname(host), load_duration);
1919
1920 rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD);
1921 + pulse_host_status(host, 0, 0); // this will detect the receiver status
1922
1923 aclk_queue_node_info(host, false);
1924
src/libnetdata/socket/nd-sock.c
+13 -13
@@ -3,19 +3,19 @@
3 #include "libnetdata/libnetdata.h"
4
5 ENUM_STR_MAP_DEFINE(ND_SOCK_ERROR) = {
6 - { .id = ND_SOCK_ERR_NONE, .name = "NONE", },
7 - { .id = ND_SOCK_ERR_CONNECTION_REFUSED, .name = "CONNECTION REFUSED", },
8 - { .id = ND_SOCK_ERR_CANNOT_RESOLVE_HOSTNAME, .name = "CANNOT RESOLVE HOSTNAME", },
9 - { .id = ND_SOCK_ERR_FAILED_TO_CREATE_SOCKET, .name = "FAILED TO CREATE SOCKET", },
10 - { .id = ND_SOCK_ERR_NO_HOST_IN_DEFINITION, .name = "NO HOST IN DEFINITION", },
11 - { .id = ND_SOCK_ERR_POLL_ERROR, .name = "POLL ERROR", },
12 - { .id = ND_SOCK_ERR_TIMEOUT, .name = "TIMEOUT", },
13 - { .id = ND_SOCK_ERR_SSL_CANT_ESTABLISH_SSL_CONNECTION, .name = "CANT ESTABLISH SSL CONNECTION", },
14 - { .id = ND_SOCK_ERR_SSL_INVALID_CERTIFICATE, .name = "INVALID SSL CERTIFICATE", },
15 - { .id = ND_SOCK_ERR_SSL_FAILED_TO_OPEN, .name = "FAILED TO OPEN SSL", },
16 - { .id = ND_SOCK_ERR_THREAD_CANCELLED, .name = "THREAD CANCELLED", },
17 - { .id = ND_SOCK_ERR_NO_DESTINATION_AVAILABLE, .name = "NO PARENT AVAILABLE", },
18 - { .id = ND_SOCK_ERR_UNKNOWN_ERROR, .name = "UNKNOWN ERROR", },
6 + { .id = ND_SOCK_ERR_NONE, .name = "no socket error", },
7 + { .id = ND_SOCK_ERR_CONNECTION_REFUSED, .name = "connection refused", },
8 + { .id = ND_SOCK_ERR_CANNOT_RESOLVE_HOSTNAME, .name = "cannot resolve hostname", },
9 + { .id = ND_SOCK_ERR_FAILED_TO_CREATE_SOCKET, .name = "cannot create socket", },
10 + { .id = ND_SOCK_ERR_NO_HOST_IN_DEFINITION, .name = "no host in definition", },
11 + { .id = ND_SOCK_ERR_POLL_ERROR, .name = "socket poll() error", },
12 + { .id = ND_SOCK_ERR_TIMEOUT, .name = "timeout", },
13 + { .id = ND_SOCK_ERR_SSL_CANT_ESTABLISH_SSL_CONNECTION, .name = "cannot establish SSL connection", },
14 + { .id = ND_SOCK_ERR_SSL_INVALID_CERTIFICATE, .name = "invalid SSL certification", },
15 + { .id = ND_SOCK_ERR_SSL_FAILED_TO_OPEN, .name = "failed to open SSL", },
16 + { .id = ND_SOCK_ERR_THREAD_CANCELLED, .name = "thread cancelled", },
17 + { .id = ND_SOCK_ERR_NO_DESTINATION_AVAILABLE, .name = "no destination available", },
18 + { .id = ND_SOCK_ERR_UNKNOWN_ERROR, .name = "unknown error", },
19
20 // terminator
21 { .name = NULL, .id = 0 }
src/libnetdata/socket/nd-sock.h
+3
@@ -19,6 +19,9 @@ typedef enum __attribute__((packed)) {
19 ND_SOCK_ERR_THREAD_CANCELLED,
20 ND_SOCK_ERR_NO_DESTINATION_AVAILABLE,
21 ND_SOCK_ERR_UNKNOWN_ERROR,
22 +
23 + // terminator
24 + ND_SOCK_ERR_MAX,
25 } ND_SOCK_ERROR;
26
27 ENUM_STR_DEFINE_FUNCTIONS_EXTERN(ND_SOCK_ERROR);
src/libnetdata/uuid/uuid.h
+1
@@ -35,6 +35,7 @@ ND_UUID_DEFINE(health_alert_notification_msgid, 0x6d, 0xb0, 0x01, 0x8e, 0x83, 0x
35 ND_UUID_DEFINE(sensors_state_transition_msgid, 0x8d, 0xda, 0xf5, 0xba, 0x33, 0xa7, 0x40, 0x78, 0xb6, 0x09, 0x25, 0x0d, 0xb1, 0xe9, 0x51, 0xf3);
36 ND_UUID_DEFINE(log_flood_protection_msgid, 0xec, 0x87, 0xa5, 0x61, 0x20, 0xd5, 0x43, 0x1b, 0xac, 0xe5, 0x1e, 0x2f, 0xb8, 0xbb, 0xa2, 0x43);
37 ND_UUID_DEFINE(netdata_startup_msgid, 0x1e, 0x60, 0x61, 0xa9, 0xfb, 0xd4, 0x45, 0x01, 0xb3, 0xcc, 0xc3, 0x68, 0x11, 0x9f, 0x2b, 0x69);
38 +ND_UUID_DEFINE(aclk_connection_msgid, 0xac, 0xb3, 0x3c, 0xb9, 0x57, 0x78, 0x47, 0x6b, 0xaa, 0xc7, 0x02, 0xeb, 0x7e, 0x4e, 0x15, 0x1d);
39
40 ND_UUID UUID_generate_from_hash(const void *payload, size_t payload_len);
41
src/plugins.d/pluginsd_parser.c
+1
@@ -207,6 +207,7 @@ static inline PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, si
207 object_state_activate(&host->state_id);
208 ml_host_start(host);
209 dyncfg_host_init(host);
210 + pulse_host_status(host, 0, 0); // this will detect the receiver status
211
212 if(host->rrdlabels) {
213 rrdlabels_migrate_to_these(host->rrdlabels, parser->user.host_define.rrdlabels);
src/plugins.d/pluginsd_parser.h
+2 -2
@@ -9,10 +9,10 @@
9 #include "streaming/stream-receiver-internals.h"
10 #endif
11
12 -#define WORKER_PARSER_FIRST_JOB 36
12 +#define WORKER_PARSER_FIRST_JOB 35
13
14 // this has to be in-sync with the same at stream-thread.c
15 -#define WORKER_RECEIVER_JOB_REPLICATION_COMPLETION 25
15 +#define WORKER_RECEIVER_JOB_REPLICATION_COMPLETION 24
16
17 // this controls the max response size of a function
18 #define PLUGINSD_MAX_DEFERRED_SIZE (100 * 1024 * 1024)
src/plugins.d/pluginsd_replication.c
+7 -4
@@ -57,7 +57,9 @@ PARSER_RC pluginsd_chart_definition_end(char **words, size_t num_words, PARSER *
57 st, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS, RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED);
58
59 if(!(old & RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS)) {
60 - rrdhost_receiver_replicating_charts_plus_one(st->rrdhost);
60 + if(rrdhost_receiver_replicating_charts_plus_one(st->rrdhost) == 1)
61 + pulse_host_status(host, PULSE_HOST_STATUS_RCV_REPLICATING, 0);
62 +
63 __atomic_add_fetch(&host->stream.rcv.status.replication.counter_in, 1, __ATOMIC_RELAXED);
64
65 #ifdef REPLICATION_TRACKING
@@ -454,9 +456,10 @@ PARSER_RC pluginsd_replay_end(char **words, size_t num_words, PARSER *parser) {
456 st, RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED,
457 RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS | RRDSET_FLAG_SYNC_CLOCK);
458
457 - if(!(old & RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED))
458 - rrdhost_receiver_replicating_charts_minus_one(st->rrdhost);
459 -
459 + if(!(old & RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED)) {
460 + if(rrdhost_receiver_replicating_charts_minus_one(st->rrdhost) == 0)
461 + pulse_host_status(host, PULSE_HOST_STATUS_RCV_RUNNING, 0);
462 + }
463 else
464 nd_log(NDLS_DAEMON, NDLP_WARNING,
465 "PLUGINSD REPLAY ERROR: 'host:%s/chart:%s' got a " PLUGINSD_KEYWORD_REPLAY_END " "
src/streaming/protocol/command-chart-definition.c
+4 -2
@@ -117,8 +117,10 @@ bool stream_sender_send_rrdset_definition(BUFFER *wb, RRDSET *st) {
117 (unsigned long long)now);
118
119 RRDSET_FLAGS old = rrdset_flag_set_and_clear(st, RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS, RRDSET_FLAG_SENDER_REPLICATION_FINISHED);
120 - if(!(old & RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS))
121 - rrdhost_sender_replicating_charts_plus_one(st->rrdhost);
120 + if(!(old & RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS)) {
121 + if(rrdhost_sender_replicating_charts_plus_one(st->rrdhost) == 1)
122 + pulse_host_status(st->rrdhost, PULSE_HOST_STATUS_SND_REPLICATING, 0);
123 + }
124
125 replication_progress = true;
126
src/streaming/stream-connector.c
+27 -36
@@ -4,7 +4,6 @@
4
5 static struct {
6 const char *response;
7 - const char *status;
7 size_t length;
8 int32_t version;
9 bool dynamic;
@@ -16,7 +15,6 @@ static struct {
15 {
16 .response = START_STREAMING_PROMPT_VN,
17 .length = sizeof(START_STREAMING_PROMPT_VN) - 1,
19 - .status = STREAM_STATUS_CONNECTED,
18 .version = STREAM_HANDSHAKE_OK_V3, // and above
19 .dynamic = true, // dynamic = we will parse the version / capabilities
20 .error = NULL,
@@ -27,7 +25,6 @@ static struct {
25 {
26 .response = START_STREAMING_PROMPT_V2,
27 .length = sizeof(START_STREAMING_PROMPT_V2) - 1,
30 - .status = STREAM_STATUS_CONNECTED,
28 .version = STREAM_HANDSHAKE_OK_V2,
29 .dynamic = false,
30 .error = NULL,
@@ -38,7 +35,6 @@ static struct {
35 {
36 .response = START_STREAMING_PROMPT_V1,
37 .length = sizeof(START_STREAMING_PROMPT_V1) - 1,
41 - .status = STREAM_STATUS_CONNECTED,
38 .version = STREAM_HANDSHAKE_OK_V1,
39 .dynamic = false,
40 .error = NULL,
@@ -49,8 +45,7 @@ static struct {
45 {
46 .response = START_STREAMING_ERROR_SAME_LOCALHOST,
47 .length = sizeof(START_STREAMING_ERROR_SAME_LOCALHOST) - 1,
52 - .status = STREAM_STATUS_LOCALHOST,
53 - .version = STREAM_HANDSHAKE_ERROR_LOCALHOST,
48 + .version = STREAM_HANDSHAKE_PARENT_IS_LOCALHOST,
49 .dynamic = false,
50 .error = "remote server rejected this stream, the host we are trying to stream is its localhost",
51 .worker_job_id = WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_BAD_HANDSHAKE,
@@ -60,8 +55,7 @@ static struct {
55 {
56 .response = START_STREAMING_ERROR_ALREADY_STREAMING,
57 .length = sizeof(START_STREAMING_ERROR_ALREADY_STREAMING) - 1,
63 - .status = STREAM_STATUS_ALREADY_CONNECTED,
64 - .version = STREAM_HANDSHAKE_ERROR_ALREADY_CONNECTED,
58 + .version = STREAM_HANDSHAKE_PARENT_NODE_ALREADY_CONNECTED,
59 .dynamic = false,
60 .error = "remote server rejected this stream, the host we are trying to stream is already streamed to it",
61 .worker_job_id = WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_BAD_HANDSHAKE,
@@ -71,8 +65,7 @@ static struct {
65 {
66 .response = START_STREAMING_ERROR_NOT_PERMITTED,
67 .length = sizeof(START_STREAMING_ERROR_NOT_PERMITTED) - 1,
74 - .status = STREAM_STATUS_PERMISSION_DENIED,
75 - .version = STREAM_HANDSHAKE_ERROR_DENIED,
68 + .version = STREAM_HANDSHAKE_PARENT_DENIED_ACCESS,
69 .dynamic = false,
70 .error = "remote server denied access, probably we don't have the right API key?",
71 .worker_job_id = WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_BAD_HANDSHAKE,
@@ -82,8 +75,7 @@ static struct {
75 {
76 .response = START_STREAMING_ERROR_BUSY_TRY_LATER,
77 .length = sizeof(START_STREAMING_ERROR_BUSY_TRY_LATER) - 1,
85 - .status = STREAM_STATUS_RATE_LIMIT,
86 - .version = STREAM_HANDSHAKE_BUSY_TRY_LATER,
78 + .version = STREAM_HANDSHAKE_PARENT_BUSY_TRY_LATER,
79 .dynamic = false,
80 .error = "remote server is currently busy, we should try later",
81 .worker_job_id = WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_BAD_HANDSHAKE,
@@ -93,8 +85,7 @@ static struct {
85 {
86 .response = START_STREAMING_ERROR_INTERNAL_ERROR,
87 .length = sizeof(START_STREAMING_ERROR_INTERNAL_ERROR) - 1,
96 - .status = STREAM_STATUS_INTERNAL_SERVER_ERROR,
97 - .version = STREAM_HANDSHAKE_INTERNAL_ERROR,
88 + .version = STREAM_HANDSHAKE_PARENT_INTERNAL_ERROR,
89 .dynamic = false,
90 .error = "remote server is encountered an internal error, we should try later",
91 .worker_job_id = WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_BAD_HANDSHAKE,
@@ -104,8 +95,7 @@ static struct {
95 {
96 .response = START_STREAMING_ERROR_INITIALIZATION,
97 .length = sizeof(START_STREAMING_ERROR_INITIALIZATION) - 1,
107 - .status = STREAM_STATUS_INITIALIZATION_IN_PROGRESS,
108 - .version = STREAM_HANDSHAKE_INITIALIZATION,
98 + .version = STREAM_HANDSHAKE_PARENT_IS_INITIALIZING,
99 .dynamic = false,
100 .error = "remote server is initializing, we should try later",
101 .worker_job_id = WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_BAD_HANDSHAKE,
@@ -117,8 +107,7 @@ static struct {
107 {
108 .response = NULL,
109 .length = 0,
120 - .status = STREAM_STATUS_BAD_HANDSHAKE,
121 - .version = STREAM_HANDSHAKE_ERROR_BAD_HANDSHAKE,
110 + .version = STREAM_HANDSHAKE_CONNECT_HANDSHAKE_FAILED,
111 .dynamic = false,
112 .error = "remote node response is not understood, is it Netdata?",
113 .worker_job_id = WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_BAD_HANDSHAKE,
@@ -216,7 +205,7 @@ err_cleanup:
205
206 static bool
207 stream_connect_validate_first_response(RRDHOST *host, struct sender_state *s, char *http, size_t http_length) {
219 - int32_t version = STREAM_HANDSHAKE_ERROR_BAD_HANDSHAKE;
208 + int32_t version = STREAM_HANDSHAKE_CONNECT_HANDSHAKE_FAILED;
209
210 int i;
211 for(i = 0; stream_responses[i].response ; i++) {
@@ -235,7 +224,7 @@ stream_connect_validate_first_response(RRDHOST *host, struct sender_state *s, ch
224 }
225
226 if(version >= STREAM_HANDSHAKE_OK_V1) {
238 - stream_parent_set_reconnect_delay(host->stream.snd.parents.current, STREAM_HANDSHAKE_CONNECTED,
227 + stream_parent_set_reconnect_delay(host->stream.snd.parents.current, STREAM_HANDSHAKE_SP_CONNECTED,
228 stream_send.parents.reconnect_delay_s);
229 s->capabilities = convert_stream_version_to_capabilities(version, host, true);
230 return true;
@@ -243,7 +232,6 @@ stream_connect_validate_first_response(RRDHOST *host, struct sender_state *s, ch
232
233 ND_LOG_FIELD_PRIORITY priority = stream_responses[i].priority;
234 const char *error = stream_responses[i].error;
246 - const char *status = stream_responses[i].status;
235 int worker_job_id = stream_responses[i].worker_job_id;
236 int delay = stream_responses[i].postpone_reconnect_seconds;
237
@@ -251,7 +239,7 @@ stream_connect_validate_first_response(RRDHOST *host, struct sender_state *s, ch
239 stream_parent_set_reconnect_delay(host->stream.snd.parents.current, version, delay);
240
241 ND_LOG_STACK lgs[] = {
254 - ND_LOG_FIELD_TXT(NDF_RESPONSE_CODE, status),
242 + ND_LOG_FIELD_I64(NDF_RESPONSE_CODE, stream_handshake_error_to_response_code(version)),
243 ND_LOG_FIELD_END(),
244 };
245 ND_LOG_STACK_PUSH(lgs);
@@ -280,14 +268,19 @@ bool stream_connect(struct sender_state *s, uint16_t default_port, time_t timeou
268 s->sock.verify_certificate = netdata_ssl_validate_certificate_sender;
269 s->sock.ctx = netdata_ssl_streaming_sender_ctx;
270
271 + pulse_host_status(s->host, PULSE_HOST_STATUS_SND_PENDING, 0);
272 if(!stream_parent_connect_to_one(
273 &s->sock, host, default_port, timeout,
274 s->remote_ip, sizeof(s->remote_ip) - 1,
275 &host->stream.snd.parents.current)) {
276
288 - if(s->sock.error != ND_SOCK_ERR_NO_DESTINATION_AVAILABLE)
277 + if(s->sock.error != ND_SOCK_ERR_NO_DESTINATION_AVAILABLE) {
278 + pulse_host_status(s->host, PULSE_HOST_STATUS_SND_OFFLINE, STREAM_HANDSHAKE_CONNECTION_FAILED);
279 nd_log(NDLS_DAEMON, NDLP_WARNING, "can't connect to a parent, last error: %s",
280 ND_SOCK_ERROR_2str(s->sock.error));
281 + }
282 + else
283 + pulse_host_status(s->host, PULSE_HOST_STATUS_SND_NO_DST, 0);
284
285 nd_sock_close(&s->sock);
286 return false;
@@ -319,16 +312,10 @@ bool stream_connect(struct sender_state *s, uint16_t default_port, time_t timeou
312 buffer_strcat(wb, "Accept: */*" HTTP_HDR_END);
313
314 if (s->parent_using_h2o && stream_connect_upgrade_prelude(host, s)) {
322 - ND_LOG_STACK lgs[] = {
323 - ND_LOG_FIELD_TXT(NDF_RESPONSE_CODE, STREAM_STATUS_CANT_UPGRADE_CONNECTION),
324 - ND_LOG_FIELD_END(),
325 - };
326 - ND_LOG_STACK_PUSH(lgs);
327 -
315 worker_is_busy(WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_CANT_UPGRADE_CONNECTION);
316 nd_sock_close(&s->sock);
317 stream_parent_set_reconnect_delay(
331 - host->stream.snd.parents.current, STREAM_HANDSHAKE_ERROR_HTTP_UPGRADE, 60);
318 + host->stream.snd.parents.current, STREAM_HANDSHAKE_SND_DISCONNECT_HTTP_UPGRADE_FAILED, 60);
319 return false;
320 }
321
@@ -336,7 +323,7 @@ bool stream_connect(struct sender_state *s, uint16_t default_port, time_t timeou
323 ssize_t bytes = nd_sock_send_timeout(&s->sock, (void *)buffer_tostring(wb), len, 0, timeout);
324 if(bytes <= 0) { // timeout is 0
325 ND_LOG_STACK lgs[] = {
339 - ND_LOG_FIELD_TXT(NDF_RESPONSE_CODE, STREAM_STATUS_TIMEOUT),
326 + ND_LOG_FIELD_I64(NDF_RESPONSE_CODE, stream_handshake_error_to_response_code(STREAM_HANDSHAKE_CONNECT_SEND_TIMEOUT)),
327 ND_LOG_FIELD_END(),
328 };
329 ND_LOG_STACK_PUSH(lgs);
@@ -349,7 +336,7 @@ bool stream_connect(struct sender_state *s, uint16_t default_port, time_t timeou
336 rrdhost_hostname(host), s->remote_ip);
337
338 stream_parent_set_reconnect_delay(
352 - host->stream.snd.parents.current, STREAM_HANDSHAKE_ERROR_SEND_TIMEOUT, 60);
339 + host->stream.snd.parents.current, STREAM_HANDSHAKE_CONNECT_SEND_TIMEOUT, 60);
340 return false;
341 }
342
@@ -359,7 +346,7 @@ bool stream_connect(struct sender_state *s, uint16_t default_port, time_t timeou
346 nd_sock_close(&s->sock);
347
348 ND_LOG_STACK lgs[] = {
362 - ND_LOG_FIELD_TXT(NDF_RESPONSE_CODE, STREAM_STATUS_TIMEOUT),
349 + ND_LOG_FIELD_I64(NDF_RESPONSE_CODE, stream_handshake_error_to_response_code(STREAM_HANDSHAKE_CONNECT_RECEIVE_TIMEOUT)),
350 ND_LOG_FIELD_END(),
351 };
352 ND_LOG_STACK_PUSH(lgs);
@@ -371,7 +358,7 @@ bool stream_connect(struct sender_state *s, uint16_t default_port, time_t timeou
358 rrdhost_hostname(host), s->remote_ip);
359
360 stream_parent_set_reconnect_delay(
374 - host->stream.snd.parents.current, STREAM_HANDSHAKE_ERROR_RECEIVE_TIMEOUT, 30);
361 + host->stream.snd.parents.current, STREAM_HANDSHAKE_CONNECT_RECEIVE_TIMEOUT, 30);
362
363 return false;
364 }
@@ -387,7 +374,7 @@ bool stream_connect(struct sender_state *s, uint16_t default_port, time_t timeou
374 log_sender_capabilities(s);
375
376 ND_LOG_STACK lgs[] = {
390 - ND_LOG_FIELD_TXT(NDF_RESPONSE_CODE, STREAM_STATUS_CONNECTED),
377 + ND_LOG_FIELD_I64(NDF_RESPONSE_CODE, HTTP_RESP_OK),
378 ND_LOG_FIELD_END(),
379 };
380 ND_LOG_STACK_PUSH(lgs);
@@ -466,6 +453,8 @@ void stream_connector_requeue(struct sender_state *s) {
453 SENDERS_SET(&sc->queue.senders, (Word_t)s, s);
454 spinlock_unlock(&sc->queue.spinlock);
455
456 + pulse_host_status(s->host, PULSE_HOST_STATUS_SND_PENDING, 0);
457 +
458 // signal the connector to catch the job
459 completion_mark_complete_a_job(&sc->completion);
460 }
@@ -504,7 +493,9 @@ static void stream_connector_remove(struct sender_state *s) {
493 "STREAM CNT '%s' [to %s]: streaming connector removed host: %s (signaled to stop)",
494 rrdhost_hostname(s->host), s->remote_ip, stream_handshake_error_to_string(s->exit.reason));
495
507 - stream_sender_remove(s);
496 + STREAM_HANDSHAKE reason = s->exit.reason ? s->exit.reason : STREAM_HANDSHAKE_DISCONNECT_SIGNALED_TO_STOP;
497 + pulse_host_status(s->host, PULSE_HOST_STATUS_SND_OFFLINE, reason);
498 + stream_sender_remove(s, reason);
499 }
500
501 static void *stream_connector_thread(void *ptr) {
src/streaming/stream-handshake.c
+64 -47
@@ -5,54 +5,58 @@
5 static struct {
6 STREAM_HANDSHAKE err;
7 const char *str;
8 + int response_code;
9 } handshake_errors[] = {
9 - {STREAM_HANDSHAKE_OK_V3, "CONNECTED"},
10 - {STREAM_HANDSHAKE_OK_V2, "CONNECTED"},
11 - {STREAM_HANDSHAKE_OK_V1, "CONNECTED"},
12 - {STREAM_HANDSHAKE_NEVER, ""},
13 - {STREAM_HANDSHAKE_ERROR_BAD_HANDSHAKE, "BAD HANDSHAKE"},
14 - {STREAM_HANDSHAKE_ERROR_LOCALHOST, "LOCALHOST"},
15 - {STREAM_HANDSHAKE_ERROR_ALREADY_CONNECTED, "ALREADY CONNECTED"},
16 - {STREAM_HANDSHAKE_ERROR_DENIED, "DENIED"},
17 - {STREAM_HANDSHAKE_ERROR_SEND_TIMEOUT, "SEND TIMEOUT"},
18 - {STREAM_HANDSHAKE_ERROR_RECEIVE_TIMEOUT, "RECEIVE TIMEOUT"},
19 - {STREAM_HANDSHAKE_ERROR_INVALID_CERTIFICATE, "INVALID CERTIFICATE"},
20 - {STREAM_HANDSHAKE_ERROR_SSL_ERROR, "SSL ERROR"},
21 - {STREAM_HANDSHAKE_ERROR_CANT_CONNECT, "CANT CONNECT"},
22 - {STREAM_HANDSHAKE_BUSY_TRY_LATER, "BUSY TRY LATER"},
23 - {STREAM_HANDSHAKE_INTERNAL_ERROR, "INTERNAL ERROR"},
24 - {STREAM_HANDSHAKE_INITIALIZATION, "REMOTE IS INITIALIZING"},
25 - {STREAM_HANDSHAKE_DISCONNECT_HOST_CLEANUP, "DISCONNECTED HOST CLEANUP"},
26 - {STREAM_HANDSHAKE_DISCONNECT_STALE_RECEIVER, "DISCONNECTED STALE RECEIVER"},
27 - {STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN, "DISCONNECTED SHUTDOWN REQUESTED"},
28 - {STREAM_HANDSHAKE_DISCONNECT_NETDATA_EXIT, "DISCONNECTED NETDATA EXIT"},
29 - {STREAM_HANDSHAKE_DISCONNECT_PARSER_EXIT, "DISCONNECTED PARSE ENDED"},
30 - {STREAM_HANDSHAKE_DISCONNECT_UNKNOWN_SOCKET_READ_ERROR, "DISCONNECTED UNKNOWN SOCKET READ ERROR"},
31 - {STREAM_HANDSHAKE_DISCONNECT_PARSER_FAILED, "DISCONNECTED PARSE ERROR"},
32 - {STREAM_HANDSHAKE_DISCONNECT_RECEIVER_LEFT, "DISCONNECTED RECEIVER LEFT"},
33 - {STREAM_HANDSHAKE_DISCONNECT_ORPHAN_HOST, "DISCONNECTED ORPHAN HOST"},
34 - {STREAM_HANDSHAKE_NON_STREAMABLE_HOST, "NON STREAMABLE HOST"},
35 - {STREAM_HANDSHAKE_DISCONNECT_NOT_SUFFICIENT_READ_BUFFER, "DISCONNECTED NOT SUFFICIENT READ BUFFER"},
36 - {STREAM_HANDSHAKE_DISCONNECT_NOT_SUFFICIENT_SENDER_COMPRESSION_FAILED, "DISCONNECTED SND COMPRESSION FAILED"},
37 - {STREAM_HANDSHAKE_DISCONNECT_NOT_SUFFICIENT_SEND_BUFFER, "DISCONNECTED NOT SUFFICIENT SEND BUFFER"},
38 - {STREAM_HANDSHAKE_DISCONNECT_SOCKET_EOF, "DISCONNECTED SOCKET EOF"},
39 - {STREAM_HANDSHAKE_DISCONNECT_SOCKET_READ_FAILED, "DISCONNECTED SOCKET READ FAILED"},
40 - {STREAM_HANDSHAKE_DISCONNECT_SOCKET_TIMEOUT, "DISCONNECTED SOCKET TIMEOUT"},
41 - {STREAM_HANDSHAKE_DISCONNECT_SOCKET_ERROR, "DISCONNECT SOCKET ERROR"},
42 - {STREAM_HANDSHAKE_DISCONNECT_SOCKET_WRITE_FAILED, "DISCONNECTED SOCKET WRITE FAILED"},
43 - {STREAM_HANDSHAKE_DISCONNECT_SOCKET_CLOSED_BY_REMOTE_END, "DISCONNECTED SOCKET CLOSED BY REMOTE END"},
44 - {STREAM_HANDSHAKE_ERROR_HTTP_UPGRADE, "HTTP UPGRADE ERROR"},
45 - {STREAM_HANDSHAKE_NO_HOST_IN_DESTINATION, "NO HOST IN DESTINATION - CONFIG ERROR"},
46 - {STREAM_HANDSHAKE_CONNECT_TIMEOUT, "CONNECT TIMEOUT"},
47 - {STREAM_HANDSHAKE_CONNECTION_REFUSED, "CONNECTION REFUSED"},
48 - {STREAM_HANDSHAKE_CANT_RESOLVE_HOSTNAME, "CANT RESOLVE HOSTNAME"},
49 - {STREAM_HANDSHAKE_PREPARING, "PREPARING"},
50 - {STREAM_HANDSHAKE_CONNECTING, "CONNECTING"},
51 - {STREAM_HANDSHAKE_CONNECTED, "CONNECTED"},
52 - {STREAM_HANDSHAKE_EXITING, "EXITING"},
53 - {STREAM_HANDSHAKE_NO_STREAM_INFO, "NO STREAM INFO"},
54 - {STREAM_HANDSHAKE_REPLICATION_STALLED, "REPLICATION STALLED"},
55 - { 0, NULL },
10 + {STREAM_HANDSHAKE_OK_V3, "CONNECTED", 200},
11 + {STREAM_HANDSHAKE_OK_V2, "CONNECTED", 200},
12 + {STREAM_HANDSHAKE_OK_V1, "CONNECTED", 200},
13 + {STREAM_HANDSHAKE_NEVER, "", 204}, // No Content
14 + {STREAM_HANDSHAKE_CONNECT_HANDSHAKE_FAILED, "BAD HANDSHAKE", 400}, // Bad Request
15 + {STREAM_HANDSHAKE_PARENT_IS_LOCALHOST, "LOCALHOST", 101}, // Switching Protocols
16 + {STREAM_HANDSHAKE_PARENT_NODE_ALREADY_CONNECTED, "ALREADY CONNECTED", 409}, // Conflict
17 + {STREAM_HANDSHAKE_PARENT_DENIED_ACCESS, "DENIED", 403}, // Forbidden
18 + {STREAM_HANDSHAKE_CONNECT_SEND_TIMEOUT, "SEND TIMEOUT", 408}, // Request Timeout
19 + {STREAM_HANDSHAKE_CONNECT_RECEIVE_TIMEOUT, "RECEIVE TIMEOUT", 504}, // Gateway Timeout
20 + {STREAM_HANDSHAKE_CONNECT_INVALID_CERTIFICATE, "INVALID CERTIFICATE", 495}, // Custom: SSL Certificate Error
21 + {STREAM_HANDSHAKE_CONNECT_SSL_ERROR, "SSL ERROR", 525}, // SSL Handshake Failure
22 + {STREAM_HANDSHAKE_CONNECTION_FAILED, "CANT CONNECT", 502}, // Bad Gateway
23 + {STREAM_HANDSHAKE_PARENT_BUSY_TRY_LATER, "BUSY TRY LATER", 503}, // Service Unavailable
24 + {STREAM_HANDSHAKE_PARENT_INTERNAL_ERROR, "INTERNAL ERROR", 500}, // Internal Server Error
25 + {STREAM_HANDSHAKE_PARENT_IS_INITIALIZING, "REMOTE IS INITIALIZING", 102}, // Processing (WebDAV)
26 +
27 + // receiver only codes
28 + {STREAM_HANDSHAKE_RCV_DISCONNECT_PARSER_FAILED, "DISCONNECTED PARSE ERROR", 400}, // Bad Request
29 + {STREAM_HANDSHAKE_RCV_DISCONNECT_STALE_RECEIVER, "DISCONNECTED STALE RECEIVER", 410}, // Gone
30 + {STREAM_HANDSHAKE_RCV_DECOMPRESSION_FAILED, "DISCONNECTED DECOMPRESSION FAILED", 415}, // Unsupported Media Type
31 +
32 + // sender only codes
33 + {STREAM_HANDSHAKE_SND_DISCONNECT_HOST_CLEANUP, "DISCONNECTED HOST CLEANUP", 202}, // Accepted
34 + {STREAM_HANDSHAKE_SND_DISCONNECT_COMPRESSION_FAILED, "DISCONNECTED SND COMPRESSION FAILED", 415}, // Unsupported Media Type
35 + {STREAM_HANDSHAKE_SND_DISCONNECT_HTTP_UPGRADE_FAILED, "HTTP UPGRADE ERROR", 426}, // Upgrade Required
36 + {STREAM_HANDSHAKE_SND_DISCONNECT_RECEIVER_LEFT, "RECEIVER LEFT", 498},
37 +
38 + // receiver and sender codes
39 + {STREAM_HANDSHAKE_DISCONNECT_SIGNALED_TO_STOP, "DISCONNECTED SIGNALED TO STOP", 499}, // Client Closed Request
40 + {STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN, "DISCONNECTED SHUTDOWN REQUESTED", 503}, // Service Unavailable
41 + {STREAM_HANDSHAKE_DISCONNECT_SOCKET_READ_FAILED, "DISCONNECTED SOCKET READ FAILED", 502}, // Bad Gateway
42 + {STREAM_HANDSHAKE_DISCONNECT_SOCKET_WRITE_FAILED, "DISCONNECTED SOCKET WRITE FAILED", 502}, // Bad Gateway
43 + {STREAM_HANDSHAKE_DISCONNECT_SOCKET_ERROR, "DISCONNECT SOCKET ERROR", 500}, // Internal Server Error
44 + {STREAM_HANDSHAKE_DISCONNECT_TIMEOUT, "DISCONNECTED TIMEOUT", 504}, // Gateway Timeout
45 + {STREAM_HANDSHAKE_DISCONNECT_SOCKET_CLOSED_BY_REMOTE, "DISCONNECTED SOCKET CLOSED BY REMOTE END", 499}, // Client Closed Request
46 + {STREAM_HANDSHAKE_DISCONNECT_BUFFER_OVERFLOW, "DISCONNECTED NOT SUFFICIENT SEND BUFFER", 413}, // Payload Too Large
47 + {STREAM_HANDSHAKE_DISCONNECT_REPLICATION_STALLED, "REPLICATION STALLED", 507}, // Insufficient Storage
48 +
49 + // sender (stream parents - SP) failures to connect
50 + {STREAM_HANDSHAKE_SP_PREPARING, "PREPARING", 102}, // Processing (WebDAV)
51 + {STREAM_HANDSHAKE_SP_NO_HOST_IN_DESTINATION, "NO HOST IN DESTINATION - CONFIG ERROR", 404}, // Not Found
52 + {STREAM_HANDSHAKE_SP_CONNECT_TIMEOUT, "CONNECT TIMEOUT", 408}, // Request Timeout
53 + {STREAM_HANDSHAKE_SP_CONNECTION_REFUSED, "CONNECTION REFUSED", 403}, // Forbidden
54 + {STREAM_HANDSHAKE_SP_CANT_RESOLVE_HOSTNAME, "CANT RESOLVE HOSTNAME", 400}, // Bad Request
55 + {STREAM_HANDSHAKE_SP_CONNECTING, "CONNECTING", 102}, // Processing (WebDAV)
56 + {STREAM_HANDSHAKE_SP_CONNECTED, "CONNECTED", 200}, // OK
57 + {STREAM_HANDSHAKE_SP_NO_STREAM_INFO, "NO STREAM INFO", 404}, // Not Found
58 +
59 + { 0, NULL, 0 },
60 };
61
62 const char *stream_handshake_error_to_string(STREAM_HANDSHAKE reason) {
@@ -67,3 +71,16 @@ const char *stream_handshake_error_to_string(STREAM_HANDSHAKE reason) {
71
72 return "UNKNOWN";
73 }
74 +
75 +int stream_handshake_error_to_response_code(STREAM_HANDSHAKE reason) {
76 + if(reason >= STREAM_HANDSHAKE_OK_V1)
77 + // handshake_error is the whole version / capabilities number
78 + return 200;
79 +
80 + for(size_t i = 0; handshake_errors[i].str ; i++) {
81 + if(reason == handshake_errors[i].err)
82 + return handshake_errors[i].response_code;
83 + }
84 +
85 + return 404;
86 +}
src/streaming/stream-handshake.h
+52 -68
@@ -17,81 +17,65 @@
17 #define START_STREAMING_ERROR_INTERNAL_ERROR "The server encountered an internal error. Try later."
18 #define START_STREAMING_ERROR_INITIALIZATION "The server is initializing. Try later."
19
20 -#define STREAM_STATUS_CONNECTED "CONNECTED"
21 -#define STREAM_STATUS_ALREADY_CONNECTED "ALREADY CONNECTED"
22 -#define STREAM_STATUS_DISCONNECTED "DISCONNECTED"
23 -#define STREAM_STATUS_RATE_LIMIT "RATE LIMIT TRY LATER"
24 -#define STREAM_STATUS_INITIALIZATION_IN_PROGRESS "INITIALIZATION IN PROGRESS RETRY LATER"
25 -#define STREAM_STATUS_INTERNAL_SERVER_ERROR "INTERNAL SERVER ERROR DROPPING CONNECTION"
26 -#define STREAM_STATUS_DUPLICATE_RECEIVER "DUPLICATE RECEIVER DROPPING CONNECTION"
27 -#define STREAM_STATUS_CANT_REPLY "CANT REPLY DROPPING CONNECTION"
28 -#define STREAM_STATUS_NO_HOSTNAME "NO HOSTNAME PERMISSION DENIED"
29 -#define STREAM_STATUS_NO_API_KEY "NO API KEY PERMISSION DENIED"
30 -#define STREAM_STATUS_INVALID_API_KEY "INVALID API KEY PERMISSION DENIED"
31 -#define STREAM_STATUS_NO_MACHINE_GUID "NO MACHINE GUID PERMISSION DENIED"
32 -#define STREAM_STATUS_MACHINE_GUID_DISABLED "MACHINE GUID DISABLED PERMISSION DENIED"
33 -#define STREAM_STATUS_INVALID_MACHINE_GUID "INVALID MACHINE GUID PERMISSION DENIED"
34 -#define STREAM_STATUS_API_KEY_DISABLED "API KEY DISABLED PERMISSION DENIED"
35 -#define STREAM_STATUS_NOT_ALLOWED_IP "NOT ALLOWED IP PERMISSION DENIED"
36 -#define STREAM_STATUS_LOCALHOST "LOCALHOST PERMISSION DENIED"
37 -#define STREAM_STATUS_PERMISSION_DENIED "PERMISSION DENIED"
38 -#define STREAM_STATUS_BAD_HANDSHAKE "BAD HANDSHAKE"
39 -#define STREAM_STATUS_TIMEOUT "TIMEOUT"
40 -#define STREAM_STATUS_CANT_UPGRADE_CONNECTION "CANT UPGRADE CONNECTION"
41 -
20 typedef enum {
43 - STREAM_HANDSHAKE_OK_V3 = 3, // v3+
44 - STREAM_HANDSHAKE_OK_V2 = 2, // v2
45 - STREAM_HANDSHAKE_OK_V1 = 1, // v1
46 - STREAM_HANDSHAKE_NEVER = 0, // never tried to connect
47 - STREAM_HANDSHAKE_ERROR_BAD_HANDSHAKE = -1,
48 - STREAM_HANDSHAKE_ERROR_LOCALHOST = -2,
49 - STREAM_HANDSHAKE_ERROR_ALREADY_CONNECTED = -3,
50 - STREAM_HANDSHAKE_ERROR_DENIED = -4,
51 - STREAM_HANDSHAKE_ERROR_SEND_TIMEOUT = -5,
52 - STREAM_HANDSHAKE_ERROR_RECEIVE_TIMEOUT = -6,
53 - STREAM_HANDSHAKE_ERROR_INVALID_CERTIFICATE = -7,
54 - STREAM_HANDSHAKE_ERROR_SSL_ERROR = -8,
55 - STREAM_HANDSHAKE_ERROR_CANT_CONNECT = -9,
56 - STREAM_HANDSHAKE_BUSY_TRY_LATER = -10,
57 - STREAM_HANDSHAKE_INTERNAL_ERROR = -11,
58 - STREAM_HANDSHAKE_INITIALIZATION = -12,
59 - STREAM_HANDSHAKE_DISCONNECT_HOST_CLEANUP = -13,
60 - STREAM_HANDSHAKE_DISCONNECT_STALE_RECEIVER = -14,
61 - STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN = -15,
62 - STREAM_HANDSHAKE_DISCONNECT_NETDATA_EXIT = -16,
63 - STREAM_HANDSHAKE_DISCONNECT_PARSER_EXIT = -17,
21 + // negotiated version values
22 + // IMPORTANT: maintain compatibility across netdata version - do not change these
23 + STREAM_HANDSHAKE_OK_V3 = 3, // v3+
24 + STREAM_HANDSHAKE_OK_V2 = 2, // v2
25 + STREAM_HANDSHAKE_OK_V1 = 1, // v1
26 + STREAM_HANDSHAKE_NEVER = 0, // never tried to connect
27 + STREAM_HANDSHAKE_CONNECT_HANDSHAKE_FAILED = -1, // sent by parent - DO NOT CHANGE
28 + STREAM_HANDSHAKE_PARENT_IS_LOCALHOST = -2, // sent by parent - DO NOT CHANGE
29 + STREAM_HANDSHAKE_PARENT_NODE_ALREADY_CONNECTED = -3, // sent by parent - DO NOT CHANGE
30 + STREAM_HANDSHAKE_PARENT_DENIED_ACCESS = -4, // sent by parent - DO NOT CHANGE
31 + STREAM_HANDSHAKE_CONNECT_SEND_TIMEOUT = -5, // generated by child
32 + STREAM_HANDSHAKE_CONNECT_RECEIVE_TIMEOUT = -6, // generated by child
33 + STREAM_HANDSHAKE_CONNECT_INVALID_CERTIFICATE = -7, // generated by child
34 + STREAM_HANDSHAKE_CONNECT_SSL_ERROR = -8, // generated by child
35 + STREAM_HANDSHAKE_CONNECTION_FAILED = -9, // generated by child
36 + STREAM_HANDSHAKE_PARENT_BUSY_TRY_LATER = -10, // sent by parent - DO NOT CHANGE
37 + STREAM_HANDSHAKE_PARENT_INTERNAL_ERROR = -11, // sent by parent - DO NOT CHANGE
38 + STREAM_HANDSHAKE_PARENT_IS_INITIALIZING = -12, // sent by parent - DO NOT CHANGE
39 +
40 + // --- internal handshake reasons ---
41 +
42 + // receiver only codes
43 + STREAM_HANDSHAKE_RCV_DISCONNECT_PARSER_FAILED = -13,
44 + STREAM_HANDSHAKE_RCV_DISCONNECT_STALE_RECEIVER = -14,
45 + STREAM_HANDSHAKE_RCV_DECOMPRESSION_FAILED = -15,
46
65 - STREAM_HANDSHAKE_DISCONNECT_UNKNOWN_SOCKET_READ_ERROR = -18,
66 - STREAM_HANDSHAKE_DISCONNECT_PARSER_FAILED = -19,
67 - STREAM_HANDSHAKE_DISCONNECT_RECEIVER_LEFT = -20,
68 - STREAM_HANDSHAKE_DISCONNECT_ORPHAN_HOST = -21,
69 - STREAM_HANDSHAKE_NON_STREAMABLE_HOST = -22,
47 + // sender only codes
48 + STREAM_HANDSHAKE_SND_DISCONNECT_HOST_CLEANUP = -16,
49 + STREAM_HANDSHAKE_SND_DISCONNECT_COMPRESSION_FAILED = -17,
50 + STREAM_HANDSHAKE_SND_DISCONNECT_HTTP_UPGRADE_FAILED = -18,
51 + STREAM_HANDSHAKE_SND_DISCONNECT_RECEIVER_LEFT = -19, // used only in pulse
52
71 - STREAM_HANDSHAKE_DISCONNECT_NOT_SUFFICIENT_READ_BUFFER = -23,
72 - STREAM_HANDSHAKE_DISCONNECT_NOT_SUFFICIENT_SEND_BUFFER = -25,
53 + // receiver and sender codes
54 + STREAM_HANDSHAKE_DISCONNECT_SIGNALED_TO_STOP = -20, // a fallback when (s|rpt)->exit.reason is not set
55 + STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN = -21,
56 + STREAM_HANDSHAKE_DISCONNECT_SOCKET_READ_FAILED = -22,
57 + STREAM_HANDSHAKE_DISCONNECT_SOCKET_WRITE_FAILED = -23,
58 + STREAM_HANDSHAKE_DISCONNECT_SOCKET_ERROR = -24,
59 + STREAM_HANDSHAKE_DISCONNECT_TIMEOUT = -25,
60 + STREAM_HANDSHAKE_DISCONNECT_SOCKET_CLOSED_BY_REMOTE = -26,
61 + STREAM_HANDSHAKE_DISCONNECT_BUFFER_OVERFLOW = -27,
62 + STREAM_HANDSHAKE_DISCONNECT_REPLICATION_STALLED = -28,
63
74 - STREAM_HANDSHAKE_DISCONNECT_NOT_SUFFICIENT_SENDER_COMPRESSION_FAILED = -24,
75 - STREAM_HANDSHAKE_DISCONNECT_SOCKET_EOF = -26,
76 - STREAM_HANDSHAKE_DISCONNECT_SOCKET_READ_FAILED = -27,
77 - STREAM_HANDSHAKE_DISCONNECT_SOCKET_TIMEOUT = -28,
78 - STREAM_HANDSHAKE_DISCONNECT_SOCKET_ERROR = -29,
79 - STREAM_HANDSHAKE_DISCONNECT_SOCKET_WRITE_FAILED = -30,
80 - STREAM_HANDSHAKE_DISCONNECT_SOCKET_CLOSED_BY_REMOTE_END = -31,
81 - STREAM_HANDSHAKE_ERROR_HTTP_UPGRADE = -32,
82 - STREAM_HANDSHAKE_NO_HOST_IN_DESTINATION = -33,
83 - STREAM_HANDSHAKE_CONNECT_TIMEOUT = -34,
84 - STREAM_HANDSHAKE_CONNECTION_REFUSED = -35,
85 - STREAM_HANDSHAKE_CANT_RESOLVE_HOSTNAME = -36,
86 - STREAM_HANDSHAKE_PREPARING = -37,
87 - STREAM_HANDSHAKE_CONNECTING = -38,
88 - STREAM_HANDSHAKE_CONNECTED = -39,
89 - STREAM_HANDSHAKE_EXITING = -40,
90 - STREAM_HANDSHAKE_NO_STREAM_INFO = -41,
91 - STREAM_HANDSHAKE_REPLICATION_STALLED = -42,
64 + // sender (stream parents - SP) failures to connect
65 + STREAM_HANDSHAKE_SP_PREPARING = -29,
66 + STREAM_HANDSHAKE_SP_NO_HOST_IN_DESTINATION = -30,
67 + STREAM_HANDSHAKE_SP_CONNECT_TIMEOUT = -31,
68 + STREAM_HANDSHAKE_SP_CONNECTION_REFUSED = -32,
69 + STREAM_HANDSHAKE_SP_CANT_RESOLVE_HOSTNAME = -33,
70 + STREAM_HANDSHAKE_SP_CONNECTING = -34,
71 + STREAM_HANDSHAKE_SP_CONNECTED = -35,
72 + STREAM_HANDSHAKE_SP_NO_STREAM_INFO = -36,
73
74 + // terminator - keep this positive, bigger than all negative values
75 + STREAM_HANDSHAKE_NEGATIVE_MAX = 37,
76 } STREAM_HANDSHAKE;
77
78 const char *stream_handshake_error_to_string(STREAM_HANDSHAKE reason);
79 +int stream_handshake_error_to_response_code(STREAM_HANDSHAKE reason);
80
81 #endif //NETDATA_STREAM_HANDSHAKE_H
src/streaming/stream-parents.c
+40 -20
@@ -84,7 +84,7 @@ static bool is_a_blocked_parent(STREAM_PARENT *d) {
84 // --------------------------------------------------------------------------------------------------------------------
85
86 STREAM_HANDSHAKE stream_parent_get_disconnect_reason(STREAM_PARENT *d) {
87 - if(!d) return STREAM_HANDSHAKE_INTERNAL_ERROR;
87 + if(!d) return STREAM_HANDSHAKE_PARENT_INTERNAL_ERROR;
88 return d->reason;
89 }
90
@@ -105,7 +105,7 @@ static inline usec_t randomize_wait_ut(time_t min, time_t max) {
105 }
106
107 void rrdhost_stream_parents_reset(RRDHOST *host, STREAM_HANDSHAKE reason) {
108 - usec_t until_ut = randomize_wait_ut(5, stream_send.parents.reconnect_delay_s);
108 + usec_t until_ut = randomize_wait_ut(stream_send.parents.reconnect_delay_s / 2, stream_send.parents.reconnect_delay_s + 5);
109 rw_spinlock_write_lock(&host->stream.snd.parents.spinlock);
110 for (STREAM_PARENT *d = host->stream.snd.parents.all; d; d = d->next) {
111 d->postpone_until_ut = until_ut;
@@ -233,39 +233,39 @@ cleanup:
233 static void stream_parent_nd_sock_error_to_reason(STREAM_PARENT *d, ND_SOCK *sock) {
234 switch (sock->error) {
235 case ND_SOCK_ERR_CONNECTION_REFUSED:
236 - d->reason = STREAM_HANDSHAKE_CONNECTION_REFUSED;
236 + d->reason = STREAM_HANDSHAKE_SP_CONNECTION_REFUSED;
237 d->postpone_until_ut = randomize_wait_ut(30, 60);
238 block_parent_for_all_nodes(d, 30);
239 break;
240
241 case ND_SOCK_ERR_CANNOT_RESOLVE_HOSTNAME:
242 - d->reason = STREAM_HANDSHAKE_CANT_RESOLVE_HOSTNAME;
242 + d->reason = STREAM_HANDSHAKE_SP_CANT_RESOLVE_HOSTNAME;
243 d->postpone_until_ut = randomize_wait_ut(30, 60);
244 block_parent_for_all_nodes(d, 30);
245 break;
246
247 case ND_SOCK_ERR_NO_HOST_IN_DEFINITION:
248 - d->reason = STREAM_HANDSHAKE_NO_HOST_IN_DESTINATION;
248 + d->reason = STREAM_HANDSHAKE_SP_NO_HOST_IN_DESTINATION;
249 d->banned_for_this_session = true;
250 d->postpone_until_ut = randomize_wait_ut(30, 60);
251 block_parent_for_all_nodes(d, 30);
252 break;
253
254 case ND_SOCK_ERR_TIMEOUT:
255 - d->reason = STREAM_HANDSHAKE_CONNECT_TIMEOUT;
255 + d->reason = STREAM_HANDSHAKE_SP_CONNECT_TIMEOUT;
256 d->postpone_until_ut = randomize_wait_ut(300, d->remote.nodes < 10 ? 600 : 900);
257 block_parent_for_all_nodes(d, 300);
258 break;
259
260 case ND_SOCK_ERR_SSL_INVALID_CERTIFICATE:
261 - d->reason = STREAM_HANDSHAKE_ERROR_INVALID_CERTIFICATE;
261 + d->reason = STREAM_HANDSHAKE_CONNECT_INVALID_CERTIFICATE;
262 d->postpone_until_ut = randomize_wait_ut(300, 600);
263 block_parent_for_all_nodes(d, 300);
264 break;
265
266 case ND_SOCK_ERR_SSL_CANT_ESTABLISH_SSL_CONNECTION:
267 case ND_SOCK_ERR_SSL_FAILED_TO_OPEN:
268 - d->reason = STREAM_HANDSHAKE_ERROR_SSL_ERROR;
268 + d->reason = STREAM_HANDSHAKE_CONNECT_SSL_ERROR;
269 d->postpone_until_ut = randomize_wait_ut(60, 180);
270 block_parent_for_all_nodes(d, 60);
271 break;
@@ -274,19 +274,21 @@ static void stream_parent_nd_sock_error_to_reason(STREAM_PARENT *d, ND_SOCK *soc
274 case ND_SOCK_ERR_POLL_ERROR:
275 case ND_SOCK_ERR_FAILED_TO_CREATE_SOCKET:
276 case ND_SOCK_ERR_UNKNOWN_ERROR:
277 - d->reason = STREAM_HANDSHAKE_INTERNAL_ERROR;
277 + d->reason = STREAM_HANDSHAKE_PARENT_INTERNAL_ERROR;
278 d->postpone_until_ut = randomize_wait_ut(30, 60);
279 break;
280
281 case ND_SOCK_ERR_THREAD_CANCELLED:
282 case ND_SOCK_ERR_NO_DESTINATION_AVAILABLE:
283 - d->reason = STREAM_HANDSHAKE_INTERNAL_ERROR;
283 + d->reason = STREAM_HANDSHAKE_PARENT_INTERNAL_ERROR;
284 d->postpone_until_ut = randomize_wait_ut(30, 60);
285 break;
286 }
287 }
288
289 int stream_info_to_json_v1(BUFFER *wb, const char *machine_guid) {
290 + pulse_parent_stream_info_received_request();
291 +
292 buffer_reset(wb);
293 buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
294
@@ -386,8 +388,10 @@ static bool stream_info_fetch(STREAM_PARENT *d, const char *uuid, int default_po
388 "STREAM PARENTS '%s': fetching stream info from '%s'...",
389 hostname, string2str(d->destination));
390
391 + pulse_stream_info_sent_request();
392 +
393 // Establish connection
390 - d->reason = STREAM_HANDSHAKE_CONNECTING;
394 + d->reason = STREAM_HANDSHAKE_SP_CONNECTING;
395 if (!nd_sock_connect_to_this(&sock, string2str(d->destination), default_port, 5, ssl)) {
396 d->selection.info = false;
397 stream_parent_nd_sock_error_to_reason(d, &sock);
@@ -424,7 +428,7 @@ static bool stream_info_fetch(STREAM_PARENT *d, const char *uuid, int default_po
428 "STREAM PARENTS '%s': stream info receive buffer is full while receiving response from '%s'",
429 hostname, string2str(d->destination));
430 d->selection.info = false;
427 - d->reason = STREAM_HANDSHAKE_INTERNAL_ERROR;
431 + d->reason = STREAM_HANDSHAKE_PARENT_INTERNAL_ERROR;
432 return false;
433 }
434
@@ -465,7 +469,7 @@ static bool stream_info_fetch(STREAM_PARENT *d, const char *uuid, int default_po
469 hostname, string2str(d->destination));
470
471 d->selection.info = false;
468 - d->reason = STREAM_HANDSHAKE_INTERNAL_ERROR;
472 + d->reason = STREAM_HANDSHAKE_PARENT_INTERNAL_ERROR;
473 return false;
474 }
475 content_length = strtoul(content_length_ptr + strlen("Content-Length: "), NULL, 10);
@@ -475,7 +479,7 @@ static bool stream_info_fetch(STREAM_PARENT *d, const char *uuid, int default_po
479 hostname, string2str(d->destination));
480
481 d->selection.info = false;
478 - d->reason = STREAM_HANDSHAKE_INTERNAL_ERROR;
482 + d->reason = STREAM_HANDSHAKE_PARENT_INTERNAL_ERROR;
483 return false;
484 }
485 }
@@ -485,7 +489,7 @@ static bool stream_info_fetch(STREAM_PARENT *d, const char *uuid, int default_po
489 CLEAN_JSON_OBJECT *jobj = json_tokener_parse(payload_start);
490 if (!jobj) {
491 d->selection.info = false;
488 - d->reason = STREAM_HANDSHAKE_NO_STREAM_INFO;
492 + d->reason = STREAM_HANDSHAKE_SP_NO_STREAM_INFO;
493 nd_log(NDLS_DAEMON, NDLP_WARNING,
494 "STREAM PARENTS '%s': failed to parse stream info response from '%s', JSON data: %s",
495 hostname, string2str(d->destination), payload_start);
@@ -496,7 +500,7 @@ static bool stream_info_fetch(STREAM_PARENT *d, const char *uuid, int default_po
500
501 if(!stream_info_json_parse_v1(jobj, "", d, error)) {
502 d->selection.info = false;
499 - d->reason = STREAM_HANDSHAKE_NO_STREAM_INFO;
503 + d->reason = STREAM_HANDSHAKE_SP_NO_STREAM_INFO;
504 nd_log(NDLS_DAEMON, NDLP_WARNING,
505 "STREAM PARENTS '%s': failed to extract fields from JSON stream info response from '%s': %s"
506 " - JSON data: %s",
@@ -605,7 +609,7 @@ bool stream_parent_connect_to_one_unsafe(
609 switch(d->remote.ingest_type) {
610 case RRDHOST_INGEST_TYPE_VIRTUAL:
611 case RRDHOST_INGEST_TYPE_LOCALHOST:
608 - d->reason = STREAM_HANDSHAKE_ERROR_LOCALHOST;
612 + d->reason = STREAM_HANDSHAKE_PARENT_IS_LOCALHOST;
613 d->since_ut = now_ut;
614 d->postpone_until_ut = randomize_wait_ut(3600, 7200);
615
@@ -619,8 +623,10 @@ bool stream_parent_connect_to_one_unsafe(
623 rrdhost_hostname(host), string2str(d->destination));
624 continue;
625 }
622 - else
626 + else {
627 + pulse_sender_stream_info_failed(string2str(d->destination), d->reason);
628 skip = true;
629 + }
630 break;
631
632 default:
@@ -631,16 +637,17 @@ bool stream_parent_connect_to_one_unsafe(
637
638 switch(d->remote.ingest_status) {
639 case RRDHOST_INGEST_STATUS_INITIALIZING:
634 - d->reason = STREAM_HANDSHAKE_INITIALIZATION;
640 + d->reason = STREAM_HANDSHAKE_PARENT_IS_INITIALIZING;
641 d->since_ut = now_ut;
642 d->postpone_until_ut = randomize_wait_ut(30, 60);
643 + pulse_sender_stream_info_failed(string2str(d->destination), d->reason);
644 skip = true;
645 break;
646
647 case RRDHOST_INGEST_STATUS_REPLICATING:
648 case RRDHOST_INGEST_STATUS_ONLINE:
642 - d->reason = STREAM_HANDSHAKE_ERROR_ALREADY_CONNECTED;
649 if(rrdhost_is_host_in_stream_path_before_us(host, d->remote.host_id, host->sender->hops)) {
650 + d->reason = STREAM_HANDSHAKE_PARENT_NODE_ALREADY_CONNECTED;
651 d->since_ut = now_ut;
652 d->postpone_until_ut = randomize_wait_ut(3600, 7200);
653 d->banned_for_this_session = true;
@@ -648,8 +655,17 @@ bool stream_parent_connect_to_one_unsafe(
655 nd_log(NDLS_DAEMON, NDLP_INFO,
656 "STREAM PARENTS '%s': destination '%s' is banned for this session, because it is in our path before us.",
657 rrdhost_hostname(host), string2str(d->destination));
658 + pulse_sender_stream_info_failed(string2str(d->destination), d->reason);
659 continue;
660 }
661 +// else {
662 +// skip = true;
663 +// if(!netdata_conf_is_parent()) {
664 +// nd_log(NDLS_DAEMON, NDLP_INFO,
665 +// "STREAM PARENTS '%s': destination '%s' reports I am already connected.",
666 +// rrdhost_hostname(host), string2str(d->destination));
667 +// }
668 +// }
669 break;
670
671 default:
@@ -657,6 +673,8 @@ bool stream_parent_connect_to_one_unsafe(
673 break;
674 }
675 }
676 + else
677 + pulse_sender_stream_info_failed(string2str(d->destination), d->reason);
678
679 if(skip) {
680 skipped_but_useful++;
@@ -786,6 +804,7 @@ bool stream_parent_connect_to_one_unsafe(
804
805 d->since_ut = now_ut;
806 d->attempts++;
807 + pulse_host_status(host, PULSE_HOST_STATUS_SND_CONNECTING, 0);
808 if (nd_sock_connect_to_this(sender_sock, string2str(d->destination),
809 default_port, timeout, stream_parent_is_ssl(d))) {
810
@@ -810,6 +829,7 @@ bool stream_parent_connect_to_one_unsafe(
829 }
830 else {
831 stream_parent_nd_sock_error_to_reason(d, sender_sock);
832 + pulse_sender_connection_failed(string2str(d->destination), d->reason);
833 nd_log(NDLS_DAEMON, NDLP_DEBUG,
834 "STREAM PARENTS '%s': stream connection to '%s' failed (default port: %d): %s",
835 rrdhost_hostname(host),
src/streaming/stream-receiver-connection.c
+49 -35
@@ -27,32 +27,37 @@ static void stream_receiver_connected_msg(RRDHOST *host, char *dst, size_t len)
27 }
28 }
29
30 -void stream_receiver_log_status(struct receiver_state *rpt, const char *msg, const char *status, ND_LOG_FIELD_PRIORITY priority) {
30 +void stream_receiver_log_status(struct receiver_state *rpt, const char *msg, STREAM_HANDSHAKE reason, ND_LOG_FIELD_PRIORITY priority) {
31 // this function may be called BEFORE we spawn the receiver thread
32 // so, we need to add the fields again (it does not harm)
33 ND_LOG_STACK lgs[] = {
34 ND_LOG_FIELD_TXT(NDF_SRC_IP, rpt->remote_ip),
35 ND_LOG_FIELD_TXT(NDF_SRC_PORT, rpt->remote_port),
36 ND_LOG_FIELD_TXT(NDF_NIDL_NODE, (rpt->hostname && *rpt->hostname) ? rpt->hostname : ""),
37 - ND_LOG_FIELD_TXT(NDF_RESPONSE_CODE, status),
37 + ND_LOG_FIELD_I64(NDF_RESPONSE_CODE, stream_handshake_error_to_response_code(reason)),
38 ND_LOG_FIELD_UUID(NDF_MESSAGE_ID, &streaming_from_child_msgid),
39 ND_LOG_FIELD_END(),
40 };
41 ND_LOG_STACK_PUSH(lgs);
42
43 - nd_log(NDLS_ACCESS, priority, "api_key:'%s' machine_guid:'%s' msg:'%s'"
43 + nd_log(NDLS_ACCESS, priority, "api_key:'%s' machine_guid:'%s' node:'%s' msg:'%s' reason:'%s'"
44 , (rpt->key && *rpt->key)? rpt->key : ""
45 , (rpt->machine_guid && *rpt->machine_guid) ? rpt->machine_guid : ""
46 - , msg);
46 + , (rpt->hostname && *rpt->hostname) ? rpt->hostname : ""
47 + , msg
48 + , stream_handshake_error_to_string(reason));
49
50 nd_log(NDLS_DAEMON, priority, "STREAM RCV '%s' [from [%s]:%s]: %s %s%s%s"
51 , (rpt->hostname && *rpt->hostname) ? rpt->hostname : ""
52 , rpt->remote_ip, rpt->remote_port
53 , msg
52 - , rpt->exit.reason != STREAM_HANDSHAKE_NEVER?" (":""
53 - , stream_handshake_error_to_string(rpt->exit.reason)
54 - , rpt->exit.reason != STREAM_HANDSHAKE_NEVER?")":""
54 + , reason != STREAM_HANDSHAKE_NEVER?" (":""
55 + , stream_handshake_error_to_string(reason)
56 + , reason != STREAM_HANDSHAKE_NEVER?")":""
57 );
58 +
59 + if(reason < 0)
60 + pulse_parent_receiver_rejected(reason);
61 }
62
63 // --------------------------------------------------------------------------------------------------------------------
@@ -175,7 +180,7 @@ static bool stream_receiver_send_first_response(struct receiver_state *rpt) {
180 stream_receiver_log_status(
181 rpt,
182 "rejecting streaming connection; failed to find or create the required host structure",
178 - STREAM_STATUS_INTERNAL_SERVER_ERROR, NDLP_ERR);
183 + STREAM_HANDSHAKE_PARENT_INTERNAL_ERROR, NDLP_ERR);
184
185 stream_send_error_on_taken_over_connection(rpt, START_STREAMING_ERROR_INTERNAL_ERROR);
186 return false;
@@ -185,7 +190,7 @@ static bool stream_receiver_send_first_response(struct receiver_state *rpt) {
190 stream_receiver_log_status(
191 rpt,
192 "rejecting streaming connection; host is initializing, retry later",
188 - STREAM_STATUS_INITIALIZATION_IN_PROGRESS, NDLP_NOTICE);
193 + STREAM_HANDSHAKE_PARENT_IS_INITIALIZING, NDLP_NOTICE);
194
195 stream_send_error_on_taken_over_connection(rpt, START_STREAMING_ERROR_INITIALIZATION);
196 return false;
@@ -196,7 +201,7 @@ static bool stream_receiver_send_first_response(struct receiver_state *rpt) {
201 // stream_receiver_log_status(
202 // rpt,
203 // "rejecting streaming connection; the system is backfilling higher tiers with high-resolution data, retry later",
199 -// STREAM_STATUS_INITIALIZATION_IN_PROGRESS, NDLP_NOTICE);
204 +// STREAM_HANDSHAKE_PARENT_IS_INITIALIZING, NDLP_NOTICE);
205 //
206 // stream_send_error_on_taken_over_connection(rpt, START_STREAMING_ERROR_INITIALIZATION);
207 // return false;
@@ -206,7 +211,7 @@ static bool stream_receiver_send_first_response(struct receiver_state *rpt) {
211 stream_receiver_log_status(
212 rpt,
213 "rejecting streaming connection; host is already served by another receiver",
209 - STREAM_STATUS_DUPLICATE_RECEIVER, NDLP_INFO);
214 + STREAM_HANDSHAKE_PARENT_NODE_ALREADY_CONNECTED, NDLP_INFO);
215
216 stream_send_error_on_taken_over_connection(rpt, START_STREAMING_ERROR_ALREADY_STREAMING);
217 return false;
@@ -231,7 +236,7 @@ static bool stream_receiver_send_first_response(struct receiver_state *rpt) {
236 stream_select_receiver_compression_algorithm(rpt);
237
238 {
234 - // netdata_log_info("STREAM %s [receive from [%s]:%s]: initializing communication...", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port);
239 + // netdata_log_info("STREAM RCV %s [from [%s]:%s]: initializing communication...", rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port);
240 char initial_response[HTTP_HEADER_SIZE];
241 if (stream_has_capability(rpt, STREAM_CAP_VCAPS)) {
242 log_receiver_capabilities(rpt);
@@ -285,8 +290,8 @@ static bool stream_receiver_send_first_response(struct receiver_state *rpt) {
290 stream_receiver_log_status(
291 rpt,
292 "cannot reply back, dropping connection",
288 - STREAM_STATUS_CANT_REPLY, NDLP_ERR);
289 - rrdhost_clear_receiver(rpt);
293 + STREAM_HANDSHAKE_CONNECT_SEND_TIMEOUT, NDLP_ERR);
294 + rrdhost_clear_receiver(rpt, STREAM_HANDSHAKE_DISCONNECT_SOCKET_WRITE_FAILED);
295 return false;
296 }
297 #ifdef ENABLE_H2O
@@ -298,6 +303,7 @@ static bool stream_receiver_send_first_response(struct receiver_state *rpt) {
303 }
304
305 int stream_receiver_accept_connection(struct web_client *w, char *decoded_query_string, void *h2o_ctx __maybe_unused) {
306 + pulse_parent_receiver_request();
307
308 if(!service_running(ABILITY_STREAMING_CONNECTIONS))
309 return stream_receiver_response_too_busy_now(w);
@@ -434,7 +440,7 @@ int stream_receiver_accept_connection(struct web_client *w, char *decoded_query_
440 stream_receiver_log_status(
441 rpt,
442 "rejecting streaming connection; request without an API key",
437 - STREAM_STATUS_NO_API_KEY, NDLP_WARNING);
443 + STREAM_HANDSHAKE_PARENT_DENIED_ACCESS, NDLP_WARNING);
444
445 stream_receiver_free(rpt);
446 return stream_receiver_response_permission_denied(w);
@@ -444,7 +450,7 @@ int stream_receiver_accept_connection(struct web_client *w, char *decoded_query_
450 stream_receiver_log_status(
451 rpt,
452 "rejecting streaming connection; request without a hostname",
447 - STREAM_STATUS_NO_HOSTNAME, NDLP_WARNING);
453 + STREAM_HANDSHAKE_PARENT_DENIED_ACCESS, NDLP_WARNING);
454
455 stream_receiver_free(rpt);
456 return stream_receiver_response_permission_denied(w);
@@ -457,7 +463,7 @@ int stream_receiver_accept_connection(struct web_client *w, char *decoded_query_
463 stream_receiver_log_status(
464 rpt,
465 "rejecting streaming connection; request without a machine UUID",
460 - STREAM_STATUS_NO_MACHINE_GUID, NDLP_WARNING);
466 + STREAM_HANDSHAKE_PARENT_DENIED_ACCESS, NDLP_WARNING);
467
468 stream_receiver_free(rpt);
469 return stream_receiver_response_permission_denied(w);
@@ -470,7 +476,7 @@ int stream_receiver_accept_connection(struct web_client *w, char *decoded_query_
476 stream_receiver_log_status(
477 rpt,
478 "rejecting streaming connection; API key is not a valid UUID (use the command uuidgen to generate one)",
473 - STREAM_STATUS_INVALID_API_KEY, NDLP_WARNING);
479 + STREAM_HANDSHAKE_PARENT_DENIED_ACCESS, NDLP_WARNING);
480
481 stream_receiver_free(rpt);
482 return stream_receiver_response_permission_denied(w);
@@ -480,7 +486,7 @@ int stream_receiver_accept_connection(struct web_client *w, char *decoded_query_
486 stream_receiver_log_status(
487 rpt,
488 "rejecting streaming connection; machine UUID is not a valid UUID",
483 - STREAM_STATUS_INVALID_MACHINE_GUID, NDLP_WARNING);
489 + STREAM_HANDSHAKE_PARENT_DENIED_ACCESS, NDLP_WARNING);
490
491 stream_receiver_free(rpt);
492 return stream_receiver_response_permission_denied(w);
@@ -491,7 +497,7 @@ int stream_receiver_accept_connection(struct web_client *w, char *decoded_query_
497 stream_receiver_log_status(
498 rpt,
499 "rejecting streaming connection; API key provided is a machine UUID (did you mix them up?)",
494 - STREAM_STATUS_INVALID_API_KEY, NDLP_WARNING);
500 + STREAM_HANDSHAKE_PARENT_DENIED_ACCESS, NDLP_WARNING);
501
502 stream_receiver_free(rpt);
503 return stream_receiver_response_permission_denied(w);
@@ -503,7 +509,7 @@ int stream_receiver_accept_connection(struct web_client *w, char *decoded_query_
509 stream_receiver_log_status(
510 rpt,
511 "rejecting streaming connection; API key is not enabled in stream.conf",
506 - STREAM_STATUS_API_KEY_DISABLED, NDLP_WARNING);
512 + STREAM_HANDSHAKE_PARENT_DENIED_ACCESS, NDLP_WARNING);
513
514 stream_receiver_free(rpt);
515 return stream_receiver_response_permission_denied(w);
@@ -513,7 +519,7 @@ int stream_receiver_accept_connection(struct web_client *w, char *decoded_query_
519 stream_receiver_log_status(
520 rpt,
521 "rejecting streaming connection; API key is not allowed from this IP",
516 - STREAM_STATUS_NOT_ALLOWED_IP, NDLP_WARNING);
522 + STREAM_HANDSHAKE_PARENT_DENIED_ACCESS, NDLP_WARNING);
523
524 stream_receiver_free(rpt);
525 return stream_receiver_response_permission_denied(w);
@@ -523,7 +529,7 @@ int stream_receiver_accept_connection(struct web_client *w, char *decoded_query_
529 stream_receiver_log_status(
530 rpt,
531 "rejecting streaming connection; machine UUID is an API key (did you mix them up?)",
526 - STREAM_STATUS_INVALID_MACHINE_GUID, NDLP_WARNING);
532 + STREAM_HANDSHAKE_PARENT_DENIED_ACCESS, NDLP_WARNING);
533
534 stream_receiver_free(rpt);
535 return stream_receiver_response_permission_denied(w);
@@ -535,7 +541,7 @@ int stream_receiver_accept_connection(struct web_client *w, char *decoded_query_
541 stream_receiver_log_status(
542 rpt,
543 "rejecting streaming connection; machine UUID is not enabled in stream.conf",
538 - STREAM_STATUS_MACHINE_GUID_DISABLED, NDLP_WARNING);
544 + STREAM_HANDSHAKE_PARENT_DENIED_ACCESS, NDLP_WARNING);
545
546 stream_receiver_free(rpt);
547 return stream_receiver_response_permission_denied(w);
@@ -545,7 +551,7 @@ int stream_receiver_accept_connection(struct web_client *w, char *decoded_query_
551 stream_receiver_log_status(
552 rpt,
553 "rejecting streaming connection; machine UUID is not allowed from this IP",
548 - STREAM_STATUS_NOT_ALLOWED_IP, NDLP_WARNING);
554 + STREAM_HANDSHAKE_PARENT_DENIED_ACCESS, NDLP_WARNING);
555
556 stream_receiver_free(rpt);
557 return stream_receiver_response_permission_denied(w);
@@ -557,7 +563,7 @@ int stream_receiver_accept_connection(struct web_client *w, char *decoded_query_
563 stream_receiver_log_status(
564 rpt,
565 "rejecting streaming connection; machine UUID is my own",
560 - STREAM_STATUS_LOCALHOST, NDLP_DEBUG);
566 + STREAM_HANDSHAKE_PARENT_IS_LOCALHOST, NDLP_DEBUG);
567
568 char initial_response[HTTP_HEADER_SIZE + 1];
569 snprintfz(initial_response, HTTP_HEADER_SIZE, "%s", START_STREAMING_ERROR_SAME_LOCALHOST);
@@ -591,7 +597,7 @@ int stream_receiver_accept_connection(struct web_client *w, char *decoded_query_
597 "rejecting streaming connection; rate limit, will accept new connection in %ld secs",
598 (long)(web_client_streaming_rate_t - (now - last_stream_accepted_t)));
599
594 - stream_receiver_log_status(rpt, msg, STREAM_STATUS_RATE_LIMIT, NDLP_NOTICE);
600 + stream_receiver_log_status(rpt, msg, STREAM_HANDSHAKE_PARENT_BUSY_TRY_LATER, NDLP_NOTICE);
601
602 stream_receiver_free(rpt);
603 return stream_receiver_response_too_busy_now(w);
@@ -634,13 +640,23 @@ int stream_receiver_accept_connection(struct web_client *w, char *decoded_query_
640 }
641 rrd_rdunlock();
642
643 + if (receiver_stale && string_strcmp(host->hostname, rpt->hostname) != 0) {
644 + stream_receiver_log_status(
645 + rpt,
646 + "rejecting streaming connection; machine GUID is connected with a different hostname",
647 + STREAM_HANDSHAKE_PARENT_DENIED_ACCESS, NDLP_WARNING);
648 +
649 + stream_receiver_free(rpt);
650 + return stream_receiver_response_permission_denied(w);
651 + }
652 +
653 if (receiver_stale &&
638 - stream_receiver_signal_to_stop_and_wait(host, STREAM_HANDSHAKE_DISCONNECT_STALE_RECEIVER)) {
654 + stream_receiver_signal_to_stop_and_wait(host, STREAM_HANDSHAKE_RCV_DISCONNECT_STALE_RECEIVER)) {
655 // we stopped the receiver
656 // we can proceed with this connection
657 receiver_stale = false;
658
643 - nd_log_daemon(NDLP_NOTICE, "STREAM '%s' [receive from [%s]:%s]: "
659 + nd_log_daemon(NDLP_NOTICE, "STREAM RCV '%s' [from [%s]:%s]: "
660 "stopped previous stale receiver to accept this one."
661 , rpt->hostname
662 , rpt->remote_ip, rpt->remote_port);
@@ -652,11 +668,11 @@ int stream_receiver_accept_connection(struct web_client *w, char *decoded_query_
668
669 char msg[200 + 1];
670 snprintfz(msg, sizeof(msg) - 1,
655 - "rejecting streaming connection; multiple connections for same host, "
671 + "rejecting streaming connection; multiple connections for the same host, "
672 "old connection was last used %ld secs ago%s",
673 age, receiver_stale ? " (signaled old receiver to stop)" : " (new connection not accepted)");
674
659 - stream_receiver_log_status(rpt, msg, STREAM_STATUS_ALREADY_CONNECTED, NDLP_DEBUG);
675 + stream_receiver_log_status(rpt, msg, STREAM_HANDSHAKE_PARENT_NODE_ALREADY_CONNECTED, NDLP_WARNING);
676
677 // Have not set WEB_CLIENT_FLAG_DONT_CLOSE_SOCKET - caller should clean up
678 buffer_flush(w->response.data);
@@ -679,9 +695,7 @@ int stream_receiver_accept_connection(struct web_client *w, char *decoded_query_
695
696 char msg[256];
697 stream_receiver_connected_msg(rpt->host, msg, sizeof(msg));
682 - stream_receiver_log_status(
683 - rpt, msg,
684 - STREAM_STATUS_CONNECTED, NDLP_INFO);
698 + stream_receiver_log_status(rpt, msg, 0, NDLP_INFO);
699
700 // in case we have cloud connection we inform cloud a new child connected
701 schedule_node_state_update(rpt->host, 300);
@@ -691,7 +705,7 @@ int stream_receiver_accept_connection(struct web_client *w, char *decoded_query_
705 rrdhost_option_set(rpt->host, RRDHOST_OPTION_EPHEMERAL_HOST);
706
707 // let it reconnect to parents asap
694 - rrdhost_stream_parents_reset(rpt->host, STREAM_HANDSHAKE_PREPARING);
708 + rrdhost_stream_parents_reset(rpt->host, STREAM_HANDSHAKE_SP_PREPARING);
709
710 // add it to a stream thread queue
711 stream_receiver_add_to_queue(rpt);
src/streaming/stream-receiver-internals.h
+2 -2
@@ -112,8 +112,8 @@ struct receiver_state {
112 #endif
113
114 bool rrdhost_set_receiver(RRDHOST *host, struct receiver_state *rpt);
115 -void rrdhost_clear_receiver(struct receiver_state *rpt);
116 -void stream_receiver_log_status(struct receiver_state *rpt, const char *msg, const char *status, ND_LOG_FIELD_PRIORITY priority);
115 +void rrdhost_clear_receiver(struct receiver_state *rpt, STREAM_HANDSHAKE reason);
116 +void stream_receiver_log_status(struct receiver_state *rpt, const char *msg, STREAM_HANDSHAKE reason, ND_LOG_FIELD_PRIORITY priority);
117
118 void stream_receiver_free(struct receiver_state *rpt);
119 bool stream_receiver_signal_to_stop_and_wait(RRDHOST *host, STREAM_HANDSHAKE reason);
src/streaming/stream-receiver.c
+68 -68
@@ -65,7 +65,7 @@ void stream_receiver_log_payload(struct receiver_state *rpt, const char *payload
65 }
66 #endif
67
68 -static void stream_receiver_remove(struct stream_thread *sth, struct receiver_state *rpt, const char *why);
68 +static void stream_receiver_remove(struct stream_thread *sth, struct receiver_state *rpt, STREAM_HANDSHAKE reason);
69
70 // When a child disconnects this is the maximum we will wait
71 // before we update the cloud that the child is offline
@@ -297,8 +297,8 @@ static void receiver_set_exit_reason(struct receiver_state *rpt, STREAM_HANDSHAK
297 }
298
299 static inline bool receiver_should_stop(struct receiver_state *rpt) {
300 - if(unlikely(__atomic_load_n(&rpt->exit.shutdown, __ATOMIC_RELAXED))) {
301 - receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN, false);
300 + if(unlikely(__atomic_load_n(&rpt->exit.shutdown, __ATOMIC_ACQUIRE))) {
301 + receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_SIGNALED_TO_STOP, false);
302 return true;
303 }
304
@@ -331,7 +331,7 @@ void stream_receiver_handle_op(struct stream_thread *sth, struct receiver_state
331 sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip, rpt->remote_port,
332 stats.bytes_size, stats.bytes_max_size, stats.bytes_outstanding, stats.bytes_available);
333
334 - stream_receiver_remove(sth, rpt, "receiver send buffer overflow");
334 + stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_DISCONNECT_BUFFER_OVERFLOW);
335 return;
336 }
337
@@ -350,19 +350,23 @@ static ssize_t send_to_child(const char *txt, void *data, STREAM_TRAFFIC_TYPE ty
350 bool was_empty = stats->bytes_outstanding == 0;
351 struct stream_opcode msg = rpt->thread.send_to_child.msg;
352 msg.opcode = STREAM_OPCODE_NONE;
353 + msg.reason = 0;
354
355 size_t size = strlen(txt);
356 ssize_t rc = (ssize_t)size;
357 if(!stream_circular_buffer_add_unsafe(scb, txt, size, size, type, true)) {
358 // should never happen, because of autoscaling
359 msg.opcode = STREAM_OPCODE_RECEIVER_BUFFER_OVERFLOW;
360 + msg.reason = STREAM_HANDSHAKE_DISCONNECT_BUFFER_OVERFLOW;
361 rc = -1;
362 }
363 else {
364 stream_receiver_log_payload(rpt, txt, type, false);
365
364 - if(was_empty)
366 + if(was_empty) {
367 msg.opcode = STREAM_OPCODE_RECEIVER_POLLOUT;
368 + msg.opcode = 0;
369 + }
370 }
371
372 spinlock_unlock(&rpt->thread.send_to_child.spinlock);
@@ -487,6 +491,8 @@ void stream_receiver_move_to_running_unsafe(struct stream_thread *sth, struct re
491 parser->h2o_ctx = rpt->h2o_ctx;
492 #endif
493
494 + pulse_host_status(rpt->host, PULSE_HOST_STATUS_RCV_RUNNING, 0);
495 +
496 // keep this last - it needs everything ready since to sends data to the child
497 stream_receiver_send_node_and_claim_id_to_child(rpt->host);
498 }
@@ -504,7 +510,7 @@ void stream_receiver_move_entire_queue_to_running_unsafe(struct stream_thread *s
510 }
511 }
512
507 -static void stream_receiver_remove(struct stream_thread *sth, struct receiver_state *rpt, const char *why) {
513 +static void stream_receiver_remove(struct stream_thread *sth, struct receiver_state *rpt, STREAM_HANDSHAKE reason) {
514 internal_fatal(sth->tid != gettid_cached(), "Function %s() should only be used by the dispatcher thread", __FUNCTION__ );
515
516 ND_LOG_STACK lgs[] = {
@@ -530,7 +536,7 @@ static void stream_receiver_remove(struct stream_thread *sth, struct receiver_st
536 , rpt->remote_ip ? rpt->remote_ip : "-"
537 , rpt->remote_port ? rpt->remote_port : "-"
538 , count
533 - , why ? why : "");
539 + , stream_handshake_error_to_string(reason));
540
541 internal_fatal(META_GET(&sth->run.meta, (Word_t)&rpt->thread.meta) == NULL,
542 "Receiver to be removed is not found in the list of receivers");
@@ -555,16 +561,17 @@ static void stream_receiver_remove(struct stream_thread *sth, struct receiver_st
561 }
562
563 stream_thread_node_removed(rpt->host);
564 + pulse_host_status(rpt->host, PULSE_HOST_STATUS_RCV_OFFLINE, reason);
565
566 // set a default exit reason, if not set
560 - receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_PARSER_EXIT, false);
567 + receiver_set_exit_reason(rpt, reason, false);
568
569 // in case we are connected to netdata cloud,
570 // we inform cloud that a child got disconnected
571 uint64_t total_reboot = rrdhost_stream_path_total_reboot_time_ms(rpt->host);
572 schedule_node_state_update(rpt->host, MIN((total_reboot * MAX_CHILD_DISC_TOLERANCE), MAX_CHILD_DISC_DELAY));
573
567 - rrdhost_clear_receiver(rpt);
574 + rrdhost_clear_receiver(rpt, reason);
575 rrdhost_set_is_parent_label();
576
577 stream_receiver_free(rpt);
@@ -617,8 +624,8 @@ stream_receive_and_process(struct stream_thread *sth, struct receiver_state *rpt
624
625 while (buffered_reader_next_line(&rpt->thread.uncompressed, rpt->thread.line_buffer)) {
626 if (unlikely(parser_action(parser, rpt->thread.line_buffer->buffer))) {
620 - receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_PARSER_FAILED, false);
621 - stream_receiver_remove(sth, rpt, "parser action failed");
627 + receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_RCV_DISCONNECT_PARSER_FAILED, false);
628 + stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_RCV_DISCONNECT_PARSER_FAILED);
629 *removed = true;
630 return -1;
631 }
@@ -631,8 +638,8 @@ stream_receive_and_process(struct stream_thread *sth, struct receiver_state *rpt
638 break;
639
640 else {
634 - receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_PARSER_FAILED, false);
635 - stream_receiver_remove(sth, rpt, "receiver decompressor failed");
641 + receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_RCV_DISCONNECT_PARSER_FAILED, false);
642 + stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_RCV_DECOMPRESSION_FAILED);
643 *removed = true;
644 return -1;
645 }
@@ -641,16 +648,17 @@ stream_receive_and_process(struct stream_thread *sth, struct receiver_state *rpt
648 else if (feed_rc == DECOMPRESS_NEED_MORE_DATA)
649 break;
650 else {
644 - receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_PARSER_FAILED, false);
645 - stream_receiver_remove(sth, rpt, "receiver compressed data invalid");
651 + receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_RCV_DISCONNECT_PARSER_FAILED, false);
652 + stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_RCV_DECOMPRESSION_FAILED);
653 *removed = true;
654 return -1;
655 }
656 }
657
658 if(receiver_should_stop(rpt)) {
652 - receiver_set_exit_reason(rpt, rpt->exit.reason, false);
653 - stream_receiver_remove(sth, rpt, "received stop signal");
659 + STREAM_HANDSHAKE reason = rpt->exit.reason ? rpt->exit.reason : STREAM_HANDSHAKE_DISCONNECT_SIGNALED_TO_STOP;
660 + receiver_set_exit_reason(rpt, reason, false);
661 + stream_receiver_remove(sth, rpt, reason);
662 *removed = true;
663 return -1;
664 }
@@ -662,8 +670,8 @@ stream_receive_and_process(struct stream_thread *sth, struct receiver_state *rpt
670
671 while(buffered_reader_next_line(&rpt->thread.uncompressed, rpt->thread.line_buffer)) {
672 if(unlikely(parser_action(parser, rpt->thread.line_buffer->buffer))) {
665 - receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_PARSER_FAILED, false);
666 - stream_receiver_remove(sth, rpt, "parser action failed");
673 + receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_RCV_DISCONNECT_PARSER_FAILED, false);
674 + stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_RCV_DISCONNECT_PARSER_FAILED);
675 *removed = true;
676 return -1;
677 }
@@ -727,32 +735,29 @@ bool stream_receiver_send_data(struct stream_thread *sth, struct receiver_state
735 spinlock_unlock(&rpt->thread.send_to_child.spinlock);
736
737 if (status == EVLOOP_STATUS_SOCKET_ERROR || status == EVLOOP_STATUS_SOCKET_CLOSED) {
730 - const char *disconnect_reason;
738 STREAM_HANDSHAKE reason;
739
740 if(status == EVLOOP_STATUS_SOCKET_ERROR) {
741 worker_is_busy(WORKER_STREAM_JOB_DISCONNECT_SEND_ERROR);
735 - disconnect_reason = "socket reports error while writing";
742 reason = STREAM_HANDSHAKE_DISCONNECT_SOCKET_WRITE_FAILED;
743 }
744 else /* if(status == EVLOOP_STATUS_SOCKET_CLOSED) */ {
745 worker_is_busy(WORKER_STREAM_JOB_DISCONNECT_REMOTE_CLOSED);
740 - disconnect_reason = "socket reports EOF (closed by child)";
741 - reason = STREAM_HANDSHAKE_DISCONNECT_SOCKET_CLOSED_BY_REMOTE_END;
746 + reason = STREAM_HANDSHAKE_DISCONNECT_SOCKET_CLOSED_BY_REMOTE;
747 }
748
749 nd_log(NDLS_DAEMON, NDLP_ERR,
750 "STREAM RCV[%zu] '%s' [from [%s]:%s]: %s (%zd, on fd %d) - closing receiver connection - "
751 "we have sent %zu bytes in %zu operations.",
752 sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip, rpt->remote_port,
748 - disconnect_reason, rc, rpt->sock.fd, stats->bytes_sent, stats->sends);
753 + stream_handshake_error_to_string(reason), rc, rpt->sock.fd, stats->bytes_sent, stats->sends);
754
755 receiver_set_exit_reason(rpt, reason, false);
756
757 if(process_opcodes_and_enable_removal) {
758 // this is not executed from the opcode handling mechanism
759 // so we can safely remove the receiver.
755 - stream_receiver_remove(sth, rpt, disconnect_reason);
760 + stream_receiver_remove(sth, rpt, reason);
761 }
762 else {
763 // protection against this case:
@@ -816,26 +821,24 @@ bool stream_receiver_receive_data(struct stream_thread *sth, struct receiver_sta
821 }
822
823 if(status == EVLOOP_STATUS_SOCKET_ERROR || status == EVLOOP_STATUS_SOCKET_CLOSED) {
819 - const char *disconnect_reason;
824 STREAM_HANDSHAKE reason;
825
826 if(status == EVLOOP_STATUS_SOCKET_ERROR) {
827 worker_is_busy(WORKER_STREAM_JOB_DISCONNECT_RECEIVE_ERROR);
828 reason = STREAM_HANDSHAKE_DISCONNECT_SOCKET_READ_FAILED;
825 - disconnect_reason = "error during receive";
829 }
830 else /* if(status == EVLOOP_STATUS_SOCKET_CLOSED) */ {
831 worker_is_busy(WORKER_STREAM_JOB_DISCONNECT_REMOTE_CLOSED);
829 - reason = STREAM_HANDSHAKE_DISCONNECT_SOCKET_CLOSED_BY_REMOTE_END;
830 - disconnect_reason = "socket reports EOF (closed by child)";
832 + reason = STREAM_HANDSHAKE_DISCONNECT_SOCKET_CLOSED_BY_REMOTE;
833 }
834
835 nd_log(NDLS_DAEMON, NDLP_ERR,
836 "STREAM RCV[%zu] '%s' [from [%s]:%s]: %s (fd %d) - closing receiver connection.",
835 - sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip, rpt->remote_port, disconnect_reason, rpt->sock.fd);
837 + sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip, rpt->remote_port,
838 + stream_handshake_error_to_string(reason), rpt->sock.fd);
839
840 receiver_set_exit_reason(rpt, reason, false);
838 - stream_receiver_remove(sth, rpt, disconnect_reason);
841 + stream_receiver_remove(sth, rpt, reason);
842 }
843 else if(status == EVLOOP_STATUS_CONTINUE && process_opcodes && stream_thread_process_opcodes(sth, &rpt->thread.meta))
844 status = EVLOOP_STATUS_OPCODE_ON_ME;
@@ -860,33 +863,26 @@ bool stream_receive_process_poll_events(struct stream_thread *sth, struct receiv
863 ND_LOG_STACK_PUSH(lgs);
864
865 if (receiver_should_stop(rpt)) {
863 - receiver_set_exit_reason(rpt, rpt->exit.reason, false);
864 - stream_receiver_remove(sth, rpt, "received stop signal");
866 + STREAM_HANDSHAKE reason = rpt->exit.reason ? rpt->exit.reason : STREAM_HANDSHAKE_DISCONNECT_SIGNALED_TO_STOP;
867 + receiver_set_exit_reason(rpt, reason, false);
868 + stream_receiver_remove(sth, rpt, reason);
869 return false;
870 }
871
872 if (unlikely(events & (ND_POLL_ERROR | ND_POLL_HUP | ND_POLL_INVALID))) {
873 // we have errors on this socket
874
871 - worker_is_busy(WORKER_STREAM_JOB_SOCKET_ERROR);
875 + worker_is_busy(WORKER_STREAM_JOB_DISCONNECT_SOCKET_ERROR);
876
873 - char *error = "unknown error";
874 -
875 - if (events & ND_POLL_ERROR)
876 - error = "socket reports errors";
877 - else if (events & ND_POLL_HUP)
878 - error = "connection closed by remote end (HUP)";
879 - else if (events & ND_POLL_INVALID)
880 - error = "connection is invalid";
881 -
882 - worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SOCKET_ERROR);
877 + STREAM_HANDSHAKE reason = events & ND_POLL_HUP ? STREAM_HANDSHAKE_DISCONNECT_SOCKET_CLOSED_BY_REMOTE : STREAM_HANDSHAKE_DISCONNECT_SOCKET_ERROR;
878
879 nd_log(NDLS_DAEMON, NDLP_ERR,
880 "STREAM RCV[%zu] '%s' [from [%s]:%s]: %s - closing connection",
886 - sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip, rpt->remote_port, error);
881 + sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip, rpt->remote_port,
882 + stream_handshake_error_to_string(reason));
883
888 - receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_SOCKET_ERROR, false);
889 - stream_receiver_remove(sth, rpt, error);
884 + receiver_set_exit_reason(rpt, reason, false);
885 + stream_receiver_remove(sth, rpt, reason);
886 return false;
887 }
888
@@ -938,7 +934,7 @@ void stream_receiver_check_all_nodes_from_poll(struct stream_thread *sth, usec_t
934 };
935 ND_LOG_STACK_PUSH(lgs);
936
941 - worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT);
937 + worker_is_busy(WORKER_STREAM_JOB_DISCONNECT_TIMEOUT);
938
939 char duration[RFC3339_MAX_LENGTH];
940 duration_snprintf(duration, sizeof(duration), (int64_t)(now_monotonic_usec() - rpt->thread.last_traffic_ut), "us", true);
@@ -954,8 +950,8 @@ void stream_receiver_check_all_nodes_from_poll(struct stream_thread *sth, usec_t
950 sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip, timeout_s,
951 stats.bytes_sent, stats.sends, duration, pending, stats.buffer_ratio);
952
957 - receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_SOCKET_TIMEOUT, false);
958 - stream_receiver_remove(sth, rpt, "timeout");
953 + receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_TIMEOUT, false);
954 + stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_DISCONNECT_TIMEOUT);
955 continue;
956 }
957
@@ -1046,8 +1042,8 @@ void stream_receiver_replication_check_from_poll(struct stream_thread *sth, usec
1042 __atomic_load_n(&host->stream.rcv.status.replication.counter_out, __ATOMIC_RELAXED),
1043 __atomic_load_n(&host->stream.rcv.status.replication.counter_in, __ATOMIC_RELAXED));
1044
1049 - receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_REPLICATION_STALLED, false);
1050 - stream_receiver_remove(sth, rpt, "replication reception stalled");
1045 + receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_REPLICATION_STALLED, false);
1046 + stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_DISCONNECT_REPLICATION_STALLED);
1047 }
1048
1049 rpt->replication.last_checked_ut = rpt->replication.last_progress_ut;
@@ -1061,8 +1057,8 @@ void stream_receiver_cleanup(struct stream_thread *sth) {
1057 m = META_NEXT(&sth->run.meta, &idx)) {
1058 if (m->type != POLLFD_TYPE_RECEIVER) continue;
1059 struct receiver_state *rpt = m->rpt;
1064 - receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN, false);
1065 - stream_receiver_remove(sth, rpt, "shutdown");
1060 + receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN, true);
1061 + stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN);
1062 }
1063 }
1064
@@ -1110,7 +1106,8 @@ bool rrdhost_set_receiver(RRDHOST *host, struct receiver_state *rpt) {
1106 host->receiver = rpt;
1107 rpt->host = host;
1108
1113 - __atomic_store_n(&rpt->exit.shutdown, false, __ATOMIC_RELAXED);
1109 + rpt->exit.reason = 0;
1110 + __atomic_store_n(&rpt->exit.shutdown, false, __ATOMIC_RELEASE);
1111 host->stream.rcv.status.last_connected = now_realtime_sec();
1112 host->stream.rcv.status.last_disconnected = 0;
1113 host->stream.rcv.status.last_chart = 0;
@@ -1139,7 +1136,7 @@ bool rrdhost_set_receiver(RRDHOST *host, struct receiver_state *rpt) {
1136 rrdhost_flag_set(rpt->host, RRDHOST_FLAG_COLLECTOR_ONLINE);
1137 aclk_queue_node_info(rpt->host, true);
1138
1142 - rrdhost_stream_parents_reset(host, STREAM_HANDSHAKE_PREPARING);
1139 + rrdhost_stream_parents_reset(host, STREAM_HANDSHAKE_SP_PREPARING);
1140
1141 set_this = true;
1142 }
@@ -1155,7 +1152,7 @@ bool rrdhost_set_receiver(RRDHOST *host, struct receiver_state *rpt) {
1152 return set_this;
1153 }
1154
1158 -void rrdhost_clear_receiver(struct receiver_state *rpt) {
1155 +void rrdhost_clear_receiver(struct receiver_state *rpt, STREAM_HANDSHAKE reason) {
1156 RRDHOST *host = rpt->host;
1157 if(!host) return;
1158
@@ -1176,13 +1173,13 @@ void rrdhost_clear_receiver(struct receiver_state *rpt) {
1173 rrdhost_set_health_evloop_iteration(host);
1174 ml_host_stop(host);
1175 stream_path_child_disconnected(host);
1179 - stream_sender_signal_to_stop_and_wait(host, STREAM_HANDSHAKE_DISCONNECT_RECEIVER_LEFT, false);
1176 + stream_sender_signal_to_stop_and_wait(host, reason, false);
1177 rrdcontext_host_child_disconnected(host);
1178
1179 if (rpt->config.health.enabled)
1180 rrdcalc_child_disconnected(host);
1181
1185 - rrdhost_stream_parents_reset(host, STREAM_HANDSHAKE_DISCONNECT_RECEIVER_LEFT);
1182 + rrdhost_stream_parents_reset(host, reason);
1183 }
1184 rrdhost_receiver_lock(host);
1185
@@ -1191,7 +1188,8 @@ void rrdhost_clear_receiver(struct receiver_state *rpt) {
1188 stream_receiver_replication_reset(host);
1189 streaming_receiver_disconnected();
1190
1194 - __atomic_store_n(&host->receiver->exit.shutdown, false, __ATOMIC_RELAXED);
1191 + host->receiver->exit.reason = 0;
1192 + __atomic_store_n(&host->receiver->exit.shutdown, false, __ATOMIC_RELEASE);
1193 host->stream.rcv.status.check_obsolete = false;
1194 host->stream.rcv.status.last_connected = 0;
1195 host->stream.rcv.status.last_disconnected = now_realtime_sec();
@@ -1215,16 +1213,18 @@ bool stream_receiver_signal_to_stop_and_wait(RRDHOST *host, STREAM_HANDSHAKE rea
1213
1214 rrdhost_receiver_lock(host);
1215
1218 - if(host->receiver) {
1219 - if(!__atomic_load_n(&host->receiver->exit.shutdown, __ATOMIC_RELAXED)) {
1220 - __atomic_store_n(&host->receiver->exit.shutdown, true, __ATOMIC_RELAXED);
1221 - receiver_set_exit_reason(host->receiver, reason, true);
1222 - shutdown(host->receiver->sock.fd, SHUT_RDWR);
1216 + struct receiver_state *rpt = host->receiver;
1217 +
1218 + if(rpt) {
1219 + if(!__atomic_load_n(&rpt->exit.shutdown, __ATOMIC_ACQUIRE)) {
1220 + receiver_set_exit_reason(rpt, reason, true);
1221 + __atomic_store_n(&rpt->exit.shutdown, true, __ATOMIC_RELEASE);
1222 + shutdown(rpt->sock.fd, SHUT_RDWR);
1223 }
1224 }
1225
1226 int count = 2000;
1227 - while (host->receiver && count-- > 0) {
1227 + while (host->receiver == rpt && count-- > 0) {
1228 rrdhost_receiver_unlock(host);
1229
1230 // let the lock for the receiver thread to exit
@@ -1233,11 +1233,11 @@ bool stream_receiver_signal_to_stop_and_wait(RRDHOST *host, STREAM_HANDSHAKE rea
1233 rrdhost_receiver_lock(host);
1234 }
1235
1236 - if(host->receiver)
1236 + if(host->receiver == rpt)
1237 netdata_log_error("STREAM RCV[x] '%s' [from [%s]:%s]: "
1238 "streaming thread takes too long to stop, giving up..."
1239 , rrdhost_hostname(host)
1240 - , host->receiver->remote_ip, host->receiver->remote_port);
1240 + , rpt->remote_ip, rpt->remote_port);
1241 else
1242 ret = true;
1243
src/streaming/stream-replication-sender.c
+2 -1
@@ -705,7 +705,8 @@ bool replication_response_execute_finalize_and_send(struct replication_query *q,
705
706 RRDSET_FLAGS old = rrdset_flag_set_and_clear(st, RRDSET_FLAG_SENDER_REPLICATION_FINISHED, RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS);
707 if(!(old & RRDSET_FLAG_SENDER_REPLICATION_FINISHED)) {
708 - rrdhost_sender_replicating_charts_minus_one(st->rrdhost);
708 + if(rrdhost_sender_replicating_charts_minus_one(st->rrdhost) == 0)
709 + pulse_host_status(st->rrdhost, PULSE_HOST_STATUS_SND_RUNNING, 0);
710
711 if(!finished_with_gap)
712 st->stream.snd.resync_time_s = 0;
src/streaming/stream-sender-api.c
+6 -4
@@ -62,7 +62,7 @@ void stream_sender_structures_free(struct rrdhost *host) {
62 if (unlikely(!host->sender)) return;
63
64 // stop a possibly running thread
65 - stream_sender_signal_to_stop_and_wait(host, STREAM_HANDSHAKE_DISCONNECT_HOST_CLEANUP, true);
65 + stream_sender_signal_to_stop_and_wait(host, STREAM_HANDSHAKE_SND_DISCONNECT_HOST_CLEANUP, true);
66 stream_circular_buffer_destroy(host->sender->scb);
67 host->sender->scb = NULL;
68 waitq_destroy(&host->sender->waitq);
@@ -110,10 +110,12 @@ void stream_sender_signal_to_stop_and_wait(struct rrdhost *host, STREAM_HANDSHAK
110 struct stream_opcode msg = host->sender->thread.msg;
111 stream_sender_unlock(host->sender);
112
113 - if(reason == STREAM_HANDSHAKE_DISCONNECT_RECEIVER_LEFT)
114 - msg.opcode = STREAM_OPCODE_SENDER_STOP_RECEIVER_LEFT;
115 - else
113 + if(reason == STREAM_HANDSHAKE_SND_DISCONNECT_HOST_CLEANUP)
114 msg.opcode = STREAM_OPCODE_SENDER_STOP_HOST_CLEANUP;
115 + else
116 + msg.opcode = STREAM_OPCODE_SENDER_STOP_RECEIVER_LEFT;
117 + msg.reason = reason;
118 +
119 stream_sender_send_opcode(host->sender, msg);
120
121 while(wait && rrdhost_flag_check(host, RRDHOST_FLAG_STREAM_SENDER_ADDED))
src/streaming/stream-sender-commit.c
+3
@@ -195,6 +195,7 @@ void sender_buffer_commit(struct sender_state *s, BUFFER *wb, struct sender_buff
195
196 if (enable_sending) {
197 msg.opcode = STREAM_OPCODE_SENDER_POLLOUT;
198 + msg.reason = 0;
199 stream_sender_send_opcode(s, msg);
200 }
201
@@ -205,6 +206,7 @@ overflow_with_lock: {
206 stream_sender_unlock(s);
207 waitq_release(&s->waitq);
208 msg.opcode = STREAM_OPCODE_SENDER_BUFFER_OVERFLOW;
209 + msg.reason = STREAM_HANDSHAKE_DISCONNECT_BUFFER_OVERFLOW;
210 stream_sender_send_opcode(s, msg);
211 nd_log_limit_static_global_var(erl, 1, 0);
212 nd_log_limit(&erl, NDLS_DAEMON, NDLP_ERR,
@@ -221,6 +223,7 @@ compression_failed_with_lock: {
223 stream_sender_unlock(s);
224 waitq_release(&s->waitq);
225 msg.opcode = STREAM_OPCODE_SENDER_RECONNECT_WITHOUT_COMPRESSION;
226 + msg.reason = STREAM_HANDSHAKE_SND_DISCONNECT_COMPRESSION_FAILED;
227 stream_sender_send_opcode(s, msg);
228 nd_log_limit_static_global_var(erl, 1, 0);
229 nd_log_limit(&erl, NDLS_DAEMON, NDLP_ERR,
src/streaming/stream-sender-internals.h
+1 -1
@@ -152,7 +152,7 @@ bool stream_connector_is_signaled_to_stop(struct sender_state *s);
152
153 void stream_sender_on_connect(struct sender_state *s);
154
155 -void stream_sender_remove(struct sender_state *s);
155 +void stream_sender_remove(struct sender_state *s, STREAM_HANDSHAKE reason);
156
157 #ifdef NETDATA_LOG_STREAM_SENDER
158 void stream_sender_log_payload(struct sender_state *s, BUFFER *payload, STREAM_TRAFFIC_TYPE type, bool inbound);
src/streaming/stream-sender.c
+47 -31
@@ -4,7 +4,7 @@
4 #include "stream-sender-internals.h"
5 #include "stream-replication-sender.h"
6
7 -static void stream_sender_move_running_to_connector_or_remove(struct stream_thread *sth, struct sender_state *s, STREAM_HANDSHAKE reason, bool reconnect);
7 +static void stream_sender_move_running_to_connector_or_remove(struct stream_thread *sth, struct sender_state *s, STREAM_HANDSHAKE reason, STREAM_HANDSHAKE receiver_reason, bool reconnect);
8
9 // --------------------------------------------------------------------------------------------------------------------
10
@@ -229,14 +229,17 @@ void stream_sender_handle_op(struct stream_thread *sth, struct sender_state *s,
229 stats.bytes_size, stats.bytes_max_size, stats.bytes_outstanding, stats.bytes_available);
230
231 stream_sender_move_running_to_connector_or_remove(
232 - sth, s, STREAM_HANDSHAKE_DISCONNECT_NOT_SUFFICIENT_SEND_BUFFER, true);
232 + sth, s, STREAM_HANDSHAKE_DISCONNECT_BUFFER_OVERFLOW, 0, true);
233 return;
234 }
235
236 if(msg->opcode & STREAM_OPCODE_SENDER_STOP_RECEIVER_LEFT) {
237 worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_RECEIVER_LEFT);
238 stream_sender_move_running_to_connector_or_remove(
239 - sth, s, STREAM_HANDSHAKE_DISCONNECT_RECEIVER_LEFT, false);
239 + sth, s, STREAM_HANDSHAKE_SND_DISCONNECT_RECEIVER_LEFT, msg->reason, false);
240 +
241 + // at this point we also have access to the receiver exit reason as msg->reason
242 +
243 return;
244 }
245
@@ -248,14 +251,14 @@ void stream_sender_handle_op(struct stream_thread *sth, struct sender_state *s,
251 sth->id, rrdhost_hostname(s->host), s->remote_ip);
252
253 stream_sender_move_running_to_connector_or_remove(
251 - sth, s, STREAM_HANDSHAKE_DISCONNECT_NOT_SUFFICIENT_SENDER_COMPRESSION_FAILED, true);
254 + sth, s, STREAM_HANDSHAKE_SND_DISCONNECT_COMPRESSION_FAILED, 0, true);
255 return;
256 }
257
258 if(msg->opcode & STREAM_OPCODE_SENDER_STOP_HOST_CLEANUP) {
259 worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_HOST_CLEANUP);
260 stream_sender_move_running_to_connector_or_remove(
258 - sth, s, STREAM_HANDSHAKE_DISCONNECT_HOST_CLEANUP, false);
261 + sth, s, STREAM_HANDSHAKE_SND_DISCONNECT_HOST_CLEANUP, 0, false);
262 return;
263 }
264
@@ -330,27 +333,35 @@ void stream_sender_move_queue_to_running_unsafe(struct stream_thread *sth) {
333 sth->id, rrdhost_hostname(s->host), s->remote_ip);
334
335 stream_sender_on_ready_to_dispatch(s);
336 +
337 + pulse_host_status(s->host, PULSE_HOST_STATUS_SND_RUNNING, 0);
338 }
339 }
340
336 -void stream_sender_remove(struct sender_state *s) {
341 +void stream_sender_remove(struct sender_state *s, STREAM_HANDSHAKE reason) {
342 // THIS FUNCTION IS USED BY THE CONNECTOR TOO
343 // when it gives up on a certain node
344
345 stream_sender_lock(s);
346
347 + if(reason == STREAM_HANDSHAKE_DISCONNECT_SIGNALED_TO_STOP && s->exit.reason) {
348 + reason = s->exit.reason;
349 + s->exit.reason = 0;
350 + }
351 +
352 __atomic_store_n(&s->exit.shutdown, false, __ATOMIC_RELAXED);
353 rrdhost_flag_clear(s->host,
344 - RRDHOST_FLAG_STREAM_SENDER_ADDED | RRDHOST_FLAG_STREAM_SENDER_CONNECTED |
345 - RRDHOST_FLAG_STREAM_SENDER_READY_4_METRICS);
354 + RRDHOST_FLAG_STREAM_SENDER_ADDED | RRDHOST_FLAG_STREAM_SENDER_CONNECTED |
355 + RRDHOST_FLAG_STREAM_SENDER_READY_4_METRICS);
356
357 s->last_state_since_t = now_realtime_sec();
348 - stream_parent_set_disconnect_reason(s->host->stream.snd.parents.current, s->exit.reason, s->last_state_since_t);
358 + stream_parent_set_disconnect_reason(s->host->stream.snd.parents.current, reason, s->last_state_since_t);
359 s->connector.id = -1;
360 + s->exit.reason = 0;
361
362 stream_sender_unlock(s);
363
353 - rrdhost_stream_parents_reset(s->host, STREAM_HANDSHAKE_EXITING);
364 + rrdhost_stream_parents_reset(s->host, reason);
365
366 #ifdef NETDATA_LOG_STREAM_SENDER
367 spinlock_lock(&s->log.spinlock);
@@ -364,21 +375,26 @@ void stream_sender_remove(struct sender_state *s) {
375 #endif
376 }
377
367 -static void stream_sender_log_disconnection(struct stream_thread *sth, struct sender_state *s, STREAM_HANDSHAKE reason) {
378 +static void stream_sender_log_disconnection(struct stream_thread *sth, struct sender_state *s, STREAM_HANDSHAKE reason, STREAM_HANDSHAKE receiver_reason) {
379 ND_LOG_STACK lgs[] = {
380 ND_LOG_FIELD_UUID(NDF_MESSAGE_ID, &streaming_to_parent_msgid),
381 ND_LOG_FIELD_END(),
382 };
383 ND_LOG_STACK_PUSH(lgs);
384
374 - nd_log(NDLS_DAEMON, NDLP_NOTICE,
375 - "STREAM SND[%zu] '%s' [to %s]: sender disconnected from parent, reason: %s (replication in: %u, out: %u, pending: %zu)",
376 - sth->id, rrdhost_hostname(s->host), s->remote_ip, stream_handshake_error_to_string(reason),
377 - s->host->stream.snd.status.replication.counter_in, s->host->stream.snd.status.replication.counter_out,
378 - dictionary_entries(s->replication.requests));
385 + if(reason == STREAM_HANDSHAKE_SND_DISCONNECT_RECEIVER_LEFT && receiver_reason)
386 + nd_log(NDLS_DAEMON, NDLP_NOTICE,
387 + "STREAM SND[%zu] '%s' [to %s]: sender disconnected from parent, reason: %s (receiver left due to: %s)",
388 + sth->id, rrdhost_hostname(s->host), s->remote_ip,
389 + stream_handshake_error_to_string(reason),
390 + stream_handshake_error_to_string(receiver_reason));
391 + else
392 + nd_log(NDLS_DAEMON, NDLP_NOTICE,
393 + "STREAM SND[%zu] '%s' [to %s]: sender disconnected from parent, reason: %s",
394 + sth->id, rrdhost_hostname(s->host), s->remote_ip, stream_handshake_error_to_string(reason));
395 }
396
381 -static void stream_sender_move_running_to_connector_or_remove(struct stream_thread *sth, struct sender_state *s, STREAM_HANDSHAKE reason, bool reconnect) {
397 +static void stream_sender_move_running_to_connector_or_remove(struct stream_thread *sth, struct sender_state *s, STREAM_HANDSHAKE reason, STREAM_HANDSHAKE receiver_reason, bool reconnect) {
398 internal_fatal(sth->tid != gettid_cached(), "Function %s() should only be used by the dispatcher thread", __FUNCTION__ );
399
400 ND_LOG_STACK lgs[] = {
@@ -412,7 +428,7 @@ static void stream_sender_move_running_to_connector_or_remove(struct stream_thre
428 s->host->stream.snd.status.tid = 0;
429 stream_sender_unlock(s);
430
415 - stream_sender_log_disconnection(sth, s, reason);
431 + stream_sender_log_disconnection(sth, s, reason, receiver_reason);
432
433 nd_sock_close(&s->sock);
434
@@ -423,8 +439,10 @@ static void stream_sender_move_running_to_connector_or_remove(struct stream_thre
439
440 stream_thread_node_removed(s->host);
441
442 + pulse_host_status(s->host, PULSE_HOST_STATUS_SND_OFFLINE, reason);
443 +
444 if (should_remove)
427 - stream_sender_remove(s);
445 + stream_sender_remove(s, reason);
446 else
447 stream_connector_requeue(s);
448 }
@@ -466,7 +484,7 @@ void stream_sender_check_all_nodes_from_poll(struct stream_thread *sth, usec_t n
484 };
485 ND_LOG_STACK_PUSH(lgs);
486
469 - worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT);
487 + worker_is_busy(WORKER_STREAM_JOB_DISCONNECT_TIMEOUT);
488
489 char duration[RFC3339_MAX_LENGTH];
490 duration_snprintf(duration, sizeof(duration), (int64_t)(now_monotonic_usec() - s->thread.last_traffic_ut), "us", true);
@@ -483,7 +501,7 @@ void stream_sender_check_all_nodes_from_poll(struct stream_thread *sth, usec_t n
501 stats.bytes_sent, stats.sends,
502 duration, pending, stats.buffer_ratio);
503
486 - stream_sender_move_running_to_connector_or_remove(sth, s, STREAM_HANDSHAKE_DISCONNECT_SOCKET_TIMEOUT, true);
504 + stream_sender_move_running_to_connector_or_remove(sth, s, STREAM_HANDSHAKE_DISCONNECT_TIMEOUT, 0, true);
505 continue;
506 }
507
@@ -594,7 +612,7 @@ void stream_sender_replication_check_from_poll(struct stream_thread *sth, usec_t
612 __atomic_load_n(&host->stream.snd.status.replication.counter_in, __ATOMIC_RELAXED),
613 __atomic_load_n(&host->stream.snd.status.replication.counter_out, __ATOMIC_RELAXED));
614
597 - stream_sender_move_running_to_connector_or_remove(sth, s, STREAM_HANDSHAKE_REPLICATION_STALLED, true);
615 + stream_sender_move_running_to_connector_or_remove(sth, s, STREAM_HANDSHAKE_DISCONNECT_REPLICATION_STALLED, 0, true);
616 }
617
618 s->replication.last_checked_ut = s->replication.last_progress_ut;
@@ -665,7 +683,7 @@ bool stream_sender_send_data(struct stream_thread *sth, struct sender_state *s,
683 else /* if(status == EVLOOP_STATUS_SOCKET_CLOSED) */ {
684 worker_is_busy(WORKER_STREAM_JOB_DISCONNECT_REMOTE_CLOSED);
685 disconnect_reason = "socket reports EOF (closed by parent)";
668 - reason = STREAM_HANDSHAKE_DISCONNECT_SOCKET_CLOSED_BY_REMOTE_END;
686 + reason = STREAM_HANDSHAKE_DISCONNECT_SOCKET_CLOSED_BY_REMOTE;
687 }
688
689 nd_log(NDLS_DAEMON, NDLP_ERR,
@@ -677,7 +695,7 @@ bool stream_sender_send_data(struct stream_thread *sth, struct sender_state *s,
695 if(process_opcodes_and_enable_removal) {
696 // this is not executed from the opcode handling mechanism
697 // so we can safely remove the sender
680 - stream_sender_move_running_to_connector_or_remove(sth, s, reason, true);
698 + stream_sender_move_running_to_connector_or_remove(sth, s, reason, 0, true);
699 }
700 else {
701 // protection against this case:
@@ -739,7 +757,7 @@ bool stream_sender_receive_data(struct stream_thread *sth, struct sender_state *
757 }
758 else /* if(status == EVLOOP_STATUS_SOCKET_CLOSED) */ {
759 worker_is_busy(WORKER_STREAM_JOB_DISCONNECT_REMOTE_CLOSED);
742 - reason = STREAM_HANDSHAKE_DISCONNECT_SOCKET_CLOSED_BY_REMOTE_END;
760 + reason = STREAM_HANDSHAKE_DISCONNECT_SOCKET_CLOSED_BY_REMOTE;
761 disconnect_reason = "socket reports EOF (closed by parent)";
762 }
763
@@ -748,7 +766,7 @@ bool stream_sender_receive_data(struct stream_thread *sth, struct sender_state *
766 sth->id, rrdhost_hostname(s->host), s->remote_ip, disconnect_reason, s->sock.fd);
767
768 stream_sender_move_running_to_connector_or_remove(
751 - sth, s, reason, true);
769 + sth, s, reason, 0, true);
770 }
771 else if(status == EVLOOP_STATUS_CONTINUE && process_opcodes && stream_thread_process_opcodes(sth, &s->thread.meta))
772 status = EVLOOP_STATUS_OPCODE_ON_ME;
@@ -775,8 +793,6 @@ bool stream_sender_process_poll_events(struct stream_thread *sth, struct sender_
793 if(unlikely(events & (ND_POLL_ERROR|ND_POLL_HUP|ND_POLL_INVALID))) {
794 // we have errors on this socket
795
778 - worker_is_busy(WORKER_STREAM_JOB_SOCKET_ERROR);
779 -
796 char *error = "unknown error";
797
798 if (events & ND_POLL_ERROR)
@@ -786,7 +802,7 @@ bool stream_sender_process_poll_events(struct stream_thread *sth, struct sender_
802 else if (events & ND_POLL_INVALID)
803 error = "connection is invalid";
804
789 - worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SOCKET_ERROR);
805 + worker_is_busy(WORKER_STREAM_JOB_DISCONNECT_SOCKET_ERROR);
806
807 stream_sender_lock(s);
808 // copy the statistics
@@ -797,7 +813,7 @@ bool stream_sender_process_poll_events(struct stream_thread *sth, struct sender_
813 "STREAM SND[%zu] '%s' [to %s]: %s restarting connection - %zu bytes transmitted in %zu operations.",
814 sth->id, rrdhost_hostname(s->host), s->remote_ip, error, stats.bytes_sent, stats.sends);
815
800 - stream_sender_move_running_to_connector_or_remove(sth, s, STREAM_HANDSHAKE_DISCONNECT_SOCKET_ERROR, true);
816 + stream_sender_move_running_to_connector_or_remove(sth, s, STREAM_HANDSHAKE_DISCONNECT_SOCKET_ERROR, 0, true);
817 return false;
818 }
819
@@ -827,6 +843,6 @@ void stream_sender_cleanup(struct stream_thread *sth) {
843
844 s->exit.reason = STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN;
845 s->exit.shutdown = true;
830 - stream_sender_move_running_to_connector_or_remove(sth, s, STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN, false);
846 + stream_sender_move_running_to_connector_or_remove(sth, s, STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN, 0, false);
847 }
848 }
src/streaming/stream-thread.c
+18 -5
@@ -142,6 +142,8 @@ void stream_receiver_send_opcode(struct receiver_state *rpt, struct stream_opcod
142 if (sth->messages.array[i].meta == &rpt->thread.meta) {
143 rpt->thread.send_to_child.msg_slot = i;
144 sth->messages.array[rpt->thread.send_to_child.msg_slot].opcode |= msg.opcode;
145 + if(msg.reason)
146 + sth->messages.array[rpt->thread.send_to_child.msg_slot].reason = msg.reason;
147 spinlock_unlock(&sth->messages.spinlock);
148 internal_fatal(true, "the stream opcode queue is full, but this receiver is already on slot %zu", i);
149 return;
@@ -158,9 +160,12 @@ void stream_receiver_send_opcode(struct receiver_state *rpt, struct stream_opcod
160 rpt->thread.send_to_child.msg_slot = sth->messages.used++;
161 sth->messages.array[rpt->thread.send_to_child.msg_slot] = msg;
162 }
161 - else
163 + else {
164 // the existing slot is good
165 sth->messages.array[rpt->thread.send_to_child.msg_slot].opcode |= msg.opcode;
166 + if(msg.reason)
167 + sth->messages.array[rpt->thread.send_to_child.msg_slot].reason = msg.reason;
168 + }
169 }
170 spinlock_unlock(&sth->messages.spinlock);
171
@@ -223,6 +228,8 @@ void stream_sender_send_opcode(struct sender_state *s, struct stream_opcode msg)
228 if (sth->messages.array[i].meta == &s->thread.meta) {
229 s->thread.msg_slot = i;
230 sth->messages.array[s->thread.msg_slot].opcode |= msg.opcode;
231 + if(msg.reason)
232 + sth->messages.array[s->thread.msg_slot].reason = msg.reason;
233 spinlock_unlock(&sth->messages.spinlock);
234 internal_fatal(true, "the dispatcher message queue is full, but this sender is already on slot %zu", i);
235 return;
@@ -239,9 +246,12 @@ void stream_sender_send_opcode(struct sender_state *s, struct stream_opcode msg)
246 s->thread.msg_slot = sth->messages.used++;
247 sth->messages.array[s->thread.msg_slot] = msg;
248 }
242 - else
249 + else {
250 // the existing slot is good
251 sth->messages.array[s->thread.msg_slot].opcode |= msg.opcode;
252 + if(msg.reason)
253 + sth->messages.array[s->thread.msg_slot].reason = msg.reason;
254 + }
255 }
256 spinlock_unlock(&sth->messages.spinlock);
257
@@ -395,7 +405,6 @@ void *stream_thread(void *ptr) {
405 // both sender and receiver
406 worker_register_job_name(WORKER_STREAM_JOB_SOCKET_RECEIVE, "receive");
407 worker_register_job_name(WORKER_STREAM_JOB_SOCKET_SEND, "send");
398 - worker_register_job_name(WORKER_STREAM_JOB_SOCKET_ERROR, "sock error");
408
409 // receiver
410 worker_register_job_name(WORKER_STREAM_JOB_COMPRESS, "compress");
@@ -409,8 +418,8 @@ void *stream_thread(void *ptr) {
418
419 // disconnection reasons
420 worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_OVERFLOW, "disconnect overflow");
412 - worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT, "disconnect timeout");
413 - worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_SOCKET_ERROR, "disconnect socket error");
421 + worker_register_job_name(WORKER_STREAM_JOB_DISCONNECT_TIMEOUT, "disconnect timeout");
422 + worker_register_job_name(WORKER_STREAM_JOB_DISCONNECT_SOCKET_ERROR, "disconnect socket error");
423 worker_register_job_name(WORKER_STREAM_JOB_DISCONNECT_REMOTE_CLOSED, "disconnect remote closed");
424 worker_register_job_name(WORKER_STREAM_JOB_DISCONNECT_RECEIVE_ERROR, "disconnect receive error");
425 worker_register_job_name(WORKER_STREAM_JOB_DISCONNECT_SEND_ERROR, "disconnect send error");
@@ -745,6 +754,8 @@ void stream_receiver_add_to_queue(struct receiver_state *rpt) {
754 RECEIVERS_SET(&sth->queue.receivers, ++sth->queue.id, rpt);
755 sth->queue.receivers_waiting++;
756 spinlock_unlock(&sth->queue.spinlock);
757 +
758 + pulse_host_status(rpt->host, PULSE_HOST_STATUS_RCV_WAITING, 0);
759 }
760
761 void stream_sender_add_to_queue(struct sender_state *s) {
@@ -759,6 +770,8 @@ void stream_sender_add_to_queue(struct sender_state *s) {
770 spinlock_lock(&sth->queue.spinlock);
771 SENDERS_SET(&sth->queue.senders, ++sth->queue.id, s);
772 spinlock_unlock(&sth->queue.spinlock);
773 +
774 + pulse_host_status(s->host, PULSE_HOST_STATUS_SND_WAITING, 0);
775 }
776
777 void stream_threads_cancel(void) {
src/streaming/stream-thread.h
+35 -31
@@ -5,6 +5,7 @@
5
6 #include "libnetdata/libnetdata.h"
7 #include "stream-circular-buffer.h"
8 +#include "stream-handshake.h"
9
10 struct stream_thread;
11 struct pollfd_slotted {
@@ -30,6 +31,7 @@ struct stream_opcode {
31 int32_t thread_slot; // the dispatcher id this message refers to
32 uint32_t session; // random number used to verify that the message the dispatcher receives is for this sender
33 STREAM_OPCODE opcode; // the actual message to be delivered
34 + STREAM_HANDSHAKE reason;
35 struct pollfd_meta *meta;
36 };
37
@@ -45,46 +47,48 @@ struct stream_opcode {
47 // socket operations
48 #define WORKER_STREAM_JOB_SOCKET_RECEIVE 5
49 #define WORKER_STREAM_JOB_SOCKET_SEND 6
48 -#define WORKER_STREAM_JOB_SOCKET_ERROR 7
50
51 // compression
51 -#define WORKER_STREAM_JOB_COMPRESS 8
52 -#define WORKER_STREAM_JOB_DECOMPRESS 9
52 +#define WORKER_STREAM_JOB_COMPRESS 7
53 +#define WORKER_STREAM_JOB_DECOMPRESS 8
54
55 // receiver events
55 -#define WORKER_RECEIVER_JOB_BYTES_READ 10
56 -#define WORKER_RECEIVER_JOB_BYTES_UNCOMPRESSED 11
56 +#define WORKER_RECEIVER_JOB_BYTES_READ 9
57 +#define WORKER_RECEIVER_JOB_BYTES_UNCOMPRESSED 10
58
59 // sender received commands
59 -#define WORKER_SENDER_JOB_EXECUTE 12
60 -#define WORKER_SENDER_JOB_EXECUTE_REPLAY 13
61 -#define WORKER_SENDER_JOB_EXECUTE_FUNCTION 14
62 -#define WORKER_SENDER_JOB_EXECUTE_META 15
63 -
64 -#define WORKER_SENDER_JOB_DISCONNECT_OVERFLOW 16
65 -#define WORKER_SENDER_JOB_DISCONNECT_TIMEOUT 17
66 -#define WORKER_SENDER_JOB_DISCONNECT_SOCKET_ERROR 18
67 -#define WORKER_STREAM_JOB_DISCONNECT_REMOTE_CLOSED 19
68 -#define WORKER_STREAM_JOB_DISCONNECT_RECEIVE_ERROR 20
69 -#define WORKER_STREAM_JOB_DISCONNECT_SEND_ERROR 21
70 -#define WORKER_SENDER_JOB_DISCONNECT_COMPRESSION_ERROR 22
71 -#define WORKER_SENDER_JOB_DISCONNECT_RECEIVER_LEFT 23
72 -#define WORKER_SENDER_JOB_DISCONNECT_HOST_CLEANUP 24
60 +#define WORKER_SENDER_JOB_EXECUTE 11
61 +#define WORKER_SENDER_JOB_EXECUTE_REPLAY 12
62 +#define WORKER_SENDER_JOB_EXECUTE_FUNCTION 13
63 +#define WORKER_SENDER_JOB_EXECUTE_META 14
64 +
65 +// disconnect reasons
66 +#define WORKER_STREAM_JOB_DISCONNECT_REMOTE_CLOSED 15
67 +#define WORKER_STREAM_JOB_DISCONNECT_RECEIVE_ERROR 16
68 +#define WORKER_STREAM_JOB_DISCONNECT_SEND_ERROR 17
69 +#define WORKER_STREAM_JOB_DISCONNECT_TIMEOUT 18
70 +#define WORKER_STREAM_JOB_DISCONNECT_SOCKET_ERROR 19
71 +
72 +// sender-only disconnect reasons
73 +#define WORKER_SENDER_JOB_DISCONNECT_OVERFLOW 20
74 +#define WORKER_SENDER_JOB_DISCONNECT_COMPRESSION_ERROR 21
75 +#define WORKER_SENDER_JOB_DISCONNECT_RECEIVER_LEFT 22
76 +#define WORKER_SENDER_JOB_DISCONNECT_HOST_CLEANUP 23
77
78 // dispatcher metrics
79 // this has to be the same at pluginsd_parser.h
76 -#define WORKER_RECEIVER_JOB_REPLICATION_COMPLETION 25
77 -#define WORKER_STREAM_METRIC_NODES 26
78 -#define WORKER_SENDER_JOB_BUFFER_RATIO 27
79 -#define WORKER_SENDER_JOB_BYTES_RECEIVED 28
80 -#define WORKER_SENDER_JOB_BYTES_SENT 29
81 -#define WORKER_SENDER_JOB_BYTES_COMPRESSED 30
82 -#define WORKER_SENDER_JOB_BYTES_UNCOMPRESSED 31
83 -#define WORKER_SENDER_JOB_BYTES_COMPRESSION_RATIO 32
84 -#define WORKER_SENDER_JOB_REPLAY_DICT_SIZE 33
85 -#define WORKER_SENDER_JOB_MESSAGES 34
86 -#define WORKER_STREAM_JOB_RECEIVERS_WAITING_LIST_SIZE 35
87 -#define WORKER_STREAM_JOB_SEND_MISSES 36
80 +#define WORKER_RECEIVER_JOB_REPLICATION_COMPLETION 24
81 +#define WORKER_STREAM_METRIC_NODES 25
82 +#define WORKER_SENDER_JOB_BUFFER_RATIO 26
83 +#define WORKER_SENDER_JOB_BYTES_RECEIVED 27
84 +#define WORKER_SENDER_JOB_BYTES_SENT 28
85 +#define WORKER_SENDER_JOB_BYTES_COMPRESSED 29
86 +#define WORKER_SENDER_JOB_BYTES_UNCOMPRESSED 30
87 +#define WORKER_SENDER_JOB_BYTES_COMPRESSION_RATIO 31
88 +#define WORKER_SENDER_JOB_REPLAY_DICT_SIZE 32
89 +#define WORKER_SENDER_JOB_MESSAGES 33
90 +#define WORKER_STREAM_JOB_RECEIVERS_WAITING_LIST_SIZE 34
91 +#define WORKER_STREAM_JOB_SEND_MISSES 35
92
93 // IMPORTANT: to add workers, you have to edit WORKER_PARSER_FIRST_JOB accordingly
94