@cryptotaxi247 / netdata-1 / commits / 3465ff45c

remove deadlock from sender (#18438)

* remove deadlock from sender * rrdpush_sender_thread_close_socket() takes sender state as param

Costa Tsaousis committed Aug 29, 2024 at 19:36 UTC 3465ff45c69fbd86e474020dea79c7ae7f04de67

4 files changed +72 -82

src/streaming/sender.c

+51 -64

@@ -130,22 +130,14 @@ void rrdpush_sender_after_connect(RRDHOST *host) {

130 rrdpush_sender_thread_send_custom_host_variables(host);

131 }

132

133 -void rrdpush_sender_disconnect_and_cleanup(RRDHOST *host) {

134 - rrdhost_flag_clear(host, RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED | RRDHOST_FLAG_RRDPUSH_SENDER_READY_4_METRICS);

135 -

136 - rrdpush_sender_thread_close_socket(host);

137 -

133 +static void rrdpush_sender_on_disconnect(RRDHOST *host) {

134 // we have been connected to this parent - let's cleanup

135

140 - // do not flush the circular buffer here

141 - // this function is called sometimes with the sender lock, sometimes without the lock

142 -

136 rrdpush_sender_charts_and_replication_reset(host);

137

138 // clear the parent's claim id

139 rrdpush_sender_clear_parent_claim_id(host);

140 rrdpush_receiver_send_node_and_claim_id_to_child(host);

148 -

141 stream_path_parent_disconnected(host);

142 }

143

@@ -179,7 +171,7 @@ static ssize_t attempt_to_send(struct sender_state *s) {

171 worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SEND_ERROR);

172 netdata_log_debug(D_STREAM, "STREAM: Send failed - closing socket...");

173 netdata_log_error("STREAM %s [send to %s]: failed to send metrics - closing connection - we have sent %zu bytes on this connection.", rrdhost_hostname(s->host), s->connected_to, s->sent_bytes_on_this_connection);

182 - rrdpush_sender_disconnect_and_cleanup(s->host);

174 + rrdpush_sender_thread_close_socket(s);

175 }

176 else

177 netdata_log_debug(D_STREAM, "STREAM: send() returned 0 -> no error but no transmission");

@@ -217,16 +209,11 @@ static ssize_t attempt_read(struct sender_state *s) {

209 netdata_log_error("STREAM %s [send to %s]: error during receive (%zd) - closing connection.", rrdhost_hostname(s->host), s->connected_to, ret);

210 }

211

220 - rrdpush_sender_disconnect_and_cleanup(s->host);

212 + rrdpush_sender_thread_close_socket(s);

213

214 return ret;

215 }

216

225 -struct rrdpush_sender_thread_data {

226 - RRDHOST *host;

227 - char *pipe_buffer;

228 -};

229 -

217 static bool rrdpush_sender_pipe_close(RRDHOST *host, int *pipe_fds, bool reopen) {

218 static netdata_mutex_t mutex = NETDATA_MUTEX_INITIALIZER;

219

@@ -354,38 +341,6 @@ static bool rrdhost_sender_should_exit(struct sender_state *s) {

341 return false;

342 }

343

357 -static void rrdpush_sender_thread_cleanup_callback(void *pptr) {

358 - struct rrdpush_sender_thread_data *s = CLEANUP_FUNCTION_GET_PTR(pptr);

359 - if(!s) return;

360 -

361 - worker_unregister();

362 -

363 - RRDHOST *host = s->host;

364 -

365 - sender_lock(host->sender);

366 - netdata_log_info("STREAM %s [send]: sending thread exits %s",

367 - rrdhost_hostname(host),

368 - host->sender->exit.reason != STREAM_HANDSHAKE_NEVER ? stream_handshake_error_to_string(host->sender->exit.reason) : "");

369 -

370 - rrdpush_sender_disconnect_and_cleanup(host);

371 - rrdpush_sender_pipe_close(host, host->sender->rrdpush_sender_pipe, false);

372 - rrdpush_sender_execute_commands_cleanup(host->sender);

373 -

374 - rrdhost_clear_sender___while_having_sender_mutex(host);

375 -

376 -#ifdef NETDATA_LOG_STREAM_SENDER

377 - if(host->sender->stream_log_fp) {

378 - fclose(host->sender->stream_log_fp);

379 - host->sender->stream_log_fp = NULL;

380 - }

381 -#endif

382 -

383 - sender_unlock(host->sender);

384 -

385 - freez(s->pipe_buffer);

386 - freez(s);

387 -}

388 -

344 void rrdpush_initialize_ssl_ctx(RRDHOST *host __maybe_unused) {

345 static SPINLOCK sp = NETDATA_SPINLOCK_INITIALIZER;

346 spinlock_lock(&sp);

@@ -546,12 +501,9 @@ void *rrdpush_sender_thread(void *ptr) {

501 return NULL;

502 }

503

549 - struct rrdpush_sender_thread_data *thread_data = callocz(1, sizeof(struct rrdpush_sender_thread_data));

550 - thread_data->pipe_buffer = mallocz(pipe_buffer_size);

551 - thread_data->host = s->host;

552 -

553 - CLEANUP_FUNCTION_REGISTER(rrdpush_sender_thread_cleanup_callback) cleanup_ptr = thread_data;

504 + char *pipe_buffer = mallocz(pipe_buffer_size);

505

506 + bool was_connected = false;

507 size_t iterations = 0;

508 time_t now_s = now_monotonic_sec();

509 while(!rrdhost_sender_should_exit(s)) {

@@ -559,6 +511,11 @@ void *rrdpush_sender_thread(void *ptr) {

511

512 // The connection attempt blocks (after which we use the socket in nonblocking)

513 if(unlikely(s->rrdpush_sender_socket == -1)) {

514 + if(was_connected) {

515 + rrdpush_sender_on_disconnect(s->host);

516 + was_connected = false;

517 + }

518 +

519 worker_is_busy(WORKER_SENDER_JOB_CONNECT);

520

521 now_s = now_monotonic_sec();

@@ -583,6 +540,7 @@ void *rrdpush_sender_thread(void *ptr) {

540 rrdpush_send_host_labels(s->host);

541 rrdpush_send_global_functions(s->host);

542 s->replication.oldest_request_after_t = 0;

543 + was_connected = true;

544

545 rrdhost_flag_set(s->host, RRDHOST_FLAG_RRDPUSH_SENDER_READY_4_METRICS);

546

@@ -603,7 +561,7 @@ void *rrdpush_sender_thread(void *ptr) {

561 )) {

562 worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT);

563 netdata_log_error("STREAM %s [send to %s]: could not send metrics for %d seconds - closing connection - we have sent %zu bytes on this connection via %zu send attempts.", rrdhost_hostname(s->host), s->connected_to, s->timeout, s->sent_bytes_on_this_connection, s->send_attempts);

606 - rrdpush_sender_disconnect_and_cleanup(s->host);

564 + rrdpush_sender_thread_close_socket(s);

565 continue;

566 }

567

@@ -632,9 +590,9 @@ void *rrdpush_sender_thread(void *ptr) {

590

591 if(unlikely(s->rrdpush_sender_pipe[PIPE_READ] == -1)) {

592 if(!rrdpush_sender_pipe_close(s->host, s->rrdpush_sender_pipe, true)) {

635 - netdata_log_error("STREAM %s [send]: cannot create inter-thread communication pipe. Disabling streaming.",

636 - rrdhost_hostname(s->host));

637 - rrdpush_sender_disconnect_and_cleanup(s->host);

593 + netdata_log_error("STREAM %s [send]: cannot create inter-thread communication pipe. "

594 + "Disabling streaming.", rrdhost_hostname(s->host));

595 + rrdpush_sender_thread_close_socket(s);

596 break;

597 }

598 }

@@ -685,7 +643,7 @@ void *rrdpush_sender_thread(void *ptr) {

643 worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_POLL_ERROR);

644 netdata_log_error("STREAM %s [send to %s]: failed to poll(). Closing socket.", rrdhost_hostname(s->host), s->connected_to);

645 rrdpush_sender_pipe_close(s->host, s->rrdpush_sender_pipe, true);

688 - rrdpush_sender_disconnect_and_cleanup(s->host);

646 + rrdpush_sender_thread_close_socket(s);

647 continue;

648 }

649

@@ -704,7 +662,7 @@ void *rrdpush_sender_thread(void *ptr) {

662 worker_is_busy(WORKER_SENDER_JOB_PIPE_READ);

663 netdata_log_debug(D_STREAM, "STREAM: Data added to send buffer (current buffer chunk %zu bytes)...", outstanding);

664

707 - if (read(fds[Collector].fd, thread_data->pipe_buffer, pipe_buffer_size) == -1)

665 + if (read(fds[Collector].fd, pipe_buffer, pipe_buffer_size) == -1)

666 netdata_log_error("STREAM %s [send to %s]: cannot read from internal pipe.", rrdhost_hostname(s->host), s->connected_to);

667 }

668

@@ -734,7 +692,7 @@ void *rrdpush_sender_thread(void *ptr) {

692 if(error) {

693 rrdpush_sender_pipe_close(s->host, s->rrdpush_sender_pipe, true);

694 netdata_log_error("STREAM %s [send to %s]: restarting internal pipe: %s.",

737 - rrdhost_hostname(s->host), s->connected_to, error);

695 + rrdhost_hostname(s->host), s->connected_to, error);

696 }

697 }

698

@@ -751,8 +709,8 @@ void *rrdpush_sender_thread(void *ptr) {

709 if(unlikely(error)) {

710 worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SOCKET_ERROR);

711 netdata_log_error("STREAM %s [send to %s]: restarting connection: %s - %zu bytes transmitted.",

754 - rrdhost_hostname(s->host), s->connected_to, error, s->sent_bytes_on_this_connection);

755 - rrdpush_sender_disconnect_and_cleanup(s->host);

712 + rrdhost_hostname(s->host), s->connected_to, error, s->sent_bytes_on_this_connection);

713 + rrdpush_sender_thread_close_socket(s);

714 }

715 }

716

@@ -761,12 +719,41 @@ void *rrdpush_sender_thread(void *ptr) {

719 worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_OVERFLOW);

720 errno_clear();

721 netdata_log_error("STREAM %s [send to %s]: buffer full (allocated %zu bytes) after sending %zu bytes. Restarting connection",

764 - rrdhost_hostname(s->host), s->connected_to, s->buffer->size, s->sent_bytes_on_this_connection);

765 - rrdpush_sender_disconnect_and_cleanup(s->host);

722 + rrdhost_hostname(s->host), s->connected_to, s->buffer->size, s->sent_bytes_on_this_connection);

723 + rrdpush_sender_thread_close_socket(s);

724 }

725

726 worker_set_metric(WORKER_SENDER_JOB_REPLAY_DICT_SIZE, (NETDATA_DOUBLE) dictionary_entries(s->replication.requests));

727 }

728

729 + if(was_connected)

730 + rrdpush_sender_on_disconnect(s->host);

731 +

732 + netdata_log_info("STREAM %s [send]: sending thread exits %s",

733 + rrdhost_hostname(s->host),

734 + s->host->sender->exit.reason != STREAM_HANDSHAKE_NEVER ? stream_handshake_error_to_string(s->host->sender->exit.reason) : "");

735 +

736 + sender_lock(s->host->sender);

737 + {

738 + rrdpush_sender_thread_close_socket(s);

739 + rrdpush_sender_pipe_close(s->host, s->rrdpush_sender_pipe, false);

740 + rrdpush_sender_execute_commands_cleanup(s);

741 +

742 + rrdhost_clear_sender___while_having_sender_mutex(s->host);

743 +

744 +#ifdef NETDATA_LOG_STREAM_SENDER

745 + if (s->host->sender->stream_log_fp) {

746 + fclose(s->host->sender->stream_log_fp);

747 + s->host->sender->stream_log_fp = NULL;

748 + }

749 +#endif

750 + }

751 + sender_unlock(s->host->sender);

752 +

753 + freez(pipe_buffer);

754 + freez(s);

755 +

756 + worker_unregister();

757 +

758 return NULL;

759 }

src/streaming/sender_commit.c

+1 -1

@@ -118,7 +118,7 @@ void sender_commit(struct sender_state *s, BUFFER *wb, STREAM_TRAFFIC_TYPE type)

118

119 worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_NO_COMPRESSION);

120 rrdpush_compression_deactivate(s);

121 - rrdpush_sender_disconnect_and_cleanup(s->host);

121 + rrdpush_sender_thread_close_socket(s);

122 sender_unlock(s);

123 return;

124 }

src/streaming/sender_connect.c

+19 -14

@@ -2,13 +2,18 @@

3 #include "sender_internals.h"

5 -void rrdpush_sender_thread_close_socket(RRDHOST *host) {

6 - netdata_ssl_close(&host->sender->ssl);

5 +void rrdpush_sender_thread_close_socket(struct sender_state *s) {

6 + rrdhost_flag_clear(s->host, RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED | RRDHOST_FLAG_RRDPUSH_SENDER_READY_4_METRICS);

7 +

8 + netdata_ssl_close(&s->ssl);

8 - if(host->sender->rrdpush_sender_socket != -1) {

9 - close(host->sender->rrdpush_sender_socket);

10 - host->sender->rrdpush_sender_socket = -1;

10 + if(s->rrdpush_sender_socket != -1) {

11 + close(s->rrdpush_sender_socket);

12 + s->rrdpush_sender_socket = -1;

13 }

14 +

15 + // do not flush the circular buffer here

16 + // this function is called sometimes with the sender lock, sometimes without the lock

17 }

19 void rrdpush_encode_variable(stream_encoded_t *se, RRDHOST *host) {

@@ -204,7 +209,7 @@ static inline bool rrdpush_sender_validate_response(RRDHOST *host, struct sender

209 int delay = stream_responses[i].postpone_reconnect_seconds;

210

211 worker_is_busy(worker_job_id);

207 - rrdpush_sender_thread_close_socket(host);

212 + rrdpush_sender_thread_close_socket(s);

213 host->destination->reason = version;

214 host->destination->postpone_reconnection_until = now_realtime_sec() + delay;

215

@@ -231,9 +236,9 @@ unsigned char alpn_proto_list[] = {

236

237 #define CONN_UPGRADE_VAL "upgrade"

238

234 -static bool rrdpush_sender_connect_ssl(struct sender_state *s __maybe_unused) {

239 +static bool rrdpush_sender_connect_ssl(struct sender_state *s) {

240 RRDHOST *host = s->host;

236 - bool ssl_required = host->destination && host->destination->ssl;

241 + bool ssl_required = host && host->destination && host->destination->ssl;

242

243 netdata_ssl_close(&host->sender->ssl);

244

@@ -251,7 +256,7 @@ static bool rrdpush_sender_connect_ssl(struct sender_state *s __maybe_unused) {

256 ND_LOG_STACK_PUSH(lgs);

257

258 worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SSL_ERROR);

254 - rrdpush_sender_thread_close_socket(host);

259 + rrdpush_sender_thread_close_socket(s);

260 host->destination->reason = STREAM_HANDSHAKE_ERROR_SSL_ERROR;

261 host->destination->postpone_reconnection_until = now_realtime_sec() + 5 * 60;

262 return false;

@@ -269,7 +274,7 @@ static bool rrdpush_sender_connect_ssl(struct sender_state *s __maybe_unused) {

274

275 worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SSL_ERROR);

276 netdata_log_error("SSL: closing the stream connection, because the server SSL certificate is not valid.");

272 - rrdpush_sender_thread_close_socket(host);

277 + rrdpush_sender_thread_close_socket(s);

278 host->destination->reason = STREAM_HANDSHAKE_ERROR_INVALID_CERTIFICATE;

279 host->destination->postpone_reconnection_until = now_realtime_sec() + 5 * 60;

280 return false;

@@ -390,7 +395,7 @@ static bool rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_p

395 };

396

397 // make sure the socket is closed

393 - rrdpush_sender_thread_close_socket(host);

398 + rrdpush_sender_thread_close_socket(s);

399

400 s->rrdpush_sender_socket = connect_to_one_of_destinations(

401 host

@@ -527,7 +532,7 @@ static bool rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_p

532 ND_LOG_STACK_PUSH(lgs);

533

534 worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_CANT_UPGRADE_CONNECTION);

530 - rrdpush_sender_thread_close_socket(host);

535 + rrdpush_sender_thread_close_socket(s);

536 host->destination->reason = STREAM_HANDSHAKE_ERROR_HTTP_UPGRADE;

537 host->destination->postpone_reconnection_until = now_realtime_sec() + 1 * 60;

538 return false;

@@ -550,7 +555,7 @@ static bool rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_p

555 ND_LOG_STACK_PUSH(lgs);

556

557 worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT);

553 - rrdpush_sender_thread_close_socket(host);

558 + rrdpush_sender_thread_close_socket(s);

559

560 nd_log(NDLS_DAEMON, NDLP_ERR,

561 "STREAM %s [send to %s]: failed to send HTTP header to remote netdata.",

@@ -577,7 +582,7 @@ static bool rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_p

582 ND_LOG_STACK_PUSH(lgs);

583

584 worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT);

580 - rrdpush_sender_thread_close_socket(host);

585 + rrdpush_sender_thread_close_socket(s);

586

587 nd_log(NDLS_DAEMON, NDLP_ERR,

588 "STREAM %s [send to %s]: remote netdata does not respond.",

src/streaming/sender_internals.h

+1 -3

@@ -44,9 +44,7 @@ extern char *netdata_ssl_ca_file;

44 bool attempt_to_connect(struct sender_state *state);

45 void rrdpush_sender_on_connect(RRDHOST *host);

46 void rrdpush_sender_after_connect(RRDHOST *host);

47 -void rrdpush_sender_thread_close_socket(RRDHOST *host);

48 -

49 -void rrdpush_sender_disconnect_and_cleanup(RRDHOST *host);

47 +void rrdpush_sender_thread_close_socket(struct sender_state *s);

49 void rrdpush_sender_execute_commands_cleanup(struct sender_state *s);

50 void rrdpush_sender_execute_commands(struct sender_state *s);