Fixes the race-hazard in streaming during the shutdown sequence (#9370)
The streaming component detects when a receiver stream has closed, and stops an attached sender on the same host. This is to support proxy configurations where the stream is passed through. During the shutdown sequence, once netdata_exit has been set no thread should touch any RRDHOST structure as the non-static threads are not joined before the database shuts down. The destruction of the thread state has been separated from the cleanup and can be called from two points. If the thread can detach itself from the host (i.e. it is not during the shutdown sequence) then it does so and destroys the state. During shutdown the thread leaves the state intact so that it can be destroyed during the host destruction, and the host destruction now cancels the thread to ensure a consistent sequence of events.
Andrew Moss committed
Jun 19, 2020 at 19:37 UTC
85752833adac111cf01ae558ee0c46b3a76aed69
4 files changed
+66
-40
database/rrdhost.c
+15
-1
@@ -666,6 +666,7 @@ void rrdhost_system_info_free(struct rrdhost_system_info *system_info) {
666
}
667
}
668
669
+void destroy_receiver_state(struct receiver_state *rpt);
670
void rrdhost_free(RRDHOST *host) {
671
if(!host) return;
672
@@ -674,12 +675,25 @@ void rrdhost_free(RRDHOST *host) {
675
rrd_check_wrlock(); // make sure the RRDs are write locked
676
677
// ------------------------------------------------------------------------
677
- // clean up the sender
678
+ // clean up streaming
679
rrdpush_sender_thread_stop(host); // stop a possibly running thread
680
cbuffer_free(host->sender->buffer);
681
buffer_free(host->sender->build);
682
freez(host->sender);
683
host->sender = NULL;
684
+ if (netdata_exit) {
685
+ netdata_mutex_lock(&host->receiver_lock);
686
+ if (host->receiver) {
687
+ if (!host->receiver->exited)
688
+ netdata_thread_cancel(host->receiver->thread);
689
+ while (!host->receiver->exited)
690
+ sleep_usec(50 * USEC_PER_MS);
691
+ destroy_receiver_state(host->receiver);
692
+ }
693
+ netdata_mutex_unlock(&host->receiver_lock);
694
+ }
695
+
696
+
697
698
rrdhost_wrlock(host); // lock this RRDHOST
699
streaming/receiver.c
+47
-36
@@ -4,14 +4,40 @@
4
5
extern struct config stream_config;
6
7
+void destroy_receiver_state(struct receiver_state *rpt) {
8
+ freez(rpt->key);
9
+ freez(rpt->hostname);
10
+ freez(rpt->registry_hostname);
11
+ freez(rpt->machine_guid);
12
+ freez(rpt->os);
13
+ freez(rpt->timezone);
14
+ freez(rpt->tags);
15
+ freez(rpt->client_ip);
16
+ freez(rpt->client_port);
17
+ freez(rpt->program_name);
18
+ freez(rpt->program_version);
19
+#ifdef ENABLE_HTTPS
20
+ if(rpt->ssl.conn){
21
+ SSL_free(rpt->ssl.conn);
22
+ }
23
+#endif
24
+ freez(rpt);
25
+}
26
+
27
static void rrdpush_receiver_thread_cleanup(void *ptr) {
28
static __thread int executed = 0;
29
if(!executed) {
30
executed = 1;
31
struct receiver_state *rpt = (struct receiver_state *) ptr;
32
+ // If the shutdown sequence has started, and this receiver is still attached to the host then we cannot touch
33
+ // the host pointer as it is unpredicable when the RRDHOST is deleted. Do the cleanup from rrdhost_free().
34
+ if (netdata_exit && rpt->host) {
35
+ rpt->exited = 1;
36
+ return;
37
+ }
38
39
// Make sure that we detach this thread and don't kill a freshly arriving receiver
14
- if (rpt->host) {
40
+ if (!netdata_exit && rpt->host) {
41
netdata_mutex_lock(&rpt->host->receiver_lock);
42
if (rpt->host->receiver == rpt)
43
rpt->host->receiver = NULL;
@@ -19,25 +45,7 @@ static void rrdpush_receiver_thread_cleanup(void *ptr) {
45
}
46
47
info("STREAM %s [receive from [%s]:%s]: receive thread ended (task id %d)", rpt->hostname, rpt->client_ip, rpt->client_port, gettid());
22
-
23
- freez(rpt->key);
24
- freez(rpt->hostname);
25
- freez(rpt->registry_hostname);
26
- freez(rpt->machine_guid);
27
- freez(rpt->os);
28
- freez(rpt->timezone);
29
- freez(rpt->tags);
30
- freez(rpt->client_ip);
31
- freez(rpt->client_port);
32
- freez(rpt->program_name);
33
- freez(rpt->program_version);
34
-#ifdef ENABLE_HTTPS
35
- if(rpt->ssl.conn){
36
- SSL_free(rpt->ssl.conn);
37
- }
38
-#endif
39
- freez(rpt);
40
-
48
+ destroy_receiver_state(rpt);
49
}
50
}
51
@@ -413,26 +421,29 @@ static int rrdpush_receive(struct receiver_state *rpt)
421
422
423
size_t count = streaming_parser(rpt, &cd, fp);
416
- //size_t count = pluginsd_process(host, &cd, fp, 1);
417
-
418
- log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rpt->host->hostname, "DISCONNECTED");
419
- error("STREAM %s [receive from [%s]:%s]: disconnected (completed %zu updates).", rpt->host->hostname, rpt->client_ip, rpt->client_port, count);
420
-
421
- netdata_mutex_lock(&rpt->host->receiver_lock);
422
- if (rpt->host->receiver == rpt) {
423
- rrdhost_wrlock(rpt->host);
424
- rpt->host->senders_disconnected_time = now_realtime_sec();
425
- rrdhost_flag_set(rpt->host, RRDHOST_FLAG_ORPHAN);
426
- if(health_enabled == CONFIG_BOOLEAN_AUTO)
427
- rpt->host->health_enabled = 0;
428
- rrdhost_unlock(rpt->host);
429
- rrdpush_sender_thread_stop(rpt->host);
424
+
425
+ log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rpt->hostname,
426
+ "DISCONNECTED");
427
+ error("STREAM %s [receive from [%s]:%s]: disconnected (completed %zu updates).", rpt->hostname, rpt->client_ip,
428
+ rpt->client_port, count);
429
+
430
+ // During a shutdown there is cleanup code in rrdhost that will cancel the sender thread
431
+ if (!netdata_exit && rpt->host) {
432
+ netdata_mutex_lock(&rpt->host->receiver_lock);
433
+ if (rpt->host->receiver == rpt) {
434
+ rrdhost_wrlock(rpt->host);
435
+ rpt->host->senders_disconnected_time = now_realtime_sec();
436
+ rrdhost_flag_set(rpt->host, RRDHOST_FLAG_ORPHAN);
437
+ if(health_enabled == CONFIG_BOOLEAN_AUTO)
438
+ rpt->host->health_enabled = 0;
439
+ rrdhost_unlock(rpt->host);
440
+ rrdpush_sender_thread_stop(rpt->host);
441
+ }
442
+ netdata_mutex_unlock(&rpt->host->receiver_lock);
443
}
431
- netdata_mutex_unlock(&rpt->host->receiver_lock);
444
445
// cleanup
446
fclose(fp);
435
-
447
return (int)count;
448
}
449
streaming/rrdpush.c
+1
-2
@@ -673,14 +673,13 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *url) {
673
}
674
675
676
- netdata_thread_t thread;
676
677
debug(D_SYSTEM, "starting STREAM receive thread.");
678
679
char tag[FILENAME_MAX + 1];
680
snprintfz(tag, FILENAME_MAX, "STREAM_RECEIVER[%s,[%s]:%s]", rpt->hostname, w->client_ip, w->client_port);
681
683
- if(netdata_thread_create(&thread, tag, NETDATA_THREAD_OPTION_DEFAULT, rrdpush_receiver_thread, (void *)rpt))
682
+ if(netdata_thread_create(&rpt->thread, tag, NETDATA_THREAD_OPTION_DEFAULT, rrdpush_receiver_thread, (void *)rpt))
683
error("Failed to create new STREAM receive thread for client.");
684
685
// prevent the caller from closing the streaming socket
streaming/rrdpush.h
+3
-1
@@ -64,6 +64,7 @@ struct sender_state {
64
65
struct receiver_state {
66
RRDHOST *host;
67
+ netdata_thread_t thread;
68
int fd;
69
char *key;
70
char *hostname;
@@ -85,7 +86,8 @@ struct receiver_state {
86
#ifdef ENABLE_HTTPS
87
struct netdata_ssl ssl;
88
#endif
88
- unsigned int shutdown:1;
89
+ unsigned int shutdown:1; // Tell the thread to exit
90
+ unsigned int exited; // Indicates that the thread has exited (NOT A BITFIELD!)
91
};
92
93