Fix SSL non-blocking retry handling in the web server (#15222)
* readers should be able to recursively acquire the lock, even when there is a writer waiting * log failure to establish connection * debug log on streaming sender * log the length of the message * log recv_timeout() * debug * debug * debug ssl read * log SSL_get_error() responses * handle EWOULDBLOCK in web_client_receive() * make the web server handle EWOULDBLOCK case for SSL * remove debug statements * handle more errno values
Costa Tsaousis committed
Jun 20, 2023 at 21:55 UTC
16a4dc97f9f9726e8d54883211add9b0592529be
5 files changed
+93
-80
libnetdata/socket/security.c
+79
-72
@@ -23,80 +23,19 @@ static SOCKET_PEERS netdata_ssl_peers(NETDATA_SSL *ssl) {
23
return socket_peers(sock_fd);
24
}
25
26
-bool netdata_ssl_open(NETDATA_SSL *ssl, SSL_CTX *ctx, int fd) {
27
- errno = 0;
28
- ssl->ssl_errno = 0;
29
-
30
- if(ssl->conn) {
31
- if(!ctx || SSL_get_SSL_CTX(ssl->conn) != ctx) {
32
- SSL_free(ssl->conn);
33
- ssl->conn = NULL;
34
- }
35
- else if (SSL_clear(ssl->conn) == 0) {
36
- netdata_ssl_log_error_queue("SSL_clear", ssl);
37
- SSL_free(ssl->conn);
38
- ssl->conn = NULL;
39
- }
40
- }
41
-
42
- if(!ssl->conn) {
43
- if(!ctx) {
44
- internal_error(true, "SSL: not CTX given");
45
- ssl->state = NETDATA_SSL_STATE_FAILED;
46
- return false;
47
- }
48
-
49
- ssl->conn = SSL_new(ctx);
50
- if (!ssl->conn) {
51
- netdata_ssl_log_error_queue("SSL_new", ssl);
52
- ssl->state = NETDATA_SSL_STATE_FAILED;
53
- return false;
54
- }
55
- }
56
-
57
- if(SSL_set_fd(ssl->conn, fd) != 1) {
58
- netdata_ssl_log_error_queue("SSL_set_fd", ssl);
59
- ssl->state = NETDATA_SSL_STATE_FAILED;
60
- return false;
61
- }
62
-
63
- ssl->state = NETDATA_SSL_STATE_INIT;
64
-
65
- ERR_clear_error();
66
-
67
- return true;
68
-}
69
-
70
-void netdata_ssl_close(NETDATA_SSL *ssl) {
71
- errno = 0;
72
- ssl->ssl_errno = 0;
73
-
74
- if(ssl->conn) {
75
- if(SSL_connection(ssl)) {
76
- int ret = SSL_shutdown(ssl->conn);
77
- if(ret == 0)
78
- SSL_shutdown(ssl->conn);
79
- }
26
+static void netdata_ssl_log_error_queue(const char *call, NETDATA_SSL *ssl, unsigned long err) {
27
+ error_limit_static_thread_var(erl, 1, 0);
28
81
- SSL_free(ssl->conn);
29
+ if(err == SSL_ERROR_NONE)
30
+ err = ERR_get_error();
31
83
- ERR_clear_error();
84
- }
85
-
86
- *ssl = NETDATA_SSL_UNSET_CONNECTION;
87
-}
32
+ if(err == SSL_ERROR_NONE)
33
+ return;
34
89
-void netdata_ssl_log_error_queue(const char *call, NETDATA_SSL *ssl) {
90
- error_limit_static_thread_var(erl, 1, 0);
91
- unsigned long err;
92
- while((err = ERR_get_error())) {
35
+ do {
36
char *code;
37
38
switch (err) {
96
- case SSL_ERROR_NONE:
97
- code = "SSL_ERROR_NONE";
98
- break;
99
-
39
case SSL_ERROR_SSL:
40
code = "SSL_ERROR_SSL";
41
ssl->state = NETDATA_SSL_STATE_FAILED;
@@ -166,7 +105,71 @@ void netdata_ssl_log_error_queue(const char *call, NETDATA_SSL *ssl) {
105
SOCKET_PEERS peers = netdata_ssl_peers(ssl);
106
error_limit(&erl, "SSL: %s() on socket local [[%s]:%d] <-> remote [[%s]:%d], returned error %lu (%s): %s",
107
call, peers.local.ip, peers.local.port, peers.peer.ip, peers.peer.port, err, code, str);
108
+
109
+ } while((err = ERR_get_error()));
110
+}
111
+
112
+bool netdata_ssl_open(NETDATA_SSL *ssl, SSL_CTX *ctx, int fd) {
113
+ errno = 0;
114
+ ssl->ssl_errno = 0;
115
+
116
+ if(ssl->conn) {
117
+ if(!ctx || SSL_get_SSL_CTX(ssl->conn) != ctx) {
118
+ SSL_free(ssl->conn);
119
+ ssl->conn = NULL;
120
+ }
121
+ else if (SSL_clear(ssl->conn) == 0) {
122
+ netdata_ssl_log_error_queue("SSL_clear", ssl, SSL_ERROR_NONE);
123
+ SSL_free(ssl->conn);
124
+ ssl->conn = NULL;
125
+ }
126
+ }
127
+
128
+ if(!ssl->conn) {
129
+ if(!ctx) {
130
+ internal_error(true, "SSL: not CTX given");
131
+ ssl->state = NETDATA_SSL_STATE_FAILED;
132
+ return false;
133
+ }
134
+
135
+ ssl->conn = SSL_new(ctx);
136
+ if (!ssl->conn) {
137
+ netdata_ssl_log_error_queue("SSL_new", ssl, SSL_ERROR_NONE);
138
+ ssl->state = NETDATA_SSL_STATE_FAILED;
139
+ return false;
140
+ }
141
}
142
+
143
+ if(SSL_set_fd(ssl->conn, fd) != 1) {
144
+ netdata_ssl_log_error_queue("SSL_set_fd", ssl, SSL_ERROR_NONE);
145
+ ssl->state = NETDATA_SSL_STATE_FAILED;
146
+ return false;
147
+ }
148
+
149
+ ssl->state = NETDATA_SSL_STATE_INIT;
150
+
151
+ ERR_clear_error();
152
+
153
+ return true;
154
+}
155
+
156
+void netdata_ssl_close(NETDATA_SSL *ssl) {
157
+ errno = 0;
158
+ ssl->ssl_errno = 0;
159
+
160
+ if(ssl->conn) {
161
+ if(SSL_connection(ssl)) {
162
+ int ret = SSL_shutdown(ssl->conn);
163
+ if(ret == 0)
164
+ SSL_shutdown(ssl->conn);
165
+ }
166
+
167
+ SSL_free(ssl->conn);
168
+
169
+ ERR_clear_error();
170
+ }
171
+
172
+ *ssl = NETDATA_SSL_UNSET_CONNECTION;
173
}
174
175
static inline bool is_handshake_complete(NETDATA_SSL *ssl, const char *op) {
@@ -231,11 +234,12 @@ ssize_t netdata_ssl_read(NETDATA_SSL *ssl, void *buf, size_t num) {
234
235
if(unlikely(bytes <= 0)) {
236
int err = SSL_get_error(ssl->conn, bytes);
234
- netdata_ssl_log_error_queue("SSL_read", ssl);
237
if (err == SSL_ERROR_WANT_READ || err == SSL_ERROR_WANT_WRITE) {
238
ssl->ssl_errno = err;
239
errno = EWOULDBLOCK;
240
}
241
+ else
242
+ netdata_ssl_log_error_queue("SSL_read", ssl, err);
243
244
bytes = -1; // according to read() or recv()
245
}
@@ -267,11 +271,12 @@ ssize_t netdata_ssl_write(NETDATA_SSL *ssl, const void *buf, size_t num) {
271
272
if(unlikely(bytes <= 0)) {
273
int err = SSL_get_error(ssl->conn, bytes);
270
- netdata_ssl_log_error_queue("SSL_write", ssl);
274
if (err == SSL_ERROR_WANT_READ || err == SSL_ERROR_WANT_WRITE) {
275
ssl->ssl_errno = err;
276
errno = EWOULDBLOCK;
277
}
278
+ else
279
+ netdata_ssl_log_error_queue("SSL_write", ssl, err);
280
281
bytes = -1; // according to write() or send()
282
}
@@ -353,7 +358,8 @@ bool netdata_ssl_connect(NETDATA_SSL *ssl) {
358
}
359
360
if (err != 1) {
356
- netdata_ssl_log_error_queue("SSL_connect", ssl);
361
+ err = SSL_get_error(ssl->conn, err);
362
+ netdata_ssl_log_error_queue("SSL_connect", ssl, err);
363
ssl->state = NETDATA_SSL_STATE_FAILED;
364
return false;
365
}
@@ -378,7 +384,8 @@ bool netdata_ssl_accept(NETDATA_SSL *ssl) {
384
}
385
386
if (err != 1) {
381
- netdata_ssl_log_error_queue("SSL_accept", ssl);
387
+ err = SSL_get_error(ssl->conn, err);
388
+ netdata_ssl_log_error_queue("SSL_accept", ssl, err);
389
ssl->state = NETDATA_SSL_STATE_FAILED;
390
return false;
391
}
libnetdata/socket/security.h
-1
@@ -65,7 +65,6 @@ bool netdata_ssl_accept(NETDATA_SSL *ssl);
65
66
bool netdata_ssl_open(NETDATA_SSL *ssl, SSL_CTX *ctx, int fd);
67
void netdata_ssl_close(NETDATA_SSL *ssl);
68
-void netdata_ssl_log_error_queue(const char *call, NETDATA_SSL *ssl);
68
69
ssize_t netdata_ssl_read(NETDATA_SSL *ssl, void *buf, size_t num);
70
ssize_t netdata_ssl_write(NETDATA_SSL *ssl, const void *buf, size_t num);
libnetdata/socket/socket.c
+5
-2
@@ -1052,19 +1052,22 @@ ssize_t recv_timeout(int sockfd, void *buf, size_t len, int flags, int timeout)
1052
return -1;
1053
}
1054
1055
- if(!retval)
1055
+ if(!retval) {
1056
// timeout
1057
return 0;
1058
+ }
1059
1060
if(fd.revents & POLLIN)
1061
break;
1062
}
1063
1064
#ifdef ENABLE_HTTPS
1064
- if (SSL_connection(ssl))
1065
+ if (SSL_connection(ssl)) {
1066
return netdata_ssl_read(ssl, buf, len);
1067
+ }
1068
#endif
1069
1070
+ internal_error(true, "%s(): calling recv()", __FUNCTION__ );
1071
return recv(sockfd, buf, len, flags);
1072
}
1073
streaming/sender.c
+3
-5
@@ -555,7 +555,7 @@ static bool rrdpush_sender_connect_ssl(struct sender_state *s) {
555
return true;
556
}
557
558
- // failed to establish connection
558
+ error("SSL: failed to establish connection.");
559
return false;
560
561
#else
@@ -709,7 +709,7 @@ static bool rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_p
709
if(!rrdpush_sender_connect_ssl(s))
710
return false;
711
712
- ssize_t bytes;
712
+ ssize_t bytes, len = strlen(http);
713
714
bytes = send_timeout(
715
#ifdef ENABLE_HTTPS
@@ -717,7 +717,7 @@ static bool rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_p
717
#endif
718
s->rrdpush_sender_socket,
719
http,
720
- strlen(http),
720
+ len,
721
0,
722
timeout);
723
@@ -731,8 +731,6 @@ static bool rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_p
731
return false;
732
}
733
734
- // info("STREAM %s [send to %s]: waiting response from remote netdata...", rrdhost_hostname(host), s->connected_to);
735
-
734
bytes = recv_timeout(
735
#ifdef ENABLE_HTTPS
736
&host->sender->ssl,
web/server/web_client.c
+6
@@ -1972,6 +1972,8 @@ ssize_t web_client_receive(struct web_client *w)
1972
// do we have any space for more data?
1973
buffer_need_bytes(w->response.data, NETDATA_WEB_REQUEST_INITIAL_SIZE);
1974
1975
+ errno = 0;
1976
+
1977
#ifdef ENABLE_HTTPS
1978
if ( (!web_client_check_unix(w)) && (netdata_ssl_web_server_ctx) ) {
1979
if (SSL_connection(&w->ssl)) {
@@ -2001,6 +2003,10 @@ ssize_t web_client_receive(struct web_client *w)
2003
debug(D_WEB_CLIENT, "%llu: Received %zd bytes.", w->id, bytes);
2004
debug(D_WEB_DATA, "%llu: Received data: '%s'.", w->id, &w->response.data->buffer[old]);
2005
}
2006
+ else if(unlikely(bytes < 0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR))) {
2007
+ web_client_enable_wait_receive(w);
2008
+ return 0;
2009
+ }
2010
else if (bytes < 0) {
2011
debug(D_WEB_CLIENT, "%llu: receive data failed.", w->id);
2012
WEB_CLIENT_IS_DEAD(w);