@cryptotaxi247 / netdata-1 / commits / a8e3e5ad2

[cloud-blocker] https_client add TLS ext. SNI + support chunked transfer encoding (#15739)

* add TLS extension host_name (SNI) * increase buffer size for hdrs * implement chunked encoding support

Timotej S committed Aug 7, 2023 at 17:14 UTC a8e3e5ad224c66888d1fb77432d0a85afedf8185
2 files changed +187 -32
aclk/aclk_otp.c
+4
@@ -414,6 +414,10 @@ int aclk_send_otp_response(const char *agent_id, const unsigned char *response,
414 aclk_parse_otp_error(resp.payload);
415 goto cleanup_response;
416 }
417 + if (resp.payload_size == 0 || resp.payload == NULL) {
418 + netdata_log_error("ACLK_OTP Password response payload is empty despite returning 201 Created!");
419 + goto cleanup_response;
420 + }
421 netdata_log_info("ACLK_OTP Got Password from Cloud");
422
423 if (parse_passwd_response(resp.payload, mqtt_auth)){
aclk/https_client.c
+183 -32
@@ -10,6 +10,8 @@
10
11 #include "daemon/global_statistics.h"
12
13 +#define DEFAULT_CHUNKED_RESPONSE_BUFFER_SIZE (4096)
14 +
15 enum http_parse_state {
16 HTTP_PARSE_INITIAL = 0,
17 HTTP_PARSE_HEADERS,
@@ -29,10 +31,27 @@ static const char *http_req_type_to_str(http_req_type_t req) {
31 }
32 }
33
34 +#define TRANSFER_ENCODING_CHUNKED (-2)
35 +
36 typedef struct {
37 enum http_parse_state state;
38 int content_length;
39 int http_code;
40 +
41 + // for chunked data only
42 + char *chunked_response;
43 + size_t chunked_response_size;
44 + size_t chunked_response_written;
45 +
46 + enum chunked_content_state {
47 + CHUNKED_CONTENT_CHUNK_SIZE = 0,
48 + CHUNKED_CONTENT_CHUNK_DATA,
49 + CHUNKED_CONTENT_CHUNK_END_CRLF,
50 + CHUNKED_CONTENT_FINAL_CRLF
51 + } chunked_content_state;
52 +
53 + size_t chunk_size;
54 + size_t chunk_got;
55 } http_parse_ctx;
56
57 #define HTTP_PARSE_CTX_INITIALIZER { .state = HTTP_PARSE_INITIAL, .content_length = -1, .http_code = 0 }
@@ -50,17 +69,40 @@ static inline void http_parse_ctx_clear(http_parse_ctx *ctx) {
69 #define HTTP_LINE_TERM "\x0D\x0A"
70 #define RESP_PROTO "HTTP/1.1 "
71 #define HTTP_KEYVAL_SEPARATOR ": "
53 -#define HTTP_HDR_BUFFER_SIZE 256
72 +#define HTTP_HDR_BUFFER_SIZE 1024
73 #define PORT_STR_MAX_BYTES 12
74
56 -static void process_http_hdr(http_parse_ctx *parse_ctx, const char *key, const char *val)
75 +static int process_http_hdr(http_parse_ctx *parse_ctx, const char *key, const char *val)
76 {
58 - // currently we care only about content-length
59 - // but in future the way this is written
60 - // it can be extended
77 + // currently we care only about specific headers
78 + // we can skip the rest
79 if (!strcmp("content-length", key)) {
80 + if (parse_ctx->content_length == TRANSFER_ENCODING_CHUNKED) {
81 + netdata_log_error("Content-length and transfer-encoding: chunked headers are mutually exclusive");
82 + return 1;
83 + }
84 + if (parse_ctx->content_length != -1) {
85 + netdata_log_error("Duplicate content-length header");
86 + return 1;
87 + }
88 parse_ctx->content_length = atoi(val);
89 + if (parse_ctx->content_length < 0) {
90 + netdata_log_error("Invalid content-length %d", parse_ctx->content_length);
91 + return 1;
92 + }
93 + return 0;
94 }
95 + if (!strcmp("transfer-encoding", key)) {
96 + if (!strcmp("chunked", val)) {
97 + if (parse_ctx->content_length != -1) {
98 + netdata_log_error("Content-length and transfer-encoding: chunked headers are mutually exclusive");
99 + return 1;
100 + }
101 + parse_ctx->content_length = TRANSFER_ENCODING_CHUNKED;
102 + }
103 + return 0;
104 + }
105 + return 0;
106 }
107
108 static int parse_http_hdr(rbuf_t buf, http_parse_ctx *parse_ctx)
@@ -100,11 +142,104 @@ static int parse_http_hdr(rbuf_t buf, http_parse_ctx *parse_ctx)
142 for (ptr = buf_key; *ptr; ptr++)
143 *ptr = tolower(*ptr);
144
103 - process_http_hdr(parse_ctx, buf_key, buf_val);
145 + if (process_http_hdr(parse_ctx, buf_key, buf_val))
146 + return 1;
147
148 return 0;
149 }
150
151 +static inline void chunked_response_buffer_grow_by(http_parse_ctx *parse_ctx, size_t size)
152 +{
153 + if (unlikely(parse_ctx->chunked_response_size == 0)) {
154 + parse_ctx->chunked_response = mallocz(size);
155 + parse_ctx->chunked_response_size = size;
156 + return;
157 + }
158 + parse_ctx->chunked_response = reallocz((void *)parse_ctx->chunked_response, parse_ctx->chunked_response_size + size);
159 + parse_ctx->chunked_response_size += size;
160 +}
161 +
162 +static int process_chunked_content(rbuf_t buf, http_parse_ctx *parse_ctx)
163 +{
164 + int idx;
165 + size_t bytes_to_copy;
166 +
167 + do {
168 + switch (parse_ctx->chunked_content_state) {
169 + case CHUNKED_CONTENT_CHUNK_SIZE:
170 + if (!rbuf_find_bytes(buf, HTTP_LINE_TERM, strlen(HTTP_LINE_TERM), &idx)) {
171 + if (rbuf_bytes_available(buf) >= rbuf_get_capacity(buf))
172 + return PARSE_ERROR;
173 + return NEED_MORE_DATA;
174 + }
175 + if (idx == 0) {
176 + parse_ctx->chunked_content_state = CHUNKED_CONTENT_FINAL_CRLF;
177 + continue;
178 + }
179 + if (idx >= HTTP_HDR_BUFFER_SIZE) {
180 + netdata_log_error("Chunk size is too long");
181 + return PARSE_ERROR;
182 + }
183 + char buf_size[HTTP_HDR_BUFFER_SIZE];
184 + rbuf_pop(buf, buf_size, idx);
185 + buf_size[idx] = 0;
186 + parse_ctx->chunk_size = strtol(buf_size, NULL, 16);
187 + if (parse_ctx->chunk_size < 0 || parse_ctx->chunk_size == LONG_MAX) {
188 + netdata_log_error("Chunk size out of range");
189 + return PARSE_ERROR;
190 + }
191 + if (parse_ctx->chunk_size == 0) {
192 + if (errno == EINVAL) {
193 + netdata_log_error("Invalid chunk size");
194 + return PARSE_ERROR;
195 + }
196 + parse_ctx->chunked_content_state = CHUNKED_CONTENT_CHUNK_END_CRLF;
197 + return 0;
198 + }
199 + parse_ctx->chunk_got = 0;
200 + chunked_response_buffer_grow_by(parse_ctx, parse_ctx->chunk_size);
201 + rbuf_bump_tail(buf, strlen(HTTP_LINE_TERM));
202 + parse_ctx->chunked_content_state = CHUNKED_CONTENT_CHUNK_DATA;
203 + // fallthrough
204 + case CHUNKED_CONTENT_CHUNK_DATA:
205 + if (!(bytes_to_copy = rbuf_bytes_available(buf)))
206 + return NEED_MORE_DATA;
207 + if (bytes_to_copy > parse_ctx->chunk_size - parse_ctx->chunk_got)
208 + bytes_to_copy = parse_ctx->chunk_size - parse_ctx->chunk_got;
209 + rbuf_pop(buf, parse_ctx->chunked_response + parse_ctx->chunked_response_written, bytes_to_copy);
210 + parse_ctx->chunk_got += bytes_to_copy;
211 + parse_ctx->chunked_response_written += bytes_to_copy;
212 + if (parse_ctx->chunk_got != parse_ctx->chunk_size)
213 + continue;
214 + parse_ctx->chunked_content_state = CHUNKED_CONTENT_CHUNK_END_CRLF;
215 + // fallthrough
216 + case CHUNKED_CONTENT_FINAL_CRLF:
217 + case CHUNKED_CONTENT_CHUNK_END_CRLF:
218 + if (rbuf_bytes_available(buf) < strlen(HTTP_LINE_TERM))
219 + return NEED_MORE_DATA;
220 + char buf_crlf[strlen(HTTP_LINE_TERM)];
221 + rbuf_pop(buf, buf_crlf, strlen(HTTP_LINE_TERM));
222 + if (memcmp(buf_crlf, HTTP_LINE_TERM, strlen(HTTP_LINE_TERM))) {
223 + netdata_log_error("CRLF expected");
224 + return PARSE_ERROR;
225 + }
226 + if (parse_ctx->chunked_content_state == CHUNKED_CONTENT_FINAL_CRLF) {
227 + if (parse_ctx->chunked_response_size != parse_ctx->chunked_response_written)
228 + netdata_log_error("Chunked response size mismatch");
229 + chunked_response_buffer_grow_by(parse_ctx, 1);
230 + parse_ctx->chunked_response[parse_ctx->chunked_response_written] = 0;
231 + return PARSE_SUCCESS;
232 + }
233 + if (parse_ctx->chunk_size == 0) {
234 + parse_ctx->chunked_content_state = CHUNKED_CONTENT_FINAL_CRLF;
235 + continue;
236 + }
237 + parse_ctx->chunked_content_state = CHUNKED_CONTENT_CHUNK_SIZE;
238 + continue;
239 + }
240 + } while(1);
241 +}
242 +
243 static int parse_http_response(rbuf_t buf, http_parse_ctx *parse_ctx)
244 {
245 int idx;
@@ -154,6 +289,9 @@ static int parse_http_response(rbuf_t buf, http_parse_ctx *parse_ctx)
289 break;
290 case HTTP_PARSE_CONTENT:
291 // replies like CONNECT etc. do not have content
292 + if (parse_ctx->content_length == TRANSFER_ENCODING_CHUNKED)
293 + return process_chunked_content(buf, parse_ctx);
294 +
295 if (parse_ctx->content_length < 0)
296 return PARSE_SUCCESS;
297
@@ -312,37 +450,39 @@ static int read_parse_response(https_req_ctx_t *ctx) {
450 }
451 ctx->poll_fd.events = 0;
452
315 - ptr = rbuf_get_linear_insert_range(ctx->buf_rx, &size);
453 + do {
454 + ptr = rbuf_get_linear_insert_range(ctx->buf_rx, &size);
455
317 - if (ctx->ssl_ctx)
318 - ret = SSL_read(ctx->ssl, ptr, size);
319 - else
320 - ret = read(ctx->sock, ptr, size);
456 + if (ctx->ssl_ctx)
457 + ret = SSL_read(ctx->ssl, ptr, size);
458 + else
459 + ret = read(ctx->sock, ptr, size);
460
322 - if (ret > 0) {
323 - rbuf_bump_head(ctx->buf_rx, ret);
324 - } else {
325 - if (ctx->ssl_ctx) {
326 - ret = SSL_get_error(ctx->ssl, ret);
327 - switch (ret) {
328 - case SSL_ERROR_WANT_READ:
329 - ctx->poll_fd.events |= POLLIN;
330 - break;
331 - case SSL_ERROR_WANT_WRITE:
332 - ctx->poll_fd.events |= POLLOUT;
333 - break;
334 - default:
335 - netdata_log_error("SSL_read Err: %s", _ssl_err_tos(ret));
336 - return 3;
337 - }
461 + if (ret > 0) {
462 + rbuf_bump_head(ctx->buf_rx, ret);
463 } else {
339 - if (errno != EAGAIN && errno != EWOULDBLOCK) {
340 - netdata_log_error("write error");
341 - return 3;
464 + if (ctx->ssl_ctx) {
465 + ret = SSL_get_error(ctx->ssl, ret);
466 + switch (ret) {
467 + case SSL_ERROR_WANT_READ:
468 + ctx->poll_fd.events |= POLLIN;
469 + break;
470 + case SSL_ERROR_WANT_WRITE:
471 + ctx->poll_fd.events |= POLLOUT;
472 + break;
473 + default:
474 + netdata_log_error("SSL_read Err: %s", _ssl_err_tos(ret));
475 + return 3;
476 + }
477 + } else {
478 + if (errno != EAGAIN && errno != EWOULDBLOCK) {
479 + netdata_log_error("write error");
480 + return 3;
481 + }
482 + ctx->poll_fd.events |= POLLIN;
483 }
343 - ctx->poll_fd.events |= POLLIN;
484 }
345 - }
485 + } while (ctx->poll_fd.events == 0 && rbuf_bytes_free(ctx->buf_rx) > 0);
486 } while (!(ret = parse_http_response(ctx->buf_rx, &ctx->parse_ctx)));
487
488 if (ret != PARSE_SUCCESS) {
@@ -435,6 +575,8 @@ static int handle_http_request(https_req_ctx_t *ctx) {
575 // Read The Response
576 if (read_parse_response(ctx)) {
577 netdata_log_error("Error reading or parsing response from server");
578 + if (ctx->parse_ctx.chunked_response)
579 + freez(ctx->parse_ctx.chunked_response);
580 rc = 4;
581 goto err_exit;
582 }
@@ -546,6 +688,11 @@ int https_request(https_req_t *request, https_req_response_t *response) {
688 goto exit_CTX;
689 }
690
691 + if (!SSL_set_tlsext_host_name(ctx->ssl, connect_host)) {
692 + netdata_log_error("Error setting TLS SNI host");
693 + goto exit_CTX;
694 + }
695 +
696 SSL_set_fd(ctx->ssl, ctx->sock);
697 ret = SSL_connect(ctx->ssl);
698 if (ret != -1 && ret != 1) {
@@ -568,6 +715,10 @@ int https_request(https_req_t *request, https_req_response_t *response) {
715 goto exit_SSL;
716 }
717 response->http_code = ctx->parse_ctx.http_code;
718 + if (ctx->parse_ctx.content_length == TRANSFER_ENCODING_CHUNKED) {
719 + response->payload_size = ctx->parse_ctx.chunked_response_size;
720 + response->payload = ctx->parse_ctx.chunked_response;
721 + }
722 if (ctx->parse_ctx.content_length > 0) {
723 response->payload_size = ctx->parse_ctx.content_length;
724 response->payload = mallocz(response->payload_size + 1);