@cryptotaxi247 / netdata
Code Commits Issues Pulls Sessions CI/CD
master
master
c 2,036 lines 78.3 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "web_client.h"
4 #include "web/websocket/websocket.h"
5 #include "web/mcp/adapters/mcp-http.h"
6 #include "web/mcp/adapters/mcp-sse.h"
7
8 // this is an async I/O implementation of the web server request parser
9 // it is used by all netdata web servers
10
11 int respect_web_browser_do_not_track_policy = 0;
12 const char *web_x_frame_options = NULL;
13
14 int web_enable_gzip = 1, web_gzip_level = 3, web_gzip_strategy = Z_DEFAULT_STRATEGY;
15
16 void web_client_set_conn_tcp(struct web_client *w) {
17 web_client_flags_clear_conn(w);
18 web_client_flag_set(w, WEB_CLIENT_FLAG_CONN_TCP);
19 }
20
21 void web_client_set_conn_unix(struct web_client *w) {
22 web_client_flags_clear_conn(w);
23 web_client_flag_set(w, WEB_CLIENT_FLAG_CONN_UNIX);
24 }
25
26 void web_client_set_conn_cloud(struct web_client *w) {
27 web_client_flags_clear_conn(w);
28 web_client_flag_set(w, WEB_CLIENT_FLAG_CONN_CLOUD);
29 }
30
31 void web_client_set_conn_webrtc(struct web_client *w) {
32 web_client_flags_clear_conn(w);
33 web_client_flag_set(w, WEB_CLIENT_FLAG_CONN_WEBRTC);
34 }
35
36 void web_client_reset_permissions(struct web_client *w) {
37 w->user_auth.method = USER_AUTH_METHOD_NONE;
38 w->user_auth.access = HTTP_ACCESS_NONE;
39 w->user_auth.user_role = HTTP_USER_ROLE_NONE;
40 web_client_clear_mcp_preview_key(w);
41 }
42
43 void web_client_set_permissions(struct web_client *w, HTTP_ACCESS access, HTTP_USER_ROLE role, USER_AUTH_METHOD type) {
44 web_client_reset_permissions(w);
45 w->user_auth.method = type;
46 w->user_auth.access = access;
47 w->user_auth.user_role = role;
48 }
49
50 inline int web_client_permission_denied_acl(struct web_client *w) {
51 w->response.data->content_type = CT_TEXT_PLAIN;
52 buffer_flush(w->response.data);
53 buffer_strcat(w->response.data, "You need to be authorized to access this resource");
54 w->response.code = HTTP_RESP_UNAVAILABLE_FOR_LEGAL_REASONS;
55 return HTTP_RESP_UNAVAILABLE_FOR_LEGAL_REASONS;
56 }
57
58 inline int web_client_permission_denied(struct web_client *w) {
59 w->response.data->content_type = CT_TEXT_PLAIN;
60 buffer_flush(w->response.data);
61
62 if(w->user_auth.access & HTTP_ACCESS_SIGNED_ID)
63 buffer_strcat(w->response.data,
64 "You don't have enough permissions to access this resource");
65 else
66 buffer_strcat(w->response.data,
67 "You need to be authorized to access this resource");
68
69 w->response.code = HTTP_ACCESS_PERMISSION_DENIED_HTTP_CODE(w->user_auth.access);
70 return w->response.code;
71 }
72
73 inline int web_client_service_unavailable(struct web_client *w) {
74 w->response.data->content_type = CT_TEXT_PLAIN;
75 buffer_flush(w->response.data);
76 buffer_strcat(w->response.data, "This service is currently unavailable.");
77 w->response.code = HTTP_RESP_SERVICE_UNAVAILABLE;
78 return HTTP_RESP_SERVICE_UNAVAILABLE;
79 }
80
81 static inline int bad_request_multiple_dashboard_versions(struct web_client *w) {
82 w->response.data->content_type = CT_TEXT_PLAIN;
83 buffer_flush(w->response.data);
84 buffer_strcat(w->response.data, "Multiple dashboard versions given at the URL.");
85 w->response.code = HTTP_RESP_BAD_REQUEST;
86 return HTTP_RESP_BAD_REQUEST;
87 }
88
89 static inline void web_client_enable_wait_from_ssl(struct web_client *w) {
90 if (w->ssl.ssl_errno == SSL_ERROR_WANT_READ)
91 web_client_enable_ssl_wait_receive(w);
92 else if (w->ssl.ssl_errno == SSL_ERROR_WANT_WRITE)
93 web_client_enable_ssl_wait_send(w);
94 else {
95 web_client_disable_ssl_wait_receive(w);
96 web_client_disable_ssl_wait_send(w);
97 }
98 }
99
100 static inline char *strip_control_characters(char *url) {
101 if(!url) return "";
102
103 for(char *s = url; *s ;s++)
104 if(iscntrl((uint8_t)*s)) *s = ' ';
105
106 return url;
107 }
108
109 static void web_client_reset_allocations(struct web_client *w, bool free_all) {
110
111 if(free_all) {
112 // the web client is to be destroyed
113
114 buffer_free(w->url_as_received);
115 w->url_as_received = NULL;
116
117 buffer_free(w->url_path_decoded);
118 w->url_path_decoded = NULL;
119
120 buffer_free(w->url_query_string_decoded);
121 w->url_query_string_decoded = NULL;
122
123 buffer_free(w->response.header_output);
124 w->response.header_output = NULL;
125
126 buffer_free(w->response.header);
127 w->response.header = NULL;
128
129 buffer_free(w->response.data);
130 w->response.data = NULL;
131
132 buffer_free(w->payload);
133 w->payload = NULL;
134 }
135 else {
136 // the web client is to be re-used
137
138 buffer_reset(w->url_as_received);
139 buffer_reset(w->url_path_decoded);
140 buffer_reset(w->url_query_string_decoded);
141
142 buffer_reset(w->response.header_output);
143 buffer_reset(w->response.header);
144 buffer_reset(w->response.data);
145
146 if(w->payload)
147 buffer_reset(w->payload);
148
149 // to add more items here,
150 // web_client_reuse_from_cache() needs to be adjusted to maintain them
151 }
152
153 freez(w->server_host);
154 w->server_host = NULL;
155
156 freez(w->forwarded_host);
157 w->forwarded_host = NULL;
158
159 freez(w->origin);
160 w->origin = NULL;
161
162 freez(w->user_agent);
163 w->user_agent = NULL;
164
165 freez(w->auth_bearer_token);
166 w->auth_bearer_token = NULL;
167
168 memset(w->mcp_session_id, 0, sizeof(w->mcp_session_id));
169
170 // Free WebSocket resources
171 freez(w->websocket.key);
172 w->websocket.key = NULL;
173
174 w->websocket.ext_flags = WS_EXTENSION_NONE;
175 w->websocket.protocol = WS_PROTOCOL_DEFAULT;
176 w->websocket.client_max_window_bits = 0;
177 w->websocket.server_max_window_bits = 0;
178
179 // if we had enabled compression, release it
180 if(w->response.zinitialized) {
181 deflateEnd(&w->response.zstream);
182 w->response.zsent = 0;
183 w->response.zhave = 0;
184 w->response.zstream.avail_in = 0;
185 w->response.zstream.avail_out = 0;
186 w->response.zstream.total_in = 0;
187 w->response.zstream.total_out = 0;
188 w->response.zinitialized = false;
189 web_client_flag_clear(w, WEB_CLIENT_CHUNKED_TRANSFER);
190 }
191
192 memset(w->transaction, 0, sizeof(w->transaction));
193 memset(&w->auth, 0, sizeof(w->auth));
194 memset(&w->user_auth, 0, sizeof(w->user_auth));
195
196 web_client_reset_permissions(w);
197 web_client_flag_clear(w, WEB_CLIENT_ENCODING_GZIP|WEB_CLIENT_ENCODING_DEFLATE);
198 web_client_flag_clear(w, WEB_CLIENT_FLAG_ACCEPT_JSON |
199 WEB_CLIENT_FLAG_ACCEPT_SSE |
200 WEB_CLIENT_FLAG_ACCEPT_TEXT);
201 web_client_reset_path_flags(w);
202 }
203
204 void web_client_log_completed_request(struct web_client *w, bool update_web_stats) {
205 struct timeval tv;
206 now_monotonic_high_precision_timeval(&tv);
207
208 size_t size = w->response.data->len;
209 size_t sent = w->response.zoutput ? (size_t)w->response.zstream.total_out : size;
210
211 usec_t prep_ut = w->timings.tv_ready.tv_sec ? dt_usec(&w->timings.tv_ready, &w->timings.tv_in) : 0;
212 usec_t sent_ut = w->timings.tv_ready.tv_sec ? dt_usec(&tv, &w->timings.tv_ready) : 0;
213 usec_t total_ut = dt_usec(&tv, &w->timings.tv_in);
214 strip_control_characters((char *)buffer_tostring(w->url_as_received));
215
216 ND_LOG_STACK lgs[] = {
217 ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id),
218 ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction),
219 ND_LOG_FIELD_TXT(NDF_NIDL_NODE, w->client_host),
220 ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, HTTP_REQUEST_MODE_2str(w->mode)),
221 ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received),
222 ND_LOG_FIELD_U64(NDF_RESPONSE_CODE, w->response.code),
223 ND_LOG_FIELD_U64(NDF_RESPONSE_SENT_BYTES, sent),
224 ND_LOG_FIELD_U64(NDF_RESPONSE_SIZE_BYTES, size),
225 ND_LOG_FIELD_U64(NDF_RESPONSE_PREPARATION_TIME_USEC, prep_ut),
226 ND_LOG_FIELD_U64(NDF_RESPONSE_SENT_TIME_USEC, sent_ut),
227 ND_LOG_FIELD_U64(NDF_RESPONSE_TOTAL_TIME_USEC, total_ut),
228 ND_LOG_FIELD_TXT(NDF_SRC_IP, w->user_auth.client_ip),
229 ND_LOG_FIELD_TXT(NDF_SRC_PORT, w->client_port),
230 ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_FOR, w->user_auth.forwarded_for),
231 ND_LOG_FIELD_UUID(NDF_ACCOUNT_ID, &w->user_auth.cloud_account_id.uuid),
232 ND_LOG_FIELD_TXT(NDF_USER_NAME, w->user_auth.client_name),
233 ND_LOG_FIELD_TXT(NDF_USER_ROLE, http_id2user_role(w->user_auth.user_role)),
234 ND_LOG_FIELD_CB(NDF_USER_ACCESS, log_cb_http_access_to_hex, &w->user_auth.access),
235 ND_LOG_FIELD_END(),
236 };
237 ND_LOG_STACK_PUSH(lgs);
238
239 ND_LOG_FIELD_PRIORITY prio = NDLP_INFO;
240 if(w->response.code >= 500)
241 prio = NDLP_EMERG;
242 else if(w->response.code >= 400)
243 prio = NDLP_WARNING;
244 else if(w->response.code >= 300)
245 prio = NDLP_NOTICE;
246
247 // cleanup progress
248 if(web_client_flag_check(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING)) {
249 web_client_flag_clear(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING);
250 query_progress_finished(&w->transaction, 0, w->response.code, total_ut, size, sent);
251 }
252
253 // access log
254 if(likely(buffer_strlen(w->url_as_received))) {
255 nd_log(NDLS_ACCESS, prio, NULL);
256
257 if(update_web_stats)
258 pulse_web_request_completed(
259 dt_usec(&tv, &w->timings.tv_in), w->statistics.received_bytes, w->statistics.sent_bytes, size, sent);
260 }
261 }
262
263 void web_client_request_done(struct web_client *w) {
264 sock_setcork(w->fd, false);
265
266 netdata_log_debug(D_WEB_CLIENT, "%llu: Resetting client.", w->id);
267
268 web_client_log_completed_request(w, true);
269 web_client_reset_allocations(w, false);
270
271 w->mode = HTTP_REQUEST_MODE_GET;
272
273 web_client_disable_donottrack(w);
274 web_client_disable_tracking_required(w);
275 web_client_disable_keepalive(w);
276
277 // Clear URL-derived flags between requests. PATH_IS_MCP is re-set
278 // during the next URL decode; clearing it here makes sure a keepalive
279 // connection cannot carry the previous request's classification into
280 // a new request that fails before URL decoding runs (e.g., malformed
281 // request line, unsupported method).
282 web_client_flag_clear(w, WEB_CLIENT_FLAG_PATH_IS_MCP);
283
284 w->header_parse_tries = 0;
285 w->header_parse_last_size = 0;
286
287 web_client_enable_wait_receive(w);
288 web_client_disable_wait_send(w);
289
290 w->response.has_cookies = false;
291 w->response.sent = 0;
292 w->response.code = 0;
293 w->response.zoutput = false;
294
295 w->statistics.received_bytes = 0;
296 w->statistics.sent_bytes = 0;
297 }
298
299 static int append_slash_to_url_and_redirect(struct web_client *w) {
300 // this function returns a relative redirect
301 // it finds the last path component on the URL and just appends / to it
302 //
303 // So, if the URL is:
304 //
305 // /path/to/file?query_string
306 //
307 // It adds a Location header like this:
308 //
309 // Location: file/?query_string\r\n
310 //
311 // The web browser already knows that it is inside /path/to/
312 // so it converts the path to /path/to/file/ and executes the
313 // request again.
314
315 buffer_strcat(w->response.header, "Location: ");
316 const char *b = buffer_tostring(w->url_as_received);
317 const char *q = strchr(b, '?');
318 if(q && q > b) {
319 const char *e = q - 1;
320 while(e > b && *e != '/') e--;
321 if(*e == '/') e++;
322
323 size_t len = q - e;
324 buffer_strncat(w->response.header, e, len);
325 buffer_strncat(w->response.header, "/", 1);
326 buffer_strcat(w->response.header, q);
327 }
328 else {
329 const char *e = &b[buffer_strlen(w->url_as_received) - 1];
330 while(e > b && *e != '/') e--;
331 if(*e == '/') e++;
332
333 buffer_strcat(w->response.header, e);
334 buffer_strncat(w->response.header, "/", 1);
335 }
336
337 buffer_strncat(w->response.header, "\r\n", 2);
338
339 w->response.data->content_type = CT_TEXT_HTML;
340 buffer_flush(w->response.data);
341 buffer_strcat(w->response.data,
342 "<!DOCTYPE html><html>"
343 "<body onload=\"window.location.href = window.location.origin + window.location.pathname + '/' + window.location.search + window.location.hash\">"
344 "Redirecting. In case your browser does not support redirection, please click "
345 "<a onclick=\"window.location.href = window.location.origin + window.location.pathname + '/' + window.location.search + window.location.hash\">here</a>."
346 "</body></html>");
347 return HTTP_RESP_MOVED_PERM;
348 }
349
350 // Work around a bug in the CMocka library by removing this function during testing.
351 #ifndef REMOVE_MYSENDFILE
352
353 static inline int dashboard_version(struct web_client *w) {
354 if(!web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_WITH_VERSION))
355 return -1;
356
357 if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_IS_V3))
358 return 3;
359 if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_IS_V2))
360 return 2;
361 if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_IS_V1))
362 return 1;
363 if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_IS_V0))
364 return 0;
365
366 return -1;
367 }
368
369 static bool find_filename_to_serve(const char *filename, char *dst, size_t dst_len, struct stat *statbuf, struct web_client *w, bool *is_dir) {
370 int d_version = dashboard_version(w);
371 bool has_extension = web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_HAS_FILE_EXTENSION);
372
373 int fallback = 0;
374
375 if(has_extension) {
376 if(d_version == -1)
377 snprintfz(dst, dst_len, "%s/%s", netdata_configured_web_dir, filename);
378 else {
379 // check if the filename or directory exists
380 // fallback to the same path without the dashboard version otherwise
381 snprintfz(dst, dst_len, "%s/v%d/%s", netdata_configured_web_dir, d_version, filename);
382 fallback = 1;
383 }
384 }
385 else if(d_version != -1) {
386 if(filename && *filename) {
387 // check if the filename exists
388 // fallback to /vN/index.html otherwise
389 snprintfz(dst, dst_len, "%s/%s", netdata_configured_web_dir, filename);
390 fallback = 2;
391 }
392 else {
393 if(filename && *filename)
394 web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_HAS_TRAILING_SLASH);
395 snprintfz(dst, dst_len, "%s/v%d", netdata_configured_web_dir, d_version);
396 }
397 }
398 else {
399 // check if filename exists
400 // this is needed to serve {filename}/index.html, in case a user puts a html file into a directory
401 // fallback to /index.html otherwise
402 snprintfz(dst, dst_len, "%s/%s", netdata_configured_web_dir, filename);
403 fallback = 3;
404 }
405
406 if (stat(dst, statbuf) != 0) {
407 if(fallback == 1) {
408 snprintfz(dst, dst_len, "%s/%s", netdata_configured_web_dir, filename);
409 if (stat(dst, statbuf) != 0)
410 return false;
411 }
412 else if(fallback == 2) {
413 if(filename && *filename)
414 web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_HAS_TRAILING_SLASH);
415 snprintfz(dst, dst_len, "%s/v%d", netdata_configured_web_dir, d_version);
416 if (stat(dst, statbuf) != 0)
417 return false;
418 }
419 else if(fallback == 3) {
420 if(filename && *filename)
421 web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_HAS_TRAILING_SLASH);
422 snprintfz(dst, dst_len, "%s", netdata_configured_web_dir);
423 if (stat(dst, statbuf) != 0)
424 return false;
425 }
426 else
427 return false;
428 }
429
430 if((statbuf->st_mode & S_IFMT) == S_IFDIR) {
431 size_t len = strlen(dst);
432 if(len > dst_len - 11)
433 return false;
434
435 strncpyz(&dst[len], "/index.html", dst_len - len);
436
437 if (stat(dst, statbuf) != 0)
438 return false;
439
440 *is_dir = true;
441 }
442
443 return true;
444 }
445
446 static int web_server_static_file(struct web_client *w, char *filename) {
447 char web_path[FILENAME_MAX];
448 snprintfz(web_path, sizeof(web_path), "%s/%s", netdata_configured_web_dir, filename);
449 #if defined(OS_WINDOWS)
450 char display_path[FILENAME_MAX];
451 netdata_log_debug(D_WEB_CLIENT, "%llu: Looking for file '%s'", w->id,
452 os_translate_path(display_path, web_path, sizeof(display_path)));
453 #else
454 netdata_log_debug(D_WEB_CLIENT, "%llu: Looking for file '%s'", w->id, web_path);
455 #endif
456
457 if(!http_can_access_dashboard(w))
458 return web_client_permission_denied_acl(w);
459
460 // skip leading slashes
461 while (*filename == '/') filename++;
462
463 // if the filename contains "strange" characters, refuse to serve it
464 char *s;
465 for(s = filename; *s ;s++) {
466 if( !isalnum((uint8_t)*s) && *s != '/' && *s != '.' && *s != '-' && *s != '_') {
467 netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not acceptable.", w->id, filename);
468 w->response.data->content_type = CT_TEXT_HTML;
469 buffer_sprintf(w->response.data, "Filename contains invalid characters: ");
470 buffer_strcat_htmlescape(w->response.data, filename);
471 return HTTP_RESP_BAD_REQUEST;
472 }
473 }
474
475 // if the filename contains a double dot refuse to serve it
476 if(strstr(filename, "..") != 0) {
477 netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not acceptable.", w->id, filename);
478 w->response.data->content_type = CT_TEXT_HTML;
479 buffer_strcat(w->response.data, "Relative filenames are not supported: ");
480 buffer_strcat_htmlescape(w->response.data, filename);
481 return HTTP_RESP_BAD_REQUEST;
482 }
483
484 // find the physical file on disk
485 bool is_dir = false;
486 char web_filename[FILENAME_MAX + 1];
487 struct stat statbuf;
488 if(!find_filename_to_serve(filename, web_filename, FILENAME_MAX, &statbuf, w, &is_dir)) {
489 w->response.data->content_type = CT_TEXT_HTML;
490 buffer_strcat(w->response.data, "File does not exist, or is not accessible: ");
491 buffer_strcat_htmlescape(w->response.data, filename);
492 return HTTP_RESP_NOT_FOUND;
493 }
494
495 if(is_dir && !web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_HAS_TRAILING_SLASH))
496 return append_slash_to_url_and_redirect(w);
497
498 buffer_flush(w->response.data);
499 buffer_need_bytes(w->response.data, (size_t)statbuf.st_size);
500 w->response.data->len = (size_t)statbuf.st_size;
501
502 // open the file
503 int fd = open(web_filename, O_RDONLY | O_CLOEXEC);
504
505 // read the file
506 if(fd != -1 && read(fd, w->response.data->buffer, statbuf.st_size) != statbuf.st_size) {
507 // cannot read the whole file
508 nd_log(NDLS_DAEMON, NDLP_ERR, "Web server failed to read file '%s'", web_filename);
509 close(fd);
510 fd = -1;
511 }
512
513 // check for failures
514 if(fd == -1) {
515 buffer_flush(w->response.data);
516
517 if(errno == EBUSY || errno == EAGAIN) {
518 netdata_log_error("%llu: File '%s' is busy, sending 307 Moved Temporarily to force retry.", w->id, web_filename);
519 w->response.data->content_type = CT_TEXT_HTML;
520 buffer_sprintf(w->response.header, "Location: /%s\r\n", filename);
521 buffer_strcat(w->response.data, "File is currently busy, please try again later: ");
522 buffer_strcat_htmlescape(w->response.data, filename);
523 return HTTP_RESP_REDIR_TEMP;
524 }
525 else {
526 netdata_log_error("%llu: Cannot open file '%s'.", w->id, web_filename);
527 w->response.data->content_type = CT_TEXT_HTML;
528 buffer_strcat(w->response.data, "Cannot open file: ");
529 buffer_strcat_htmlescape(w->response.data, filename);
530 return HTTP_RESP_NOT_FOUND;
531 }
532 }
533 else
534 close(fd);
535
536 w->response.data->content_type = contenttype_for_filename(web_filename);
537 netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Sending file '%s' (%"PRId64" bytes, fd %d).", w->id, web_filename, (int64_t)statbuf.st_size, w->fd);
538
539 w->mode = HTTP_REQUEST_MODE_GET;
540 web_client_enable_wait_send(w);
541 web_client_disable_wait_receive(w);
542
543 #ifdef __APPLE__
544 w->response.data->date = statbuf.st_mtimespec.tv_sec;
545 #else
546 w->response.data->date = statbuf.st_mtim.tv_sec;
547 #endif
548 w->response.data->expires = now_realtime_sec() + 86400;
549
550 buffer_cacheable(w->response.data);
551
552 return HTTP_RESP_OK;
553 }
554 #endif
555
556 static inline int check_host_and_call(RRDHOST *host, struct web_client *w, char *url, int (*func)(RRDHOST *, struct web_client *, char *)) {
557 return func(host, w, url);
558 }
559
560 int web_client_api_request(RRDHOST *host, struct web_client *w, char *url_path_fragment) {
561 ND_LOG_STACK lgs[] = {
562 ND_LOG_FIELD_TXT(NDF_SRC_IP, w->user_auth.client_ip),
563 ND_LOG_FIELD_TXT(NDF_SRC_PORT, w->client_port),
564 ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_HOST, w->forwarded_host),
565 ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_FOR, w->user_auth.forwarded_for),
566 ND_LOG_FIELD_TXT(NDF_NIDL_NODE, w->client_host),
567 ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, HTTP_REQUEST_MODE_2str(w->mode)),
568 ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received),
569 ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id),
570 ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction),
571 ND_LOG_FIELD_UUID(NDF_ACCOUNT_ID, &w->user_auth.cloud_account_id.uuid),
572 ND_LOG_FIELD_TXT(NDF_USER_NAME, w->user_auth.client_name),
573 ND_LOG_FIELD_TXT(NDF_USER_ROLE, http_id2user_role(w->user_auth.user_role)),
574 ND_LOG_FIELD_CB(NDF_USER_ACCESS, log_cb_http_access_to_hex, &w->user_auth.access),
575 ND_LOG_FIELD_END(),
576 };
577 ND_LOG_STACK_PUSH(lgs);
578
579 if(!web_client_flag_check(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING)) {
580 web_client_flag_set(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING);
581 query_progress_start_or_update(&w->transaction, 0, w->mode, w->acl,
582 buffer_tostring(w->url_as_received),
583 w->payload,
584 w->user_auth.forwarded_for[0] ? w->user_auth.forwarded_for : w->user_auth.client_ip);
585 }
586
587 // get the api version
588 char *tok = strsep_skip_consecutive_separators(&url_path_fragment, "/");
589 if(tok && *tok) {
590 if(strcmp(tok, "v3") == 0)
591 return web_client_api_request_v3(host, w, url_path_fragment);
592 else if(strcmp(tok, "v2") == 0)
593 return web_client_api_request_v2(host, w, url_path_fragment);
594 else if(strcmp(tok, "v1") == 0)
595 return web_client_api_request_v1(host, w, url_path_fragment);
596 else {
597 buffer_flush(w->response.data);
598 w->response.data->content_type = CT_TEXT_HTML;
599 buffer_strcat(w->response.data, "Unsupported API version: ");
600 buffer_strcat_htmlescape(w->response.data, tok);
601 return HTTP_RESP_NOT_FOUND;
602 }
603 }
604 else {
605 buffer_flush(w->response.data);
606 buffer_sprintf(w->response.data, "Which API version?");
607 return HTTP_RESP_BAD_REQUEST;
608 }
609 }
610
611
612 /**
613 * Valid Method
614 *
615 * Netdata accepts only three methods, including one of these three(STREAM) is an internal method.
616 *
617 * @param w is the structure with the client request
618 * @param s is the start string to parse
619 *
620 * @return it returns the next address to parse case the method is valid and NULL otherwise.
621 */
622 static inline char *web_client_valid_method(struct web_client *w, char *s) {
623 // is is a valid request?
624 if(!strncmp(s, "GET ", 4)) {
625 s = &s[4];
626 w->mode = HTTP_REQUEST_MODE_GET;
627 }
628 else if(!strncmp(s, "OPTIONS ", 8)) {
629 s = &s[8];
630 w->mode = HTTP_REQUEST_MODE_OPTIONS;
631 }
632 else if(!strncmp(s, "POST ", 5)) {
633 s = &s[5];
634 w->mode = HTTP_REQUEST_MODE_POST;
635 }
636 else if(!strncmp(s, "PUT ", 4)) {
637 s = &s[4];
638 w->mode = HTTP_REQUEST_MODE_PUT;
639 }
640 else if(!strncmp(s, "DELETE ", 7)) {
641 s = &s[7];
642 w->mode = HTTP_REQUEST_MODE_DELETE;
643 }
644 else if(!strncmp(s, "STREAM ", 7)) {
645 s = &s[7];
646
647 if (!SSL_connection(&w->ssl) && http_is_using_ssl_force(w)) {
648 w->header_parse_tries = 0;
649 w->header_parse_last_size = 0;
650 web_client_disable_wait_receive(w);
651
652 char hostname[256];
653 char *copyme = strstr(s,"hostname=");
654 if ( copyme ){
655 copyme += 9;
656 char *end = strchr(copyme,'&');
657 if(end){
658 size_t length = MIN(255, end - copyme);
659 memcpy(hostname,copyme,length);
660 hostname[length] = 0X00;
661 }
662 else{
663 memcpy(hostname,"not available",13);
664 hostname[13] = 0x00;
665 }
666 }
667 else{
668 memcpy(hostname,"not available",13);
669 hostname[13] = 0x00;
670 }
671 netdata_log_error("The server is configured to always use encrypted connections, please enable the SSL on child with hostname '%s'.",hostname);
672 s = NULL;
673 }
674
675 w->mode = HTTP_REQUEST_MODE_STREAM;
676 }
677 else {
678 s = NULL;
679 }
680
681 return s;
682 }
683
684 /**
685 * Request validate
686 *
687 * @param w is the structure with the client request
688 *
689 * @return It returns HTTP_VALIDATION_OK on success and another code present
690 * in the enum HTTP_VALIDATION otherwise.
691 */
692 HTTP_VALIDATION http_request_validate(struct web_client *w) {
693 char *s = (char *)buffer_tostring(w->response.data), *encoded_url = NULL;
694
695 size_t last_pos = w->header_parse_last_size;
696
697 w->header_parse_tries++;
698 w->header_parse_last_size = buffer_strlen(w->response.data);
699
700 int is_it_valid;
701 if(w->header_parse_tries > 1) {
702 if(last_pos > 4) last_pos -= 4; // allow searching for \r\n\r\n
703 else last_pos = 0;
704
705 if(w->header_parse_last_size <= last_pos)
706 last_pos = 0;
707
708 is_it_valid = url_is_request_complete_and_extract_payload(s, &s[last_pos],
709 w->header_parse_last_size, &w->payload);
710
711 if(!is_it_valid) {
712 if(w->header_parse_tries > HTTP_REQ_MAX_HEADER_FETCH_TRIES) {
713 netdata_log_info("Disabling slow client after %zu attempts to read the request (%zu bytes received)", w->header_parse_tries, buffer_strlen(w->response.data));
714 w->header_parse_tries = 0;
715 w->header_parse_last_size = 0;
716 web_client_disable_wait_receive(w);
717 return HTTP_VALIDATION_TOO_MANY_READ_RETRIES;
718 }
719
720 return HTTP_VALIDATION_INCOMPLETE;
721 }
722
723 is_it_valid = 1;
724 } else {
725 last_pos = w->header_parse_last_size;
726 is_it_valid =
727 url_is_request_complete_and_extract_payload(s, &s[last_pos], w->header_parse_last_size, &w->payload);
728 }
729
730 s = web_client_valid_method(w, s);
731 if (!s) {
732 w->header_parse_tries = 0;
733 w->header_parse_last_size = 0;
734 web_client_disable_wait_receive(w);
735
736 return HTTP_VALIDATION_NOT_SUPPORTED;
737 } else if (!is_it_valid) {
738 web_client_enable_wait_receive(w);
739 return HTTP_VALIDATION_INCOMPLETE;
740 }
741
742 //After the method we have the path and query string together
743 encoded_url = s;
744
745 //we search for the position where we have " HTTP/", because it finishes the user request
746 s = url_find_protocol(s);
747
748 // incomplete requests
749 if(unlikely(!*s)) {
750 web_client_enable_wait_receive(w);
751 return HTTP_VALIDATION_INCOMPLETE;
752 }
753
754 // we have the end of encoded_url - remember it
755 char *ue = s;
756
757 // make sure we have complete request
758 // complete requests contain: \r\n\r\n
759 while(*s) {
760 // find a line feed
761 while(*s && *s++ != '\r');
762
763 // did we reach the end?
764 if(unlikely(!*s)) break;
765
766 // is it \r\n ?
767 if(likely(*s++ == '\n')) {
768
769 // is it again \r\n ? (header end)
770 if(unlikely(*s == '\r' && s[1] == '\n')) {
771 // a valid complete HTTP request found
772
773 char c = *ue;
774 *ue = '\0';
775 web_client_decode_path_and_query_string(w, encoded_url);
776 *ue = c;
777
778 if ( (web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx) ) {
779 if (!w->ssl.conn && (http_is_using_ssl_force(w) || http_is_using_ssl_default(w)) && (w->mode != HTTP_REQUEST_MODE_STREAM)) {
780 w->header_parse_tries = 0;
781 w->header_parse_last_size = 0;
782 web_client_disable_wait_receive(w);
783 return HTTP_VALIDATION_REDIRECT;
784 }
785 }
786
787 w->header_parse_tries = 0;
788 w->header_parse_last_size = 0;
789 web_client_disable_wait_receive(w);
790 return HTTP_VALIDATION_OK;
791 }
792
793 // another header line
794 s = http_header_parse_line(w, s);
795 }
796 }
797
798 // incomplete request
799 web_client_enable_wait_receive(w);
800 return HTTP_VALIDATION_INCOMPLETE;
801 }
802
803 static inline ssize_t web_client_send_data(struct web_client *w,const void *buf,size_t len, int flags)
804 {
805 do {
806 errno_clear();
807
808 ssize_t bytes;
809 if ((web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx)) {
810 if (SSL_connection(&w->ssl)) {
811 bytes = netdata_ssl_write(&w->ssl, buf, len);
812 web_client_enable_wait_from_ssl(w);
813 } else
814 bytes = send(w->fd, buf, len, flags);
815 } else if (web_client_check_conn_tcp(w) || web_client_check_conn_unix(w))
816 bytes = send(w->fd, buf, len, flags);
817 else
818 bytes = -999;
819
820 if(bytes < 0 && errno == EAGAIN) {
821 tinysleep();
822 continue;
823 }
824
825 return bytes;
826 } while(true);
827 }
828
829 void web_client_build_http_header(struct web_client *w) {
830 if(unlikely(w->response.code != HTTP_RESP_OK))
831 buffer_no_cacheable(w->response.data);
832
833 if(unlikely(!w->response.data->date))
834 w->response.data->date = now_realtime_sec();
835
836 // set a proper expiration date, if not already set
837 if(unlikely(!w->response.data->expires))
838 w->response.data->expires = w->response.data->date +
839 ((w->response.data->options & WB_CONTENT_NO_CACHEABLE) ? 0 : 86400);
840
841 // prepare the HTTP response header
842 netdata_log_debug(D_WEB_CLIENT, "%llu: Generating HTTP header with response %d.", w->id, w->response.code);
843
844 const char *code_msg = http_response_code2string(w->response.code);
845
846 // prepare the last modified and expiration dates
847 char rfc7231_date[RFC7231_MAX_LENGTH], rfc7231_expires[RFC7231_MAX_LENGTH];
848 rfc7231_datetime(rfc7231_date, sizeof(rfc7231_date), w->response.data->date);
849 rfc7231_datetime(rfc7231_expires, sizeof(rfc7231_expires), w->response.data->expires);
850
851 if (w->response.code == HTTP_RESP_HTTPS_UPGRADE) {
852 buffer_sprintf(w->response.header_output,
853 "HTTP/1.1 %d %s\r\n"
854 "Location: https://%s%s\r\n",
855 w->response.code, code_msg,
856 w->server_host ? w->server_host : "",
857 buffer_tostring(w->url_as_received));
858 w->response.code = HTTP_RESP_MOVED_PERM;
859 }
860 else {
861 buffer_sprintf(w->response.header_output,
862 "HTTP/1.1 %d %s\r\n"
863 "Connection: %s\r\n"
864 "Server: Netdata Embedded HTTP Server %s\r\n"
865 "Access-Control-Allow-Origin: %s\r\n"
866 "Access-Control-Allow-Credentials: true\r\n"
867 "Date: %s\r\n",
868 w->response.code,
869 code_msg,
870 web_client_has_keepalive(w)?"keep-alive":"close",
871 NETDATA_VERSION,
872 w->origin ? w->origin : "*",
873 rfc7231_date);
874
875 http_header_content_type(w->response.header_output, w->response.data->content_type);
876 }
877
878 // MCP-specific CORS: widen the allowlist + advertise exposed
879 // headers only for MCP transport endpoints (/mcp, /sse). The flag
880 // is set once during URL decoding; see WEB_CLIENT_FLAG_PATH_IS_MCP.
881 bool is_mcp_path = web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_IS_MCP);
882
883 if(is_mcp_path && w->mode != HTTP_REQUEST_MODE_OPTIONS)
884 buffer_strcat(w->response.header_output,
885 "Access-Control-Expose-Headers: Mcp-Session-Id\r\n");
886
887 if(unlikely(web_x_frame_options))
888 buffer_sprintf(w->response.header_output, "X-Frame-Options: %s\r\n", web_x_frame_options);
889
890 if(w->response.has_cookies) {
891 if(respect_web_browser_do_not_track_policy)
892 buffer_sprintf(w->response.header_output,
893 "Tk: T;cookies\r\n");
894 }
895 else {
896 if(respect_web_browser_do_not_track_policy) {
897 if(web_client_has_tracking_required(w))
898 buffer_sprintf(w->response.header_output,
899 "Tk: T;cookies\r\n");
900 else
901 buffer_sprintf(w->response.header_output,
902 "Tk: N\r\n");
903 }
904 }
905
906 if(w->mode == HTTP_REQUEST_MODE_OPTIONS) {
907 // Methods, max-age, and the base header allowlist are identical
908 // for every OPTIONS preflight. MCP preflights append the extra
909 // request headers the SDK uses: mcp-protocol-version,
910 // mcp-session-id, last-event-id (SSE resumption), authorization
911 // (bearer tokens). DELETE is *not* advertised — the MCP handlers
912 // currently return 405 for it, and advertising it would let the
913 // preflight succeed only for the real request to fail. Add DELETE
914 // here when the handlers learn session teardown.
915 buffer_strcat(w->response.header_output,
916 "Access-Control-Allow-Methods: GET, POST, OPTIONS\r\n"
917 "Access-Control-Allow-Headers: accept, x-requested-with, origin, content-type, cookie, pragma, cache-control, x-auth-token, x-netdata-auth, x-transaction-id");
918
919 if(is_mcp_path)
920 buffer_strcat(w->response.header_output,
921 ", authorization, mcp-protocol-version, mcp-session-id, last-event-id");
922
923 buffer_strcat(w->response.header_output,
924 "\r\n"
925 "Access-Control-Max-Age: 1209600\r\n" // 86400 * 14
926 );
927 }
928 else {
929 buffer_sprintf(w->response.header_output,
930 "Cache-Control: %s\r\n"
931 "Expires: %s\r\n",
932 (w->response.data->options & WB_CONTENT_NO_CACHEABLE)?"no-cache, no-store, must-revalidate\r\nPragma: no-cache":"public",
933 rfc7231_expires);
934 }
935
936 // copy a possibly available custom header
937 if(unlikely(buffer_strlen(w->response.header)))
938 buffer_strcat(w->response.header_output, buffer_tostring(w->response.header));
939
940 // headers related to the transfer method
941 if(likely(w->response.zoutput))
942 buffer_strcat(w->response.header_output, "Content-Encoding: gzip\r\n");
943
944 if(likely(w->flags & WEB_CLIENT_CHUNKED_TRANSFER))
945 buffer_strcat(w->response.header_output, "Transfer-Encoding: chunked\r\n");
946 else {
947 if(likely(w->response.data->len)) {
948 // we know the content length, put it
949 buffer_sprintf(w->response.header_output, "Content-Length: %zu\r\n", (size_t)w->response.data->len);
950 }
951 else {
952 // we don't know the content length, disable keep-alive
953 web_client_disable_keepalive(w);
954 }
955 }
956
957 char uuid[UUID_COMPACT_STR_LEN];
958 uuid_unparse_lower_compact(w->transaction, uuid);
959 buffer_sprintf(w->response.header_output,
960 "X-Transaction-ID: %s\r\n", uuid);
961
962 // end of HTTP header
963 buffer_strcat(w->response.header_output, "\r\n");
964 }
965
966 static inline void web_client_send_http_header(struct web_client *w) {
967 // For WebSocket handshake, the header is already fully prepared in websocket_handle_handshake
968 // For standard HTTP responses, we need to build the header
969 if (w->response.code != HTTP_RESP_WEBSOCKET_HANDSHAKE) {
970 web_client_build_http_header(w);
971 }
972
973 // sent the HTTP header
974 netdata_log_debug(D_WEB_DATA, "%llu: Sending response HTTP header of size %zu: '%s'"
975 , w->id
976 , buffer_strlen(w->response.header_output)
977 , buffer_tostring(w->response.header_output)
978 );
979
980 sock_setcork(w->fd, true);
981
982 size_t count = 0;
983 ssize_t bytes;
984
985 if ( (web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx) ) {
986 if (SSL_connection(&w->ssl)) {
987 bytes = netdata_ssl_write(&w->ssl, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output));
988 web_client_enable_wait_from_ssl(w);
989 }
990 else {
991 while((bytes = send(w->fd, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output), 0)) == -1) {
992 count++;
993
994 if(count > 100 || (errno != EAGAIN && errno != EWOULDBLOCK)) {
995 netdata_log_error("Cannot send HTTP headers to web client.");
996 break;
997 }
998 }
999 }
1000 }
1001 else if(web_client_check_conn_tcp(w) || web_client_check_conn_unix(w)) {
1002 while((bytes = send(w->fd, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output), 0)) == -1) {
1003 count++;
1004
1005 if(count > 100 || (errno != EAGAIN && errno != EWOULDBLOCK)) {
1006 netdata_log_error("Cannot send HTTP headers to web client.");
1007 break;
1008 }
1009 }
1010 }
1011 else
1012 bytes = -999;
1013
1014 if(bytes != (ssize_t) buffer_strlen(w->response.header_output)) {
1015 if(bytes > 0)
1016 w->statistics.sent_bytes += bytes;
1017
1018 if (bytes < 0) {
1019 netdata_log_error("HTTP headers failed to be sent (I sent %zu bytes but the system sent %zd bytes). Closing web client."
1020 , buffer_strlen(w->response.header_output)
1021 , bytes);
1022
1023 WEB_CLIENT_IS_DEAD(w);
1024 return;
1025 }
1026 }
1027 else
1028 w->statistics.sent_bytes += bytes;
1029 }
1030
1031 static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, char *url, bool nodeid, int (*func)(RRDHOST *, struct web_client *, char *)) {
1032 static uint32_t hash_localhost = 0;
1033
1034 if(unlikely(!hash_localhost)) {
1035 hash_localhost = simple_hash("localhost");
1036 }
1037
1038 if(host != localhost) {
1039 buffer_flush(w->response.data);
1040 buffer_strcat(w->response.data, "Nesting of hosts is not allowed.");
1041 return HTTP_RESP_BAD_REQUEST;
1042 }
1043
1044 char *tok = strsep_skip_consecutive_separators(&url, "/");
1045 if(tok && *tok) {
1046 netdata_log_debug(D_WEB_CLIENT, "%llu: Searching for host with name '%s'.", w->id, tok);
1047
1048 if(nodeid) {
1049 host = rrdhost_find_by_node_id(tok);
1050 if(!host) {
1051 host = rrdhost_find_by_guid(tok);
1052 if (!host)
1053 host = rrdhost_find_by_hostname(tok);
1054 }
1055 }
1056 else {
1057 host = rrdhost_find_by_guid(tok);
1058 if(!host) {
1059 host = rrdhost_find_by_node_id(tok);
1060 if (!host)
1061 host = rrdhost_find_by_hostname(tok);
1062 }
1063 }
1064
1065 if(!host) {
1066 // we didn't find it, but it may be a uuid case mismatch for MACHINE_GUID
1067 // so, recreate the machine guid in lower-case.
1068 nd_uuid_t uuid;
1069 char txt[UUID_STR_LEN];
1070 if (uuid_parse(tok, uuid) == 0) {
1071 uuid_unparse_lower(uuid, txt);
1072 host = rrdhost_find_by_guid(txt);
1073 }
1074 }
1075
1076 if (host) {
1077 if(!url)
1078 //no delim found
1079 return append_slash_to_url_and_redirect(w);
1080
1081 buffer_flush(w->url_path_decoded);
1082 buffer_strcat(w->url_path_decoded, "/");
1083 buffer_strcat(w->url_path_decoded, url);
1084 char *mutable_path = strdupz(buffer_tostring(w->url_path_decoded));
1085 int rc = func(host, w, mutable_path);
1086 freez(mutable_path);
1087 return rc;
1088 }
1089 }
1090
1091 buffer_flush(w->response.data);
1092 w->response.data->content_type = CT_TEXT_HTML;
1093 buffer_strcat(w->response.data, "This netdata does not maintain a database for host: ");
1094 buffer_strcat_htmlescape(w->response.data, tok?tok:"");
1095 return HTTP_RESP_NOT_FOUND;
1096 }
1097
1098 int web_client_api_request_with_node_selection(RRDHOST *host, struct web_client *w, char *decoded_url_path) {
1099 // entry point for all API requests
1100
1101 ND_LOG_STACK lgs[] = {
1102 ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, HTTP_REQUEST_MODE_2str(w->mode)),
1103 ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received),
1104 ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id),
1105 ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction),
1106 ND_LOG_FIELD_UUID(NDF_ACCOUNT_ID, &w->user_auth.cloud_account_id.uuid),
1107 ND_LOG_FIELD_TXT(NDF_USER_NAME, w->user_auth.client_name),
1108 ND_LOG_FIELD_TXT(NDF_USER_ROLE, http_id2user_role(w->user_auth.user_role)),
1109 ND_LOG_FIELD_CB(NDF_USER_ACCESS, log_cb_http_access_to_hex, &w->user_auth.access),
1110 ND_LOG_FIELD_END(),
1111 };
1112 ND_LOG_STACK_PUSH(lgs);
1113
1114 // give a new transaction id to the request
1115 if(uuid_is_null(w->transaction))
1116 uuid_generate_random(w->transaction);
1117
1118 static uint32_t
1119 hash_api = 0,
1120 hash_host = 0,
1121 hash_node = 0;
1122
1123 if(unlikely(!hash_api)) {
1124 hash_api = simple_hash("api");
1125 hash_host = simple_hash("host");
1126 hash_node = simple_hash("node");
1127 }
1128
1129 char *tok = strsep_skip_consecutive_separators(&decoded_url_path, "/?");
1130 if(likely(tok && *tok)) {
1131 uint32_t hash = simple_hash(tok);
1132
1133 if(unlikely(hash == hash_api && strcmp(tok, "api") == 0)) {
1134 // current API
1135 netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: API request ...", w->id);
1136 return check_host_and_call(host, w, decoded_url_path, web_client_api_request);
1137 }
1138 else if(unlikely((hash == hash_host && strcmp(tok, "host") == 0) || (hash == hash_node && strcmp(tok, "node") == 0))) {
1139 // host switching
1140 netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: host switch request ...", w->id);
1141 return web_client_switch_host(host, w, decoded_url_path, hash == hash_node, web_client_api_request_with_node_selection);
1142 }
1143 }
1144
1145 buffer_flush(w->response.data);
1146 buffer_strcat(w->response.data, "Unknown API endpoint.");
1147 w->response.data->content_type = CT_TEXT_HTML;
1148 return HTTP_RESP_NOT_FOUND;
1149 }
1150
1151 static inline int web_client_process_url(RRDHOST *host, struct web_client *w, char *decoded_url_path) {
1152 if(unlikely(!service_running(ABILITY_WEB_REQUESTS)))
1153 return web_client_service_unavailable(w);
1154
1155 static uint32_t
1156 hash_api = 0,
1157 hash_netdata_conf = 0,
1158 hash_host = 0,
1159 hash_node = 0,
1160 hash_v0 = 0,
1161 hash_v1 = 0,
1162 hash_v2 = 0,
1163 hash_v3 = 0,
1164 hash_mcp = 0,
1165 hash_sse = 0;
1166
1167 #ifdef NETDATA_INTERNAL_CHECKS
1168 static uint32_t hash_exit = 0, hash_debug = 0, hash_mirror = 0;
1169 #endif
1170
1171 if(unlikely(!hash_api)) {
1172 hash_api = simple_hash("api");
1173 hash_netdata_conf = simple_hash("netdata.conf");
1174 hash_host = simple_hash("host");
1175 hash_node = simple_hash("node");
1176 hash_v0 = simple_hash("v0");
1177 hash_v1 = simple_hash("v1");
1178 hash_v2 = simple_hash("v2");
1179 hash_v3 = simple_hash("v3");
1180 hash_mcp = simple_hash("mcp");
1181 hash_sse = simple_hash("sse");
1182 #ifdef NETDATA_INTERNAL_CHECKS
1183 hash_exit = simple_hash("exit");
1184 hash_debug = simple_hash("debug");
1185 hash_mirror = simple_hash("mirror");
1186 #endif
1187 }
1188
1189 // keep a copy of the decoded path, in case we need to serve it as a filename
1190 char filename[FILENAME_MAX + 1];
1191 strncpyz(filename, decoded_url_path ? decoded_url_path : "", FILENAME_MAX);
1192
1193 char *tok = strsep_skip_consecutive_separators(&decoded_url_path, "/?");
1194 if(likely(tok && *tok)) {
1195 uint32_t hash = simple_hash(tok);
1196 netdata_log_debug(D_WEB_CLIENT, "%llu: Processing command '%s'.", w->id, tok);
1197
1198 if(likely(hash == hash_api && strcmp(tok, "api") == 0)) { // current API
1199 netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: API request ...", w->id);
1200 return check_host_and_call(host, w, decoded_url_path, web_client_api_request);
1201 }
1202 else if(likely(hash == hash_mcp && strcmp(tok, "mcp") == 0)) {
1203 if(unlikely(!http_can_access_dashboard(w)))
1204 return web_client_permission_denied_acl(w);
1205 return mcp_http_handle_request(host, w);
1206 }
1207 else if(likely(hash == hash_sse && strcmp(tok, "sse") == 0)) {
1208 if(unlikely(!http_can_access_dashboard(w)))
1209 return web_client_permission_denied_acl(w);
1210 return mcp_sse_handle_request(host, w);
1211 }
1212 else if(unlikely((hash == hash_host && strcmp(tok, "host") == 0) || (hash == hash_node && strcmp(tok, "node") == 0))) { // host switching
1213 netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: host switch request ...", w->id);
1214 return web_client_switch_host(host, w, decoded_url_path, hash == hash_node, web_client_process_url);
1215 }
1216 else if(unlikely(hash == hash_v3 && strcmp(tok, "v3") == 0)) {
1217 if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_WITH_VERSION))
1218 return bad_request_multiple_dashboard_versions(w);
1219 web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_IS_V3);
1220 return web_client_process_url(host, w, decoded_url_path);
1221 }
1222 else if(unlikely(hash == hash_v2 && strcmp(tok, "v2") == 0)) {
1223 if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_WITH_VERSION))
1224 return bad_request_multiple_dashboard_versions(w);
1225 web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_IS_V2);
1226 return web_client_process_url(host, w, decoded_url_path);
1227 }
1228 else if(unlikely(hash == hash_v1 && strcmp(tok, "v1") == 0)) {
1229 if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_WITH_VERSION))
1230 return bad_request_multiple_dashboard_versions(w);
1231 web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_IS_V1);
1232 return web_client_process_url(host, w, decoded_url_path);
1233 }
1234 else if(unlikely(hash == hash_v0 && strcmp(tok, "v0") == 0)) {
1235 if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_WITH_VERSION))
1236 return bad_request_multiple_dashboard_versions(w);
1237 web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_IS_V0);
1238 return web_client_process_url(host, w, decoded_url_path);
1239 }
1240 else if(unlikely(hash == hash_netdata_conf && strcmp(tok, "netdata.conf") == 0)) { // netdata.conf
1241 if(unlikely(!http_can_access_netdataconf(w)))
1242 return web_client_permission_denied_acl(w);
1243
1244 netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: generating netdata.conf ...", w->id);
1245 w->response.data->content_type = CT_TEXT_PLAIN;
1246 buffer_flush(w->response.data);
1247
1248 inicfg_generate(&netdata_config, w->response.data, 0, true);
1249 return HTTP_RESP_OK;
1250 }
1251 #ifdef NETDATA_INTERNAL_CHECKS
1252 else if(unlikely(hash == hash_exit && strcmp(tok, "exit") == 0)) {
1253 if(unlikely(!http_can_access_netdataconf(w)))
1254 return web_client_permission_denied_acl(w);
1255
1256 w->response.data->content_type = CT_TEXT_PLAIN;
1257 buffer_flush(w->response.data);
1258
1259 if(!exit_initiated_get())
1260 buffer_strcat(w->response.data, "ok, will do...");
1261 else
1262 buffer_strcat(w->response.data, "I am doing it already");
1263
1264 netdata_log_error("web request to exit received.");
1265 netdata_exit_gracefully(EXIT_REASON_API_QUIT, true);
1266 return HTTP_RESP_OK;
1267 }
1268 else if(unlikely(hash == hash_debug && strcmp(tok, "debug") == 0)) {
1269 if(unlikely(!http_can_access_netdataconf(w)))
1270 return web_client_permission_denied_acl(w);
1271
1272 buffer_flush(w->response.data);
1273
1274 // get the name of the data to show
1275 tok = strsep_skip_consecutive_separators(&decoded_url_path, "&");
1276 if(tok && *tok) {
1277 netdata_log_debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok);
1278
1279 // do we have such a data set?
1280 RRDSET *st = rrdset_find_byname(host, tok);
1281 if(!st) st = rrdset_find(host, tok, false);
1282 if(!st) {
1283 w->response.data->content_type = CT_TEXT_HTML;
1284 buffer_strcat(w->response.data, "Chart is not found: ");
1285 buffer_strcat_htmlescape(w->response.data, tok);
1286 netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: %s is not found.", w->id, tok);
1287 return HTTP_RESP_NOT_FOUND;
1288 }
1289
1290 if(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))
1291 rrdset_flag_clear(st, RRDSET_FLAG_DEBUG);
1292 else
1293 rrdset_flag_set(st, RRDSET_FLAG_DEBUG);
1294
1295 w->response.data->content_type = CT_TEXT_HTML;
1296 buffer_sprintf(w->response.data, "Chart has now debug %s: ", rrdset_flag_check(st, RRDSET_FLAG_DEBUG)?"enabled":"disabled");
1297 buffer_strcat_htmlescape(w->response.data, tok);
1298 netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: debug for %s is %s.", w->id, tok, rrdset_flag_check(st, RRDSET_FLAG_DEBUG)?"enabled":"disabled");
1299 return HTTP_RESP_OK;
1300 }
1301
1302 buffer_flush(w->response.data);
1303 buffer_strcat(w->response.data, "debug which chart?\r\n");
1304 return HTTP_RESP_BAD_REQUEST;
1305 }
1306 else if(unlikely(hash == hash_mirror && strcmp(tok, "mirror") == 0)) {
1307 if(unlikely(!http_can_access_netdataconf(w)))
1308 return web_client_permission_denied_acl(w);
1309
1310 netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Mirroring...", w->id);
1311
1312 // replace the zero bytes with spaces
1313 buffer_char_replace(w->response.data, '\0', ' ');
1314
1315 // just leave the buffer as-is
1316 // it will be copied back to the client
1317
1318 return HTTP_RESP_OK;
1319 }
1320 #endif /* NETDATA_INTERNAL_CHECKS */
1321 }
1322
1323 buffer_flush(w->response.data);
1324 return web_server_static_file(w, filename);
1325 }
1326
1327 static bool web_server_log_transport(BUFFER *wb, void *ptr) {
1328 struct web_client *w = ptr;
1329 if(!w)
1330 return false;
1331
1332 buffer_strcat(wb, SSL_connection(&w->ssl) ? "https" : "http");
1333 return true;
1334 }
1335
1336 void web_client_process_request_from_web_server(struct web_client *w) {
1337 // entry point for web server requests
1338
1339 ND_LOG_STACK lgs[] = {
1340 ND_LOG_FIELD_CB(NDF_SRC_TRANSPORT, web_server_log_transport, w),
1341 ND_LOG_FIELD_TXT(NDF_SRC_IP, w->user_auth.client_ip),
1342 ND_LOG_FIELD_TXT(NDF_SRC_PORT, w->client_port),
1343 ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_HOST, w->forwarded_host),
1344 ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_FOR, w->user_auth.forwarded_for),
1345 ND_LOG_FIELD_TXT(NDF_NIDL_NODE, w->client_host),
1346 ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, HTTP_REQUEST_MODE_2str(w->mode)),
1347 ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received),
1348 ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id),
1349 ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction),
1350 ND_LOG_FIELD_UUID(NDF_ACCOUNT_ID, &w->user_auth.cloud_account_id.uuid),
1351 ND_LOG_FIELD_TXT(NDF_USER_NAME, w->user_auth.client_name),
1352 ND_LOG_FIELD_TXT(NDF_USER_ROLE, http_id2user_role(w->user_auth.user_role)),
1353 ND_LOG_FIELD_CB(NDF_USER_ACCESS, log_cb_http_access_to_hex, &w->user_auth.access),
1354 ND_LOG_FIELD_END(),
1355 };
1356 ND_LOG_STACK_PUSH(lgs);
1357
1358 // give a new transaction id to the request
1359 if(uuid_is_null(w->transaction))
1360 uuid_generate_random(w->transaction);
1361
1362 // start timing us
1363 web_client_timeout_checkpoint_init(w);
1364
1365 switch(http_request_validate(w)) {
1366 case HTTP_VALIDATION_OK:
1367 if(!web_client_flag_check(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING)) {
1368 web_client_flag_set(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING);
1369 query_progress_start_or_update(&w->transaction, 0, w->mode, w->acl,
1370 buffer_tostring(w->url_as_received),
1371 w->payload,
1372 w->user_auth.forwarded_for[0] ? w->user_auth.forwarded_for : w->user_auth.client_ip);
1373 }
1374
1375 // Check if this is a WebSocket upgrade request
1376 // The full WebSocket handshake detection will happen in the header parsing,
1377 // but we need to set the initial mode to GET for processing to continue
1378 if (w->mode == HTTP_REQUEST_MODE_GET && web_client_has_websocket_handshake(w) && web_client_is_websocket(w)) {
1379 w->mode = HTTP_REQUEST_MODE_WEBSOCKET;
1380 netdata_log_debug(D_WEB_CLIENT, "%llu: Detected WebSocket handshake request", w->id);
1381 }
1382
1383 switch(w->mode) {
1384 case HTTP_REQUEST_MODE_STREAM:
1385 if(unlikely(!http_can_access_stream(w))) {
1386 web_client_permission_denied_acl(w);
1387 return;
1388 }
1389
1390 w->response.code = stream_receiver_accept_connection(
1391 w, (char *)buffer_tostring(w->url_query_string_decoded));
1392 return;
1393
1394 case HTTP_REQUEST_MODE_WEBSOCKET:
1395 if(unlikely(!http_can_access_dashboard(w))) {
1396 web_client_permission_denied_acl(w);
1397 return;
1398 }
1399
1400 // Handle WebSocket handshake - this will take over the socket
1401 // similar to how stream_receiver_accept_connection works
1402 w->response.code = websocket_handle_handshake(w);
1403
1404 // After this point the socket has been taken over
1405 // No need to send a response as the WebSocket handler
1406 // has already sent the handshake response
1407 return;
1408
1409 case HTTP_REQUEST_MODE_OPTIONS:
1410 if(unlikely(
1411 !http_can_access_dashboard(w) &&
1412 !http_can_access_registry(w) &&
1413 !http_can_access_badges(w) &&
1414 !http_can_access_mgmt(w) &&
1415 !http_can_access_netdataconf(w)
1416 )) {
1417 web_client_permission_denied_acl(w);
1418 break;
1419 }
1420
1421 w->response.data->content_type = CT_TEXT_PLAIN;
1422 buffer_flush(w->response.data);
1423 buffer_strcat(w->response.data, "OK");
1424 w->response.code = HTTP_RESP_OK;
1425 break;
1426
1427 case HTTP_REQUEST_MODE_POST:
1428 case HTTP_REQUEST_MODE_GET:
1429 case HTTP_REQUEST_MODE_PUT:
1430 case HTTP_REQUEST_MODE_DELETE:
1431 if(unlikely(
1432 !http_can_access_dashboard(w) &&
1433 !http_can_access_registry(w) &&
1434 !http_can_access_badges(w) &&
1435 !http_can_access_mgmt(w) &&
1436 !http_can_access_netdataconf(w)
1437 )) {
1438 web_client_permission_denied_acl(w);
1439 break;
1440 }
1441
1442 web_client_reset_path_flags(w);
1443
1444 // find if the URL path has a filename extension
1445 char path[FILENAME_MAX + 1];
1446 strncpyz(path, buffer_tostring(w->url_path_decoded), FILENAME_MAX);
1447 char *s = path, *e = path;
1448
1449 // remove the query string and find the last char
1450 for (; *e ; e++) {
1451 if (*e == '?')
1452 break;
1453 }
1454
1455 if(e == s || (*(e - 1) == '/'))
1456 web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_HAS_TRAILING_SLASH);
1457
1458 // check if there is a filename extension
1459 while (--e > s) {
1460 if (*e == '/')
1461 break;
1462 if(*e == '.') {
1463 web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_HAS_FILE_EXTENSION);
1464 break;
1465 }
1466 }
1467
1468 w->response.code = (short)web_client_process_url(localhost, w, path);
1469 break;
1470
1471 default:
1472 web_client_permission_denied_acl(w);
1473 return;
1474 }
1475 break;
1476
1477 case HTTP_VALIDATION_INCOMPLETE:
1478 if(w->response.data->len > NETDATA_WEB_REQUEST_MAX_SIZE) {
1479 buffer_flush(w->url_as_received);
1480 buffer_strcat(w->url_as_received, "too big request");
1481
1482 netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Received request is too big (%zu bytes).", w->id, (size_t)w->response.data->len);
1483
1484 size_t len = w->response.data->len;
1485 buffer_flush(w->response.data);
1486 buffer_sprintf(w->response.data, "Received request is too big (received %zu bytes, max is %zu bytes).\r\n", len, (size_t)NETDATA_WEB_REQUEST_MAX_SIZE);
1487 w->response.code = HTTP_RESP_BAD_REQUEST;
1488 }
1489 else {
1490 // wait for more data
1491 // set to normal to prevent web_server_rcv_callback
1492 // from going into stream mode
1493 if (w->mode == HTTP_REQUEST_MODE_STREAM || w->mode == HTTP_REQUEST_MODE_WEBSOCKET)
1494 w->mode = HTTP_REQUEST_MODE_GET;
1495 return;
1496 }
1497 break;
1498
1499 case HTTP_VALIDATION_REDIRECT:
1500 {
1501 buffer_flush(w->response.data);
1502 w->response.data->content_type = CT_TEXT_HTML;
1503 buffer_strcat(w->response.data,
1504 "<!DOCTYPE html><!-- SPDX-License-Identifier: GPL-3.0-or-later --><html>"
1505 "<body onload=\"window.location.href ='https://'+ window.location.hostname +"
1506 " ':' + window.location.port + window.location.pathname + window.location.search\">"
1507 "Redirecting to safety connection, case your browser does not support redirection, please"
1508 " click <a onclick=\"window.location.href ='https://'+ window.location.hostname + ':' "
1509 " + window.location.port + window.location.pathname + window.location.search\">here</a>."
1510 "</body></html>");
1511 w->response.code = HTTP_RESP_HTTPS_UPGRADE;
1512 break;
1513 }
1514
1515 case HTTP_VALIDATION_MALFORMED_URL:
1516 netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Malformed URL '%s'.", w->id, w->response.data->buffer);
1517
1518 buffer_flush(w->response.data);
1519 buffer_strcat(w->response.data, "Malformed URL...\r\n");
1520 w->response.code = HTTP_RESP_BAD_REQUEST;
1521 break;
1522 case HTTP_VALIDATION_TOO_MANY_READ_RETRIES:
1523 netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Too many retries to read request '%s'.", w->id, w->response.data->buffer);
1524
1525 buffer_flush(w->response.data);
1526 buffer_strcat(w->response.data, "Too many retries to read request.\r\n");
1527 w->response.code = HTTP_RESP_BAD_REQUEST;
1528 break;
1529 case HTTP_VALIDATION_NOT_SUPPORTED:
1530 netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: HTTP method requested is not supported '%s'.", w->id, w->response.data->buffer);
1531
1532 buffer_flush(w->response.data);
1533 buffer_strcat(w->response.data, "HTTP method requested is not supported...\r\n");
1534 w->response.code = HTTP_RESP_BAD_REQUEST;
1535 break;
1536 }
1537
1538 // keep track of the processing time
1539 web_client_timeout_checkpoint_response_ready(w, NULL);
1540
1541 w->response.sent = 0;
1542
1543 web_client_send_http_header(w);
1544
1545 // enable sending immediately if we have data
1546 if(w->response.data->len) web_client_enable_wait_send(w);
1547 else web_client_disable_wait_send(w);
1548
1549 switch(w->mode) {
1550 case HTTP_REQUEST_MODE_STREAM:
1551 netdata_log_debug(D_WEB_CLIENT, "%llu: STREAM done.", w->id);
1552 break;
1553
1554 case HTTP_REQUEST_MODE_WEBSOCKET:
1555 netdata_log_debug(D_WEB_CLIENT, "%llu: Done preparing the WEBSOCKET response..", w->id);
1556 break;
1557
1558 case HTTP_REQUEST_MODE_OPTIONS:
1559 netdata_log_debug(D_WEB_CLIENT,
1560 "%llu: Done preparing the OPTIONS response. Sending data (%zu bytes) to client.",
1561 w->id, (size_t)w->response.data->len);
1562 break;
1563
1564 case HTTP_REQUEST_MODE_POST:
1565 case HTTP_REQUEST_MODE_GET:
1566 case HTTP_REQUEST_MODE_PUT:
1567 case HTTP_REQUEST_MODE_DELETE:
1568 netdata_log_debug(D_WEB_CLIENT,
1569 "%llu: Done preparing the response. Sending data (%zu bytes) to client.",
1570 w->id, (size_t)w->response.data->len);
1571 break;
1572
1573 default:
1574 fatal("%llu: Unknown client mode %u.", w->id, w->mode);
1575 break;
1576 }
1577 }
1578
1579 ssize_t web_client_send_chunk_header(struct web_client *w, size_t len)
1580 {
1581 netdata_log_debug(D_DEFLATE, "%llu: OPEN CHUNK of %zu bytes (hex: %zx).", w->id, len, len);
1582 char buf[24];
1583 ssize_t bytes;
1584 bytes = (ssize_t)sprintf(buf, "%zX\r\n", len);
1585 buf[bytes] = 0x00;
1586
1587 bytes = web_client_send_data(w,buf,strlen(buf),0);
1588 if(bytes > 0) {
1589 netdata_log_debug(D_DEFLATE, "%llu: Sent chunk header %zd bytes.", w->id, bytes);
1590 w->statistics.sent_bytes += bytes;
1591 }
1592
1593 else if(bytes == 0) {
1594 netdata_log_debug(D_WEB_CLIENT, "%llu: Did not send chunk header to the client.", w->id);
1595 }
1596 else {
1597 netdata_log_debug(D_WEB_CLIENT, "%llu: Failed to send chunk header to client.", w->id);
1598 WEB_CLIENT_IS_DEAD(w);
1599 }
1600
1601 return bytes;
1602 }
1603
1604 ssize_t web_client_send_chunk_close(struct web_client *w)
1605 {
1606 //debug(D_DEFLATE, "%llu: CLOSE CHUNK.", w->id);
1607
1608 ssize_t bytes;
1609 bytes = web_client_send_data(w,"\r\n",2,0);
1610 if(bytes > 0) {
1611 netdata_log_debug(D_DEFLATE, "%llu: Sent chunk suffix %zd bytes.", w->id, bytes);
1612 w->statistics.sent_bytes += bytes;
1613 }
1614
1615 else if(bytes == 0) {
1616 netdata_log_debug(D_WEB_CLIENT, "%llu: Did not send chunk suffix to the client.", w->id);
1617 }
1618 else {
1619 netdata_log_debug(D_WEB_CLIENT, "%llu: Failed to send chunk suffix to client.", w->id);
1620 WEB_CLIENT_IS_DEAD(w);
1621 }
1622
1623 return bytes;
1624 }
1625
1626 ssize_t web_client_send_chunk_finalize(struct web_client *w)
1627 {
1628 //debug(D_DEFLATE, "%llu: FINALIZE CHUNK.", w->id);
1629
1630 ssize_t bytes;
1631 bytes = web_client_send_data(w,"\r\n0\r\n\r\n",7,0);
1632 if(bytes > 0) {
1633 netdata_log_debug(D_DEFLATE, "%llu: Sent chunk suffix %zd bytes.", w->id, bytes);
1634 w->statistics.sent_bytes += bytes;
1635 }
1636
1637 else if(bytes == 0) {
1638 netdata_log_debug(D_WEB_CLIENT, "%llu: Did not send chunk finalize suffix to the client.", w->id);
1639 }
1640 else {
1641 netdata_log_debug(D_WEB_CLIENT, "%llu: Failed to send chunk finalize suffix to client.", w->id);
1642 WEB_CLIENT_IS_DEAD(w);
1643 }
1644
1645 return bytes;
1646 }
1647
1648 ssize_t web_client_send_deflate(struct web_client *w)
1649 {
1650 ssize_t len = 0, t = 0;
1651
1652 // when using compression,
1653 // w->response.sent is the amount of bytes passed through compression
1654
1655 netdata_log_debug(D_DEFLATE,
1656 "%llu: web_client_send_deflate(): w->response.data->len = %zu, w->response.sent = %zu, w->response.zhave = %zu, w->response.zsent = %zu, w->response.zstream.avail_in = %u, w->response.zstream.avail_out = %u, w->response.zstream.total_in = %lu, w->response.zstream.total_out = %lu.",
1657 w->id, (size_t)w->response.data->len, w->response.sent, w->response.zhave, w->response.zsent, w->response.zstream.avail_in, w->response.zstream.avail_out, w->response.zstream.total_in, w->response.zstream.total_out);
1658
1659 if(w->response.data->len - w->response.sent == 0 && w->response.zstream.avail_in == 0 && w->response.zhave == w->response.zsent && w->response.zstream.avail_out != 0) {
1660 // there is nothing to send
1661
1662 netdata_log_debug(D_WEB_CLIENT, "%llu: Out of output data.", w->id);
1663
1664 // finalize the chunk
1665 if(w->response.sent != 0) {
1666 t = web_client_send_chunk_finalize(w);
1667 if(t < 0) return t;
1668 }
1669
1670 if(unlikely(!web_client_has_keepalive(w))) {
1671 netdata_log_debug(D_WEB_CLIENT, "%llu: Closing (keep-alive is not enabled). %zu bytes sent.", w->id, w->response.sent);
1672 WEB_CLIENT_IS_DEAD(w);
1673 return t;
1674 }
1675
1676 // reset the client
1677 web_client_request_done(w);
1678 netdata_log_debug(D_WEB_CLIENT, "%llu: Done sending all data on socket.", w->id);
1679 return t;
1680 }
1681
1682 if(w->response.zhave == w->response.zsent) {
1683 // compress more input data
1684
1685 // close the previous open chunk
1686 if(w->response.sent != 0) {
1687 t = web_client_send_chunk_close(w);
1688 if(t < 0) return t;
1689 }
1690
1691 netdata_log_debug(D_DEFLATE, "%llu: Compressing %zu new bytes starting from %zu (and %u left behind).", w->id, (w->response.data->len - w->response.sent), w->response.sent, w->response.zstream.avail_in);
1692
1693 // give the compressor all the data not passed through the compressor yet
1694 if(w->response.data->len > w->response.sent) {
1695 w->response.zstream.next_in = (Bytef *)&w->response.data->buffer[w->response.sent - w->response.zstream.avail_in];
1696 w->response.zstream.avail_in += (uInt) (w->response.data->len - w->response.sent);
1697 }
1698
1699 // reset the compressor output buffer
1700 w->response.zstream.next_out = w->response.zbuffer;
1701 w->response.zstream.avail_out = NETDATA_WEB_RESPONSE_ZLIB_CHUNK_SIZE;
1702
1703 // ask for FINISH if we have all the input
1704 int flush = Z_SYNC_FLUSH;
1705 if((w->mode == HTTP_REQUEST_MODE_GET ||
1706 w->mode == HTTP_REQUEST_MODE_POST ||
1707 w->mode == HTTP_REQUEST_MODE_PUT ||
1708 w->mode == HTTP_REQUEST_MODE_DELETE)) {
1709 flush = Z_FINISH;
1710 netdata_log_debug(D_DEFLATE, "%llu: Requesting Z_FINISH, if possible.", w->id);
1711 }
1712 else {
1713 netdata_log_debug(D_DEFLATE, "%llu: Requesting Z_SYNC_FLUSH.", w->id);
1714 }
1715
1716 // compress
1717 if(deflate(&w->response.zstream, flush) == Z_STREAM_ERROR) {
1718 netdata_log_error("%llu: Compression failed. Closing down client.", w->id);
1719 web_client_request_done(w);
1720 return(-1);
1721 }
1722
1723 w->response.zhave = NETDATA_WEB_RESPONSE_ZLIB_CHUNK_SIZE - w->response.zstream.avail_out;
1724 w->response.zsent = 0;
1725
1726 // keep track of the bytes passed through the compressor
1727 w->response.sent = w->response.data->len;
1728
1729 netdata_log_debug(D_DEFLATE, "%llu: Compression produced %zu bytes.", w->id, w->response.zhave);
1730
1731 // open a new chunk
1732 ssize_t t2 = web_client_send_chunk_header(w, w->response.zhave);
1733 if(t2 < 0) return t2;
1734 t += t2;
1735 }
1736
1737 netdata_log_debug(D_WEB_CLIENT, "%llu: Sending %zu bytes of data (+%zd of chunk header).", w->id, w->response.zhave - w->response.zsent, t);
1738
1739 len = web_client_send_data(w,&w->response.zbuffer[w->response.zsent], (size_t) (w->response.zhave - w->response.zsent), MSG_DONTWAIT);
1740 if(len > 0) {
1741 w->statistics.sent_bytes += len;
1742 w->response.zsent += len;
1743 len += t;
1744 netdata_log_debug(D_WEB_CLIENT, "%llu: Sent %zd bytes.", w->id, len);
1745 }
1746 else if(len == 0) {
1747 netdata_log_debug(D_WEB_CLIENT, "%llu: Did not send any bytes to the client (zhave = %zu, zsent = %zu, need to send = %zu).",
1748 w->id, w->response.zhave, w->response.zsent, w->response.zhave - w->response.zsent);
1749
1750 }
1751 else {
1752 netdata_log_debug(D_WEB_CLIENT, "%llu: Failed to send data to client.", w->id);
1753 WEB_CLIENT_IS_DEAD(w);
1754 }
1755
1756 return(len);
1757 }
1758
1759 ssize_t web_client_send(struct web_client *w) {
1760 if(likely(w->response.zoutput)) return web_client_send_deflate(w);
1761
1762 ssize_t bytes;
1763
1764 if(unlikely(w->response.data->len - w->response.sent == 0)) {
1765 // there is nothing to send
1766
1767 netdata_log_debug(D_WEB_CLIENT, "%llu: Out of output data.", w->id);
1768
1769 // there can be two cases for this
1770 // A. we have done everything
1771 // B. we temporarily have nothing to send, waiting for the buffer to be filled by ifd
1772
1773 if(unlikely(!web_client_has_keepalive(w))) {
1774 netdata_log_debug(D_WEB_CLIENT, "%llu: Closing (keep-alive is not enabled). %zu bytes sent.", w->id, w->response.sent);
1775 WEB_CLIENT_IS_DEAD(w);
1776 return 0;
1777 }
1778
1779 web_client_request_done(w);
1780 netdata_log_debug(D_WEB_CLIENT, "%llu: Done sending all data on socket. Waiting for next request on the same socket.", w->id);
1781 return 0;
1782 }
1783
1784 bytes = web_client_send_data(w,&w->response.data->buffer[w->response.sent], w->response.data->len - w->response.sent, MSG_DONTWAIT);
1785 if(likely(bytes > 0)) {
1786 w->statistics.sent_bytes += bytes;
1787 w->response.sent += bytes;
1788 netdata_log_debug(D_WEB_CLIENT, "%llu: Sent %zd bytes.", w->id, bytes);
1789 }
1790 else if(likely(bytes == 0)) {
1791 netdata_log_debug(D_WEB_CLIENT, "%llu: Did not send any bytes to the client.", w->id);
1792 }
1793 else {
1794 netdata_log_debug(D_WEB_CLIENT, "%llu: Failed to send data to client.", w->id);
1795 WEB_CLIENT_IS_DEAD(w);
1796 }
1797
1798 return(bytes);
1799 }
1800
1801 ssize_t web_client_receive(struct web_client *w) {
1802 ssize_t bytes;
1803
1804 // do we have any space for more data?
1805 buffer_need_bytes(w->response.data, NETDATA_WEB_REQUEST_INITIAL_SIZE);
1806
1807 ssize_t left = (ssize_t)(w->response.data->size - w->response.data->len);
1808
1809 errno_clear();
1810
1811 if ( (web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx) ) {
1812 if (SSL_connection(&w->ssl)) {
1813 bytes = netdata_ssl_read(&w->ssl, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1));
1814 web_client_enable_wait_from_ssl(w);
1815 }
1816 else {
1817 bytes = recv(w->fd, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1), MSG_DONTWAIT);
1818 }
1819 }
1820 else if(web_client_check_conn_tcp(w) || web_client_check_conn_unix(w)) {
1821 bytes = recv(w->fd, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1), MSG_DONTWAIT);
1822 }
1823 else // other connection methods
1824 bytes = -1;
1825
1826 if(likely(bytes > 0)) {
1827 w->statistics.received_bytes += bytes;
1828
1829 size_t old = w->response.data->len;
1830 (void)old;
1831
1832 w->response.data->len += bytes;
1833 w->response.data->buffer[w->response.data->len] = '\0';
1834
1835 netdata_log_debug(D_WEB_CLIENT, "%llu: Received %zd bytes.", w->id, bytes);
1836 netdata_log_debug(D_WEB_DATA, "%llu: Received data: '%s'.", w->id, &w->response.data->buffer[old]);
1837 }
1838 else if(unlikely(bytes < 0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR))) {
1839 web_client_enable_wait_receive(w);
1840 return 0;
1841 }
1842 else if (bytes < 0) {
1843 netdata_log_debug(D_WEB_CLIENT, "%llu: receive data failed.", w->id);
1844 WEB_CLIENT_IS_DEAD(w);
1845 } else
1846 netdata_log_debug(D_WEB_CLIENT, "%llu: Received %zd bytes.", w->id, bytes);
1847
1848 return(bytes);
1849 }
1850
1851 void web_client_decode_path_and_query_string(struct web_client *w, const char *path_and_query_string) {
1852 char buffer[NETDATA_WEB_REQUEST_URL_SIZE + 2];
1853 buffer[0] = '\0';
1854
1855 buffer_flush(w->url_path_decoded);
1856 buffer_flush(w->url_query_string_decoded);
1857
1858 if(buffer_strlen(w->url_as_received) == 0)
1859 // do not overwrite this if it is already filled
1860 buffer_strcat(w->url_as_received, path_and_query_string);
1861
1862 // PATH_IS_MCP is a function of the URL alone; clear and re-derive on
1863 // every decode so keepalived connections reusing the same web_client
1864 // for a different URL see a fresh value.
1865 web_client_flag_clear(w, WEB_CLIENT_FLAG_PATH_IS_MCP);
1866
1867 if(w->mode == HTTP_REQUEST_MODE_STREAM) {
1868 // in stream mode, there is no path
1869
1870 url_decode_r(buffer, path_and_query_string, NETDATA_WEB_REQUEST_URL_SIZE + 1);
1871
1872 buffer[NETDATA_WEB_REQUEST_URL_SIZE + 1] = '\0';
1873 buffer_strcat(w->url_query_string_decoded, buffer);
1874 }
1875 else {
1876 // in non-stream mode, there is a path
1877 // FIXME - the way this is implemented, query string params never accept the symbol &, not even encoded as %26
1878 // To support the symbol & in query string params, we need to turn the url_query_string_decoded into a
1879 // dictionary and decode each of the parameters individually.
1880 // OR: in url_query_string_decoded use as separator a control character that cannot appear in the URL.
1881
1882 url_decode_r(buffer, path_and_query_string, NETDATA_WEB_REQUEST_URL_SIZE + 1);
1883
1884 char *question_mark_start = strchr(buffer, '?');
1885 if (question_mark_start) {
1886 buffer_strcat(w->url_query_string_decoded, question_mark_start);
1887 char c = *question_mark_start;
1888 *question_mark_start = '\0';
1889 buffer_strcat(w->url_path_decoded, buffer);
1890 *question_mark_start = c;
1891 } else {
1892 buffer_strcat(w->url_query_string_decoded, "");
1893 buffer_strcat(w->url_path_decoded, buffer);
1894 }
1895
1896 // Classify path: set PATH_IS_MCP when the URL addresses one of
1897 // Netdata's MCP transport endpoints (/mcp or /sse, and their
1898 // subpaths). Done here — at URL-decoding time — so the flag is
1899 // available later both to the URL dispatcher and to the response
1900 // header builder, including for OPTIONS preflights which bypass
1901 // the dispatcher. Matching requires a path-segment boundary so a
1902 // hypothetical /mcpfoo does not leak through.
1903 const char *decoded_path = buffer_tostring(w->url_path_decoded);
1904 size_t decoded_path_len = buffer_strlen(w->url_path_decoded);
1905 if(decoded_path_len >= 4
1906 && (memcmp(decoded_path, "/mcp", 4) == 0 || memcmp(decoded_path, "/sse", 4) == 0)
1907 && (decoded_path_len == 4 || decoded_path[4] == '/'))
1908 web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_IS_MCP);
1909 }
1910 }
1911
1912 void web_client_reuse_from_cache(struct web_client *w) {
1913 // zero everything about it - but keep the buffers
1914
1915 web_client_reset_allocations(w, false);
1916
1917 // remember the pointers to the buffers
1918 BUFFER *b1 = w->response.data;
1919 BUFFER *b2 = w->response.header;
1920 BUFFER *b3 = w->response.header_output;
1921 BUFFER *b4 = w->url_path_decoded;
1922 BUFFER *b5 = w->url_as_received;
1923 BUFFER *b6 = w->url_query_string_decoded;
1924 BUFFER *b7 = w->payload;
1925
1926 NETDATA_SSL ssl = w->ssl;
1927
1928 size_t use_count = w->use_count;
1929 size_t *statistics_memory_accounting = w->statistics.memory_accounting;
1930
1931 // zero everything
1932 memset(w, 0, sizeof(struct web_client));
1933
1934 w->fd = -1;
1935 w->statistics.memory_accounting = statistics_memory_accounting;
1936 w->use_count = use_count;
1937
1938 w->ssl = ssl;
1939
1940 // restore the pointers of the buffers
1941 w->response.data = b1;
1942 w->response.header = b2;
1943 w->response.header_output = b3;
1944 w->url_path_decoded = b4;
1945 w->url_as_received = b5;
1946 w->url_query_string_decoded = b6;
1947 w->payload = b7;
1948 }
1949
1950 struct web_client *web_client_create(size_t *statistics_memory_accounting) {
1951 struct web_client *w = (struct web_client *)callocz(1, sizeof(struct web_client));
1952
1953 w->ssl = NETDATA_SSL_UNSET_CONNECTION;
1954
1955 w->use_count = 1;
1956 w->statistics.memory_accounting = statistics_memory_accounting;
1957
1958 w->url_as_received = buffer_create(NETDATA_WEB_DECODED_URL_INITIAL_SIZE, w->statistics.memory_accounting);
1959 w->url_path_decoded = buffer_create(NETDATA_WEB_DECODED_URL_INITIAL_SIZE, w->statistics.memory_accounting);
1960 w->url_query_string_decoded = buffer_create(NETDATA_WEB_DECODED_URL_INITIAL_SIZE, w->statistics.memory_accounting);
1961 w->response.data = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, w->statistics.memory_accounting);
1962 w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_INITIAL_SIZE, w->statistics.memory_accounting);
1963 w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_INITIAL_SIZE, w->statistics.memory_accounting);
1964
1965 __atomic_add_fetch(w->statistics.memory_accounting, sizeof(struct web_client), __ATOMIC_RELAXED);
1966
1967 return w;
1968 }
1969
1970 void web_client_free(struct web_client *w) {
1971 netdata_ssl_close(&w->ssl);
1972
1973 web_client_reset_allocations(w, true);
1974
1975 __atomic_sub_fetch(w->statistics.memory_accounting, sizeof(struct web_client), __ATOMIC_RELAXED);
1976 freez(w);
1977 }
1978
1979 inline void web_client_timeout_checkpoint_init(struct web_client *w) {
1980 now_monotonic_high_precision_timeval(&w->timings.tv_in);
1981 }
1982
1983 inline void web_client_timeout_checkpoint_set(struct web_client *w, int timeout_ms) {
1984 w->timings.timeout_ut = timeout_ms * USEC_PER_MS;
1985
1986 if(!w->timings.tv_in.tv_sec)
1987 web_client_timeout_checkpoint_init(w);
1988
1989 if(!w->timings.tv_timeout_last_checkpoint.tv_sec)
1990 w->timings.tv_timeout_last_checkpoint = w->timings.tv_in;
1991 }
1992
1993 inline usec_t web_client_timeout_checkpoint(struct web_client *w) {
1994 struct timeval now;
1995 now_monotonic_high_precision_timeval(&now);
1996
1997 if (!w->timings.tv_timeout_last_checkpoint.tv_sec)
1998 w->timings.tv_timeout_last_checkpoint = w->timings.tv_in;
1999
2000 usec_t since_last_check_ut = dt_usec(&w->timings.tv_timeout_last_checkpoint, &now);
2001
2002 w->timings.tv_timeout_last_checkpoint = now;
2003
2004 return since_last_check_ut;
2005 }
2006
2007 inline usec_t web_client_timeout_checkpoint_response_ready(struct web_client *w, usec_t *usec_since_last_checkpoint) {
2008 usec_t since_last_check_ut = web_client_timeout_checkpoint(w);
2009 if(usec_since_last_checkpoint)
2010 *usec_since_last_checkpoint = since_last_check_ut;
2011
2012 w->timings.tv_ready = w->timings.tv_timeout_last_checkpoint;
2013
2014 // return the total time of the query
2015 return dt_usec(&w->timings.tv_in, &w->timings.tv_ready);
2016 }
2017
2018 inline bool web_client_timeout_checkpoint_and_check(struct web_client *w, usec_t *usec_since_last_checkpoint) {
2019
2020 usec_t since_last_check_ut = web_client_timeout_checkpoint(w);
2021 if(usec_since_last_checkpoint)
2022 *usec_since_last_checkpoint = since_last_check_ut;
2023
2024 if(!w->timings.timeout_ut)
2025 return false;
2026
2027 usec_t since_reception_ut = dt_usec(&w->timings.tv_in, &w->timings.tv_timeout_last_checkpoint);
2028 if (since_reception_ut >= w->timings.timeout_ut) {
2029 buffer_flush(w->response.data);
2030 buffer_strcat(w->response.data, "Query timeout exceeded");
2031 w->response.code = HTTP_RESP_GATEWAY_TIMEOUT;
2032 return true;
2033 }
2034
2035 return false;
2036 }