web_client.c · cryptotaxi247/netdata-1

Code Commits Issues Pulls Sessions CI/CD

master

c 2,036 lines 78.3 KB

Raw

1	// SPDX-License-Identifier: GPL-3.0-or-later
2
3	#include "web_client.h"
4	#include "web/websocket/websocket.h"
5	#include "web/mcp/adapters/mcp-http.h"
6	#include "web/mcp/adapters/mcp-sse.h"
7
8	// this is an async I/O implementation of the web server request parser
9	// it is used by all netdata web servers
10
11	int respect_web_browser_do_not_track_policy = 0;
12	const char *web_x_frame_options = NULL;
13
14	int web_enable_gzip = 1, web_gzip_level = 3, web_gzip_strategy = Z_DEFAULT_STRATEGY;
15
16	void web_client_set_conn_tcp(struct web_client *w) {
17	web_client_flags_clear_conn(w);
18	web_client_flag_set(w, WEB_CLIENT_FLAG_CONN_TCP);
19	}
20
21	void web_client_set_conn_unix(struct web_client *w) {
22	web_client_flags_clear_conn(w);
23	web_client_flag_set(w, WEB_CLIENT_FLAG_CONN_UNIX);
24	}
25
26	void web_client_set_conn_cloud(struct web_client *w) {
27	web_client_flags_clear_conn(w);
28	web_client_flag_set(w, WEB_CLIENT_FLAG_CONN_CLOUD);
29	}
30
31	void web_client_set_conn_webrtc(struct web_client *w) {
32	web_client_flags_clear_conn(w);
33	web_client_flag_set(w, WEB_CLIENT_FLAG_CONN_WEBRTC);
34	}
35
36	void web_client_reset_permissions(struct web_client *w) {
37	w->user_auth.method = USER_AUTH_METHOD_NONE;
38	w->user_auth.access = HTTP_ACCESS_NONE;
39	w->user_auth.user_role = HTTP_USER_ROLE_NONE;
40	web_client_clear_mcp_preview_key(w);
41	}
42
43	void web_client_set_permissions(struct web_client *w, HTTP_ACCESS access, HTTP_USER_ROLE role, USER_AUTH_METHOD type) {
44	web_client_reset_permissions(w);
45	w->user_auth.method = type;
46	w->user_auth.access = access;
47	w->user_auth.user_role = role;
48	}
49
50	inline int web_client_permission_denied_acl(struct web_client *w) {
51	w->response.data->content_type = CT_TEXT_PLAIN;
52	buffer_flush(w->response.data);
53	buffer_strcat(w->response.data, "You need to be authorized to access this resource");
54	w->response.code = HTTP_RESP_UNAVAILABLE_FOR_LEGAL_REASONS;
55	return HTTP_RESP_UNAVAILABLE_FOR_LEGAL_REASONS;
56	}
57
58	inline int web_client_permission_denied(struct web_client *w) {
59	w->response.data->content_type = CT_TEXT_PLAIN;
60	buffer_flush(w->response.data);
61
62	if(w->user_auth.access & HTTP_ACCESS_SIGNED_ID)
63	buffer_strcat(w->response.data,
64	"You don't have enough permissions to access this resource");
65	else
66	buffer_strcat(w->response.data,
67	"You need to be authorized to access this resource");
68
69	w->response.code = HTTP_ACCESS_PERMISSION_DENIED_HTTP_CODE(w->user_auth.access);
70	return w->response.code;
71	}
72
73	inline int web_client_service_unavailable(struct web_client *w) {
74	w->response.data->content_type = CT_TEXT_PLAIN;
75	buffer_flush(w->response.data);
76	buffer_strcat(w->response.data, "This service is currently unavailable.");
77	w->response.code = HTTP_RESP_SERVICE_UNAVAILABLE;
78	return HTTP_RESP_SERVICE_UNAVAILABLE;
79	}
80
81	static inline int bad_request_multiple_dashboard_versions(struct web_client *w) {
82	w->response.data->content_type = CT_TEXT_PLAIN;
83	buffer_flush(w->response.data);
84	buffer_strcat(w->response.data, "Multiple dashboard versions given at the URL.");
85	w->response.code = HTTP_RESP_BAD_REQUEST;
86	return HTTP_RESP_BAD_REQUEST;
87	}
88
89	static inline void web_client_enable_wait_from_ssl(struct web_client *w) {
90	if (w->ssl.ssl_errno == SSL_ERROR_WANT_READ)
91	web_client_enable_ssl_wait_receive(w);
92	else if (w->ssl.ssl_errno == SSL_ERROR_WANT_WRITE)
93	web_client_enable_ssl_wait_send(w);
94	else {
95	web_client_disable_ssl_wait_receive(w);
96	web_client_disable_ssl_wait_send(w);
97	}
98	}
99
100	static inline char strip_control_characters(char url) {
101	if(!url) return "";
102
103	for(char s = url; s ;s++)
104	if(iscntrl((uint8_t)s)) s = ' ';
105
106	return url;
107	}
108
109	static void web_client_reset_allocations(struct web_client *w, bool free_all) {
110
111	if(free_all) {
112	// the web client is to be destroyed
113
114	buffer_free(w->url_as_received);
115	w->url_as_received = NULL;
116
117	buffer_free(w->url_path_decoded);
118	w->url_path_decoded = NULL;
119
120	buffer_free(w->url_query_string_decoded);
121	w->url_query_string_decoded = NULL;
122
123	buffer_free(w->response.header_output);
124	w->response.header_output = NULL;
125
126	buffer_free(w->response.header);
127	w->response.header = NULL;
128
129	buffer_free(w->response.data);
130	w->response.data = NULL;
131
132	buffer_free(w->payload);
133	w->payload = NULL;
134	}
135	else {
136	// the web client is to be re-used
137
138	buffer_reset(w->url_as_received);
139	buffer_reset(w->url_path_decoded);
140	buffer_reset(w->url_query_string_decoded);
141
142	buffer_reset(w->response.header_output);
143	buffer_reset(w->response.header);
144	buffer_reset(w->response.data);
145
146	if(w->payload)
147	buffer_reset(w->payload);
148
149	// to add more items here,
150	// web_client_reuse_from_cache() needs to be adjusted to maintain them
151	}
152
153	freez(w->server_host);
154	w->server_host = NULL;
155
156	freez(w->forwarded_host);
157	w->forwarded_host = NULL;
158
159	freez(w->origin);
160	w->origin = NULL;
161
162	freez(w->user_agent);
163	w->user_agent = NULL;
164
165	freez(w->auth_bearer_token);
166	w->auth_bearer_token = NULL;
167
168	memset(w->mcp_session_id, 0, sizeof(w->mcp_session_id));
169
170	// Free WebSocket resources
171	freez(w->websocket.key);
172	w->websocket.key = NULL;
173
174	w->websocket.ext_flags = WS_EXTENSION_NONE;
175	w->websocket.protocol = WS_PROTOCOL_DEFAULT;
176	w->websocket.client_max_window_bits = 0;
177	w->websocket.server_max_window_bits = 0;
178
179	// if we had enabled compression, release it
180	if(w->response.zinitialized) {
181	deflateEnd(&w->response.zstream);
182	w->response.zsent = 0;
183	w->response.zhave = 0;
184	w->response.zstream.avail_in = 0;
185	w->response.zstream.avail_out = 0;
186	w->response.zstream.total_in = 0;
187	w->response.zstream.total_out = 0;
188	w->response.zinitialized = false;
189	web_client_flag_clear(w, WEB_CLIENT_CHUNKED_TRANSFER);
190	}
191
192	memset(w->transaction, 0, sizeof(w->transaction));
193	memset(&w->auth, 0, sizeof(w->auth));
194	memset(&w->user_auth, 0, sizeof(w->user_auth));
195
196	web_client_reset_permissions(w);
197	web_client_flag_clear(w, WEB_CLIENT_ENCODING_GZIP\|WEB_CLIENT_ENCODING_DEFLATE);
198	web_client_flag_clear(w, WEB_CLIENT_FLAG_ACCEPT_JSON \|
199	WEB_CLIENT_FLAG_ACCEPT_SSE \|
200	WEB_CLIENT_FLAG_ACCEPT_TEXT);
201	web_client_reset_path_flags(w);
202	}
203
204	void web_client_log_completed_request(struct web_client *w, bool update_web_stats) {
205	struct timeval tv;
206	now_monotonic_high_precision_timeval(&tv);
207
208	size_t size = w->response.data->len;
209	size_t sent = w->response.zoutput ? (size_t)w->response.zstream.total_out : size;
210
211	usec_t prep_ut = w->timings.tv_ready.tv_sec ? dt_usec(&w->timings.tv_ready, &w->timings.tv_in) : 0;
212	usec_t sent_ut = w->timings.tv_ready.tv_sec ? dt_usec(&tv, &w->timings.tv_ready) : 0;
213	usec_t total_ut = dt_usec(&tv, &w->timings.tv_in);
214	strip_control_characters((char *)buffer_tostring(w->url_as_received));
215
216	ND_LOG_STACK lgs[] = {
217	ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id),
218	ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction),
219	ND_LOG_FIELD_TXT(NDF_NIDL_NODE, w->client_host),
220	ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, HTTP_REQUEST_MODE_2str(w->mode)),
221	ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received),
222	ND_LOG_FIELD_U64(NDF_RESPONSE_CODE, w->response.code),
223	ND_LOG_FIELD_U64(NDF_RESPONSE_SENT_BYTES, sent),
224	ND_LOG_FIELD_U64(NDF_RESPONSE_SIZE_BYTES, size),
225	ND_LOG_FIELD_U64(NDF_RESPONSE_PREPARATION_TIME_USEC, prep_ut),
226	ND_LOG_FIELD_U64(NDF_RESPONSE_SENT_TIME_USEC, sent_ut),
227	ND_LOG_FIELD_U64(NDF_RESPONSE_TOTAL_TIME_USEC, total_ut),
228	ND_LOG_FIELD_TXT(NDF_SRC_IP, w->user_auth.client_ip),
229	ND_LOG_FIELD_TXT(NDF_SRC_PORT, w->client_port),
230	ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_FOR, w->user_auth.forwarded_for),
231	ND_LOG_FIELD_UUID(NDF_ACCOUNT_ID, &w->user_auth.cloud_account_id.uuid),
232	ND_LOG_FIELD_TXT(NDF_USER_NAME, w->user_auth.client_name),
233	ND_LOG_FIELD_TXT(NDF_USER_ROLE, http_id2user_role(w->user_auth.user_role)),
234	ND_LOG_FIELD_CB(NDF_USER_ACCESS, log_cb_http_access_to_hex, &w->user_auth.access),
235	ND_LOG_FIELD_END(),
236	};
237	ND_LOG_STACK_PUSH(lgs);
238
239	ND_LOG_FIELD_PRIORITY prio = NDLP_INFO;
240	if(w->response.code >= 500)
241	prio = NDLP_EMERG;
242	else if(w->response.code >= 400)
243	prio = NDLP_WARNING;
244	else if(w->response.code >= 300)
245	prio = NDLP_NOTICE;
246
247	// cleanup progress
248	if(web_client_flag_check(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING)) {
249	web_client_flag_clear(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING);
250	query_progress_finished(&w->transaction, 0, w->response.code, total_ut, size, sent);
251	}
252
253	// access log
254	if(likely(buffer_strlen(w->url_as_received))) {
255	nd_log(NDLS_ACCESS, prio, NULL);
256
257	if(update_web_stats)
258	pulse_web_request_completed(
259	dt_usec(&tv, &w->timings.tv_in), w->statistics.received_bytes, w->statistics.sent_bytes, size, sent);
260	}
261	}
262
263	void web_client_request_done(struct web_client *w) {
264	sock_setcork(w->fd, false);
265
266	netdata_log_debug(D_WEB_CLIENT, "%llu: Resetting client.", w->id);
267
268	web_client_log_completed_request(w, true);
269	web_client_reset_allocations(w, false);
270
271	w->mode = HTTP_REQUEST_MODE_GET;
272
273	web_client_disable_donottrack(w);
274	web_client_disable_tracking_required(w);
275	web_client_disable_keepalive(w);
276
277	// Clear URL-derived flags between requests. PATH_IS_MCP is re-set
278	// during the next URL decode; clearing it here makes sure a keepalive
279	// connection cannot carry the previous request's classification into
280	// a new request that fails before URL decoding runs (e.g., malformed
281	// request line, unsupported method).
282	web_client_flag_clear(w, WEB_CLIENT_FLAG_PATH_IS_MCP);
283
284	w->header_parse_tries = 0;
285	w->header_parse_last_size = 0;
286
287	web_client_enable_wait_receive(w);
288	web_client_disable_wait_send(w);
289
290	w->response.has_cookies = false;
291	w->response.sent = 0;
292	w->response.code = 0;
293	w->response.zoutput = false;
294
295	w->statistics.received_bytes = 0;
296	w->statistics.sent_bytes = 0;
297	}
298
299	static int append_slash_to_url_and_redirect(struct web_client *w) {
300	// this function returns a relative redirect
301	// it finds the last path component on the URL and just appends / to it
302	//
303	// So, if the URL is:
304	//
305	// /path/to/file?query_string
306	//
307	// It adds a Location header like this:
308	//
309	// Location: file/?query_string\r\n
310	//
311	// The web browser already knows that it is inside /path/to/
312	// so it converts the path to /path/to/file/ and executes the
313	// request again.
314
315	buffer_strcat(w->response.header, "Location: ");
316	const char *b = buffer_tostring(w->url_as_received);
317	const char *q = strchr(b, '?');
318	if(q && q > b) {
319	const char *e = q - 1;
320	while(e > b && *e != '/') e--;
321	if(*e == '/') e++;
322
323	size_t len = q - e;
324	buffer_strncat(w->response.header, e, len);
325	buffer_strncat(w->response.header, "/", 1);
326	buffer_strcat(w->response.header, q);
327	}
328	else {
329	const char *e = &b[buffer_strlen(w->url_as_received) - 1];
330	while(e > b && *e != '/') e--;
331	if(*e == '/') e++;
332
333	buffer_strcat(w->response.header, e);
334	buffer_strncat(w->response.header, "/", 1);
335	}
336
337	buffer_strncat(w->response.header, "\r\n", 2);
338
339	w->response.data->content_type = CT_TEXT_HTML;
340	buffer_flush(w->response.data);
341	buffer_strcat(w->response.data,
342	"<!DOCTYPE html><html>"
343	"<body onload=\"window.location.href = window.location.origin + window.location.pathname + '/' + window.location.search + window.location.hash\">"
344	"Redirecting. In case your browser does not support redirection, please click "
345	"<a onclick=\"window.location.href = window.location.origin + window.location.pathname + '/' + window.location.search + window.location.hash\">here</a>."
346	"</body></html>");
347	return HTTP_RESP_MOVED_PERM;
348	}
349
350	// Work around a bug in the CMocka library by removing this function during testing.
351	#ifndef REMOVE_MYSENDFILE
352
353	static inline int dashboard_version(struct web_client *w) {
354	if(!web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_WITH_VERSION))
355	return -1;
356
357	if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_IS_V3))
358	return 3;
359	if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_IS_V2))
360	return 2;
361	if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_IS_V1))
362	return 1;
363	if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_IS_V0))
364	return 0;
365
366	return -1;
367	}
368
369	static bool find_filename_to_serve(const char filename, char dst, size_t dst_len, struct stat statbuf, struct web_client w, bool *is_dir) {
370	int d_version = dashboard_version(w);
371	bool has_extension = web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_HAS_FILE_EXTENSION);
372
373	int fallback = 0;
374
375	if(has_extension) {
376	if(d_version == -1)
377	snprintfz(dst, dst_len, "%s/%s", netdata_configured_web_dir, filename);
378	else {
379	// check if the filename or directory exists
380	// fallback to the same path without the dashboard version otherwise
381	snprintfz(dst, dst_len, "%s/v%d/%s", netdata_configured_web_dir, d_version, filename);
382	fallback = 1;
383	}
384	}
385	else if(d_version != -1) {
386	if(filename && *filename) {
387	// check if the filename exists
388	// fallback to /vN/index.html otherwise
389	snprintfz(dst, dst_len, "%s/%s", netdata_configured_web_dir, filename);
390	fallback = 2;
391	}
392	else {
393	if(filename && *filename)
394	web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_HAS_TRAILING_SLASH);
395	snprintfz(dst, dst_len, "%s/v%d", netdata_configured_web_dir, d_version);
396	}
397	}
398	else {
399	// check if filename exists
400	// this is needed to serve {filename}/index.html, in case a user puts a html file into a directory
401	// fallback to /index.html otherwise
402	snprintfz(dst, dst_len, "%s/%s", netdata_configured_web_dir, filename);
403	fallback = 3;
404	}
405
406	if (stat(dst, statbuf) != 0) {
407	if(fallback == 1) {
408	snprintfz(dst, dst_len, "%s/%s", netdata_configured_web_dir, filename);
409	if (stat(dst, statbuf) != 0)
410	return false;
411	}
412	else if(fallback == 2) {
413	if(filename && *filename)
414	web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_HAS_TRAILING_SLASH);
415	snprintfz(dst, dst_len, "%s/v%d", netdata_configured_web_dir, d_version);
416	if (stat(dst, statbuf) != 0)
417	return false;
418	}
419	else if(fallback == 3) {
420	if(filename && *filename)
421	web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_HAS_TRAILING_SLASH);
422	snprintfz(dst, dst_len, "%s", netdata_configured_web_dir);
423	if (stat(dst, statbuf) != 0)
424	return false;
425	}
426	else
427	return false;
428	}
429
430	if((statbuf->st_mode & S_IFMT) == S_IFDIR) {
431	size_t len = strlen(dst);
432	if(len > dst_len - 11)
433	return false;
434
435	strncpyz(&dst[len], "/index.html", dst_len - len);
436
437	if (stat(dst, statbuf) != 0)
438	return false;
439
440	*is_dir = true;
441	}
442
443	return true;
444	}
445
446	static int web_server_static_file(struct web_client w, char filename) {
447	char web_path[FILENAME_MAX];
448	snprintfz(web_path, sizeof(web_path), "%s/%s", netdata_configured_web_dir, filename);
449	#if defined(OS_WINDOWS)
450	char display_path[FILENAME_MAX];
451	netdata_log_debug(D_WEB_CLIENT, "%llu: Looking for file '%s'", w->id,
452	os_translate_path(display_path, web_path, sizeof(display_path)));
453	#else
454	netdata_log_debug(D_WEB_CLIENT, "%llu: Looking for file '%s'", w->id, web_path);
455	#endif
456
457	if(!http_can_access_dashboard(w))
458	return web_client_permission_denied_acl(w);
459
460	// skip leading slashes
461	while (*filename == '/') filename++;
462
463	// if the filename contains "strange" characters, refuse to serve it
464	char *s;
465	for(s = filename; *s ;s++) {
466	if( !isalnum((uint8_t)s) && s != '/' && s != '.' && s != '-' && *s != '_') {
467	netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not acceptable.", w->id, filename);
468	w->response.data->content_type = CT_TEXT_HTML;
469	buffer_sprintf(w->response.data, "Filename contains invalid characters: ");
470	buffer_strcat_htmlescape(w->response.data, filename);
471	return HTTP_RESP_BAD_REQUEST;
472	}
473	}
474
475	// if the filename contains a double dot refuse to serve it
476	if(strstr(filename, "..") != 0) {
477	netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: File '%s' is not acceptable.", w->id, filename);
478	w->response.data->content_type = CT_TEXT_HTML;
479	buffer_strcat(w->response.data, "Relative filenames are not supported: ");
480	buffer_strcat_htmlescape(w->response.data, filename);
481	return HTTP_RESP_BAD_REQUEST;
482	}
483
484	// find the physical file on disk
485	bool is_dir = false;
486	char web_filename[FILENAME_MAX + 1];
487	struct stat statbuf;
488	if(!find_filename_to_serve(filename, web_filename, FILENAME_MAX, &statbuf, w, &is_dir)) {
489	w->response.data->content_type = CT_TEXT_HTML;
490	buffer_strcat(w->response.data, "File does not exist, or is not accessible: ");
491	buffer_strcat_htmlescape(w->response.data, filename);
492	return HTTP_RESP_NOT_FOUND;
493	}
494
495	if(is_dir && !web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_HAS_TRAILING_SLASH))
496	return append_slash_to_url_and_redirect(w);
497
498	buffer_flush(w->response.data);
499	buffer_need_bytes(w->response.data, (size_t)statbuf.st_size);
500	w->response.data->len = (size_t)statbuf.st_size;
501
502	// open the file
503	int fd = open(web_filename, O_RDONLY \| O_CLOEXEC);
504
505	// read the file
506	if(fd != -1 && read(fd, w->response.data->buffer, statbuf.st_size) != statbuf.st_size) {
507	// cannot read the whole file
508	nd_log(NDLS_DAEMON, NDLP_ERR, "Web server failed to read file '%s'", web_filename);
509	close(fd);
510	fd = -1;
511	}
512
513	// check for failures
514	if(fd == -1) {
515	buffer_flush(w->response.data);
516
517	if(errno == EBUSY \|\| errno == EAGAIN) {
518	netdata_log_error("%llu: File '%s' is busy, sending 307 Moved Temporarily to force retry.", w->id, web_filename);
519	w->response.data->content_type = CT_TEXT_HTML;
520	buffer_sprintf(w->response.header, "Location: /%s\r\n", filename);
521	buffer_strcat(w->response.data, "File is currently busy, please try again later: ");
522	buffer_strcat_htmlescape(w->response.data, filename);
523	return HTTP_RESP_REDIR_TEMP;
524	}
525	else {
526	netdata_log_error("%llu: Cannot open file '%s'.", w->id, web_filename);
527	w->response.data->content_type = CT_TEXT_HTML;
528	buffer_strcat(w->response.data, "Cannot open file: ");
529	buffer_strcat_htmlescape(w->response.data, filename);
530	return HTTP_RESP_NOT_FOUND;
531	}
532	}
533	else
534	close(fd);
535
536	w->response.data->content_type = contenttype_for_filename(web_filename);
537	netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Sending file '%s' (%"PRId64" bytes, fd %d).", w->id, web_filename, (int64_t)statbuf.st_size, w->fd);
538
539	w->mode = HTTP_REQUEST_MODE_GET;
540	web_client_enable_wait_send(w);
541	web_client_disable_wait_receive(w);
542
543	#ifdef __APPLE__
544	w->response.data->date = statbuf.st_mtimespec.tv_sec;
545	#else
546	w->response.data->date = statbuf.st_mtim.tv_sec;
547	#endif
548	w->response.data->expires = now_realtime_sec() + 86400;
549
550	buffer_cacheable(w->response.data);
551
552	return HTTP_RESP_OK;
553	}
554	#endif
555
556	static inline int check_host_and_call(RRDHOST host, struct web_client w, char url, int (func)(RRDHOST , struct web_client , char *)) {
557	return func(host, w, url);
558	}
559
560	int web_client_api_request(RRDHOST host, struct web_client w, char *url_path_fragment) {
561	ND_LOG_STACK lgs[] = {
562	ND_LOG_FIELD_TXT(NDF_SRC_IP, w->user_auth.client_ip),
563	ND_LOG_FIELD_TXT(NDF_SRC_PORT, w->client_port),
564	ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_HOST, w->forwarded_host),
565	ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_FOR, w->user_auth.forwarded_for),
566	ND_LOG_FIELD_TXT(NDF_NIDL_NODE, w->client_host),
567	ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, HTTP_REQUEST_MODE_2str(w->mode)),
568	ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received),
569	ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id),
570	ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction),
571	ND_LOG_FIELD_UUID(NDF_ACCOUNT_ID, &w->user_auth.cloud_account_id.uuid),
572	ND_LOG_FIELD_TXT(NDF_USER_NAME, w->user_auth.client_name),
573	ND_LOG_FIELD_TXT(NDF_USER_ROLE, http_id2user_role(w->user_auth.user_role)),
574	ND_LOG_FIELD_CB(NDF_USER_ACCESS, log_cb_http_access_to_hex, &w->user_auth.access),
575	ND_LOG_FIELD_END(),
576	};
577	ND_LOG_STACK_PUSH(lgs);
578
579	if(!web_client_flag_check(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING)) {
580	web_client_flag_set(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING);
581	query_progress_start_or_update(&w->transaction, 0, w->mode, w->acl,
582	buffer_tostring(w->url_as_received),
583	w->payload,
584	w->user_auth.forwarded_for[0] ? w->user_auth.forwarded_for : w->user_auth.client_ip);
585	}
586
587	// get the api version
588	char *tok = strsep_skip_consecutive_separators(&url_path_fragment, "/");
589	if(tok && *tok) {
590	if(strcmp(tok, "v3") == 0)
591	return web_client_api_request_v3(host, w, url_path_fragment);
592	else if(strcmp(tok, "v2") == 0)
593	return web_client_api_request_v2(host, w, url_path_fragment);
594	else if(strcmp(tok, "v1") == 0)
595	return web_client_api_request_v1(host, w, url_path_fragment);
596	else {
597	buffer_flush(w->response.data);
598	w->response.data->content_type = CT_TEXT_HTML;
599	buffer_strcat(w->response.data, "Unsupported API version: ");
600	buffer_strcat_htmlescape(w->response.data, tok);
601	return HTTP_RESP_NOT_FOUND;
602	}
603	}
604	else {
605	buffer_flush(w->response.data);
606	buffer_sprintf(w->response.data, "Which API version?");
607	return HTTP_RESP_BAD_REQUEST;
608	}
609	}
610
611
612	/**
613	* Valid Method
614	*
615	* Netdata accepts only three methods, including one of these three(STREAM) is an internal method.
616	*
617	* @param w is the structure with the client request
618	* @param s is the start string to parse
619	*
620	* @return it returns the next address to parse case the method is valid and NULL otherwise.
621	*/
622	static inline char web_client_valid_method(struct web_client w, char *s) {
623	// is is a valid request?
624	if(!strncmp(s, "GET ", 4)) {
625	s = &s[4];
626	w->mode = HTTP_REQUEST_MODE_GET;
627	}
628	else if(!strncmp(s, "OPTIONS ", 8)) {
629	s = &s[8];
630	w->mode = HTTP_REQUEST_MODE_OPTIONS;
631	}
632	else if(!strncmp(s, "POST ", 5)) {
633	s = &s[5];
634	w->mode = HTTP_REQUEST_MODE_POST;
635	}
636	else if(!strncmp(s, "PUT ", 4)) {
637	s = &s[4];
638	w->mode = HTTP_REQUEST_MODE_PUT;
639	}
640	else if(!strncmp(s, "DELETE ", 7)) {
641	s = &s[7];
642	w->mode = HTTP_REQUEST_MODE_DELETE;
643	}
644	else if(!strncmp(s, "STREAM ", 7)) {
645	s = &s[7];
646
647	if (!SSL_connection(&w->ssl) && http_is_using_ssl_force(w)) {
648	w->header_parse_tries = 0;
649	w->header_parse_last_size = 0;
650	web_client_disable_wait_receive(w);
651
652	char hostname[256];
653	char *copyme = strstr(s,"hostname=");
654	if ( copyme ){
655	copyme += 9;
656	char *end = strchr(copyme,'&');
657	if(end){
658	size_t length = MIN(255, end - copyme);
659	memcpy(hostname,copyme,length);
660	hostname[length] = 0X00;
661	}
662	else{
663	memcpy(hostname,"not available",13);
664	hostname[13] = 0x00;
665	}
666	}
667	else{
668	memcpy(hostname,"not available",13);
669	hostname[13] = 0x00;
670	}
671	netdata_log_error("The server is configured to always use encrypted connections, please enable the SSL on child with hostname '%s'.",hostname);
672	s = NULL;
673	}
674
675	w->mode = HTTP_REQUEST_MODE_STREAM;
676	}
677	else {
678	s = NULL;
679	}
680
681	return s;
682	}
683
684	/**
685	* Request validate
686	*
687	* @param w is the structure with the client request
688	*
689	* @return It returns HTTP_VALIDATION_OK on success and another code present
690	* in the enum HTTP_VALIDATION otherwise.
691	*/
692	HTTP_VALIDATION http_request_validate(struct web_client *w) {
693	char s = (char )buffer_tostring(w->response.data), *encoded_url = NULL;
694
695	size_t last_pos = w->header_parse_last_size;
696
697	w->header_parse_tries++;
698	w->header_parse_last_size = buffer_strlen(w->response.data);
699
700	int is_it_valid;
701	if(w->header_parse_tries > 1) {
702	if(last_pos > 4) last_pos -= 4; // allow searching for \r\n\r\n
703	else last_pos = 0;
704
705	if(w->header_parse_last_size <= last_pos)
706	last_pos = 0;
707
708	is_it_valid = url_is_request_complete_and_extract_payload(s, &s[last_pos],
709	w->header_parse_last_size, &w->payload);
710
711	if(!is_it_valid) {
712	if(w->header_parse_tries > HTTP_REQ_MAX_HEADER_FETCH_TRIES) {
713	netdata_log_info("Disabling slow client after %zu attempts to read the request (%zu bytes received)", w->header_parse_tries, buffer_strlen(w->response.data));
714	w->header_parse_tries = 0;
715	w->header_parse_last_size = 0;
716	web_client_disable_wait_receive(w);
717	return HTTP_VALIDATION_TOO_MANY_READ_RETRIES;
718	}
719
720	return HTTP_VALIDATION_INCOMPLETE;
721	}
722
723	is_it_valid = 1;
724	} else {
725	last_pos = w->header_parse_last_size;
726	is_it_valid =
727	url_is_request_complete_and_extract_payload(s, &s[last_pos], w->header_parse_last_size, &w->payload);
728	}
729
730	s = web_client_valid_method(w, s);
731	if (!s) {
732	w->header_parse_tries = 0;
733	w->header_parse_last_size = 0;
734	web_client_disable_wait_receive(w);
735
736	return HTTP_VALIDATION_NOT_SUPPORTED;
737	} else if (!is_it_valid) {
738	web_client_enable_wait_receive(w);
739	return HTTP_VALIDATION_INCOMPLETE;
740	}
741
742	//After the method we have the path and query string together
743	encoded_url = s;
744
745	//we search for the position where we have " HTTP/", because it finishes the user request
746	s = url_find_protocol(s);
747
748	// incomplete requests
749	if(unlikely(!*s)) {
750	web_client_enable_wait_receive(w);
751	return HTTP_VALIDATION_INCOMPLETE;
752	}
753
754	// we have the end of encoded_url - remember it
755	char *ue = s;
756
757	// make sure we have complete request
758	// complete requests contain: \r\n\r\n
759	while(*s) {
760	// find a line feed
761	while(s && s++ != '\r');
762
763	// did we reach the end?
764	if(unlikely(!*s)) break;
765
766	// is it \r\n ?
767	if(likely(*s++ == '\n')) {
768
769	// is it again \r\n ? (header end)
770	if(unlikely(*s == '\r' && s[1] == '\n')) {
771	// a valid complete HTTP request found
772
773	char c = *ue;
774	*ue = '\0';
775	web_client_decode_path_and_query_string(w, encoded_url);
776	*ue = c;
777
778	if ( (web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx) ) {
779	if (!w->ssl.conn && (http_is_using_ssl_force(w) \|\| http_is_using_ssl_default(w)) && (w->mode != HTTP_REQUEST_MODE_STREAM)) {
780	w->header_parse_tries = 0;
781	w->header_parse_last_size = 0;
782	web_client_disable_wait_receive(w);
783	return HTTP_VALIDATION_REDIRECT;
784	}
785	}
786
787	w->header_parse_tries = 0;
788	w->header_parse_last_size = 0;
789	web_client_disable_wait_receive(w);
790	return HTTP_VALIDATION_OK;
791	}
792
793	// another header line
794	s = http_header_parse_line(w, s);
795	}
796	}
797
798	// incomplete request
799	web_client_enable_wait_receive(w);
800	return HTTP_VALIDATION_INCOMPLETE;
801	}
802
803	static inline ssize_t web_client_send_data(struct web_client w,const void buf,size_t len, int flags)
804	{
805	do {
806	errno_clear();
807
808	ssize_t bytes;
809	if ((web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx)) {
810	if (SSL_connection(&w->ssl)) {
811	bytes = netdata_ssl_write(&w->ssl, buf, len);
812	web_client_enable_wait_from_ssl(w);
813	} else
814	bytes = send(w->fd, buf, len, flags);
815	} else if (web_client_check_conn_tcp(w) \|\| web_client_check_conn_unix(w))
816	bytes = send(w->fd, buf, len, flags);
817	else
818	bytes = -999;
819
820	if(bytes < 0 && errno == EAGAIN) {
821	tinysleep();
822	continue;
823	}
824
825	return bytes;
826	} while(true);
827	}
828
829	void web_client_build_http_header(struct web_client *w) {
830	if(unlikely(w->response.code != HTTP_RESP_OK))
831	buffer_no_cacheable(w->response.data);
832
833	if(unlikely(!w->response.data->date))
834	w->response.data->date = now_realtime_sec();
835
836	// set a proper expiration date, if not already set
837	if(unlikely(!w->response.data->expires))
838	w->response.data->expires = w->response.data->date +
839	((w->response.data->options & WB_CONTENT_NO_CACHEABLE) ? 0 : 86400);
840
841	// prepare the HTTP response header
842	netdata_log_debug(D_WEB_CLIENT, "%llu: Generating HTTP header with response %d.", w->id, w->response.code);
843
844	const char *code_msg = http_response_code2string(w->response.code);
845
846	// prepare the last modified and expiration dates
847	char rfc7231_date[RFC7231_MAX_LENGTH], rfc7231_expires[RFC7231_MAX_LENGTH];
848	rfc7231_datetime(rfc7231_date, sizeof(rfc7231_date), w->response.data->date);
849	rfc7231_datetime(rfc7231_expires, sizeof(rfc7231_expires), w->response.data->expires);
850
851	if (w->response.code == HTTP_RESP_HTTPS_UPGRADE) {
852	buffer_sprintf(w->response.header_output,
853	"HTTP/1.1 %d %s\r\n"
854	"Location: https://%s%s\r\n",
855	w->response.code, code_msg,
856	w->server_host ? w->server_host : "",
857	buffer_tostring(w->url_as_received));
858	w->response.code = HTTP_RESP_MOVED_PERM;
859	}
860	else {
861	buffer_sprintf(w->response.header_output,
862	"HTTP/1.1 %d %s\r\n"
863	"Connection: %s\r\n"
864	"Server: Netdata Embedded HTTP Server %s\r\n"
865	"Access-Control-Allow-Origin: %s\r\n"
866	"Access-Control-Allow-Credentials: true\r\n"
867	"Date: %s\r\n",
868	w->response.code,
869	code_msg,
870	web_client_has_keepalive(w)?"keep-alive":"close",
871	NETDATA_VERSION,
872	w->origin ? w->origin : "*",
873	rfc7231_date);
874
875	http_header_content_type(w->response.header_output, w->response.data->content_type);
876	}
877
878	// MCP-specific CORS: widen the allowlist + advertise exposed
879	// headers only for MCP transport endpoints (/mcp, /sse). The flag
880	// is set once during URL decoding; see WEB_CLIENT_FLAG_PATH_IS_MCP.
881	bool is_mcp_path = web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_IS_MCP);
882
883	if(is_mcp_path && w->mode != HTTP_REQUEST_MODE_OPTIONS)
884	buffer_strcat(w->response.header_output,
885	"Access-Control-Expose-Headers: Mcp-Session-Id\r\n");
886
887	if(unlikely(web_x_frame_options))
888	buffer_sprintf(w->response.header_output, "X-Frame-Options: %s\r\n", web_x_frame_options);
889
890	if(w->response.has_cookies) {
891	if(respect_web_browser_do_not_track_policy)
892	buffer_sprintf(w->response.header_output,
893	"Tk: T;cookies\r\n");
894	}
895	else {
896	if(respect_web_browser_do_not_track_policy) {
897	if(web_client_has_tracking_required(w))
898	buffer_sprintf(w->response.header_output,
899	"Tk: T;cookies\r\n");
900	else
901	buffer_sprintf(w->response.header_output,
902	"Tk: N\r\n");
903	}
904	}
905
906	if(w->mode == HTTP_REQUEST_MODE_OPTIONS) {
907	// Methods, max-age, and the base header allowlist are identical
908	// for every OPTIONS preflight. MCP preflights append the extra
909	// request headers the SDK uses: mcp-protocol-version,
910	// mcp-session-id, last-event-id (SSE resumption), authorization
911	// (bearer tokens). DELETE is not advertised — the MCP handlers
912	// currently return 405 for it, and advertising it would let the
913	// preflight succeed only for the real request to fail. Add DELETE
914	// here when the handlers learn session teardown.
915	buffer_strcat(w->response.header_output,
916	"Access-Control-Allow-Methods: GET, POST, OPTIONS\r\n"
917	"Access-Control-Allow-Headers: accept, x-requested-with, origin, content-type, cookie, pragma, cache-control, x-auth-token, x-netdata-auth, x-transaction-id");
918
919	if(is_mcp_path)
920	buffer_strcat(w->response.header_output,
921	", authorization, mcp-protocol-version, mcp-session-id, last-event-id");
922
923	buffer_strcat(w->response.header_output,
924	"\r\n"
925	"Access-Control-Max-Age: 1209600\r\n" // 86400 * 14
926	);
927	}
928	else {
929	buffer_sprintf(w->response.header_output,
930	"Cache-Control: %s\r\n"
931	"Expires: %s\r\n",
932	(w->response.data->options & WB_CONTENT_NO_CACHEABLE)?"no-cache, no-store, must-revalidate\r\nPragma: no-cache":"public",
933	rfc7231_expires);
934	}
935
936	// copy a possibly available custom header
937	if(unlikely(buffer_strlen(w->response.header)))
938	buffer_strcat(w->response.header_output, buffer_tostring(w->response.header));
939
940	// headers related to the transfer method
941	if(likely(w->response.zoutput))
942	buffer_strcat(w->response.header_output, "Content-Encoding: gzip\r\n");
943
944	if(likely(w->flags & WEB_CLIENT_CHUNKED_TRANSFER))
945	buffer_strcat(w->response.header_output, "Transfer-Encoding: chunked\r\n");
946	else {
947	if(likely(w->response.data->len)) {
948	// we know the content length, put it
949	buffer_sprintf(w->response.header_output, "Content-Length: %zu\r\n", (size_t)w->response.data->len);
950	}
951	else {
952	// we don't know the content length, disable keep-alive
953	web_client_disable_keepalive(w);
954	}
955	}
956
957	char uuid[UUID_COMPACT_STR_LEN];
958	uuid_unparse_lower_compact(w->transaction, uuid);
959	buffer_sprintf(w->response.header_output,
960	"X-Transaction-ID: %s\r\n", uuid);
961
962	// end of HTTP header
963	buffer_strcat(w->response.header_output, "\r\n");
964	}
965
966	static inline void web_client_send_http_header(struct web_client *w) {
967	// For WebSocket handshake, the header is already fully prepared in websocket_handle_handshake
968	// For standard HTTP responses, we need to build the header
969	if (w->response.code != HTTP_RESP_WEBSOCKET_HANDSHAKE) {
970	web_client_build_http_header(w);
971	}
972
973	// sent the HTTP header
974	netdata_log_debug(D_WEB_DATA, "%llu: Sending response HTTP header of size %zu: '%s'"
975	, w->id
976	, buffer_strlen(w->response.header_output)
977	, buffer_tostring(w->response.header_output)
978	);
979
980	sock_setcork(w->fd, true);
981
982	size_t count = 0;
983	ssize_t bytes;
984
985	if ( (web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx) ) {
986	if (SSL_connection(&w->ssl)) {
987	bytes = netdata_ssl_write(&w->ssl, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output));
988	web_client_enable_wait_from_ssl(w);
989	}
990	else {
991	while((bytes = send(w->fd, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output), 0)) == -1) {
992	count++;
993
994	if(count > 100 \|\| (errno != EAGAIN && errno != EWOULDBLOCK)) {
995	netdata_log_error("Cannot send HTTP headers to web client.");
996	break;
997	}
998	}
999	}
1000	}
1001	else if(web_client_check_conn_tcp(w) \|\| web_client_check_conn_unix(w)) {
1002	while((bytes = send(w->fd, buffer_tostring(w->response.header_output), buffer_strlen(w->response.header_output), 0)) == -1) {
1003	count++;
1004
1005	if(count > 100 \|\| (errno != EAGAIN && errno != EWOULDBLOCK)) {
1006	netdata_log_error("Cannot send HTTP headers to web client.");
1007	break;
1008	}
1009	}
1010	}
1011	else
1012	bytes = -999;
1013
1014	if(bytes != (ssize_t) buffer_strlen(w->response.header_output)) {
1015	if(bytes > 0)
1016	w->statistics.sent_bytes += bytes;
1017
1018	if (bytes < 0) {
1019	netdata_log_error("HTTP headers failed to be sent (I sent %zu bytes but the system sent %zd bytes). Closing web client."
1020	, buffer_strlen(w->response.header_output)
1021	, bytes);
1022
1023	WEB_CLIENT_IS_DEAD(w);
1024	return;
1025	}
1026	}
1027	else
1028	w->statistics.sent_bytes += bytes;
1029	}
1030
1031	static inline int web_client_switch_host(RRDHOST host, struct web_client w, char url, bool nodeid, int (func)(RRDHOST , struct web_client , char *)) {
1032	static uint32_t hash_localhost = 0;
1033
1034	if(unlikely(!hash_localhost)) {
1035	hash_localhost = simple_hash("localhost");
1036	}
1037
1038	if(host != localhost) {
1039	buffer_flush(w->response.data);
1040	buffer_strcat(w->response.data, "Nesting of hosts is not allowed.");
1041	return HTTP_RESP_BAD_REQUEST;
1042	}
1043
1044	char *tok = strsep_skip_consecutive_separators(&url, "/");
1045	if(tok && *tok) {
1046	netdata_log_debug(D_WEB_CLIENT, "%llu: Searching for host with name '%s'.", w->id, tok);
1047
1048	if(nodeid) {
1049	host = rrdhost_find_by_node_id(tok);
1050	if(!host) {
1051	host = rrdhost_find_by_guid(tok);
1052	if (!host)
1053	host = rrdhost_find_by_hostname(tok);
1054	}
1055	}
1056	else {
1057	host = rrdhost_find_by_guid(tok);
1058	if(!host) {
1059	host = rrdhost_find_by_node_id(tok);
1060	if (!host)
1061	host = rrdhost_find_by_hostname(tok);
1062	}
1063	}
1064
1065	if(!host) {
1066	// we didn't find it, but it may be a uuid case mismatch for MACHINE_GUID
1067	// so, recreate the machine guid in lower-case.
1068	nd_uuid_t uuid;
1069	char txt[UUID_STR_LEN];
1070	if (uuid_parse(tok, uuid) == 0) {
1071	uuid_unparse_lower(uuid, txt);
1072	host = rrdhost_find_by_guid(txt);
1073	}
1074	}
1075
1076	if (host) {
1077	if(!url)
1078	//no delim found
1079	return append_slash_to_url_and_redirect(w);
1080
1081	buffer_flush(w->url_path_decoded);
1082	buffer_strcat(w->url_path_decoded, "/");
1083	buffer_strcat(w->url_path_decoded, url);
1084	char *mutable_path = strdupz(buffer_tostring(w->url_path_decoded));
1085	int rc = func(host, w, mutable_path);
1086	freez(mutable_path);
1087	return rc;
1088	}
1089	}
1090
1091	buffer_flush(w->response.data);
1092	w->response.data->content_type = CT_TEXT_HTML;
1093	buffer_strcat(w->response.data, "This netdata does not maintain a database for host: ");
1094	buffer_strcat_htmlescape(w->response.data, tok?tok:"");
1095	return HTTP_RESP_NOT_FOUND;
1096	}
1097
1098	int web_client_api_request_with_node_selection(RRDHOST host, struct web_client w, char *decoded_url_path) {
1099	// entry point for all API requests
1100
1101	ND_LOG_STACK lgs[] = {
1102	ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, HTTP_REQUEST_MODE_2str(w->mode)),
1103	ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received),
1104	ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id),
1105	ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction),
1106	ND_LOG_FIELD_UUID(NDF_ACCOUNT_ID, &w->user_auth.cloud_account_id.uuid),
1107	ND_LOG_FIELD_TXT(NDF_USER_NAME, w->user_auth.client_name),
1108	ND_LOG_FIELD_TXT(NDF_USER_ROLE, http_id2user_role(w->user_auth.user_role)),
1109	ND_LOG_FIELD_CB(NDF_USER_ACCESS, log_cb_http_access_to_hex, &w->user_auth.access),
1110	ND_LOG_FIELD_END(),
1111	};
1112	ND_LOG_STACK_PUSH(lgs);
1113
1114	// give a new transaction id to the request
1115	if(uuid_is_null(w->transaction))
1116	uuid_generate_random(w->transaction);
1117
1118	static uint32_t
1119	hash_api = 0,
1120	hash_host = 0,
1121	hash_node = 0;
1122
1123	if(unlikely(!hash_api)) {
1124	hash_api = simple_hash("api");
1125	hash_host = simple_hash("host");
1126	hash_node = simple_hash("node");
1127	}
1128
1129	char *tok = strsep_skip_consecutive_separators(&decoded_url_path, "/?");
1130	if(likely(tok && *tok)) {
1131	uint32_t hash = simple_hash(tok);
1132
1133	if(unlikely(hash == hash_api && strcmp(tok, "api") == 0)) {
1134	// current API
1135	netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: API request ...", w->id);
1136	return check_host_and_call(host, w, decoded_url_path, web_client_api_request);
1137	}
1138	else if(unlikely((hash == hash_host && strcmp(tok, "host") == 0) \|\| (hash == hash_node && strcmp(tok, "node") == 0))) {
1139	// host switching
1140	netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: host switch request ...", w->id);
1141	return web_client_switch_host(host, w, decoded_url_path, hash == hash_node, web_client_api_request_with_node_selection);
1142	}
1143	}
1144
1145	buffer_flush(w->response.data);
1146	buffer_strcat(w->response.data, "Unknown API endpoint.");
1147	w->response.data->content_type = CT_TEXT_HTML;
1148	return HTTP_RESP_NOT_FOUND;
1149	}
1150
1151	static inline int web_client_process_url(RRDHOST host, struct web_client w, char *decoded_url_path) {
1152	if(unlikely(!service_running(ABILITY_WEB_REQUESTS)))
1153	return web_client_service_unavailable(w);
1154
1155	static uint32_t
1156	hash_api = 0,
1157	hash_netdata_conf = 0,
1158	hash_host = 0,
1159	hash_node = 0,
1160	hash_v0 = 0,
1161	hash_v1 = 0,
1162	hash_v2 = 0,
1163	hash_v3 = 0,
1164	hash_mcp = 0,
1165	hash_sse = 0;
1166
1167	#ifdef NETDATA_INTERNAL_CHECKS
1168	static uint32_t hash_exit = 0, hash_debug = 0, hash_mirror = 0;
1169	#endif
1170
1171	if(unlikely(!hash_api)) {
1172	hash_api = simple_hash("api");
1173	hash_netdata_conf = simple_hash("netdata.conf");
1174	hash_host = simple_hash("host");
1175	hash_node = simple_hash("node");
1176	hash_v0 = simple_hash("v0");
1177	hash_v1 = simple_hash("v1");
1178	hash_v2 = simple_hash("v2");
1179	hash_v3 = simple_hash("v3");
1180	hash_mcp = simple_hash("mcp");
1181	hash_sse = simple_hash("sse");
1182	#ifdef NETDATA_INTERNAL_CHECKS
1183	hash_exit = simple_hash("exit");
1184	hash_debug = simple_hash("debug");
1185	hash_mirror = simple_hash("mirror");
1186	#endif
1187	}
1188
1189	// keep a copy of the decoded path, in case we need to serve it as a filename
1190	char filename[FILENAME_MAX + 1];
1191	strncpyz(filename, decoded_url_path ? decoded_url_path : "", FILENAME_MAX);
1192
1193	char *tok = strsep_skip_consecutive_separators(&decoded_url_path, "/?");
1194	if(likely(tok && *tok)) {
1195	uint32_t hash = simple_hash(tok);
1196	netdata_log_debug(D_WEB_CLIENT, "%llu: Processing command '%s'.", w->id, tok);
1197
1198	if(likely(hash == hash_api && strcmp(tok, "api") == 0)) { // current API
1199	netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: API request ...", w->id);
1200	return check_host_and_call(host, w, decoded_url_path, web_client_api_request);
1201	}
1202	else if(likely(hash == hash_mcp && strcmp(tok, "mcp") == 0)) {
1203	if(unlikely(!http_can_access_dashboard(w)))
1204	return web_client_permission_denied_acl(w);
1205	return mcp_http_handle_request(host, w);
1206	}
1207	else if(likely(hash == hash_sse && strcmp(tok, "sse") == 0)) {
1208	if(unlikely(!http_can_access_dashboard(w)))
1209	return web_client_permission_denied_acl(w);
1210	return mcp_sse_handle_request(host, w);
1211	}
1212	else if(unlikely((hash == hash_host && strcmp(tok, "host") == 0) \|\| (hash == hash_node && strcmp(tok, "node") == 0))) { // host switching
1213	netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: host switch request ...", w->id);
1214	return web_client_switch_host(host, w, decoded_url_path, hash == hash_node, web_client_process_url);
1215	}
1216	else if(unlikely(hash == hash_v3 && strcmp(tok, "v3") == 0)) {
1217	if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_WITH_VERSION))
1218	return bad_request_multiple_dashboard_versions(w);
1219	web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_IS_V3);
1220	return web_client_process_url(host, w, decoded_url_path);
1221	}
1222	else if(unlikely(hash == hash_v2 && strcmp(tok, "v2") == 0)) {
1223	if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_WITH_VERSION))
1224	return bad_request_multiple_dashboard_versions(w);
1225	web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_IS_V2);
1226	return web_client_process_url(host, w, decoded_url_path);
1227	}
1228	else if(unlikely(hash == hash_v1 && strcmp(tok, "v1") == 0)) {
1229	if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_WITH_VERSION))
1230	return bad_request_multiple_dashboard_versions(w);
1231	web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_IS_V1);
1232	return web_client_process_url(host, w, decoded_url_path);
1233	}
1234	else if(unlikely(hash == hash_v0 && strcmp(tok, "v0") == 0)) {
1235	if(web_client_flag_check(w, WEB_CLIENT_FLAG_PATH_WITH_VERSION))
1236	return bad_request_multiple_dashboard_versions(w);
1237	web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_IS_V0);
1238	return web_client_process_url(host, w, decoded_url_path);
1239	}
1240	else if(unlikely(hash == hash_netdata_conf && strcmp(tok, "netdata.conf") == 0)) { // netdata.conf
1241	if(unlikely(!http_can_access_netdataconf(w)))
1242	return web_client_permission_denied_acl(w);
1243
1244	netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: generating netdata.conf ...", w->id);
1245	w->response.data->content_type = CT_TEXT_PLAIN;
1246	buffer_flush(w->response.data);
1247
1248	inicfg_generate(&netdata_config, w->response.data, 0, true);
1249	return HTTP_RESP_OK;
1250	}
1251	#ifdef NETDATA_INTERNAL_CHECKS
1252	else if(unlikely(hash == hash_exit && strcmp(tok, "exit") == 0)) {
1253	if(unlikely(!http_can_access_netdataconf(w)))
1254	return web_client_permission_denied_acl(w);
1255
1256	w->response.data->content_type = CT_TEXT_PLAIN;
1257	buffer_flush(w->response.data);
1258
1259	if(!exit_initiated_get())
1260	buffer_strcat(w->response.data, "ok, will do...");
1261	else
1262	buffer_strcat(w->response.data, "I am doing it already");
1263
1264	netdata_log_error("web request to exit received.");
1265	netdata_exit_gracefully(EXIT_REASON_API_QUIT, true);
1266	return HTTP_RESP_OK;
1267	}
1268	else if(unlikely(hash == hash_debug && strcmp(tok, "debug") == 0)) {
1269	if(unlikely(!http_can_access_netdataconf(w)))
1270	return web_client_permission_denied_acl(w);
1271
1272	buffer_flush(w->response.data);
1273
1274	// get the name of the data to show
1275	tok = strsep_skip_consecutive_separators(&decoded_url_path, "&");
1276	if(tok && *tok) {
1277	netdata_log_debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok);
1278
1279	// do we have such a data set?
1280	RRDSET *st = rrdset_find_byname(host, tok);
1281	if(!st) st = rrdset_find(host, tok, false);
1282	if(!st) {
1283	w->response.data->content_type = CT_TEXT_HTML;
1284	buffer_strcat(w->response.data, "Chart is not found: ");
1285	buffer_strcat_htmlescape(w->response.data, tok);
1286	netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: %s is not found.", w->id, tok);
1287	return HTTP_RESP_NOT_FOUND;
1288	}
1289
1290	if(rrdset_flag_check(st, RRDSET_FLAG_DEBUG))
1291	rrdset_flag_clear(st, RRDSET_FLAG_DEBUG);
1292	else
1293	rrdset_flag_set(st, RRDSET_FLAG_DEBUG);
1294
1295	w->response.data->content_type = CT_TEXT_HTML;
1296	buffer_sprintf(w->response.data, "Chart has now debug %s: ", rrdset_flag_check(st, RRDSET_FLAG_DEBUG)?"enabled":"disabled");
1297	buffer_strcat_htmlescape(w->response.data, tok);
1298	netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: debug for %s is %s.", w->id, tok, rrdset_flag_check(st, RRDSET_FLAG_DEBUG)?"enabled":"disabled");
1299	return HTTP_RESP_OK;
1300	}
1301
1302	buffer_flush(w->response.data);
1303	buffer_strcat(w->response.data, "debug which chart?\r\n");
1304	return HTTP_RESP_BAD_REQUEST;
1305	}
1306	else if(unlikely(hash == hash_mirror && strcmp(tok, "mirror") == 0)) {
1307	if(unlikely(!http_can_access_netdataconf(w)))
1308	return web_client_permission_denied_acl(w);
1309
1310	netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Mirroring...", w->id);
1311
1312	// replace the zero bytes with spaces
1313	buffer_char_replace(w->response.data, '\0', ' ');
1314
1315	// just leave the buffer as-is
1316	// it will be copied back to the client
1317
1318	return HTTP_RESP_OK;
1319	}
1320	#endif /* NETDATA_INTERNAL_CHECKS */
1321	}
1322
1323	buffer_flush(w->response.data);
1324	return web_server_static_file(w, filename);
1325	}
1326
1327	static bool web_server_log_transport(BUFFER wb, void ptr) {
1328	struct web_client *w = ptr;
1329	if(!w)
1330	return false;
1331
1332	buffer_strcat(wb, SSL_connection(&w->ssl) ? "https" : "http");
1333	return true;
1334	}
1335
1336	void web_client_process_request_from_web_server(struct web_client *w) {
1337	// entry point for web server requests
1338
1339	ND_LOG_STACK lgs[] = {
1340	ND_LOG_FIELD_CB(NDF_SRC_TRANSPORT, web_server_log_transport, w),
1341	ND_LOG_FIELD_TXT(NDF_SRC_IP, w->user_auth.client_ip),
1342	ND_LOG_FIELD_TXT(NDF_SRC_PORT, w->client_port),
1343	ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_HOST, w->forwarded_host),
1344	ND_LOG_FIELD_TXT(NDF_SRC_FORWARDED_FOR, w->user_auth.forwarded_for),
1345	ND_LOG_FIELD_TXT(NDF_NIDL_NODE, w->client_host),
1346	ND_LOG_FIELD_TXT(NDF_REQUEST_METHOD, HTTP_REQUEST_MODE_2str(w->mode)),
1347	ND_LOG_FIELD_BFR(NDF_REQUEST, w->url_as_received),
1348	ND_LOG_FIELD_U64(NDF_CONNECTION_ID, w->id),
1349	ND_LOG_FIELD_UUID(NDF_TRANSACTION_ID, &w->transaction),
1350	ND_LOG_FIELD_UUID(NDF_ACCOUNT_ID, &w->user_auth.cloud_account_id.uuid),
1351	ND_LOG_FIELD_TXT(NDF_USER_NAME, w->user_auth.client_name),
1352	ND_LOG_FIELD_TXT(NDF_USER_ROLE, http_id2user_role(w->user_auth.user_role)),
1353	ND_LOG_FIELD_CB(NDF_USER_ACCESS, log_cb_http_access_to_hex, &w->user_auth.access),
1354	ND_LOG_FIELD_END(),
1355	};
1356	ND_LOG_STACK_PUSH(lgs);
1357
1358	// give a new transaction id to the request
1359	if(uuid_is_null(w->transaction))
1360	uuid_generate_random(w->transaction);
1361
1362	// start timing us
1363	web_client_timeout_checkpoint_init(w);
1364
1365	switch(http_request_validate(w)) {
1366	case HTTP_VALIDATION_OK:
1367	if(!web_client_flag_check(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING)) {
1368	web_client_flag_set(w, WEB_CLIENT_FLAG_PROGRESS_TRACKING);
1369	query_progress_start_or_update(&w->transaction, 0, w->mode, w->acl,
1370	buffer_tostring(w->url_as_received),
1371	w->payload,
1372	w->user_auth.forwarded_for[0] ? w->user_auth.forwarded_for : w->user_auth.client_ip);
1373	}
1374
1375	// Check if this is a WebSocket upgrade request
1376	// The full WebSocket handshake detection will happen in the header parsing,
1377	// but we need to set the initial mode to GET for processing to continue
1378	if (w->mode == HTTP_REQUEST_MODE_GET && web_client_has_websocket_handshake(w) && web_client_is_websocket(w)) {
1379	w->mode = HTTP_REQUEST_MODE_WEBSOCKET;
1380	netdata_log_debug(D_WEB_CLIENT, "%llu: Detected WebSocket handshake request", w->id);
1381	}
1382
1383	switch(w->mode) {
1384	case HTTP_REQUEST_MODE_STREAM:
1385	if(unlikely(!http_can_access_stream(w))) {
1386	web_client_permission_denied_acl(w);
1387	return;
1388	}
1389
1390	w->response.code = stream_receiver_accept_connection(
1391	w, (char *)buffer_tostring(w->url_query_string_decoded));
1392	return;
1393
1394	case HTTP_REQUEST_MODE_WEBSOCKET:
1395	if(unlikely(!http_can_access_dashboard(w))) {
1396	web_client_permission_denied_acl(w);
1397	return;
1398	}
1399
1400	// Handle WebSocket handshake - this will take over the socket
1401	// similar to how stream_receiver_accept_connection works
1402	w->response.code = websocket_handle_handshake(w);
1403
1404	// After this point the socket has been taken over
1405	// No need to send a response as the WebSocket handler
1406	// has already sent the handshake response
1407	return;
1408
1409	case HTTP_REQUEST_MODE_OPTIONS:
1410	if(unlikely(
1411	!http_can_access_dashboard(w) &&
1412	!http_can_access_registry(w) &&
1413	!http_can_access_badges(w) &&
1414	!http_can_access_mgmt(w) &&
1415	!http_can_access_netdataconf(w)
1416	)) {
1417	web_client_permission_denied_acl(w);
1418	break;
1419	}
1420
1421	w->response.data->content_type = CT_TEXT_PLAIN;
1422	buffer_flush(w->response.data);
1423	buffer_strcat(w->response.data, "OK");
1424	w->response.code = HTTP_RESP_OK;
1425	break;
1426
1427	case HTTP_REQUEST_MODE_POST:
1428	case HTTP_REQUEST_MODE_GET:
1429	case HTTP_REQUEST_MODE_PUT:
1430	case HTTP_REQUEST_MODE_DELETE:
1431	if(unlikely(
1432	!http_can_access_dashboard(w) &&
1433	!http_can_access_registry(w) &&
1434	!http_can_access_badges(w) &&
1435	!http_can_access_mgmt(w) &&
1436	!http_can_access_netdataconf(w)
1437	)) {
1438	web_client_permission_denied_acl(w);
1439	break;
1440	}
1441
1442	web_client_reset_path_flags(w);
1443
1444	// find if the URL path has a filename extension
1445	char path[FILENAME_MAX + 1];
1446	strncpyz(path, buffer_tostring(w->url_path_decoded), FILENAME_MAX);
1447	char s = path, e = path;
1448
1449	// remove the query string and find the last char
1450	for (; *e ; e++) {
1451	if (*e == '?')
1452	break;
1453	}
1454
1455	if(e == s \|\| (*(e - 1) == '/'))
1456	web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_HAS_TRAILING_SLASH);
1457
1458	// check if there is a filename extension
1459	while (--e > s) {
1460	if (*e == '/')
1461	break;
1462	if(*e == '.') {
1463	web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_HAS_FILE_EXTENSION);
1464	break;
1465	}
1466	}
1467
1468	w->response.code = (short)web_client_process_url(localhost, w, path);
1469	break;
1470
1471	default:
1472	web_client_permission_denied_acl(w);
1473	return;
1474	}
1475	break;
1476
1477	case HTTP_VALIDATION_INCOMPLETE:
1478	if(w->response.data->len > NETDATA_WEB_REQUEST_MAX_SIZE) {
1479	buffer_flush(w->url_as_received);
1480	buffer_strcat(w->url_as_received, "too big request");
1481
1482	netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Received request is too big (%zu bytes).", w->id, (size_t)w->response.data->len);
1483
1484	size_t len = w->response.data->len;
1485	buffer_flush(w->response.data);
1486	buffer_sprintf(w->response.data, "Received request is too big (received %zu bytes, max is %zu bytes).\r\n", len, (size_t)NETDATA_WEB_REQUEST_MAX_SIZE);
1487	w->response.code = HTTP_RESP_BAD_REQUEST;
1488	}
1489	else {
1490	// wait for more data
1491	// set to normal to prevent web_server_rcv_callback
1492	// from going into stream mode
1493	if (w->mode == HTTP_REQUEST_MODE_STREAM \|\| w->mode == HTTP_REQUEST_MODE_WEBSOCKET)
1494	w->mode = HTTP_REQUEST_MODE_GET;
1495	return;
1496	}
1497	break;
1498
1499	case HTTP_VALIDATION_REDIRECT:
1500	{
1501	buffer_flush(w->response.data);
1502	w->response.data->content_type = CT_TEXT_HTML;
1503	buffer_strcat(w->response.data,
1504	"<!DOCTYPE html><!-- SPDX-License-Identifier: GPL-3.0-or-later --><html>"
1505	"<body onload=\"window.location.href ='https://'+ window.location.hostname +"
1506	" ':' + window.location.port + window.location.pathname + window.location.search\">"
1507	"Redirecting to safety connection, case your browser does not support redirection, please"
1508	" click <a onclick=\"window.location.href ='https://'+ window.location.hostname + ':' "
1509	" + window.location.port + window.location.pathname + window.location.search\">here</a>."
1510	"</body></html>");
1511	w->response.code = HTTP_RESP_HTTPS_UPGRADE;
1512	break;
1513	}
1514
1515	case HTTP_VALIDATION_MALFORMED_URL:
1516	netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Malformed URL '%s'.", w->id, w->response.data->buffer);
1517
1518	buffer_flush(w->response.data);
1519	buffer_strcat(w->response.data, "Malformed URL...\r\n");
1520	w->response.code = HTTP_RESP_BAD_REQUEST;
1521	break;
1522	case HTTP_VALIDATION_TOO_MANY_READ_RETRIES:
1523	netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: Too many retries to read request '%s'.", w->id, w->response.data->buffer);
1524
1525	buffer_flush(w->response.data);
1526	buffer_strcat(w->response.data, "Too many retries to read request.\r\n");
1527	w->response.code = HTTP_RESP_BAD_REQUEST;
1528	break;
1529	case HTTP_VALIDATION_NOT_SUPPORTED:
1530	netdata_log_debug(D_WEB_CLIENT_ACCESS, "%llu: HTTP method requested is not supported '%s'.", w->id, w->response.data->buffer);
1531
1532	buffer_flush(w->response.data);
1533	buffer_strcat(w->response.data, "HTTP method requested is not supported...\r\n");
1534	w->response.code = HTTP_RESP_BAD_REQUEST;
1535	break;
1536	}
1537
1538	// keep track of the processing time
1539	web_client_timeout_checkpoint_response_ready(w, NULL);
1540
1541	w->response.sent = 0;
1542
1543	web_client_send_http_header(w);
1544
1545	// enable sending immediately if we have data
1546	if(w->response.data->len) web_client_enable_wait_send(w);
1547	else web_client_disable_wait_send(w);
1548
1549	switch(w->mode) {
1550	case HTTP_REQUEST_MODE_STREAM:
1551	netdata_log_debug(D_WEB_CLIENT, "%llu: STREAM done.", w->id);
1552	break;
1553
1554	case HTTP_REQUEST_MODE_WEBSOCKET:
1555	netdata_log_debug(D_WEB_CLIENT, "%llu: Done preparing the WEBSOCKET response..", w->id);
1556	break;
1557
1558	case HTTP_REQUEST_MODE_OPTIONS:
1559	netdata_log_debug(D_WEB_CLIENT,
1560	"%llu: Done preparing the OPTIONS response. Sending data (%zu bytes) to client.",
1561	w->id, (size_t)w->response.data->len);
1562	break;
1563
1564	case HTTP_REQUEST_MODE_POST:
1565	case HTTP_REQUEST_MODE_GET:
1566	case HTTP_REQUEST_MODE_PUT:
1567	case HTTP_REQUEST_MODE_DELETE:
1568	netdata_log_debug(D_WEB_CLIENT,
1569	"%llu: Done preparing the response. Sending data (%zu bytes) to client.",
1570	w->id, (size_t)w->response.data->len);
1571	break;
1572
1573	default:
1574	fatal("%llu: Unknown client mode %u.", w->id, w->mode);
1575	break;
1576	}
1577	}
1578
1579	ssize_t web_client_send_chunk_header(struct web_client *w, size_t len)
1580	{
1581	netdata_log_debug(D_DEFLATE, "%llu: OPEN CHUNK of %zu bytes (hex: %zx).", w->id, len, len);
1582	char buf[24];
1583	ssize_t bytes;
1584	bytes = (ssize_t)sprintf(buf, "%zX\r\n", len);
1585	buf[bytes] = 0x00;
1586
1587	bytes = web_client_send_data(w,buf,strlen(buf),0);
1588	if(bytes > 0) {
1589	netdata_log_debug(D_DEFLATE, "%llu: Sent chunk header %zd bytes.", w->id, bytes);
1590	w->statistics.sent_bytes += bytes;
1591	}
1592
1593	else if(bytes == 0) {
1594	netdata_log_debug(D_WEB_CLIENT, "%llu: Did not send chunk header to the client.", w->id);
1595	}
1596	else {
1597	netdata_log_debug(D_WEB_CLIENT, "%llu: Failed to send chunk header to client.", w->id);
1598	WEB_CLIENT_IS_DEAD(w);
1599	}
1600
1601	return bytes;
1602	}
1603
1604	ssize_t web_client_send_chunk_close(struct web_client *w)
1605	{
1606	//debug(D_DEFLATE, "%llu: CLOSE CHUNK.", w->id);
1607
1608	ssize_t bytes;
1609	bytes = web_client_send_data(w,"\r\n",2,0);
1610	if(bytes > 0) {
1611	netdata_log_debug(D_DEFLATE, "%llu: Sent chunk suffix %zd bytes.", w->id, bytes);
1612	w->statistics.sent_bytes += bytes;
1613	}
1614
1615	else if(bytes == 0) {
1616	netdata_log_debug(D_WEB_CLIENT, "%llu: Did not send chunk suffix to the client.", w->id);
1617	}
1618	else {
1619	netdata_log_debug(D_WEB_CLIENT, "%llu: Failed to send chunk suffix to client.", w->id);
1620	WEB_CLIENT_IS_DEAD(w);
1621	}
1622
1623	return bytes;
1624	}
1625
1626	ssize_t web_client_send_chunk_finalize(struct web_client *w)
1627	{
1628	//debug(D_DEFLATE, "%llu: FINALIZE CHUNK.", w->id);
1629
1630	ssize_t bytes;
1631	bytes = web_client_send_data(w,"\r\n0\r\n\r\n",7,0);
1632	if(bytes > 0) {
1633	netdata_log_debug(D_DEFLATE, "%llu: Sent chunk suffix %zd bytes.", w->id, bytes);
1634	w->statistics.sent_bytes += bytes;
1635	}
1636
1637	else if(bytes == 0) {
1638	netdata_log_debug(D_WEB_CLIENT, "%llu: Did not send chunk finalize suffix to the client.", w->id);
1639	}
1640	else {
1641	netdata_log_debug(D_WEB_CLIENT, "%llu: Failed to send chunk finalize suffix to client.", w->id);
1642	WEB_CLIENT_IS_DEAD(w);
1643	}
1644
1645	return bytes;
1646	}
1647
1648	ssize_t web_client_send_deflate(struct web_client *w)
1649	{
1650	ssize_t len = 0, t = 0;
1651
1652	// when using compression,
1653	// w->response.sent is the amount of bytes passed through compression
1654
1655	netdata_log_debug(D_DEFLATE,
1656	"%llu: web_client_send_deflate(): w->response.data->len = %zu, w->response.sent = %zu, w->response.zhave = %zu, w->response.zsent = %zu, w->response.zstream.avail_in = %u, w->response.zstream.avail_out = %u, w->response.zstream.total_in = %lu, w->response.zstream.total_out = %lu.",
1657	w->id, (size_t)w->response.data->len, w->response.sent, w->response.zhave, w->response.zsent, w->response.zstream.avail_in, w->response.zstream.avail_out, w->response.zstream.total_in, w->response.zstream.total_out);
1658
1659	if(w->response.data->len - w->response.sent == 0 && w->response.zstream.avail_in == 0 && w->response.zhave == w->response.zsent && w->response.zstream.avail_out != 0) {
1660	// there is nothing to send
1661
1662	netdata_log_debug(D_WEB_CLIENT, "%llu: Out of output data.", w->id);
1663
1664	// finalize the chunk
1665	if(w->response.sent != 0) {
1666	t = web_client_send_chunk_finalize(w);
1667	if(t < 0) return t;
1668	}
1669
1670	if(unlikely(!web_client_has_keepalive(w))) {
1671	netdata_log_debug(D_WEB_CLIENT, "%llu: Closing (keep-alive is not enabled). %zu bytes sent.", w->id, w->response.sent);
1672	WEB_CLIENT_IS_DEAD(w);
1673	return t;
1674	}
1675
1676	// reset the client
1677	web_client_request_done(w);
1678	netdata_log_debug(D_WEB_CLIENT, "%llu: Done sending all data on socket.", w->id);
1679	return t;
1680	}
1681
1682	if(w->response.zhave == w->response.zsent) {
1683	// compress more input data
1684
1685	// close the previous open chunk
1686	if(w->response.sent != 0) {
1687	t = web_client_send_chunk_close(w);
1688	if(t < 0) return t;
1689	}
1690
1691	netdata_log_debug(D_DEFLATE, "%llu: Compressing %zu new bytes starting from %zu (and %u left behind).", w->id, (w->response.data->len - w->response.sent), w->response.sent, w->response.zstream.avail_in);
1692
1693	// give the compressor all the data not passed through the compressor yet
1694	if(w->response.data->len > w->response.sent) {
1695	w->response.zstream.next_in = (Bytef *)&w->response.data->buffer[w->response.sent - w->response.zstream.avail_in];
1696	w->response.zstream.avail_in += (uInt) (w->response.data->len - w->response.sent);
1697	}
1698
1699	// reset the compressor output buffer
1700	w->response.zstream.next_out = w->response.zbuffer;
1701	w->response.zstream.avail_out = NETDATA_WEB_RESPONSE_ZLIB_CHUNK_SIZE;
1702
1703	// ask for FINISH if we have all the input
1704	int flush = Z_SYNC_FLUSH;
1705	if((w->mode == HTTP_REQUEST_MODE_GET \|\|
1706	w->mode == HTTP_REQUEST_MODE_POST \|\|
1707	w->mode == HTTP_REQUEST_MODE_PUT \|\|
1708	w->mode == HTTP_REQUEST_MODE_DELETE)) {
1709	flush = Z_FINISH;
1710	netdata_log_debug(D_DEFLATE, "%llu: Requesting Z_FINISH, if possible.", w->id);
1711	}
1712	else {
1713	netdata_log_debug(D_DEFLATE, "%llu: Requesting Z_SYNC_FLUSH.", w->id);
1714	}
1715
1716	// compress
1717	if(deflate(&w->response.zstream, flush) == Z_STREAM_ERROR) {
1718	netdata_log_error("%llu: Compression failed. Closing down client.", w->id);
1719	web_client_request_done(w);
1720	return(-1);
1721	}
1722
1723	w->response.zhave = NETDATA_WEB_RESPONSE_ZLIB_CHUNK_SIZE - w->response.zstream.avail_out;
1724	w->response.zsent = 0;
1725
1726	// keep track of the bytes passed through the compressor
1727	w->response.sent = w->response.data->len;
1728
1729	netdata_log_debug(D_DEFLATE, "%llu: Compression produced %zu bytes.", w->id, w->response.zhave);
1730
1731	// open a new chunk
1732	ssize_t t2 = web_client_send_chunk_header(w, w->response.zhave);
1733	if(t2 < 0) return t2;
1734	t += t2;
1735	}
1736
1737	netdata_log_debug(D_WEB_CLIENT, "%llu: Sending %zu bytes of data (+%zd of chunk header).", w->id, w->response.zhave - w->response.zsent, t);
1738
1739	len = web_client_send_data(w,&w->response.zbuffer[w->response.zsent], (size_t) (w->response.zhave - w->response.zsent), MSG_DONTWAIT);
1740	if(len > 0) {
1741	w->statistics.sent_bytes += len;
1742	w->response.zsent += len;
1743	len += t;
1744	netdata_log_debug(D_WEB_CLIENT, "%llu: Sent %zd bytes.", w->id, len);
1745	}
1746	else if(len == 0) {
1747	netdata_log_debug(D_WEB_CLIENT, "%llu: Did not send any bytes to the client (zhave = %zu, zsent = %zu, need to send = %zu).",
1748	w->id, w->response.zhave, w->response.zsent, w->response.zhave - w->response.zsent);
1749
1750	}
1751	else {
1752	netdata_log_debug(D_WEB_CLIENT, "%llu: Failed to send data to client.", w->id);
1753	WEB_CLIENT_IS_DEAD(w);
1754	}
1755
1756	return(len);
1757	}
1758
1759	ssize_t web_client_send(struct web_client *w) {
1760	if(likely(w->response.zoutput)) return web_client_send_deflate(w);
1761
1762	ssize_t bytes;
1763
1764	if(unlikely(w->response.data->len - w->response.sent == 0)) {
1765	// there is nothing to send
1766
1767	netdata_log_debug(D_WEB_CLIENT, "%llu: Out of output data.", w->id);
1768
1769	// there can be two cases for this
1770	// A. we have done everything
1771	// B. we temporarily have nothing to send, waiting for the buffer to be filled by ifd
1772
1773	if(unlikely(!web_client_has_keepalive(w))) {
1774	netdata_log_debug(D_WEB_CLIENT, "%llu: Closing (keep-alive is not enabled). %zu bytes sent.", w->id, w->response.sent);
1775	WEB_CLIENT_IS_DEAD(w);
1776	return 0;
1777	}
1778
1779	web_client_request_done(w);
1780	netdata_log_debug(D_WEB_CLIENT, "%llu: Done sending all data on socket. Waiting for next request on the same socket.", w->id);
1781	return 0;
1782	}
1783
1784	bytes = web_client_send_data(w,&w->response.data->buffer[w->response.sent], w->response.data->len - w->response.sent, MSG_DONTWAIT);
1785	if(likely(bytes > 0)) {
1786	w->statistics.sent_bytes += bytes;
1787	w->response.sent += bytes;
1788	netdata_log_debug(D_WEB_CLIENT, "%llu: Sent %zd bytes.", w->id, bytes);
1789	}
1790	else if(likely(bytes == 0)) {
1791	netdata_log_debug(D_WEB_CLIENT, "%llu: Did not send any bytes to the client.", w->id);
1792	}
1793	else {
1794	netdata_log_debug(D_WEB_CLIENT, "%llu: Failed to send data to client.", w->id);
1795	WEB_CLIENT_IS_DEAD(w);
1796	}
1797
1798	return(bytes);
1799	}
1800
1801	ssize_t web_client_receive(struct web_client *w) {
1802	ssize_t bytes;
1803
1804	// do we have any space for more data?
1805	buffer_need_bytes(w->response.data, NETDATA_WEB_REQUEST_INITIAL_SIZE);
1806
1807	ssize_t left = (ssize_t)(w->response.data->size - w->response.data->len);
1808
1809	errno_clear();
1810
1811	if ( (web_client_check_conn_tcp(w)) && (netdata_ssl_web_server_ctx) ) {
1812	if (SSL_connection(&w->ssl)) {
1813	bytes = netdata_ssl_read(&w->ssl, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1));
1814	web_client_enable_wait_from_ssl(w);
1815	}
1816	else {
1817	bytes = recv(w->fd, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1), MSG_DONTWAIT);
1818	}
1819	}
1820	else if(web_client_check_conn_tcp(w) \|\| web_client_check_conn_unix(w)) {
1821	bytes = recv(w->fd, &w->response.data->buffer[w->response.data->len], (size_t) (left - 1), MSG_DONTWAIT);
1822	}
1823	else // other connection methods
1824	bytes = -1;
1825
1826	if(likely(bytes > 0)) {
1827	w->statistics.received_bytes += bytes;
1828
1829	size_t old = w->response.data->len;
1830	(void)old;
1831
1832	w->response.data->len += bytes;
1833	w->response.data->buffer[w->response.data->len] = '\0';
1834
1835	netdata_log_debug(D_WEB_CLIENT, "%llu: Received %zd bytes.", w->id, bytes);
1836	netdata_log_debug(D_WEB_DATA, "%llu: Received data: '%s'.", w->id, &w->response.data->buffer[old]);
1837	}
1838	else if(unlikely(bytes < 0 && (errno == EAGAIN \|\| errno == EWOULDBLOCK \|\| errno == EINTR))) {
1839	web_client_enable_wait_receive(w);
1840	return 0;
1841	}
1842	else if (bytes < 0) {
1843	netdata_log_debug(D_WEB_CLIENT, "%llu: receive data failed.", w->id);
1844	WEB_CLIENT_IS_DEAD(w);
1845	} else
1846	netdata_log_debug(D_WEB_CLIENT, "%llu: Received %zd bytes.", w->id, bytes);
1847
1848	return(bytes);
1849	}
1850
1851	void web_client_decode_path_and_query_string(struct web_client w, const char path_and_query_string) {
1852	char buffer[NETDATA_WEB_REQUEST_URL_SIZE + 2];
1853	buffer[0] = '\0';
1854
1855	buffer_flush(w->url_path_decoded);
1856	buffer_flush(w->url_query_string_decoded);
1857
1858	if(buffer_strlen(w->url_as_received) == 0)
1859	// do not overwrite this if it is already filled
1860	buffer_strcat(w->url_as_received, path_and_query_string);
1861
1862	// PATH_IS_MCP is a function of the URL alone; clear and re-derive on
1863	// every decode so keepalived connections reusing the same web_client
1864	// for a different URL see a fresh value.
1865	web_client_flag_clear(w, WEB_CLIENT_FLAG_PATH_IS_MCP);
1866
1867	if(w->mode == HTTP_REQUEST_MODE_STREAM) {
1868	// in stream mode, there is no path
1869
1870	url_decode_r(buffer, path_and_query_string, NETDATA_WEB_REQUEST_URL_SIZE + 1);
1871
1872	buffer[NETDATA_WEB_REQUEST_URL_SIZE + 1] = '\0';
1873	buffer_strcat(w->url_query_string_decoded, buffer);
1874	}
1875	else {
1876	// in non-stream mode, there is a path
1877	// FIXME - the way this is implemented, query string params never accept the symbol &, not even encoded as %26
1878	// To support the symbol & in query string params, we need to turn the url_query_string_decoded into a
1879	// dictionary and decode each of the parameters individually.
1880	// OR: in url_query_string_decoded use as separator a control character that cannot appear in the URL.
1881
1882	url_decode_r(buffer, path_and_query_string, NETDATA_WEB_REQUEST_URL_SIZE + 1);
1883
1884	char *question_mark_start = strchr(buffer, '?');
1885	if (question_mark_start) {
1886	buffer_strcat(w->url_query_string_decoded, question_mark_start);
1887	char c = *question_mark_start;
1888	*question_mark_start = '\0';
1889	buffer_strcat(w->url_path_decoded, buffer);
1890	*question_mark_start = c;
1891	} else {
1892	buffer_strcat(w->url_query_string_decoded, "");
1893	buffer_strcat(w->url_path_decoded, buffer);
1894	}
1895
1896	// Classify path: set PATH_IS_MCP when the URL addresses one of
1897	// Netdata's MCP transport endpoints (/mcp or /sse, and their
1898	// subpaths). Done here — at URL-decoding time — so the flag is
1899	// available later both to the URL dispatcher and to the response
1900	// header builder, including for OPTIONS preflights which bypass
1901	// the dispatcher. Matching requires a path-segment boundary so a
1902	// hypothetical /mcpfoo does not leak through.
1903	const char *decoded_path = buffer_tostring(w->url_path_decoded);
1904	size_t decoded_path_len = buffer_strlen(w->url_path_decoded);
1905	if(decoded_path_len >= 4
1906	&& (memcmp(decoded_path, "/mcp", 4) == 0 \|\| memcmp(decoded_path, "/sse", 4) == 0)
1907	&& (decoded_path_len == 4 \|\| decoded_path[4] == '/'))
1908	web_client_flag_set(w, WEB_CLIENT_FLAG_PATH_IS_MCP);
1909	}
1910	}
1911
1912	void web_client_reuse_from_cache(struct web_client *w) {
1913	// zero everything about it - but keep the buffers
1914
1915	web_client_reset_allocations(w, false);
1916
1917	// remember the pointers to the buffers
1918	BUFFER *b1 = w->response.data;
1919	BUFFER *b2 = w->response.header;
1920	BUFFER *b3 = w->response.header_output;
1921	BUFFER *b4 = w->url_path_decoded;
1922	BUFFER *b5 = w->url_as_received;
1923	BUFFER *b6 = w->url_query_string_decoded;
1924	BUFFER *b7 = w->payload;
1925
1926	NETDATA_SSL ssl = w->ssl;
1927
1928	size_t use_count = w->use_count;
1929	size_t *statistics_memory_accounting = w->statistics.memory_accounting;
1930
1931	// zero everything
1932	memset(w, 0, sizeof(struct web_client));
1933
1934	w->fd = -1;
1935	w->statistics.memory_accounting = statistics_memory_accounting;
1936	w->use_count = use_count;
1937
1938	w->ssl = ssl;
1939
1940	// restore the pointers of the buffers
1941	w->response.data = b1;
1942	w->response.header = b2;
1943	w->response.header_output = b3;
1944	w->url_path_decoded = b4;
1945	w->url_as_received = b5;
1946	w->url_query_string_decoded = b6;
1947	w->payload = b7;
1948	}
1949
1950	struct web_client web_client_create(size_t statistics_memory_accounting) {
1951	struct web_client w = (struct web_client )callocz(1, sizeof(struct web_client));
1952
1953	w->ssl = NETDATA_SSL_UNSET_CONNECTION;
1954
1955	w->use_count = 1;
1956	w->statistics.memory_accounting = statistics_memory_accounting;
1957
1958	w->url_as_received = buffer_create(NETDATA_WEB_DECODED_URL_INITIAL_SIZE, w->statistics.memory_accounting);
1959	w->url_path_decoded = buffer_create(NETDATA_WEB_DECODED_URL_INITIAL_SIZE, w->statistics.memory_accounting);
1960	w->url_query_string_decoded = buffer_create(NETDATA_WEB_DECODED_URL_INITIAL_SIZE, w->statistics.memory_accounting);
1961	w->response.data = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, w->statistics.memory_accounting);
1962	w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_INITIAL_SIZE, w->statistics.memory_accounting);
1963	w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_INITIAL_SIZE, w->statistics.memory_accounting);
1964
1965	__atomic_add_fetch(w->statistics.memory_accounting, sizeof(struct web_client), __ATOMIC_RELAXED);
1966
1967	return w;
1968	}
1969
1970	void web_client_free(struct web_client *w) {
1971	netdata_ssl_close(&w->ssl);
1972
1973	web_client_reset_allocations(w, true);
1974
1975	__atomic_sub_fetch(w->statistics.memory_accounting, sizeof(struct web_client), __ATOMIC_RELAXED);
1976	freez(w);
1977	}
1978
1979	inline void web_client_timeout_checkpoint_init(struct web_client *w) {
1980	now_monotonic_high_precision_timeval(&w->timings.tv_in);
1981	}
1982
1983	inline void web_client_timeout_checkpoint_set(struct web_client *w, int timeout_ms) {
1984	w->timings.timeout_ut = timeout_ms * USEC_PER_MS;
1985
1986	if(!w->timings.tv_in.tv_sec)
1987	web_client_timeout_checkpoint_init(w);
1988
1989	if(!w->timings.tv_timeout_last_checkpoint.tv_sec)
1990	w->timings.tv_timeout_last_checkpoint = w->timings.tv_in;
1991	}
1992
1993	inline usec_t web_client_timeout_checkpoint(struct web_client *w) {
1994	struct timeval now;
1995	now_monotonic_high_precision_timeval(&now);
1996
1997	if (!w->timings.tv_timeout_last_checkpoint.tv_sec)
1998	w->timings.tv_timeout_last_checkpoint = w->timings.tv_in;
1999
2000	usec_t since_last_check_ut = dt_usec(&w->timings.tv_timeout_last_checkpoint, &now);
2001
2002	w->timings.tv_timeout_last_checkpoint = now;
2003
2004	return since_last_check_ut;
2005	}
2006
2007	inline usec_t web_client_timeout_checkpoint_response_ready(struct web_client w, usec_t usec_since_last_checkpoint) {
2008	usec_t since_last_check_ut = web_client_timeout_checkpoint(w);
2009	if(usec_since_last_checkpoint)
2010	*usec_since_last_checkpoint = since_last_check_ut;
2011
2012	w->timings.tv_ready = w->timings.tv_timeout_last_checkpoint;
2013
2014	// return the total time of the query
2015	return dt_usec(&w->timings.tv_in, &w->timings.tv_ready);
2016	}
2017
2018	inline bool web_client_timeout_checkpoint_and_check(struct web_client w, usec_t usec_since_last_checkpoint) {
2019
2020	usec_t since_last_check_ut = web_client_timeout_checkpoint(w);
2021	if(usec_since_last_checkpoint)
2022	*usec_since_last_checkpoint = since_last_check_ut;
2023
2024	if(!w->timings.timeout_ut)
2025	return false;
2026
2027	usec_t since_reception_ut = dt_usec(&w->timings.tv_in, &w->timings.tv_timeout_last_checkpoint);
2028	if (since_reception_ut >= w->timings.timeout_ut) {
2029	buffer_flush(w->response.data);
2030	buffer_strcat(w->response.data, "Query timeout exceeded");
2031	w->response.code = HTTP_RESP_GATEWAY_TIMEOUT;
2032	return true;
2033	}
2034
2035	return false;
2036	}