@cryptotaxi247 / netdata-1 / commits / c3d70ffcb

WEBRTC for communication between agents and browsers (#14874)

* initial webrtc setup * missing files * rewrite of webrtc integration * initialization and cleanup of webrtc connections * make it compile without libdatachannel * add missing webrtc_initialize() function when webrtc is not enabled * make c++17 optional * add build/m4/ax_compiler_vendor.m4 * add ax_cxx_compile_stdcxx.m4 * added new m4 files to makefile.am * id all webrtc connections * show warning when webrtc is disabled * fixed message * moved all webrtc error checking inside webrtc.cpp * working webrtc connection establishment and cleanup * remove obsolete code * rewrote webrtc code in C to remove dependency for c++17 * fixed left-over reference * detect binary and text messages * minor fix * naming of webrtc threads * added webrtc configuration * fix for thread_get_name_np() * smaller web_client memory footprint * universal web clients cache * free web clients every 100 uses * webrtc is now enabled by default only when compiled with internal checks * webrtc responses to /api/ requests, including LZ4 compression * fix for binary and text messages * web_client_cache is now global * unification of the internal web server API, for web requests, aclk request, webrtc requests * more cleanup and unification of web client timings * fixed compiler warnings * update sent and received bytes * eliminated of almost all big buffers in web client * registry now uses the new json generation * cookies are now an array; fixed redirects * fix redirects, again * write cookies directly to the header buffer, eliminating the need for cookie structures in web client * reset the has_cookies flag * gathered all web client cleanup to one function * fixes redirects * added summary.globals in /api/v2/data response * ars to arc in /api/v2/data * properly handle host impersonation * set the context of mem.numa_nodes

Costa Tsaousis committed Apr 20, 2023 at 20:49 UTC c3d70ffcb43b62c95d71334ed49ad345ddf4360d
55 files changed +3117 -1267
CMakeLists.txt
+2
@@ -863,6 +863,8 @@ set(API_PLUGIN_FILES
863 web/api/formatters/rrdset2json.c
864 web/api/formatters/rrdset2json.h
865 web/api/health/health_cmdapi.c
866 + web/rtc/webrtc.c
867 + web/rtc/webrtc.h
868 )
869
870 set(STREAMING_PLUGIN_FILES
Makefile.am
+6
@@ -37,6 +37,8 @@ EXTRA_DIST = \
37 build/m4/ax_c_mallopt.m4 \
38 build/m4/tcmalloc.m4 \
39 build/m4/ax_c__generic.m4 \
40 + build/m4/ax_compiler_vendor.m4 \
41 + build/m4/ax_cxx_compile_stdcxx.m4 \
42 ml/dlib \
43 README.md \
44 LICENSE \
@@ -119,6 +121,7 @@ AM_CFLAGS = \
121 $(OPTIONAL_UUID_CFLAGS) \
122 $(OPTIONAL_MQTT_CFLAGS) \
123 $(OPTIONAL_LIBCAP_LIBS) \
124 + $(OPTIONAL_DATACHANNEL_CFLAGS) \
125 $(OPTIONAL_IPMIMONITORING_CFLAGS) \
126 $(OPTIONAL_CUPS_CFLAGS) \
127 $(OPTIONAL_XENSTAT_CFLAGS) \
@@ -659,6 +662,8 @@ STATSD_PLUGIN_FILES = \
662 $(NULL)
663
664 WEB_PLUGIN_FILES = \
665 + web/rtc/webrtc.c \
666 + web/rtc/webrtc.h \
667 web/server/web_client.c \
668 web/server/web_client.h \
669 web/server/web_server.c \
@@ -980,6 +985,7 @@ NETDATA_COMMON_LIBS = \
985 $(OPTIONAL_MQTT_LIBS) \
986 $(OPTIONAL_UV_LIBS) \
987 $(OPTIONAL_LZ4_LIBS) \
988 + $(OPTIONAL_DATACHANNEL_LIBS) \
989 libjudy.a \
990 $(OPTIONAL_SSL_LIBS) \
991 $(OPTIONAL_JSONC_LIBS) \
aclk/aclk_query.c
+46 -141
@@ -3,6 +3,7 @@
3 #include "aclk_query.h"
4 #include "aclk_stats.h"
5 #include "aclk_tx_msgs.h"
6 +#include "../../web/server/web_client_cache.h"
7
8 #define WEB_HDR_ACCEPT_ENC "Accept-Encoding:"
9 #define ACLK_MAX_WEB_RESPONSE_SIZE (30 * 1024 * 1024)
@@ -12,71 +13,11 @@ pthread_mutex_t query_lock_wait = PTHREAD_MUTEX_INITIALIZER;
13 #define QUERY_THREAD_LOCK pthread_mutex_lock(&query_lock_wait)
14 #define QUERY_THREAD_UNLOCK pthread_mutex_unlock(&query_lock_wait)
15
15 -static usec_t aclk_web_api_request(RRDHOST *host, struct web_client *w, char *url, const size_t api_version)
16 -{
17 - usec_t t;
18 -
19 - t = now_monotonic_high_precision_usec();
20 -
21 - if(api_version == 2)
22 - w->response.code = web_client_api_request_v2(host, w, url);
23 - else
24 - w->response.code = web_client_api_request_v1(host, w, url);
25 -
26 - if(buffer_strlen(w->response.data) > ACLK_MAX_WEB_RESPONSE_SIZE) {
27 - buffer_flush(w->response.data);
28 - buffer_strcat(w->response.data, "response is too big");
29 - w->response.data->content_type = CT_TEXT_PLAIN;
30 - w->response.code = HTTP_RESP_CONTENT_TOO_LONG;
31 - }
32 -
33 - t = now_monotonic_high_precision_usec() - t;
34 -
35 - if (aclk_stats_enabled) {
36 - ACLK_STATS_LOCK;
37 - aclk_metrics_per_sample.cloud_q_process_total += t;
38 - aclk_metrics_per_sample.cloud_q_process_count++;
39 - if (aclk_metrics_per_sample.cloud_q_process_max < t)
40 - aclk_metrics_per_sample.cloud_q_process_max = t;
41 - ACLK_STATS_UNLOCK;
42 - }
43 -
44 - return t;
45 -}
46 -
47 -static RRDHOST *node_id_2_rrdhost(const char *node_id)
48 -{
49 - int res;
50 - uuid_t node_id_bin, host_id_bin;
51 -
52 - RRDHOST *host = find_host_by_node_id((char *)node_id);
53 - if (host)
54 - return host;
55 -
56 - char host_id[UUID_STR_LEN];
57 - if (uuid_parse(node_id, node_id_bin)) {
58 - error("Couldn't parse UUID %s", node_id);
59 - return NULL;
60 - }
61 - if ((res = get_host_id(&node_id_bin, &host_id_bin))) {
62 - error("node not found rc=%d", res);
63 - return NULL;
64 - }
65 - uuid_unparse_lower(host_id_bin, host_id);
66 - return rrdhost_find_by_guid(host_id);
67 -}
68 -
69 -#define NODE_ID_QUERY "/node/"
70 -// TODO this function should be quarantied and written nicely
71 -// lots of skeletons from initial ACLK Legacy impl.
72 -// quick and dirty from the start
73 -static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
74 -{
16 +static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query) {
17 int retval = 0;
76 - usec_t t;
18 BUFFER *local_buffer = NULL;
78 - BUFFER *log_buffer = buffer_create(NETDATA_WEB_REQUEST_URL_SIZE, &netdata_buffers_statistics.buffers_aclk);
79 - RRDHOST *query_host = localhost;
19 + size_t size = 0;
20 + size_t sent = 0;
21
22 #ifdef NETDATA_WITH_ZLIB
23 int z_ret;
@@ -84,83 +25,52 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
25 char *start, *end;
26 #endif
27
87 - struct web_client *w = (struct web_client *)callocz(1, sizeof(struct web_client));
88 - w->response.data = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_aclk);
89 - w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_INITIAL_SIZE, &netdata_buffers_statistics.buffers_aclk);
90 - w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_INITIAL_SIZE, &netdata_buffers_statistics.buffers_aclk);
91 - strcpy(w->origin, "*"); // Simulate web_client_create_on_fd()
92 - w->cookie1[0] = 0; // Simulate web_client_create_on_fd()
93 - w->cookie2[0] = 0; // Simulate web_client_create_on_fd()
28 + struct web_client *w = web_client_get_from_cache();
29 w->acl = WEB_CLIENT_ACL_ACLK;
30 + w->mode = WEB_CLIENT_MODE_GET;
31 + w->timings.tv_in = query->created_tv;
32
96 - buffer_strcat(log_buffer, query->data.http_api_v2.query);
97 - size_t size = 0;
98 - size_t sent = 0;
99 - w->tv_in = query->created_tv;
100 - now_realtime_timeval(&w->tv_ready);
101 -
102 - if (query->timeout) {
103 - int in_queue = (int) (dt_usec(&w->tv_in, &w->tv_ready) / 1000);
104 - if (in_queue > query->timeout) {
105 - log_access("QUERY CANCELED: QUEUE TIME EXCEEDED %d ms (LIMIT %d ms)", in_queue, query->timeout);
106 - retval = 1;
107 - w->response.code = HTTP_RESP_BACKEND_FETCH_FAILED;
108 - aclk_http_msg_v2_err(query_thr->client, query->callback_topic, query->msg_id, w->response.code, CLOUD_EC_SND_TIMEOUT, CLOUD_EMSG_SND_TIMEOUT, NULL, 0);
109 - goto cleanup;
110 - }
33 + usec_t t;
34 + web_client_timeout_checkpoint_set(w, query->timeout);
35 + if(web_client_timeout_checkpoint_and_check(w, &t)) {
36 + log_access("QUERY CANCELED: QUEUE TIME EXCEEDED %llu ms (LIMIT %d ms)", t / USEC_PER_MS, query->timeout);
37 + retval = 1;
38 + w->response.code = HTTP_RESP_BACKEND_FETCH_FAILED;
39 + aclk_http_msg_v2_err(query_thr->client, query->callback_topic, query->msg_id, w->response.code, CLOUD_EC_SND_TIMEOUT, CLOUD_EMSG_SND_TIMEOUT, NULL, 0);
40 + goto cleanup;
41 }
42
113 - if (!strncmp(query->data.http_api_v2.query, NODE_ID_QUERY, strlen(NODE_ID_QUERY))) {
114 - char *node_uuid = query->data.http_api_v2.query + strlen(NODE_ID_QUERY);
115 - char nodeid[UUID_STR_LEN];
116 - if (strlen(node_uuid) < (UUID_STR_LEN - 1)) {
117 - error_report(CLOUD_EMSG_MALFORMED_NODE_ID);
118 - retval = 1;
119 - w->response.code = 404;
120 - aclk_http_msg_v2_err(query_thr->client, query->callback_topic, query->msg_id, w->response.code, CLOUD_EC_MALFORMED_NODE_ID, CLOUD_EMSG_MALFORMED_NODE_ID, NULL, 0);
121 - goto cleanup;
122 - }
123 - strncpyz(nodeid, node_uuid, UUID_STR_LEN - 1);
124 -
125 - query_host = node_id_2_rrdhost(nodeid);
126 - if (!query_host) {
127 - error_report("Host with node_id \"%s\" not found! Returning 404 to Cloud!", nodeid);
128 - retval = 1;
129 - w->response.code = 404;
130 - aclk_http_msg_v2_err(query_thr->client, query->callback_topic, query->msg_id, w->response.code, CLOUD_EC_NODE_NOT_FOUND, CLOUD_EMSG_NODE_NOT_FOUND, NULL, 0);
131 - goto cleanup;
132 - }
133 - }
43 + web_client_decode_path_and_query_string(w, query->data.http_api_v2.query);
44 + char *path = (char *)buffer_tostring(w->url_path_decoded);
45
135 - size_t api_version = 1;
136 - {
137 - char *s = strstr(query->data.http_api_v2.query, "/api/v");
138 - if(s && s[6]) {
139 - api_version = str2u(&s[6]);
140 - if(api_version != 1 && api_version != 2)
141 - api_version = 1;
142 - }
46 + if (aclk_stats_enabled) {
47 + char *url_path_endpoint = strrchr(path, '/');
48 + ACLK_STATS_LOCK;
49 + int stat_idx = aclk_cloud_req_http_type_to_idx(url_path_endpoint ? url_path_endpoint + 1 : "other");
50 + aclk_metrics_per_sample.cloud_req_http_by_type[stat_idx]++;
51 + ACLK_STATS_UNLOCK;
52 }
53
145 - char *mysep = strchr(query->data.http_api_v2.query, '?');
146 - if (mysep) {
147 - url_decode_r(w->decoded_query_string, mysep, NETDATA_WEB_REQUEST_URL_SIZE + 1);
148 - *mysep = '\0';
149 - } else
150 - url_decode_r(w->decoded_query_string, query->data.http_api_v2.query, NETDATA_WEB_REQUEST_URL_SIZE + 1);
54 + w->response.code = web_client_api_request_with_node_selection(localhost, w, path);
55 + web_client_timeout_checkpoint_response_ready(w, &t);
56
152 - mysep = strrchr(query->data.http_api_v2.query, '/');
57 + if(buffer_strlen(w->response.data) > ACLK_MAX_WEB_RESPONSE_SIZE) {
58 + buffer_flush(w->response.data);
59 + buffer_strcat(w->response.data, "response is too big");
60 + w->response.data->content_type = CT_TEXT_PLAIN;
61 + w->response.code = HTTP_RESP_CONTENT_TOO_LONG;
62 + }
63
64 if (aclk_stats_enabled) {
65 ACLK_STATS_LOCK;
156 - int stat_idx = aclk_cloud_req_http_type_to_idx(mysep ? mysep + 1 : "other");
157 - aclk_metrics_per_sample.cloud_req_http_by_type[stat_idx]++;
66 + aclk_metrics_per_sample.cloud_q_process_total += t;
67 + aclk_metrics_per_sample.cloud_q_process_count++;
68 + if (aclk_metrics_per_sample.cloud_q_process_max < t)
69 + aclk_metrics_per_sample.cloud_q_process_max = t;
70 ACLK_STATS_UNLOCK;
71 }
72
161 - // execute the query
162 - t = aclk_web_api_request(query_host, w, mysep ? mysep + 1 : "noop", api_version);
163 - size = (w->mode == WEB_CLIENT_MODE_FILECOPY) ? w->response.rlen : w->response.data->len;
73 + size = w->response.data->len;
74 sent = size;
75
76 #ifdef NETDATA_WITH_ZLIB
@@ -175,8 +85,8 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
85 w->response.zstream.zfree = Z_NULL;
86 w->response.zstream.opaque = Z_NULL;
87 if(deflateInit2(&w->response.zstream, web_gzip_level, Z_DEFLATED, 15 + 16, 8, web_gzip_strategy) == Z_OK) {
178 - w->response.zinitialized = 1;
179 - w->response.zoutput = 1;
88 + w->response.zinitialized = true;
89 + w->response.zoutput = true;
90 } else
91 error("Failed to initialize zlib. Proceeding without compression.");
92 }
@@ -212,7 +122,7 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
122 }
123 #endif
124
215 - w->response.data->date = w->tv_ready.tv_sec;
125 + w->response.data->date = w->timings.tv_ready.tv_sec;
126 web_client_build_http_header(w);
127 local_buffer = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_aclk);
128 local_buffer->content_type = CT_APPLICATION_JSON;
@@ -240,7 +150,7 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
150 struct timeval tv;
151
152 cleanup:
243 - now_realtime_timeval(&tv);
153 + now_monotonic_high_precision_timeval(&tv);
154 log_access("%llu: %d '[ACLK]:%d' '%s' (sent/all = %zu/%zu bytes %0.0f%%, prep/sent/total = %0.2f/%0.2f/%0.2f ms) %d '%s'",
155 w->id
156 , gettid()
@@ -249,24 +159,19 @@ cleanup:
159 , sent
160 , size
161 , size > sent ? -(((size - sent) / (double)size) * 100.0) : ((size > 0) ? (((sent - size ) / (double)size) * 100.0) : 0.0)
252 - , dt_usec(&w->tv_ready, &w->tv_in) / 1000.0
253 - , dt_usec(&tv, &w->tv_ready) / 1000.0
254 - , dt_usec(&tv, &w->tv_in) / 1000.0
162 + , dt_usec(&w->timings.tv_ready, &w->timings.tv_in) / 1000.0
163 + , dt_usec(&tv, &w->timings.tv_ready) / 1000.0
164 + , dt_usec(&tv, &w->timings.tv_in) / 1000.0
165 , w->response.code
256 - , strip_control_characters((char *)buffer_tostring(log_buffer))
166 + , strip_control_characters((char *)buffer_tostring(w->url_as_received))
167 );
168
169 + web_client_release_to_cache(w);
170 +
171 #ifdef NETDATA_WITH_ZLIB
260 - if(w->response.zinitialized)
261 - deflateEnd(&w->response.zstream);
172 buffer_free(z_buffer);
173 #endif
264 - buffer_free(w->response.data);
265 - buffer_free(w->response.header);
266 - buffer_free(w->response.header_output);
267 - freez(w);
174 buffer_free(local_buffer);
269 - buffer_free(log_buffer);
175 return retval;
176 }
177
aclk/aclk_query_queue.c
+1 -1
@@ -20,7 +20,7 @@ static struct aclk_query_queue {
20
21 static inline int _aclk_queue_query(aclk_query_t query)
22 {
23 - now_realtime_timeval(&query->created_tv);
23 + now_monotonic_high_precision_timeval(&query->created_tv);
24 query->created = now_realtime_usec();
25
26 ACLK_QUEUE_LOCK;
build/m4/ax_compiler_vendor.m4 new
+119
@@ -0,0 +1,119 @@
1 +# ===========================================================================
2 +# https://www.gnu.org/software/autoconf-archive/ax_compiler_vendor.html
3 +# ===========================================================================
4 +#
5 +# SYNOPSIS
6 +#
7 +# AX_COMPILER_VENDOR
8 +#
9 +# DESCRIPTION
10 +#
11 +# Determine the vendor of the C, C++ or Fortran compiler. The vendor is
12 +# returned in the cache variable $ax_cv_c_compiler_vendor for C,
13 +# $ax_cv_cxx_compiler_vendor for C++ or $ax_cv_fc_compiler_vendor for
14 +# (modern) Fortran. The value is one of "intel", "ibm", "pathscale",
15 +# "clang" (LLVM), "cray", "fujitsu", "sdcc", "sx", "nvhpc" (NVIDIA HPC
16 +# Compiler), "portland" (PGI), "gnu" (GCC), "sun" (Oracle Developer
17 +# Studio), "hp", "dec", "borland", "comeau", "kai", "lcc", "sgi",
18 +# "microsoft", "metrowerks", "watcom", "tcc" (Tiny CC) or "unknown" (if
19 +# the compiler cannot be determined).
20 +#
21 +# To check for a Fortran compiler, you must first call AC_FC_PP_SRCEXT
22 +# with an appropriate preprocessor-enabled extension. For example:
23 +#
24 +# AC_LANG_PUSH([Fortran])
25 +# AC_PROG_FC
26 +# AC_FC_PP_SRCEXT([F])
27 +# AX_COMPILER_VENDOR
28 +# AC_LANG_POP([Fortran])
29 +#
30 +# LICENSE
31 +#
32 +# Copyright (c) 2008 Steven G. Johnson <stevenj@alum.mit.edu>
33 +# Copyright (c) 2008 Matteo Frigo
34 +# Copyright (c) 2018-19 John Zaitseff <J.Zaitseff@zap.org.au>
35 +#
36 +# This program is free software: you can redistribute it and/or modify it
37 +# under the terms of the GNU General Public License as published by the
38 +# Free Software Foundation, either version 3 of the License, or (at your
39 +# option) any later version.
40 +#
41 +# This program is distributed in the hope that it will be useful, but
42 +# WITHOUT ANY WARRANTY; without even the implied warranty of
43 +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
44 +# Public License for more details.
45 +#
46 +# You should have received a copy of the GNU General Public License along
47 +# with this program. If not, see <https://www.gnu.org/licenses/>.
48 +#
49 +# As a special exception, the respective Autoconf Macro's copyright owner
50 +# gives unlimited permission to copy, distribute and modify the configure
51 +# scripts that are the output of Autoconf when processing the Macro. You
52 +# need not follow the terms of the GNU General Public License when using
53 +# or distributing such scripts, even though portions of the text of the
54 +# Macro appear in them. The GNU General Public License (GPL) does govern
55 +# all other use of the material that constitutes the Autoconf Macro.
56 +#
57 +# This special exception to the GPL applies to versions of the Autoconf
58 +# Macro released by the Autoconf Archive. When you make and distribute a
59 +# modified version of the Autoconf Macro, you may extend this special
60 +# exception to the GPL to apply to your modified version as well.
61 +
62 +#serial 32
63 +
64 +AC_DEFUN([AX_COMPILER_VENDOR], [dnl
65 + AC_CACHE_CHECK([for _AC_LANG compiler vendor], ax_cv_[]_AC_LANG_ABBREV[]_compiler_vendor, [dnl
66 + dnl If you modify this list of vendors, please add similar support
67 + dnl to ax_compiler_version.m4 if at all possible.
68 + dnl
69 + dnl Note: Do NOT check for GCC first since some other compilers
70 + dnl define __GNUC__ to remain compatible with it. Compilers that
71 + dnl are very slow to start (such as Intel) are listed first.
72 +
73 + vendors="
74 + intel: __ICC,__ECC,__INTEL_COMPILER
75 + ibm: __xlc__,__xlC__,__IBMC__,__IBMCPP__,__ibmxl__
76 + pathscale: __PATHCC__,__PATHSCALE__
77 + clang: __clang__
78 + cray: _CRAYC
79 + fujitsu: __FUJITSU
80 + sdcc: SDCC,__SDCC
81 + sx: _SX
82 + nvhpc: __NVCOMPILER
83 + portland: __PGI
84 + gnu: __GNUC__
85 + sun: __SUNPRO_C,__SUNPRO_CC,__SUNPRO_F90,__SUNPRO_F95
86 + hp: __HP_cc,__HP_aCC
87 + dec: __DECC,__DECCXX,__DECC_VER,__DECCXX_VER
88 + borland: __BORLANDC__,__CODEGEARC__,__TURBOC__
89 + comeau: __COMO__
90 + kai: __KCC
91 + lcc: __LCC__
92 + sgi: __sgi,sgi
93 + microsoft: _MSC_VER
94 + metrowerks: __MWERKS__
95 + watcom: __WATCOMC__
96 + tcc: __TINYC__
97 + unknown: UNKNOWN
98 + "
99 + for ventest in $vendors; do
100 + case $ventest in
101 + *:)
102 + vendor=$ventest
103 + continue
104 + ;;
105 + *)
106 + vencpp="defined("`echo $ventest | sed 's/,/) || defined(/g'`")"
107 + ;;
108 + esac
109 +
110 + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [[
111 +#if !($vencpp)
112 + thisisanerror;
113 +#endif
114 + ]])], [break])
115 + done
116 +
117 + ax_cv_[]_AC_LANG_ABBREV[]_compiler_vendor=`echo $vendor | cut -d: -f1`
118 + ])
119 +])dnl
build/m4/ax_cxx_compile_stdcxx.m4 new
+1018
@@ -0,0 +1,1018 @@
1 +# ===========================================================================
2 +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html
3 +# ===========================================================================
4 +#
5 +# SYNOPSIS
6 +#
7 +# AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional])
8 +#
9 +# DESCRIPTION
10 +#
11 +# Check for baseline language coverage in the compiler for the specified
12 +# version of the C++ standard. If necessary, add switches to CXX and
13 +# CXXCPP to enable support. VERSION may be '11', '14', '17', or '20' for
14 +# the respective C++ standard version.
15 +#
16 +# The second argument, if specified, indicates whether you insist on an
17 +# extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g.
18 +# -std=c++11). If neither is specified, you get whatever works, with
19 +# preference for no added switch, and then for an extended mode.
20 +#
21 +# The third argument, if specified 'mandatory' or if left unspecified,
22 +# indicates that baseline support for the specified C++ standard is
23 +# required and that the macro should error out if no mode with that
24 +# support is found. If specified 'optional', then configuration proceeds
25 +# regardless, after defining HAVE_CXX${VERSION} if and only if a
26 +# supporting mode is found.
27 +#
28 +# LICENSE
29 +#
30 +# Copyright (c) 2008 Benjamin Kosnik <bkoz@redhat.com>
31 +# Copyright (c) 2012 Zack Weinberg <zackw@panix.com>
32 +# Copyright (c) 2013 Roy Stogner <roystgnr@ices.utexas.edu>
33 +# Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov <sokolov@google.com>
34 +# Copyright (c) 2015 Paul Norman <penorman@mac.com>
35 +# Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu>
36 +# Copyright (c) 2016, 2018 Krzesimir Nowak <qdlacz@gmail.com>
37 +# Copyright (c) 2019 Enji Cooper <yaneurabeya@gmail.com>
38 +# Copyright (c) 2020 Jason Merrill <jason@redhat.com>
39 +# Copyright (c) 2021 Jörn Heusipp <osmanx@problemloesungsmaschine.de>
40 +#
41 +# Copying and distribution of this file, with or without modification, are
42 +# permitted in any medium without royalty provided the copyright notice
43 +# and this notice are preserved. This file is offered as-is, without any
44 +# warranty.
45 +
46 +#serial 18
47 +
48 +dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro
49 +dnl (serial version number 13).
50 +
51 +AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl
52 + m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"],
53 + [$1], [14], [ax_cxx_compile_alternatives="14 1y"],
54 + [$1], [17], [ax_cxx_compile_alternatives="17 1z"],
55 + [$1], [20], [ax_cxx_compile_alternatives="20"],
56 + [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl
57 + m4_if([$2], [], [],
58 + [$2], [ext], [],
59 + [$2], [noext], [],
60 + [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl
61 + m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true],
62 + [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true],
63 + [$3], [optional], [ax_cxx_compile_cxx$1_required=false],
64 + [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])])
65 + AC_LANG_PUSH([C++])dnl
66 + ac_success=no
67 +
68 + m4_if([$2], [], [dnl
69 + AC_CACHE_CHECK(whether $CXX supports C++$1 features by default,
70 + ax_cv_cxx_compile_cxx$1,
71 + [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])],
72 + [ax_cv_cxx_compile_cxx$1=yes],
73 + [ax_cv_cxx_compile_cxx$1=no])])
74 + if test x$ax_cv_cxx_compile_cxx$1 = xyes; then
75 + ac_success=yes
76 + fi])
77 +
78 + m4_if([$2], [noext], [], [dnl
79 + if test x$ac_success = xno; then
80 + for alternative in ${ax_cxx_compile_alternatives}; do
81 + switch="-std=gnu++${alternative}"
82 + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch])
83 + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch,
84 + $cachevar,
85 + [ac_save_CXX="$CXX"
86 + CXX="$CXX $switch"
87 + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])],
88 + [eval $cachevar=yes],
89 + [eval $cachevar=no])
90 + CXX="$ac_save_CXX"])
91 + if eval test x\$$cachevar = xyes; then
92 + CXX="$CXX $switch"
93 + if test -n "$CXXCPP" ; then
94 + CXXCPP="$CXXCPP $switch"
95 + fi
96 + ac_success=yes
97 + break
98 + fi
99 + done
100 + fi])
101 +
102 + m4_if([$2], [ext], [], [dnl
103 + if test x$ac_success = xno; then
104 + dnl HP's aCC needs +std=c++11 according to:
105 + dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf
106 + dnl Cray's crayCC needs "-h std=c++11"
107 + dnl MSVC needs -std:c++NN for C++17 and later (default is C++14)
108 + for alternative in ${ax_cxx_compile_alternatives}; do
109 + for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}" MSVC; do
110 + if test x"$switch" = xMSVC; then
111 + dnl AS_TR_SH maps both `:` and `=` to `_` so -std:c++17 would collide
112 + dnl with -std=c++17. We suffix the cache variable name with _MSVC to
113 + dnl avoid this.
114 + switch=-std:c++${alternative}
115 + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_${switch}_MSVC])
116 + else
117 + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch])
118 + fi
119 + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch,
120 + $cachevar,
121 + [ac_save_CXX="$CXX"
122 + CXX="$CXX $switch"
123 + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])],
124 + [eval $cachevar=yes],
125 + [eval $cachevar=no])
126 + CXX="$ac_save_CXX"])
127 + if eval test x\$$cachevar = xyes; then
128 + CXX="$CXX $switch"
129 + if test -n "$CXXCPP" ; then
130 + CXXCPP="$CXXCPP $switch"
131 + fi
132 + ac_success=yes
133 + break
134 + fi
135 + done
136 + if test x$ac_success = xyes; then
137 + break
138 + fi
139 + done
140 + fi])
141 + AC_LANG_POP([C++])
142 + if test x$ax_cxx_compile_cxx$1_required = xtrue; then
143 + if test x$ac_success = xno; then
144 + AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.])
145 + fi
146 + fi
147 + if test x$ac_success = xno; then
148 + HAVE_CXX$1=0
149 + AC_MSG_NOTICE([No compiler with C++$1 support was found])
150 + else
151 + HAVE_CXX$1=1
152 + AC_DEFINE(HAVE_CXX$1,1,
153 + [define if the compiler supports basic C++$1 syntax])
154 + fi
155 + AC_SUBST(HAVE_CXX$1)
156 +])
157 +
158 +
159 +dnl Test body for checking C++11 support
160 +
161 +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11],
162 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11
163 +)
164 +
165 +dnl Test body for checking C++14 support
166 +
167 +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14],
168 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11
169 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14
170 +)
171 +
172 +dnl Test body for checking C++17 support
173 +
174 +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17],
175 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11
176 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14
177 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17
178 +)
179 +
180 +dnl Test body for checking C++20 support
181 +
182 +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_20],
183 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11
184 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14
185 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17
186 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_20
187 +)
188 +
189 +
190 +dnl Tests for new features in C++11
191 +
192 +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[
193 +
194 +// If the compiler admits that it is not ready for C++11, why torture it?
195 +// Hopefully, this will speed up the test.
196 +
197 +#ifndef __cplusplus
198 +
199 +#error "This is not a C++ compiler"
200 +
201 +// MSVC always sets __cplusplus to 199711L in older versions; newer versions
202 +// only set it correctly if /Zc:__cplusplus is specified as well as a
203 +// /std:c++NN switch:
204 +// https://devblogs.microsoft.com/cppblog/msvc-now-correctly-reports-__cplusplus/
205 +#elif __cplusplus < 201103L && !defined _MSC_VER
206 +
207 +#error "This is not a C++11 compiler"
208 +
209 +#else
210 +
211 +namespace cxx11
212 +{
213 +
214 + namespace test_static_assert
215 + {
216 +
217 + template <typename T>
218 + struct check
219 + {
220 + static_assert(sizeof(int) <= sizeof(T), "not big enough");
221 + };
222 +
223 + }
224 +
225 + namespace test_final_override
226 + {
227 +
228 + struct Base
229 + {
230 + virtual ~Base() {}
231 + virtual void f() {}
232 + };
233 +
234 + struct Derived : public Base
235 + {
236 + virtual ~Derived() override {}
237 + virtual void f() override {}
238 + };
239 +
240 + }
241 +
242 + namespace test_double_right_angle_brackets
243 + {
244 +
245 + template < typename T >
246 + struct check {};
247 +
248 + typedef check<void> single_type;
249 + typedef check<check<void>> double_type;
250 + typedef check<check<check<void>>> triple_type;
251 + typedef check<check<check<check<void>>>> quadruple_type;
252 +
253 + }
254 +
255 + namespace test_decltype
256 + {
257 +
258 + int
259 + f()
260 + {
261 + int a = 1;
262 + decltype(a) b = 2;
263 + return a + b;
264 + }
265 +
266 + }
267 +
268 + namespace test_type_deduction
269 + {
270 +
271 + template < typename T1, typename T2 >
272 + struct is_same
273 + {
274 + static const bool value = false;
275 + };
276 +
277 + template < typename T >
278 + struct is_same<T, T>
279 + {
280 + static const bool value = true;
281 + };
282 +
283 + template < typename T1, typename T2 >
284 + auto
285 + add(T1 a1, T2 a2) -> decltype(a1 + a2)
286 + {
287 + return a1 + a2;
288 + }
289 +
290 + int
291 + test(const int c, volatile int v)
292 + {
293 + static_assert(is_same<int, decltype(0)>::value == true, "");
294 + static_assert(is_same<int, decltype(c)>::value == false, "");
295 + static_assert(is_same<int, decltype(v)>::value == false, "");
296 + auto ac = c;
297 + auto av = v;
298 + auto sumi = ac + av + 'x';
299 + auto sumf = ac + av + 1.0;
300 + static_assert(is_same<int, decltype(ac)>::value == true, "");
301 + static_assert(is_same<int, decltype(av)>::value == true, "");
302 + static_assert(is_same<int, decltype(sumi)>::value == true, "");
303 + static_assert(is_same<int, decltype(sumf)>::value == false, "");
304 + static_assert(is_same<int, decltype(add(c, v))>::value == true, "");
305 + return (sumf > 0.0) ? sumi : add(c, v);
306 + }
307 +
308 + }
309 +
310 + namespace test_noexcept
311 + {
312 +
313 + int f() { return 0; }
314 + int g() noexcept { return 0; }
315 +
316 + static_assert(noexcept(f()) == false, "");
317 + static_assert(noexcept(g()) == true, "");
318 +
319 + }
320 +
321 + namespace test_constexpr
322 + {
323 +
324 + template < typename CharT >
325 + unsigned long constexpr
326 + strlen_c_r(const CharT *const s, const unsigned long acc) noexcept
327 + {
328 + return *s ? strlen_c_r(s + 1, acc + 1) : acc;
329 + }
330 +
331 + template < typename CharT >
332 + unsigned long constexpr
333 + strlen_c(const CharT *const s) noexcept
334 + {
335 + return strlen_c_r(s, 0UL);
336 + }
337 +
338 + static_assert(strlen_c("") == 0UL, "");
339 + static_assert(strlen_c("1") == 1UL, "");
340 + static_assert(strlen_c("example") == 7UL, "");
341 + static_assert(strlen_c("another\0example") == 7UL, "");
342 +
343 + }
344 +
345 + namespace test_rvalue_references
346 + {
347 +
348 + template < int N >
349 + struct answer
350 + {
351 + static constexpr int value = N;
352 + };
353 +
354 + answer<1> f(int&) { return answer<1>(); }
355 + answer<2> f(const int&) { return answer<2>(); }
356 + answer<3> f(int&&) { return answer<3>(); }
357 +
358 + void
359 + test()
360 + {
361 + int i = 0;
362 + const int c = 0;
363 + static_assert(decltype(f(i))::value == 1, "");
364 + static_assert(decltype(f(c))::value == 2, "");
365 + static_assert(decltype(f(0))::value == 3, "");
366 + }
367 +
368 + }
369 +
370 + namespace test_uniform_initialization
371 + {
372 +
373 + struct test
374 + {
375 + static const int zero {};
376 + static const int one {1};
377 + };
378 +
379 + static_assert(test::zero == 0, "");
380 + static_assert(test::one == 1, "");
381 +
382 + }
383 +
384 + namespace test_lambdas
385 + {
386 +
387 + void
388 + test1()
389 + {
390 + auto lambda1 = [](){};
391 + auto lambda2 = lambda1;
392 + lambda1();
393 + lambda2();
394 + }
395 +
396 + int
397 + test2()
398 + {
399 + auto a = [](int i, int j){ return i + j; }(1, 2);
400 + auto b = []() -> int { return '0'; }();
401 + auto c = [=](){ return a + b; }();
402 + auto d = [&](){ return c; }();
403 + auto e = [a, &b](int x) mutable {
404 + const auto identity = [](int y){ return y; };
405 + for (auto i = 0; i < a; ++i)
406 + a += b--;
407 + return x + identity(a + b);
408 + }(0);
409 + return a + b + c + d + e;
410 + }
411 +
412 + int
413 + test3()
414 + {
415 + const auto nullary = [](){ return 0; };
416 + const auto unary = [](int x){ return x; };
417 + using nullary_t = decltype(nullary);
418 + using unary_t = decltype(unary);
419 + const auto higher1st = [](nullary_t f){ return f(); };
420 + const auto higher2nd = [unary](nullary_t f1){
421 + return [unary, f1](unary_t f2){ return f2(unary(f1())); };
422 + };
423 + return higher1st(nullary) + higher2nd(nullary)(unary);
424 + }
425 +
426 + }
427 +
428 + namespace test_variadic_templates
429 + {
430 +
431 + template <int...>
432 + struct sum;
433 +
434 + template <int N0, int... N1toN>
435 + struct sum<N0, N1toN...>
436 + {
437 + static constexpr auto value = N0 + sum<N1toN...>::value;
438 + };
439 +
440 + template <>
441 + struct sum<>
442 + {
443 + static constexpr auto value = 0;
444 + };
445 +
446 + static_assert(sum<>::value == 0, "");
447 + static_assert(sum<1>::value == 1, "");
448 + static_assert(sum<23>::value == 23, "");
449 + static_assert(sum<1, 2>::value == 3, "");
450 + static_assert(sum<5, 5, 11>::value == 21, "");
451 + static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, "");
452 +
453 + }
454 +
455 + // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae
456 + // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function
457 + // because of this.
458 + namespace test_template_alias_sfinae
459 + {
460 +
461 + struct foo {};
462 +
463 + template<typename T>
464 + using member = typename T::member_type;
465 +
466 + template<typename T>
467 + void func(...) {}
468 +
469 + template<typename T>
470 + void func(member<T>*) {}
471 +
472 + void test();
473 +
474 + void test() { func<foo>(0); }
475 +
476 + }
477 +
478 +} // namespace cxx11
479 +
480 +#endif // __cplusplus >= 201103L
481 +
482 +]])
483 +
484 +
485 +dnl Tests for new features in C++14
486 +
487 +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_14], [[
488 +
489 +// If the compiler admits that it is not ready for C++14, why torture it?
490 +// Hopefully, this will speed up the test.
491 +
492 +#ifndef __cplusplus
493 +
494 +#error "This is not a C++ compiler"
495 +
496 +#elif __cplusplus < 201402L && !defined _MSC_VER
497 +
498 +#error "This is not a C++14 compiler"
499 +
500 +#else
501 +
502 +namespace cxx14
503 +{
504 +
505 + namespace test_polymorphic_lambdas
506 + {
507 +
508 + int
509 + test()
510 + {
511 + const auto lambda = [](auto&&... args){
512 + const auto istiny = [](auto x){
513 + return (sizeof(x) == 1UL) ? 1 : 0;
514 + };
515 + const int aretiny[] = { istiny(args)... };
516 + return aretiny[0];
517 + };
518 + return lambda(1, 1L, 1.0f, '1');
519 + }
520 +
521 + }
522 +
523 + namespace test_binary_literals
524 + {
525 +
526 + constexpr auto ivii = 0b0000000000101010;
527 + static_assert(ivii == 42, "wrong value");
528 +
529 + }
530 +
531 + namespace test_generalized_constexpr
532 + {
533 +
534 + template < typename CharT >
535 + constexpr unsigned long
536 + strlen_c(const CharT *const s) noexcept
537 + {
538 + auto length = 0UL;
539 + for (auto p = s; *p; ++p)
540 + ++length;
541 + return length;
542 + }
543 +
544 + static_assert(strlen_c("") == 0UL, "");
545 + static_assert(strlen_c("x") == 1UL, "");
546 + static_assert(strlen_c("test") == 4UL, "");
547 + static_assert(strlen_c("another\0test") == 7UL, "");
548 +
549 + }
550 +
551 + namespace test_lambda_init_capture
552 + {
553 +
554 + int
555 + test()
556 + {
557 + auto x = 0;
558 + const auto lambda1 = [a = x](int b){ return a + b; };
559 + const auto lambda2 = [a = lambda1(x)](){ return a; };
560 + return lambda2();
561 + }
562 +
563 + }
564 +
565 + namespace test_digit_separators
566 + {
567 +
568 + constexpr auto ten_million = 100'000'000;
569 + static_assert(ten_million == 100000000, "");
570 +
571 + }
572 +
573 + namespace test_return_type_deduction
574 + {
575 +
576 + auto f(int& x) { return x; }
577 + decltype(auto) g(int& x) { return x; }
578 +
579 + template < typename T1, typename T2 >
580 + struct is_same
581 + {
582 + static constexpr auto value = false;
583 + };
584 +
585 + template < typename T >
586 + struct is_same<T, T>
587 + {
588 + static constexpr auto value = true;
589 + };
590 +
591 + int
592 + test()
593 + {
594 + auto x = 0;
595 + static_assert(is_same<int, decltype(f(x))>::value, "");
596 + static_assert(is_same<int&, decltype(g(x))>::value, "");
597 + return x;
598 + }
599 +
600 + }
601 +
602 +} // namespace cxx14
603 +
604 +#endif // __cplusplus >= 201402L
605 +
606 +]])
607 +
608 +
609 +dnl Tests for new features in C++17
610 +
611 +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_17], [[
612 +
613 +// If the compiler admits that it is not ready for C++17, why torture it?
614 +// Hopefully, this will speed up the test.
615 +
616 +#ifndef __cplusplus
617 +
618 +#error "This is not a C++ compiler"
619 +
620 +#elif __cplusplus < 201703L && !defined _MSC_VER
621 +
622 +#error "This is not a C++17 compiler"
623 +
624 +#else
625 +
626 +#include <initializer_list>
627 +#include <utility>
628 +#include <type_traits>
629 +
630 +namespace cxx17
631 +{
632 +
633 + namespace test_constexpr_lambdas
634 + {
635 +
636 + constexpr int foo = [](){return 42;}();
637 +
638 + }
639 +
640 + namespace test::nested_namespace::definitions
641 + {
642 +
643 + }
644 +
645 + namespace test_fold_expression
646 + {
647 +
648 + template<typename... Args>
649 + int multiply(Args... args)
650 + {
651 + return (args * ... * 1);
652 + }
653 +
654 + template<typename... Args>
655 + bool all(Args... args)
656 + {
657 + return (args && ...);
658 + }
659 +
660 + }
661 +
662 + namespace test_extended_static_assert
663 + {
664 +
665 + static_assert (true);
666 +
667 + }
668 +
669 + namespace test_auto_brace_init_list
670 + {
671 +
672 + auto foo = {5};
673 + auto bar {5};
674 +
675 + static_assert(std::is_same<std::initializer_list<int>, decltype(foo)>::value);
676 + static_assert(std::is_same<int, decltype(bar)>::value);
677 + }
678 +
679 + namespace test_typename_in_template_template_parameter
680 + {
681 +
682 + template<template<typename> typename X> struct D;
683 +
684 + }
685 +
686 + namespace test_fallthrough_nodiscard_maybe_unused_attributes
687 + {
688 +
689 + int f1()
690 + {
691 + return 42;
692 + }
693 +
694 + [[nodiscard]] int f2()
695 + {
696 + [[maybe_unused]] auto unused = f1();
697 +
698 + switch (f1())
699 + {
700 + case 17:
701 + f1();
702 + [[fallthrough]];
703 + case 42:
704 + f1();
705 + }
706 + return f1();
707 + }
708 +
709 + }
710 +
711 + namespace test_extended_aggregate_initialization
712 + {
713 +
714 + struct base1
715 + {
716 + int b1, b2 = 42;
717 + };
718 +
719 + struct base2
720 + {
721 + base2() {
722 + b3 = 42;
723 + }
724 + int b3;
725 + };
726 +
727 + struct derived : base1, base2
728 + {
729 + int d;
730 + };
731 +
732 + derived d1 {{1, 2}, {}, 4}; // full initialization
733 + derived d2 {{}, {}, 4}; // value-initialized bases
734 +
735 + }
736 +
737 + namespace test_general_range_based_for_loop
738 + {
739 +
740 + struct iter
741 + {
742 + int i;
743 +
744 + int& operator* ()
745 + {
746 + return i;
747 + }
748 +
749 + const int& operator* () const
750 + {
751 + return i;
752 + }
753 +
754 + iter& operator++()
755 + {
756 + ++i;
757 + return *this;
758 + }
759 + };
760 +
761 + struct sentinel
762 + {
763 + int i;
764 + };
765 +
766 + bool operator== (const iter& i, const sentinel& s)
767 + {
768 + return i.i == s.i;
769 + }
770 +
771 + bool operator!= (const iter& i, const sentinel& s)
772 + {
773 + return !(i == s);
774 + }
775 +
776 + struct range
777 + {
778 + iter begin() const
779 + {
780 + return {0};
781 + }
782 +
783 + sentinel end() const
784 + {
785 + return {5};
786 + }
787 + };
788 +
789 + void f()
790 + {
791 + range r {};
792 +
793 + for (auto i : r)
794 + {
795 + [[maybe_unused]] auto v = i;
796 + }
797 + }
798 +
799 + }
800 +
801 + namespace test_lambda_capture_asterisk_this_by_value
802 + {
803 +
804 + struct t
805 + {
806 + int i;
807 + int foo()
808 + {
809 + return [*this]()
810 + {
811 + return i;
812 + }();
813 + }
814 + };
815 +
816 + }
817 +
818 + namespace test_enum_class_construction
819 + {
820 +
821 + enum class byte : unsigned char
822 + {};
823 +
824 + byte foo {42};
825 +
826 + }
827 +
828 + namespace test_constexpr_if
829 + {
830 +
831 + template <bool cond>
832 + int f ()
833 + {
834 + if constexpr(cond)
835 + {
836 + return 13;
837 + }
838 + else
839 + {
840 + return 42;
841 + }
842 + }
843 +
844 + }
845 +
846 + namespace test_selection_statement_with_initializer
847 + {
848 +
849 + int f()
850 + {
851 + return 13;
852 + }
853 +
854 + int f2()
855 + {
856 + if (auto i = f(); i > 0)
857 + {
858 + return 3;
859 + }
860 +
861 + switch (auto i = f(); i + 4)
862 + {
863 + case 17:
864 + return 2;
865 +
866 + default:
867 + return 1;
868 + }
869 + }
870 +
871 + }
872 +
873 + namespace test_template_argument_deduction_for_class_templates
874 + {
875 +
876 + template <typename T1, typename T2>
877 + struct pair
878 + {
879 + pair (T1 p1, T2 p2)
880 + : m1 {p1},
881 + m2 {p2}
882 + {}
883 +
884 + T1 m1;
885 + T2 m2;
886 + };
887 +
888 + void f()
889 + {
890 + [[maybe_unused]] auto p = pair{13, 42u};
891 + }
892 +
893 + }
894 +
895 + namespace test_non_type_auto_template_parameters
896 + {
897 +
898 + template <auto n>
899 + struct B
900 + {};
901 +
902 + B<5> b1;
903 + B<'a'> b2;
904 +
905 + }
906 +
907 + namespace test_structured_bindings
908 + {
909 +
910 + int arr[2] = { 1, 2 };
911 + std::pair<int, int> pr = { 1, 2 };
912 +
913 + auto f1() -> int(&)[2]
914 + {
915 + return arr;
916 + }
917 +
918 + auto f2() -> std::pair<int, int>&
919 + {
920 + return pr;
921 + }
922 +
923 + struct S
924 + {
925 + int x1 : 2;
926 + volatile double y1;
927 + };
928 +
929 + S f3()
930 + {
931 + return {};
932 + }
933 +
934 + auto [ x1, y1 ] = f1();
935 + auto& [ xr1, yr1 ] = f1();
936 + auto [ x2, y2 ] = f2();
937 + auto& [ xr2, yr2 ] = f2();
938 + const auto [ x3, y3 ] = f3();
939 +
940 + }
941 +
942 + namespace test_exception_spec_type_system
943 + {
944 +
945 + struct Good {};
946 + struct Bad {};
947 +
948 + void g1() noexcept;
949 + void g2();
950 +
951 + template<typename T>
952 + Bad
953 + f(T*, T*);
954 +
955 + template<typename T1, typename T2>
956 + Good
957 + f(T1*, T2*);
958 +
959 + static_assert (std::is_same_v<Good, decltype(f(g1, g2))>);
960 +
961 + }
962 +
963 + namespace test_inline_variables
964 + {
965 +
966 + template<class T> void f(T)
967 + {}
968 +
969 + template<class T> inline T g(T)
970 + {
971 + return T{};
972 + }
973 +
974 + template<> inline void f<>(int)
975 + {}
976 +
977 + template<> int g<>(int)
978 + {
979 + return 5;
980 + }
981 +
982 + }
983 +
984 +} // namespace cxx17
985 +
986 +#endif // __cplusplus < 201703L && !defined _MSC_VER
987 +
988 +]])
989 +
990 +
991 +dnl Tests for new features in C++20
992 +
993 +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_20], [[
994 +
995 +#ifndef __cplusplus
996 +
997 +#error "This is not a C++ compiler"
998 +
999 +#elif __cplusplus < 202002L && !defined _MSC_VER
1000 +
1001 +#error "This is not a C++20 compiler"
1002 +
1003 +#else
1004 +
1005 +#include <version>
1006 +
1007 +namespace cxx20
1008 +{
1009 +
1010 +// As C++20 supports feature test macros in the standard, there is no
1011 +// immediate need to actually test for feature availability on the
1012 +// Autoconf side.
1013 +
1014 +} // namespace cxx20
1015 +
1016 +#endif // __cplusplus < 202002L && !defined _MSC_VER
1017 +
1018 +]])
collectors/cgroups.plugin/sys_fs_cgroup.c
+2 -2
@@ -1771,11 +1771,11 @@ static inline void substitute_dots_in_id(char *s) {
1771
1772 char *cgroup_parse_resolved_name_and_labels(DICTIONARY *labels, char *data) {
1773 // the first word, up to the first space is the name
1774 - char *name = mystrsep(&data, " ");
1774 + char *name = strsep_skip_consecutive_separators(&data, " ");
1775
1776 // the rest are key=value pairs separated by comma
1777 while(data) {
1778 - char *pair = mystrsep(&data, ",");
1778 + char *pair = strsep_skip_consecutive_separators(&data, ",");
1779 rrdlabels_add_pair(labels, pair, RRDLABEL_SRC_AUTO| RRDLABEL_SRC_K8S);
1780 }
1781
collectors/proc.plugin/sys_devices_system_node.c
+1 -1
@@ -105,7 +105,7 @@ int do_proc_sys_devices_system_node(int update_every, usec_t dt) {
105 , m->name
106 , NULL
107 , "numa"
108 - , NULL
108 + , "mem.numa_nodes"
109 , "NUMA events"
110 , "events/s"
111 , PLUGIN_PROC_NAME
collectors/proc.plugin/sys_fs_btrfs.c
+4 -4
@@ -135,9 +135,9 @@ static inline int collect_btrfs_error_stats(BTRFS_DEVICE *device){
135
136 char *p = buffer;
137 while(p){
138 - char *val = mystrsep(&p, "\n");
138 + char *val = strsep_skip_consecutive_separators(&p, "\n");
139 if(unlikely(!val || !*val)) break;
140 - char *key = mystrsep(&val, " ");
140 + char *key = strsep_skip_consecutive_separators(&val, " ");
141
142 if(!strcmp(key, "write_errs")) device->write_errs = str2ull(val, NULL);
143 else if(!strcmp(key, "read_errs")) device->read_errs = str2ull(val, NULL);
@@ -166,9 +166,9 @@ static inline int collect_btrfs_commits_stats(BTRFS_NODE *node, int update_every
166
167 char *p = buffer;
168 while(p){
169 - char *val = mystrsep(&p, "\n");
169 + char *val = strsep_skip_consecutive_separators(&p, "\n");
170 if(unlikely(!val || !*val)) break;
171 - char *key = mystrsep(&val, " ");
171 + char *key = strsep_skip_consecutive_separators(&val, " ");
172
173 if(!strcmp(key, "commits")){
174 long long commits_total_new = str2ull(val, NULL);
configure.ac
+65 -20
@@ -233,6 +233,55 @@ if test "${enable_cloud}" = "no"; then
233 AC_DEFINE([DISABLE_CLOUD], [1], [disable netdata cloud functionality])
234 fi
235
236 +# -----------------------------------------------------------------------------
237 +# C++ version check
238 +
239 +# Check for C++17 support (optional)
240 +# AX_CXX_COMPILE_STDCXX(17, noext, optional)
241 +
242 +if test "x$HAVE_CXX17" != "x1"; then
243 + # Check for C++11 support (optional)
244 + AX_CXX_COMPILE_STDCXX(11, noext, optional)
245 +fi
246 +
247 +AC_MSG_CHECKING([c++ standard to use])
248 +if test "x$HAVE_CXX17" = "x1"; then
249 + have_cxx17="yes"
250 + have_cxx11="yes"
251 + CPP_STD_FLAG="-std=c++17"
252 + cpp_std_to_use="c++17"
253 + AM_CONDITIONAL([HAVE_CXX17], [true])
254 + AM_CONDITIONAL([HAVE_CXX11], [true])
255 +elif test "x$HAVE_CXX11" = "x1"; then
256 + have_cxx17="no"
257 + have_cxx11="yes"
258 + CPP_STD_FLAG="-std=c++11"
259 + cpp_std_to_use="c++11"
260 + AM_CONDITIONAL([HAVE_CXX17], [false])
261 + AM_CONDITIONAL([HAVE_CXX11], [true])
262 +else
263 + have_cxx17="no"
264 + have_cxx11="no"
265 + CPP_STD_FLAG=""
266 + cpp_std_to_use="no c++"
267 + AM_CONDITIONAL([HAVE_CXX17], [false])
268 + AM_CONDITIONAL([HAVE_CXX11], [false])
269 +fi
270 +
271 +# PPC64LE needs -std=gnu++11 in order to build dlib. However, the rest of
272 +# the agent's components use and have been tested only with -std=c++11.
273 +# Skip ML compilation on that CPU until we reorganize and test the C++ flags.
274 +if test "${host_cpu}" = "powerpc64le"; then
275 + have_cxx17="no"
276 + have_cxx11="no"
277 + CPP_STD_FLAG=""
278 + cpp_std_to_use="no c++ on powerpc64le"
279 + AM_CONDITIONAL([HAVE_CXX17], [false])
280 + AM_CONDITIONAL([HAVE_CXX11], [false])
281 +fi
282 +
283 +AC_MSG_RESULT([${cpp_std_to_use}])
284 +
285 # -----------------------------------------------------------------------------
286 # netdata required checks
287
@@ -347,6 +396,17 @@ AC_CHECK_LIB(
396 [AC_DEFINE([HAVE_PTHREAD_GETNAME_NP], [1], [Is set if pthread_getname_np is available])]
397 )
398
399 +# -----------------------------------------------------------------------------
400 +# libdatachannel
401 +
402 +AC_CHECK_LIB([datachannel], [rtcCreatePeerConnection],
403 + [LIBDATACHANNEL_FOUND=yes],
404 + [LIBDATACHANNEL_FOUND=no])
405 +
406 +if test "x$LIBDATACHANNEL_FOUND" = "xyes"; then
407 + AC_DEFINE([HAVE_LIBDATACHANNEL], [1], [libdatachannel usability])
408 + OPTIONAL_DATACHANNEL_LIBS="-ldatachannel"
409 +fi
410
411 # -----------------------------------------------------------------------------
412 # libm
@@ -858,7 +918,6 @@ if test "$enable_cloud" != "no"; then
918 AC_DEFINE([ENABLE_ACLK], [1], [netdata ACLK])
919 OPTIONAL_ACLK_CFLAGS="-I \$(abs_top_srcdir)/mqtt_websockets/src/include -I \$(abs_top_srcdir)/mqtt_websockets/c-rbuf/include -I \$(abs_top_srcdir)/aclk/aclk-schemas"
920 OPTIONAL_PROTOBUF_CFLAGS="${PROTOBUF_CFLAGS}"
861 - CXX11FLAG="-std=c++11"
921 OPTIONAL_PROTOBUF_LIBS="${PROTOBUF_LIBS}"
922 fi
923 fi
@@ -1195,22 +1254,8 @@ if test "${enable_ml}" = "yes" -a "${have_ml_submodules}" = "no"; then
1254 AC_MSG_ERROR([You have explicitly requested --enable-ml functionality but it cannot be built because the required git submodules are missing.])
1255 fi
1256
1198 -# Check if C++ toolchain does not support C++11. Fail if ML was explicitly requested.
1199 -AC_LANG_PUSH([C++])
1200 -AX_CHECK_COMPILE_FLAG([-std=c++11], [have_cxx11=yes], [have_cxx11=no])
1201 -AC_LANG_POP([C++])
1202 -
1203 -# PPC64LE needs -std=gnu++11 in order to build dlib. However, the rest of
1204 -# the agent's components use and have been tested only with -std=c++11.
1205 -# Skip ML compilation on that CPU until we reorganize and test the C++ flags.
1206 -if test "${host_cpu}" = "powerpc64le"; then
1207 - have_cxx11="no"
1208 -fi
1209 -
1257 if test "${enable_ml}" = "yes" -a "${have_cxx11}" = "no"; then
1258 AC_MSG_ERROR([You have explicitly requested --enable-ml functionality but it cannot be built without a C++11 toolchain.])
1212 -else
1213 - CXX11FLAG="$CXX11FLAG -std=c++11"
1259 fi
1260
1261 # Decide if we should build ML
@@ -1360,7 +1405,7 @@ if test "${enable_exporting_kinesis}" != "no" -a "${have_libaws_cpp_sdk_kinesis}
1405 enable_exporting_kinesis="yes"
1406 AC_DEFINE([HAVE_KINESIS], [1], [libaws-cpp-sdk-kinesis usability])
1407 OPTIONAL_KINESIS_CFLAGS="${LIBCRYPTO_CFLAGS} ${LIBSSL_CFLAGS} ${LIBCURL_CFLAGS}"
1363 - CXX11FLAG="${AWS_CPP_SDK_KINESIS_CFLAGS} ${AWS_CPP_SDK_CORE_CFLAGS}"
1408 + OPTIONAL_KINESIS_CXXFLAGS="${AWS_CPP_SDK_KINESIS_CFLAGS} ${AWS_CPP_SDK_CORE_CFLAGS}"
1409 OPTIONAL_KINESIS_LIBS="${AWS_CPP_SDK_KINESIS_LIBS} ${AWS_CPP_SDK_CORE_LIBS} \
1410 ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}"
1411 else
@@ -1411,7 +1456,6 @@ if test "${enable_exporting_pubsub}" != "no" -a "${have_pubsub_protos}" = "yes"
1456 enable_exporting_pubsub="yes"
1457 AC_DEFINE([ENABLE_EXPORTING_PUBSUB], [1], [Pub/Sub API usability])
1458 OPTIONAL_PUBSUB_CFLAGS="${GRPC_CFLAGS} ${PUBSUB_CFLAGS}"
1414 - CXX11FLAG="-std=c++11"
1459 OPTIONAL_PUBSUB_LIBS="${GRPC_LIBS} ${PUBSUB_LIBS}"
1460 else
1461 enable_pubsub="no"
@@ -1431,7 +1475,7 @@ AC_MSG_CHECKING([for snappy::RawCompress in -lsnappy])
1475 save_LIBS="${LIBS}"
1476 LIBS="-lsnappy"
1477 save_CXXFLAGS="${CXXFLAGS}"
1434 - CXXFLAGS="${CXXFLAGS} -std=c++11"
1478 + CXXFLAGS="${CXXFLAGS} ${CPP_STD_FLAG}"
1479
1480 AC_TRY_LINK(
1481 [
@@ -1477,7 +1521,6 @@ if test "${enable_exporting_prometheus_remote_write}" != "no" -a "${have_libprot
1521 enable_exporting_prometheus_remote_write="yes"
1522 AC_DEFINE([ENABLE_PROMETHEUS_REMOTE_WRITE], [1], [Prometheus remote write API usability])
1523 OPTIONAL_PROMETHEUS_REMOTE_WRITE_CFLAGS="${SNAPPY_CFLAGS} -I \$(abs_top_srcdir)/exporting/prometheus/remote_write"
1480 - CXX11FLAG="-std=c++11"
1524 OPTIONAL_PROMETHEUS_REMOTE_WRITE_LIBS="${SNAPPY_LIBS}"
1525 OPTIONAL_PROTOBUF_CFLAGS="${PROTOBUF_CFLAGS}"
1526 OPTIONAL_PROTOBUF_LIBS="${PROTOBUF_LIBS}"
@@ -1648,7 +1691,7 @@ CFLAGS="${originalCFLAGS} ${OPTIONAL_LTO_CFLAGS} ${OPTIONAL_PROTOBUF_CFLAGS} ${O
1691 ${OPTIONAL_MONGOC_CFLAGS} ${LWS_CFLAGS} ${OPTIONAL_JSONC_STATIC_CFLAGS} ${OPTIONAL_YAML_STATIC_CFLAGS} ${OPTIONAL_BPF_CFLAGS} ${JUDY_CFLAGS} \
1692 ${OPTIONAL_ACLK_CFLAGS} ${OPTIONAL_ML_CFLAGS} ${OPTIONAL_OS_DEP_CFLAGS}"
1693
1651 -CXXFLAGS="${CFLAGS} ${CXX11FLAG}"
1694 +CXXFLAGS="${CFLAGS} ${OPTIONAL_KINESIS_CXXFLAGS} ${CPP_STD_FLAG}"
1695
1696 CPPFLAGS="\
1697 -DVARLIB_DIR=\"\\\"${varlibdir}\\\"\" \
@@ -1663,6 +1706,7 @@ CPPFLAGS="\
1706
1707 AC_SUBST([OPTIONAL_MATH_CFLAGS])
1708 AC_SUBST([OPTIONAL_MATH_LIBS])
1709 +AC_SUBST([OPTIONAL_DATACHANNEL_LIBS])
1710 AC_SUBST([OPTIONAL_UV_LIBS])
1711 AC_SUBST([OPTIONAL_LZ4_LIBS])
1712 AC_SUBST([OPTIONAL_SSL_LIBS])
@@ -1842,6 +1886,7 @@ AC_CONFIG_FILES([
1886 web/api/health/Makefile
1887 web/gui/Makefile
1888 web/gui/dashboard/Makefile
1889 + web/rtc/Makefile
1890 web/server/Makefile
1891 web/server/static/Makefile
1892 claim/Makefile
daemon/common.h
+1
@@ -59,6 +59,7 @@
59
60 // the netdata API
61 #include "web/server/web_client.h"
62 +#include "web/rtc/webrtc.h"
63
64 // all data collection plugins
65 #include "collectors/all.h"
daemon/main.c
+15
@@ -309,6 +309,8 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {
309 timeout = false; \
310 }
311
312 +void web_client_cache_destroy(void);
313 +
314 void netdata_cleanup_and_exit(int ret) {
315 usec_t started_ut = now_monotonic_usec();
316 usec_t last_ut = started_ut;
@@ -336,6 +338,10 @@ void netdata_cleanup_and_exit(int ret) {
338 }
339 #endif
340
341 + delta_shutdown_time("close webrtc connections");
342 +
343 + webrtc_close_all_connections();
344 +
345 delta_shutdown_time("disable ML detection and training threads");
346
347 ml_stop_threads();
@@ -380,6 +386,10 @@ void netdata_cleanup_and_exit(int ret) {
386 SERVICE_MAINTENANCE
387 , 3 * USEC_PER_SEC);
388
389 + delta_shutdown_time("clear web client cache");
390 +
391 + web_client_cache_destroy();
392 +
393 delta_shutdown_time("clean rrdhost database");
394
395 rrdhost_cleanup_all();
@@ -2136,6 +2146,11 @@ int main(int argc, char **argv) {
2146 }
2147 #endif
2148
2149 + // ------------------------------------------------------------------------
2150 + // initialize WebRTC
2151 +
2152 + webrtc_initialize();
2153 +
2154 // ------------------------------------------------------------------------
2155 // unblock signals
2156
database/rrd.h
+1
@@ -1295,6 +1295,7 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unitt
1295
1296 RRDHOST *rrdhost_find_by_hostname(const char *hostname);
1297 RRDHOST *rrdhost_find_by_guid(const char *guid);
1298 +RRDHOST *find_host_by_node_id(char *node_id);
1299
1300 RRDHOST *rrdhost_find_or_create(
1301 const char *hostname
database/rrdhost.c
+16
@@ -41,6 +41,22 @@ bool is_storage_engine_shared(STORAGE_INSTANCE *engine __maybe_unused) {
41 return false;
42 }
43
44 +RRDHOST *find_host_by_node_id(char *node_id) {
45 + uuid_t node_uuid;
46 + if (unlikely(!node_id || uuid_parse(node_id, node_uuid)))
47 + return NULL;
48 +
49 + RRDHOST *host, *ret = NULL;
50 + dfe_start_read(rrdhost_root_index, host) {
51 + if (host->node_id && !(uuid_memcmp(host->node_id, &node_uuid))) {
52 + ret = host;
53 + break;
54 + }
55 + }
56 + dfe_done(host);
57 +
58 + return ret;
59 +}
60
61 // ----------------------------------------------------------------------------
62 // RRDHOST indexes management
database/sqlite/sqlite_aclk.h
-20
@@ -82,26 +82,6 @@ struct aclk_sync_host_config {
82 uint64_t alerts_ack_sequence_id; // last sequence_id ack'ed from cloud via sendsnapshot message
83 };
84
85 -static inline RRDHOST *find_host_by_node_id(char *node_id)
86 -{
87 - uuid_t node_uuid;
88 - if (unlikely(!node_id || uuid_parse(node_id, node_uuid)))
89 - return NULL;
90 -
91 - rrd_rdlock();
92 - RRDHOST *host, *ret = NULL;
93 - rrdhost_foreach_read(host) {
94 - if (host->node_id && !(uuid_memcmp(host->node_id, &node_uuid))) {
95 - ret = host;
96 - break;
97 - }
98 - }
99 - rrd_unlock();
100 -
101 - return ret;
102 -}
103 -
104 -
85 extern sqlite3 *db_meta;
86
87 int aclk_database_enq_cmd_noblock(struct aclk_database_cmd *cmd);
health/health_json.c
+1 -1
@@ -309,7 +309,7 @@ void health_aggregate_alarms(RRDHOST *host, BUFFER *wb, BUFFER* contexts, RRDCAL
309
310 if (contexts) {
311 p = (char*)buffer_tostring(contexts);
312 - while(p && *p && (tok = mystrsep(&p, ", |"))) {
312 + while(p && *p && (tok = strsep_skip_consecutive_separators(&p, ", |"))) {
313 if(!*tok) continue;
314
315 STRING *tok_string = string_strdupz(tok);
health/notifications/alarm-notify.sh.in
+1 -1
@@ -201,7 +201,7 @@ fi
201 [ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@"
202 [ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="@cachedir_POST@"
203 [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"
204 -[ -z "${NETDATA_REGISTRY_CLOUD_BASE_URL}" ] && NETDATA_REGISTRY_CLOUD_BASE_URL="https://api.netdata.cloud"
204 +[ -z "${NETDATA_REGISTRY_CLOUD_BASE_URL}" ] && NETDATA_REGISTRY_CLOUD_BASE_URL="https://app.netdata.cloud"
205
206 # -----------------------------------------------------------------------------
207 # parse command line parameters
libnetdata/buffer/buffer.h
+22
@@ -201,6 +201,28 @@ static inline void buffer_strcat(BUFFER *wb, const char *txt) {
201 buffer_overflow_check(wb);
202 }
203
204 +static inline void buffer_strncat(BUFFER *wb, const char *txt, size_t len) {
205 + if(unlikely(!txt || !*txt)) return;
206 +
207 + const char *t = txt;
208 + while(*t) {
209 + buffer_need_bytes(wb, len);
210 + char *s = &wb->buffer[wb->len];
211 + char *d = s;
212 + const char *e = &wb->buffer[wb->len + len];
213 +
214 + while(*t && d < e)
215 + *d++ = *t++;
216 +
217 + wb->len += d - s;
218 + }
219 +
220 + buffer_need_bytes(wb, 1);
221 + wb->buffer[wb->len] = '\0';
222 +
223 + buffer_overflow_check(wb);
224 +}
225 +
226 static inline void buffer_json_strcat(BUFFER *wb, const char *txt) {
227 if(unlikely(!txt || !*txt)) return;
228
libnetdata/config/appconfig.c
+8 -7
@@ -845,13 +845,14 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed)
845 else if(!strcmp(co->name, CONFIG_SECTION_ML)) pri = 8;
846 else if(!strcmp(co->name, CONFIG_SECTION_HEALTH)) pri = 9;
847 else if(!strcmp(co->name, CONFIG_SECTION_WEB)) pri = 10;
848 - // by default, new sections will get pri = 11 (set at the end, below)
849 - else if(!strcmp(co->name, CONFIG_SECTION_REGISTRY)) pri = 12;
850 - else if(!strcmp(co->name, CONFIG_SECTION_GLOBAL_STATISTICS)) pri = 13;
851 - else if(!strcmp(co->name, CONFIG_SECTION_PLUGINS)) pri = 14;
852 - else if(!strcmp(co->name, CONFIG_SECTION_STATSD)) pri = 15;
853 - else if(!strncmp(co->name, "plugin:", 7)) pri = 16; // << change the loop too if you change this
854 - else pri = 11; // this is used for any new (currently unknown) sections
848 + else if(!strcmp(co->name, CONFIG_SECTION_WEBRTC)) pri = 11;
849 + // by default, new sections will get pri = 12 (set at the end, below)
850 + else if(!strcmp(co->name, CONFIG_SECTION_REGISTRY)) pri = 13;
851 + else if(!strcmp(co->name, CONFIG_SECTION_GLOBAL_STATISTICS)) pri = 14;
852 + else if(!strcmp(co->name, CONFIG_SECTION_PLUGINS)) pri = 15;
853 + else if(!strcmp(co->name, CONFIG_SECTION_STATSD)) pri = 16;
854 + else if(!strncmp(co->name, "plugin:", 7)) pri = 17; // << change the loop too if you change this
855 + else pri = 12; // this is used for any new (currently unknown) sections
856
857 if(i == pri) {
858 int loaded = 0;
libnetdata/config/appconfig.h
+1
@@ -88,6 +88,7 @@
88 #define CONFIG_SECTION_ENV_VARS "environment variables"
89 #define CONFIG_SECTION_SQLITE "sqlite"
90 #define CONFIG_SECTION_WEB "web"
91 +#define CONFIG_SECTION_WEBRTC "webrtc"
92 #define CONFIG_SECTION_STATSD "statsd"
93 #define CONFIG_SECTION_PLUGINS "plugins"
94 #define CONFIG_SECTION_CLOUD "cloud"
libnetdata/inlined.h
+58
@@ -514,4 +514,62 @@ static inline int uuid_memcmp(const uuid_t *uu1, const uuid_t *uu2) {
514 return memcmp(uu1, uu2, sizeof(uuid_t));
515 }
516
517 +static inline char *strsep_skip_consecutive_separators(char **ptr, char *s) {
518 + char *p = (char *)"";
519 + while (p && !p[0] && *ptr) p = strsep(ptr, s);
520 + return (p);
521 +}
522 +
523 +// remove leading and trailing spaces; may return NULL
524 +static inline char *trim(char *s) {
525 + // skip leading spaces
526 + while (*s && isspace(*s)) s++;
527 + if (!*s) return NULL;
528 +
529 + // skip tailing spaces
530 + // this way is way faster. Writes only one NUL char.
531 + ssize_t l = (ssize_t)strlen(s);
532 + if (--l >= 0) {
533 + char *p = s + l;
534 + while (p > s && isspace(*p)) p--;
535 + *++p = '\0';
536 + }
537 +
538 + if (!*s) return NULL;
539 +
540 + return s;
541 +}
542 +
543 +// like trim(), but also remove duplicate spaces inside the string; may return NULL
544 +static inline char *trim_all(char *buffer) {
545 + char *d = buffer, *s = buffer;
546 +
547 + // skip spaces
548 + while(isspace(*s)) s++;
549 +
550 + while(*s) {
551 + // copy the non-space part
552 + while(*s && !isspace(*s)) *d++ = *s++;
553 +
554 + // add a space if we have to
555 + if(*s && isspace(*s)) {
556 + *d++ = ' ';
557 + s++;
558 + }
559 +
560 + // skip spaces
561 + while(isspace(*s)) s++;
562 + }
563 +
564 + *d = '\0';
565 +
566 + if(d > buffer) {
567 + d--;
568 + if(isspace(*d)) *d = '\0';
569 + }
570 +
571 + if(!buffer[0]) return NULL;
572 + return buffer;
573 +}
574 +
575 #endif //NETDATA_INLINED_H
libnetdata/libnetdata.c
-165
@@ -1042,171 +1042,6 @@ void netdata_fix_chart_id(char *s) {
1042 while ((*s = netdata_map_chart_ids[(unsigned char) *s])) s++;
1043 }
1044
1045 -/*
1046 -// http://stackoverflow.com/questions/7666509/hash-function-for-string
1047 -uint32_t simple_hash(const char *name)
1048 -{
1049 - const char *s = name;
1050 - uint32_t hash = 5381;
1051 - int i;
1052 -
1053 - while((i = *s++)) hash = ((hash << 5) + hash) + i;
1054 -
1055 - // fprintf(stderr, "HASH: %lu %s\n", hash, name);
1056 -
1057 - return hash;
1058 -}
1059 -*/
1060 -
1061 -/*
1062 -// http://isthe.com/chongo/tech/comp/fnv/#FNV-1a
1063 -uint32_t simple_hash(const char *name) {
1064 - unsigned char *s = (unsigned char *) name;
1065 - uint32_t hval = 0x811c9dc5;
1066 -
1067 - // FNV-1a algorithm
1068 - while (*s) {
1069 - // multiply by the 32 bit FNV magic prime mod 2^32
1070 - // NOTE: No need to optimize with left shifts.
1071 - // GCC will use imul instruction anyway.
1072 - // Tested with 'gcc -O3 -S'
1073 - //hval += (hval<<1) + (hval<<4) + (hval<<7) + (hval<<8) + (hval<<24);
1074 - hval *= 16777619;
1075 -
1076 - // xor the bottom with the current octet
1077 - hval ^= (uint32_t) *s++;
1078 - }
1079 -
1080 - // fprintf(stderr, "HASH: %u = %s\n", hval, name);
1081 - return hval;
1082 -}
1083 -
1084 -uint32_t simple_uhash(const char *name) {
1085 - unsigned char *s = (unsigned char *) name;
1086 - uint32_t hval = 0x811c9dc5, c;
1087 -
1088 - // FNV-1a algorithm
1089 - while ((c = *s++)) {
1090 - if (unlikely(c >= 'A' && c <= 'Z')) c += 'a' - 'A';
1091 - hval *= 16777619;
1092 - hval ^= c;
1093 - }
1094 - return hval;
1095 -}
1096 -*/
1097 -
1098 -/*
1099 -// http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx
1100 -// one at a time hash
1101 -uint32_t simple_hash(const char *name) {
1102 - unsigned char *s = (unsigned char *)name;
1103 - uint32_t h = 0;
1104 -
1105 - while(*s) {
1106 - h += *s++;
1107 - h += (h << 10);
1108 - h ^= (h >> 6);
1109 - }
1110 -
1111 - h += (h << 3);
1112 - h ^= (h >> 11);
1113 - h += (h << 15);
1114 -
1115 - // fprintf(stderr, "HASH: %u = %s\n", h, name);
1116 -
1117 - return h;
1118 -}
1119 -*/
1120 -
1121 -void strreverse(char *begin, char *end) {
1122 - while (end > begin) {
1123 - // clearer code.
1124 - char aux = *end;
1125 - *end-- = *begin;
1126 - *begin++ = aux;
1127 - }
1128 -}
1129 -
1130 -char *strsep_on_1char(char **ptr, char c) {
1131 - if(unlikely(!ptr || !*ptr))
1132 - return NULL;
1133 -
1134 - // remember the position we started
1135 - char *s = *ptr;
1136 -
1137 - // skip separators in front
1138 - while(*s == c) s++;
1139 - char *ret = s;
1140 -
1141 - // find the next separator
1142 - while(*s++) {
1143 - if(unlikely(*s == c)) {
1144 - *s++ = '\0';
1145 - *ptr = s;
1146 - return ret;
1147 - }
1148 - }
1149 -
1150 - *ptr = NULL;
1151 - return ret;
1152 -}
1153 -
1154 -char *mystrsep(char **ptr, char *s) {
1155 - char *p = "";
1156 - while (p && !p[0] && *ptr) p = strsep(ptr, s);
1157 - return (p);
1158 -}
1159 -
1160 -char *trim(char *s) {
1161 - // skip leading spaces
1162 - while (*s && isspace(*s)) s++;
1163 - if (!*s) return NULL;
1164 -
1165 - // skip tailing spaces
1166 - // this way is way faster. Writes only one NUL char.
1167 - ssize_t l = strlen(s);
1168 - if (--l >= 0) {
1169 - char *p = s + l;
1170 - while (p > s && isspace(*p)) p--;
1171 - *++p = '\0';
1172 - }
1173 -
1174 - if (!*s) return NULL;
1175 -
1176 - return s;
1177 -}
1178 -
1179 -inline char *trim_all(char *buffer) {
1180 - char *d = buffer, *s = buffer;
1181 -
1182 - // skip spaces
1183 - while(isspace(*s)) s++;
1184 -
1185 - while(*s) {
1186 - // copy the non-space part
1187 - while(*s && !isspace(*s)) *d++ = *s++;
1188 -
1189 - // add a space if we have to
1190 - if(*s && isspace(*s)) {
1191 - *d++ = ' ';
1192 - s++;
1193 - }
1194 -
1195 - // skip spaces
1196 - while(isspace(*s)) s++;
1197 - }
1198 -
1199 - *d = '\0';
1200 -
1201 - if(d > buffer) {
1202 - d--;
1203 - if(isspace(*d)) *d = '\0';
1204 - }
1205 -
1206 - if(!buffer[0]) return NULL;
1207 - return buffer;
1208 -}
1209 -
1045 static int memory_file_open(const char *filename, size_t size) {
1046 // info("memory_file_open('%s', %zu", filename, size);
1047
libnetdata/libnetdata.h
+2 -8
@@ -491,11 +491,6 @@ typedef struct storage_point {
491 void netdata_fix_chart_id(char *s);
492 void netdata_fix_chart_name(char *s);
493
494 -void strreverse(char* begin, char* end);
495 -char *mystrsep(char **ptr, char *s);
496 -char *trim(char *s); // remove leading and trailing spaces; may return NULL
497 -char *trim_all(char *buffer); // like trim(), but also remove duplicate spaces inside the string; may return NULL
498 -
494 int madvise_sequential(void *mem, size_t len);
495 int madvise_random(void *mem, size_t len);
496 int madvise_dontfork(void *mem, size_t len);
@@ -673,9 +668,8 @@ extern char *netdata_configured_host_prefix;
668 #include "parser/parser.h"
669 #include "yaml.h"
670
676 -// BEWARE: Outside of the C code this also exists in alarm-notify.sh
677 -#define DEFAULT_CLOUD_BASE_URL "https://api.netdata.cloud"
678 -#define DEFAULT_CLOUD_UI_URL "https://app.netdata.cloud"
671 +// BEWARE: this exists in alarm-notify.sh
672 +#define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud"
673
674 #define RRD_STORAGE_TIERS 5
675
libnetdata/socket/socket.c
+2 -2
@@ -1149,8 +1149,8 @@ int accept4(int sock, struct sockaddr *addr, socklen_t *addrlen, int flags) {
1149 * update the client_host if uninitialized - ensure the hostsize is the number
1150 * of *writable* bytes (i.e. be aware of the strdup used to compact the pollinfo).
1151 */
1152 -extern int connection_allowed(int fd, char *client_ip, char *client_host, size_t hostsize, SIMPLE_PATTERN *access_list,
1153 - const char *patname, int allow_dns)
1152 +int connection_allowed(int fd, char *client_ip, char *client_host, size_t hostsize, SIMPLE_PATTERN *access_list,
1153 + const char *patname, int allow_dns)
1154 {
1155 debug(D_LISTENER,"checking %s... (allow_dns=%d)", patname, allow_dns);
1156 if (!access_list)
libnetdata/socket/socket.h
+3
@@ -22,8 +22,11 @@ typedef enum web_client_acl {
22 WEB_CLIENT_ACL_SSL_FORCE = (1 << 7),
23 WEB_CLIENT_ACL_SSL_DEFAULT = (1 << 8),
24 WEB_CLIENT_ACL_ACLK = (1 << 9),
25 + WEB_CLIENT_ACL_WEBRTC = (1 << 10),
26 } WEB_CLIENT_ACL;
27
28 +#define WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC (WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK | WEB_CLIENT_ACL_WEBRTC)
29 +
30 #define WEB_CLIENT_ACL_ALL 0xFFFF
31
32 #define web_client_can_access_dashboard(w) ((w)->acl & WEB_CLIENT_ACL_DASHBOARD)
libnetdata/threads/threads.c
+58 -1
@@ -21,8 +21,63 @@ inline int netdata_thread_tag_exists(void) {
21 return (netdata_thread && netdata_thread->tag && *netdata_thread->tag);
22 }
23
24 +static const char *thread_name_get(bool recheck) {
25 + static __thread char threadname[NETDATA_THREAD_NAME_MAX + 1] = "";
26 +
27 + if(netdata_thread_tag_exists())
28 + strncpyz(threadname, netdata_thread->tag, NETDATA_THREAD_NAME_MAX + 1);
29 + else {
30 + if(!recheck && threadname[0])
31 + return threadname;
32 +
33 +#if defined(__FreeBSD__)
34 + pthread_get_name_np(pthread_self(), threadname, NETDATA_THREAD_NAME_MAX + 1);
35 + if(strcmp(threadname, "netdata") == 0)
36 + strncpyz(threadname, "MAIN", NETDATA_THREAD_NAME_MAX + 1);
37 +#elif defined(__APPLE__)
38 + strncpyz(threadname, "MAIN", NETDATA_THREAD_NAME_MAX + 1);
39 +#elif defined(HAVE_PTHREAD_GETNAME_NP)
40 + pthread_getname_np(pthread_self(), threadname, NETDATA_THREAD_NAME_MAX + 1);
41 + if(strcmp(threadname, "netdata") == 0)
42 + strncpyz(threadname, "MAIN", NETDATA_THREAD_NAME_MAX + 1);
43 +#else
44 + strncpyz(threadname, "MAIN", NETDATA_THREAD_NAME_MAX + 1);
45 +#endif
46 + }
47 +
48 + return threadname;
49 +}
50 +
51 const char *netdata_thread_tag(void) {
25 - return (netdata_thread_tag_exists() ? netdata_thread->tag : "MAIN");
52 + return thread_name_get(false);
53 +}
54 +
55 +static size_t webrtc_id = 0;
56 +static __thread bool webrtc_name_set = false;
57 +void webrtc_set_thread_name(void) {
58 + if(!netdata_thread && !webrtc_name_set) {
59 + webrtc_name_set = true;
60 + char threadname[NETDATA_THREAD_NAME_MAX + 1];
61 +
62 +#if defined(__FreeBSD__)
63 + snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "WEBRTC[%zu]", __atomic_fetch_add(&webrtc_id, 1, __ATOMIC_RELAXED));
64 + pthread_set_name_np(pthread_self(), threadname);
65 +#elif defined(__APPLE__)
66 + snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "WEBRTC[%zu]", __atomic_fetch_add(&webrtc_id, 1, __ATOMIC_RELAXED));
67 + pthread_setname_np(threadname);
68 +#elif defined(HAVE_PTHREAD_GETNAME_NP)
69 + pthread_getname_np(pthread_self(), threadname, NETDATA_THREAD_NAME_MAX+1);
70 + if(strcmp(threadname, "netdata") == 0) {
71 + snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "WEBRTC[%zu]", __atomic_fetch_add(&webrtc_id, 1, __ATOMIC_RELAXED));
72 + pthread_setname_np(pthread_self(), threadname);
73 + }
74 +#else
75 + snprintfz(threadname, NETDATA_THREAD_NAME_MAX, "WEBRTC[%zu]", __atomic_fetch_add(&webrtc_id, 1, __ATOMIC_RELAXED));
76 + pthread_setname_np(pthread_self(), threadname);
77 +#endif
78 +
79 + thread_name_get(true);
80 + }
81 }
82
83 // ----------------------------------------------------------------------------
@@ -173,6 +228,8 @@ void uv_thread_set_name_np(uv_thread_t ut, const char* name) {
228 ret = pthread_setname_np(ut, threadname);
229 #endif
230
231 + thread_name_get(true);
232 +
233 if (ret)
234 info("cannot set libuv thread name to %s. Err: %d", threadname, ret);
235 }
libnetdata/threads/threads.h
+2
@@ -43,6 +43,8 @@ int netdata_thread_detach(pthread_t thread);
43 void uv_thread_set_name_np(uv_thread_t ut, const char* name);
44 void os_thread_get_current_name_np(char threadname[NETDATA_THREAD_NAME_MAX + 1]);
45
46 +void webrtc_set_thread_name(void);
47 +
48 #define netdata_thread_self pthread_self
49 #define netdata_thread_testcancel pthread_testcancel
50
libnetdata/url/url.c
+44 -136
@@ -17,7 +17,7 @@ char to_hex(char code) {
17 return hex[code & 15];
18 }
19
20 -/* Returns a url-encoded version of str */
20 +/* Returns an url-encoded version of str */
21 /* IMPORTANT: be sure to free() the returned string after use */
22 char *url_encode(char *str) {
23 char *buf, *pbuf;
@@ -33,8 +33,8 @@ char *url_encode(char *str) {
33
34 else{
35 *pbuf++ = '%';
36 - *pbuf++ = to_hex(*str >> 4);
37 - *pbuf++ = to_hex(*str & 15);
36 + *pbuf++ = to_hex((char)(*str >> 4));
37 + *pbuf++ = to_hex((char)(*str & 15));
38 }
39
40 str++;
@@ -55,9 +55,9 @@ char *url_encode(char *str) {
55 *
56 * @return The character decoded on success and 0 otherwise
57 */
58 -char url_percent_escape_decode(char *s) {
58 +char url_percent_escape_decode(const char *s) {
59 if(likely(s[1] && s[2]))
60 - return from_hex(s[1]) << 4 | from_hex(s[2]);
60 + return (char)(from_hex(s[1]) << 4 | from_hex(s[2]));
61 return 0;
62 }
63
@@ -98,7 +98,7 @@ char url_utf8_get_byte_length(char c) {
98 *
99 * @return count of bytes written to *d
100 */
101 -char url_decode_multibyte_utf8(char *s, char *d, char *d_end) {
101 +char url_decode_multibyte_utf8(const char *s, char *d, const char *d_end) {
102 char first_byte = url_percent_escape_decode(s);
103
104 if(unlikely(!first_byte || !IS_UTF8_STARTBYTE(first_byte)))
@@ -189,9 +189,9 @@ unsigned char *utf8_check(unsigned char *s)
189 return NULL;
190 }
191
192 -char *url_decode_r(char *to, char *url, size_t size) {
193 - char *s = url, // source
194 - *d = to, // destination
192 +char *url_decode_r(char *to, const char *url, size_t size) {
193 + const char *s = url; // source
194 + char *d = to, // destination
195 *e = &to[size - 1]; // destination end
196
197 while(*s && d < e) {
@@ -236,31 +236,45 @@ fail_cleanup:
236 return NULL;
237 }
238
239 -/**
240 - * Is request complete?
241 - *
242 - * Check whether the request is complete.
243 - * This function cannot check all the requests METHODS, for example, case you are working with POST, it will fail.
244 - *
245 - * @param begin is the first character of the sequence to analyse.
246 - * @param end is the last character of the sequence
247 - * @param length is the length of the total of bytes read, it is not the difference between end and begin.
248 - *
249 - * @return It returns 1 when the request is complete and 0 otherwise.
250 - */
251 -inline int url_is_request_complete(char *begin, char *end, size_t length) {
239 +inline bool url_is_request_complete(char *begin, char *end, size_t length, char **post_payload, size_t *post_payload_size) {
240 + if (begin == end || length < 4)
241 + return false;
242
253 - if ( begin == end) {
254 - //Message cannot be complete when first and last address are the same
255 - return 0;
243 + if(likely(strncmp(begin, "GET ", 4)) == 0) {
244 + return strstr(end - 4, "\r\n\r\n");
245 }
246 + else if(unlikely(strncmp(begin, "POST ", 5) == 0)) {
247 + char *cl = strstr(begin, "Content-Length: ");
248 + if(!cl) return false;
249 + cl = &cl[16];
250
258 - //This math to verify the last is valid, because we are discarding the POST
259 - if (length > 4) {
260 - begin = end - 4;
261 - }
251 + size_t content_length = str2ul(cl);
252 +
253 + char *payload = strstr(cl, "\r\n\r\n");
254 + if(!payload) return false;
255 + payload += 4;
256 +
257 + size_t payload_length = length - (payload - begin);
258 +
259 + if(payload_length == content_length) {
260 + if(post_payload && post_payload_size) {
261 + if (*post_payload)
262 + freez(*post_payload);
263 +
264 + *post_payload = mallocz(payload_length + 1);
265 + memcpy(*post_payload, payload, payload_length);
266 + (*post_payload)[payload_length] = '\0';
267 +
268 + *post_payload_size = payload_length;
269 + }
270 + return true;
271 + }
272
263 - return (strstr(begin, "\r\n\r\n"))?1:0;
273 + return false;
274 + }
275 + else {
276 + return strstr(end - 4, "\r\n\r\n");
277 + }
278 }
279
280 /**
@@ -283,109 +297,3 @@ inline char *url_find_protocol(char *s) {
297
298 return s;
299 }
286 -
287 -/**
288 - * Map query string
289 - *
290 - * Map the query string fields that will be decoded.
291 - * This functions must be called after to check the presence of query strings,
292 - * here we are assuming that you already tested this.
293 - *
294 - * @param out the pointer to pointers that will be used to map
295 - * @param url the input url that we are decoding.
296 - *
297 - * @return It returns the number of total variables in the query string.
298 - */
299 -int url_map_query_string(char **out, char *url) {
300 - (void)out;
301 - (void)url;
302 - int count = 0;
303 -
304 - //First we try to parse considering that there was not URL encode process
305 - char *moveme = url;
306 - char *ptr;
307 -
308 - //We always we have at least one here, so I can set this.
309 - out[count++] = moveme;
310 - while(moveme) {
311 - ptr = strchr((moveme+1), '&');
312 - if(ptr) {
313 - out[count++] = ptr;
314 - }
315 -
316 - moveme = ptr;
317 - }
318 -
319 - //I could not find any '&', so I am assuming now it is like '%26'
320 - if (count == 1) {
321 - moveme = url;
322 - while(moveme) {
323 - ptr = strchr((moveme+1), '%');
324 - if(ptr) {
325 - char *test = (ptr+1);
326 - if (!strncmp(test, "3f", 2) || !strncmp(test, "3F", 2)) {
327 - out[count++] = ptr;
328 - }
329 - }
330 - moveme = ptr;
331 - }
332 - }
333 -
334 - return count;
335 -}
336 -
337 -/**
338 - * Parse query string
339 - *
340 - * Parse the query string mapped and store it inside output.
341 - *
342 - * @param output is a vector where I will store the string.
343 - * @param max is the maximum length of the output
344 - * @param map the map done by the function url_map_query_string.
345 - * @param total the total number of variables inside map
346 - *
347 - * @return It returns 0 on success and -1 otherwise
348 - */
349 -int url_parse_query_string(char *output, size_t max, char **map, int total) {
350 - if(!total) {
351 - return 0;
352 - }
353 -
354 - int counter, next;
355 - size_t length;
356 - char *end;
357 - char *begin = map[0];
358 - char save;
359 - size_t copied = 0;
360 - for(counter = 0, next=1 ; next <= total ; ++counter, ++next) {
361 - if (next != total) {
362 - end = map[next];
363 - length = (size_t) (end - begin);
364 - save = *end;
365 - *end = 0x00;
366 - } else {
367 - length = strlen(begin);
368 - end = NULL;
369 - }
370 - length++;
371 -
372 - if (length > (max - copied)) {
373 - error("Parsing query string: we cannot parse a query string so big");
374 - break;
375 - }
376 -
377 - if(!url_decode_r(output, begin, length)) {
378 - return -1;
379 - }
380 - length = strlen(output);
381 - copied += length;
382 - output += length;
383 -
384 - begin = end;
385 - if (begin) {
386 - *begin = save;
387 - }
388 - }
389 -
390 - return 0;
391 -}
libnetdata/url/url.h
+2 -6
@@ -23,13 +23,9 @@ char *url_encode(char *str);
23 /* IMPORTANT: be sure to free() the returned string after use */
24 char *url_decode(char *str);
25
26 -char *url_decode_r(char *to, char *url, size_t size);
26 +char *url_decode_r(char *to, const char *url, size_t size);
27
28 -#define WEB_FIELDS_MAX 400
29 -int url_map_query_string(char **out, char *url);
30 -int url_parse_query_string(char *output, size_t max, char **map, int total);
31 -
32 -int url_is_request_complete(char *begin,char *end,size_t length);
28 +bool url_is_request_complete(char *begin, char *end, size_t length, char **post_payload, size_t *post_payload_length);
29 char *url_find_protocol(char *s);
30
31 #endif /* NETDATA_URL_H */
registry/registry.c
+69 -76
@@ -4,6 +4,7 @@
4 #include "registry_internals.h"
5
6 #define REGISTRY_STATUS_OK "ok"
7 +#define REGISTRY_STATUS_REDIRECT "redirect"
8 #define REGISTRY_STATUS_FAILED "failed"
9 #define REGISTRY_STATUS_DISABLED "disabled"
10
@@ -18,35 +19,26 @@ static inline void registry_unlock(void) {
19 netdata_mutex_unlock(&registry.lock);
20 }
21
21 -
22 // ----------------------------------------------------------------------------
23 // COOKIES
24
25 static void registry_set_cookie(struct web_client *w, const char *guid) {
26 - char edate[100], domain[512];
26 + char edate[100];
27 time_t et = now_realtime_sec() + registry.persons_expiration;
28 struct tm etmbuf, *etm = gmtime_r(&et, &etmbuf);
29 strftime(edate, sizeof(edate), "%a, %d %b %Y %H:%M:%S %Z", etm);
30
31 - snprintfz(w->cookie1, NETDATA_WEB_REQUEST_COOKIE_SIZE, NETDATA_REGISTRY_COOKIE_NAME "=%s; Expires=%s", guid, edate);
32 -
33 - if(registry.registry_domain && registry.registry_domain[0])
34 - snprintfz(domain, 511, "Domain=%s", registry.registry_domain);
35 - else
36 - domain[0]='\0';
37 -
38 - int length = snprintfz(w->cookie2, NETDATA_WEB_REQUEST_COOKIE_SIZE,
39 - NETDATA_REGISTRY_COOKIE_NAME "=%s; Expires=%s; %s",
40 - guid, edate, domain);
31 + buffer_sprintf(w->response.header, "Set-Cookie: " NETDATA_REGISTRY_COOKIE_NAME "=%s; Expires=%s\r\n", guid, edate);
32 + if(registry.enable_cookies_samesite_secure)
33 + buffer_sprintf(w->response.header, "Set-Cookie: " NETDATA_REGISTRY_COOKIE_NAME "=%s; Expires=%s; SameSite=None; Secure\r\n", guid, edate);
34
42 - size_t remaining_length = NETDATA_WEB_REQUEST_COOKIE_SIZE - length;
43 - // 25 is the necessary length to add new cookies
44 - if (registry.enable_cookies_samesite_secure) {
45 - if (length > 0 && remaining_length > 25)
46 - snprintfz(&w->cookie2[length], remaining_length, "; SameSite=None; Secure");
47 - else
48 - error("Netdata does not have enough space to store cookies SameSite and Secure");
35 + if(registry.registry_domain && *registry.registry_domain) {
36 + buffer_sprintf(w->response.header, "Set-Cookie: " NETDATA_REGISTRY_COOKIE_NAME "=%s; Expires=%s; Domain=%s\r\n", guid, edate, registry.registry_domain);
37 + if(registry.enable_cookies_samesite_secure)
38 + buffer_sprintf(w->response.header, "Set-Cookie: " NETDATA_REGISTRY_COOKIE_NAME "=%s; Expires=%s; Domain=%s; SameSite=None; Secure\r\n", guid, edate, registry.registry_domain);
39 }
40 +
41 + w->response.has_cookies = true;
42 }
43
44 static inline void registry_set_person_cookie(struct web_client *w, REGISTRY_PERSON *p) {
@@ -60,22 +52,24 @@ static inline void registry_set_person_cookie(struct web_client *w, REGISTRY_PER
52 static inline void registry_json_header(RRDHOST *host, struct web_client *w, const char *action, const char *status) {
53 buffer_flush(w->response.data);
54 w->response.data->content_type = CT_APPLICATION_JSON;
63 - buffer_sprintf(w->response.data, "{\n\t\"action\": \"%s\",\n\t\"status\": \"%s\",\n\t\"hostname\": \"%s\",\n\t\"machine_guid\": \"%s\"",
64 - action, status, rrdhost_registry_hostname(host), host->machine_guid);
55 + buffer_json_initialize(w->response.data, "\"", "\"", 0, true, false);
56 + buffer_json_member_add_string(w->response.data, "action", action);
57 + buffer_json_member_add_string(w->response.data, "status", status);
58 + buffer_json_member_add_string(w->response.data, "hostname", rrdhost_registry_hostname(host));
59 + buffer_json_member_add_string(w->response.data, "machine_guid", host->machine_guid);
60 }
61
62 static inline void registry_json_footer(struct web_client *w) {
68 - buffer_strcat(w->response.data, "\n}\n");
63 + buffer_json_finalize(w->response.data);
64 }
65
66 static inline int registry_json_disabled(RRDHOST *host, struct web_client *w, const char *action) {
67 registry_json_header(host, w, action, REGISTRY_STATUS_DISABLED);
68
74 - buffer_sprintf(w->response.data, ",\n\t\"registry\": \"%s\"",
75 - registry.registry_to_announce);
69 + buffer_json_member_add_string(w->response.data, "registry", registry.registry_to_announce);
70
71 registry_json_footer(w);
78 - return 200;
72 + return HTTP_RESP_OK;
73 }
74
75
@@ -97,14 +91,16 @@ static int registry_json_person_url_callback(void *entry, void *data) {
91 struct web_client *w = c->w;
92
93 if (!strcmp(pu->url->url,"***")) return 0;
100 -
101 - if(unlikely(c->count++))
102 - buffer_strcat(w->response.data, ",");
94
104 - buffer_sprintf(w->response.data, "\n\t\t[ \"%s\", \"%s\", %u000, %u, \"%s\" ]",
105 - pu->machine->guid, pu->url->url, pu->last_t, pu->usages, pu->machine_name);
95 + buffer_json_add_array_item_array(w->response.data);
96 + buffer_json_add_array_item_string(w->response.data, pu->machine->guid);
97 + buffer_json_add_array_item_string(w->response.data, pu->url->url);
98 + buffer_json_add_array_item_uint64(w->response.data, pu->last_t * 1000);
99 + buffer_json_add_array_item_uint64(w->response.data, pu->usages);
100 + buffer_json_add_array_item_string(w->response.data, pu->machine_name);
101 + buffer_json_array_close(w->response.data);
102
107 - return 0;
103 + return 1;
104 }
105
106 // callback for rendering MACHINE_URLs
@@ -114,13 +110,14 @@ static int registry_json_machine_url_callback(const DICTIONARY_ITEM *item __mayb
110 struct web_client *w = c->w;
111 REGISTRY_MACHINE *m = c->m;
112
117 - if (!strcmp(mu->url->url,"***")) return 1;
113 + if (!strcmp(mu->url->url,"***")) return 0;
114
119 - if(unlikely(c->count++))
120 - buffer_strcat(w->response.data, ",");
121 -
122 - buffer_sprintf(w->response.data, "\n\t\t[ \"%s\", \"%s\", %u000, %u ]",
123 - m->guid, mu->url->url, mu->last_t, mu->usages);
115 + buffer_json_add_array_item_array(w->response.data);
116 + buffer_json_add_array_item_string(w->response.data, m->guid);
117 + buffer_json_add_array_item_string(w->response.data, mu->url->url);
118 + buffer_json_add_array_item_uint64(w->response.data, mu->last_t * 1000);
119 + buffer_json_add_array_item_uint64(w->response.data, mu->usages);
120 + buffer_json_array_close(w->response.data);
121
122 return 1;
123 }
@@ -149,30 +146,23 @@ static inline int registry_person_url_callback_verify_machine_exists(void *entry
146 // The registry does not seem to be designed to support this and I cannot see any concurrency protection
147 // that could make this safe, so try to be as atomic as possible.
148
152 -void registry_update_cloud_base_url()
153 -{
154 - // This is guaranteed to be set early in main via post_conf_load()
155 - registry.cloud_base_url = appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", NULL);
156 - if (registry.cloud_base_url == NULL)
157 - fatal("Do not move the cloud base url out of post_conf_load!!");
158 -
149 +void registry_update_cloud_base_url() {
150 + registry.cloud_base_url = appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", DEFAULT_CLOUD_BASE_URL);
151 setenv("NETDATA_REGISTRY_CLOUD_BASE_URL", registry.cloud_base_url, 1);
152 }
153 +
154 // ----------------------------------------------------------------------------
155 // public HELLO request
156
157 int registry_request_hello_json(RRDHOST *host, struct web_client *w) {
158 registry_json_header(host, w, "hello", REGISTRY_STATUS_OK);
159
167 - const char *cloud_ui_url = appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud ui url", DEFAULT_CLOUD_UI_URL);
168 -
169 - buffer_sprintf(w->response.data,
170 - ",\n\t\"registry\": \"%s\",\n\t\"cloud_base_url\": \"%s\",\n\t\"anonymous_statistics\": %s",
171 - registry.registry_to_announce,
172 - cloud_ui_url, netdata_anonymous_statistics_enabled?"true":"false");
160 + buffer_json_member_add_string(w->response.data, "registry", registry.registry_to_announce);
161 + buffer_json_member_add_string(w->response.data, "cloud_base_url", registry.cloud_base_url);
162 + buffer_json_member_add_boolean(w->response.data, "anonymous_statistics", netdata_anonymous_statistics_enabled);
163
164 registry_json_footer(w);
175 - return 200;
165 + return HTTP_RESP_OK;
166 }
167
168 // ----------------------------------------------------------------------------
@@ -192,8 +182,10 @@ int registry_request_access_json(RRDHOST *host, struct web_client *w, char *pers
182 buffer_flush(w->response.data);
183 registry_set_cookie(w, REGISTRY_VERIFY_COOKIES_GUID);
184 w->response.data->content_type = CT_APPLICATION_JSON;
195 - buffer_sprintf(w->response.data, "{ \"status\": \"redirect\", \"registry\": \"%s\" }", registry.registry_to_announce);
196 - return 200;
185 + registry_json_header(host, w, "access", REGISTRY_STATUS_REDIRECT);
186 + buffer_json_member_add_string(w->response.data, "registry", registry.registry_to_announce);
187 + registry_json_footer(w);
188 + return HTTP_RESP_OK;
189 }
190
191 if(unlikely(person_guid[0] && !strcmp(person_guid, REGISTRY_VERIFY_COOKIES_GUID)))
@@ -208,7 +200,7 @@ int registry_request_access_json(RRDHOST *host, struct web_client *w, char *pers
200 registry_json_header(host, w, "access", REGISTRY_STATUS_FAILED);
201 registry_json_footer(w);
202 registry_unlock();
211 - return 412;
203 + return HTTP_RESP_PRECOND_FAIL;
204 }
205
206 // set the cookie
@@ -216,15 +208,16 @@ int registry_request_access_json(RRDHOST *host, struct web_client *w, char *pers
208
209 // generate the response
210 registry_json_header(host, w, "access", REGISTRY_STATUS_OK);
211 + buffer_json_member_add_string(w->response.data, "person_guid", p->guid);
212 + buffer_json_member_add_array(w->response.data, "urls");
213
220 - buffer_sprintf(w->response.data, ",\n\t\"person_guid\": \"%s\",\n\t\"urls\": [", p->guid);
214 struct registry_json_walk_person_urls_callback c = { p, NULL, w, 0 };
215 avl_traverse(&p->person_urls, registry_json_person_url_callback, &c);
223 - buffer_strcat(w->response.data, "\n\t]\n");
216 + buffer_json_array_close(w->response.data); // urls
217
218 registry_json_footer(w);
219 registry_unlock();
227 - return 200;
220 + return HTTP_RESP_OK;
221 }
222
223 // ----------------------------------------------------------------------------
@@ -242,14 +235,14 @@ int registry_request_delete_json(RRDHOST *host, struct web_client *w, char *pers
235 registry_json_header(host, w, "delete", REGISTRY_STATUS_FAILED);
236 registry_json_footer(w);
237 registry_unlock();
245 - return 412;
238 + return HTTP_RESP_PRECOND_FAIL;
239 }
240
241 // generate the response
242 registry_json_header(host, w, "delete", REGISTRY_STATUS_OK);
243 registry_json_footer(w);
244 registry_unlock();
252 - return 200;
245 + return HTTP_RESP_OK;
246 }
247
248 // ----------------------------------------------------------------------------
@@ -267,19 +260,19 @@ int registry_request_search_json(RRDHOST *host, struct web_client *w, char *pers
260 registry_json_header(host, w, "search", REGISTRY_STATUS_FAILED);
261 registry_json_footer(w);
262 registry_unlock();
270 - return 404;
263 + return HTTP_RESP_NOT_FOUND;
264 }
265
266 registry_json_header(host, w, "search", REGISTRY_STATUS_OK);
267
275 - buffer_strcat(w->response.data, ",\n\t\"urls\": [");
268 + buffer_json_member_add_array(w->response.data, "urls");
269 struct registry_json_walk_person_urls_callback c = { NULL, m, w, 0 };
270 dictionary_walkthrough_read(m->machine_urls, registry_json_machine_url_callback, &c);
278 - buffer_strcat(w->response.data, "\n\t]\n");
271 + buffer_json_array_close(w->response.data);
272
273 registry_json_footer(w);
274 registry_unlock();
282 - return 200;
275 + return HTTP_RESP_OK;
276 }
277
278 // ----------------------------------------------------------------------------
@@ -346,11 +339,11 @@ int registry_request_switch_json(RRDHOST *host, struct web_client *w, char *pers
339
340 // generate the response
341 registry_json_header(host, w, "switch", REGISTRY_STATUS_OK);
349 - buffer_sprintf(w->response.data, ",\n\t\"person_guid\": \"%s\"", np->guid);
342 + buffer_json_member_add_string(w->response.data, "person_guid", np->guid);
343 registry_json_footer(w);
344
345 registry_unlock();
353 - return 200;
346 + return HTTP_RESP_OK;
347 }
348
349 // ----------------------------------------------------------------------------
@@ -380,7 +373,7 @@ void registry_statistics(void) {
373 rrddim_add(sts, "sessions", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
374 }
375
383 - rrddim_set(sts, "sessions", registry.usages_count);
376 + rrddim_set(sts, "sessions", (collected_number)registry.usages_count);
377 rrdset_done(sts);
378
379 // ------------------------------------------------------------------------
@@ -408,11 +401,11 @@ void registry_statistics(void) {
401 rrddim_add(stc, "machines_urls", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
402 }
403
411 - rrddim_set(stc, "persons", registry.persons_count);
412 - rrddim_set(stc, "machines", registry.machines_count);
413 - rrddim_set(stc, "urls", registry.urls_count);
414 - rrddim_set(stc, "persons_urls", registry.persons_urls_count);
415 - rrddim_set(stc, "machines_urls", registry.machines_urls_count);
404 + rrddim_set(stc, "persons", (collected_number)registry.persons_count);
405 + rrddim_set(stc, "machines", (collected_number)registry.machines_count);
406 + rrddim_set(stc, "urls", (collected_number)registry.urls_count);
407 + rrddim_set(stc, "persons_urls", (collected_number)registry.persons_urls_count);
408 + rrddim_set(stc, "machines_urls", (collected_number)registry.machines_urls_count);
409 rrdset_done(stc);
410
411 // ------------------------------------------------------------------------
@@ -440,10 +433,10 @@ void registry_statistics(void) {
433 rrddim_add(stm, "machines_urls", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);
434 }
435
443 - rrddim_set(stm, "persons", registry.persons_memory + dictionary_stats_for_registry(registry.persons));
444 - rrddim_set(stm, "machines", registry.machines_memory + dictionary_stats_for_registry(registry.machines));
445 - rrddim_set(stm, "urls", registry.urls_memory);
446 - rrddim_set(stm, "persons_urls", registry.persons_urls_memory);
447 - rrddim_set(stm, "machines_urls", registry.machines_urls_memory);
436 + rrddim_set(stm, "persons", (collected_number)registry.persons_memory + dictionary_stats_for_registry(registry.persons));
437 + rrddim_set(stm, "machines", (collected_number)registry.machines_memory + dictionary_stats_for_registry(registry.machines));
438 + rrddim_set(stm, "urls", (collected_number)registry.urls_memory);
439 + rrddim_set(stm, "persons_urls", (collected_number)registry.persons_urls_memory);
440 + rrddim_set(stm, "machines_urls", (collected_number)registry.machines_urls_memory);
441 rrdset_done(stm);
442 }
streaming/rrdpush.c
+4 -4
@@ -725,7 +725,7 @@ int rrdpush_receiver_too_busy_now(struct web_client *w) {
725 }
726
727 void *rrdpush_receiver_thread(void *ptr);
728 -int rrdpush_receiver_thread_spawn(struct web_client *w, char *url) {
728 +int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_string) {
729
730 if(!service_running(ABILITY_STREAMING_CONNECTIONS))
731 return rrdpush_receiver_too_busy_now(w);
@@ -757,11 +757,11 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *url) {
757
758 // parse the parameters and fill rpt and rpt->system_info
759
760 - while(url) {
761 - char *value = mystrsep(&url, "&");
760 + while(decoded_query_string) {
761 + char *value = strsep_skip_consecutive_separators(&decoded_query_string, "&");
762 if(!value || !*value) continue;
763
764 - char *name = mystrsep(&value, "=");
764 + char *name = strsep_skip_consecutive_separators(&value, "=");
765 if(!name || !*name) continue;
766 if(!value || !*value) continue;
767
streaming/rrdpush.h
+1 -1
@@ -331,7 +331,7 @@ void rrdpush_claimed_id(RRDHOST *host);
331 #define THREAD_TAG_STREAM_RECEIVER "RCVR" // "[host]" is appended
332 #define THREAD_TAG_STREAM_SENDER "SNDR" // "[host]" is appended
333
334 -int rrdpush_receiver_thread_spawn(struct web_client *w, char *url);
334 +int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_string);
335 void rrdpush_sender_thread_stop(RRDHOST *host, const char *reason, bool wait);
336
337 void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, const RRDVAR_ACQUIRED *rva);
web/Makefile.am
+1
@@ -7,6 +7,7 @@ SUBDIRS = \
7 api \
8 gui \
9 server \
10 + rtc \
11 $(NULL)
12
13 usersslconfigdir=$(configdir)/ssl
web/api/badges/web_buffer_svg.c
+2 -2
@@ -898,10 +898,10 @@ int web_client_api_request_v1_badge(RRDHOST *host, struct web_client *w, char *u
898 RRDSET *st = NULL;
899
900 while(url) {
901 - char *value = mystrsep(&url, "&");
901 + char *value = strsep_skip_consecutive_separators(&url, "&");
902 if(!value || !*value) continue;
903
904 - char *name = mystrsep(&value, "=");
904 + char *name = strsep_skip_consecutive_separators(&value, "=");
905 if(!name || !*name) continue;
906 if(!value || !*value) continue;
907
web/api/exporters/allmetrics.c
+2 -2
@@ -39,10 +39,10 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client
39 prometheus_prefix = global_exporting_prefix;
40
41 while(url) {
42 - char *value = mystrsep(&url, "&");
42 + char *value = strsep_skip_consecutive_separators(&url, "&");
43 if (!value || !*value) continue;
44
45 - char *name = mystrsep(&value, "=");
45 + char *name = strsep_skip_consecutive_separators(&value, "=");
46 if(!name || !*name) continue;
47 if(!value || !*value) continue;
48
web/api/formatters/json_wrapper.c
+8 -3
@@ -226,7 +226,7 @@ static inline void query_target_points_statistics(BUFFER *wb, QUERY_TARGET *qt,
226 }
227
228 if(sp->anomaly_count != 0)
229 - buffer_json_member_add_double(wb, "ars", storage_point_anomaly_rate(*sp));
229 + buffer_json_member_add_uint64(wb, "arc", sp->anomaly_count);
230 }
231 else {
232 NETDATA_DOUBLE avg = (sp->count) ? sp->sum / (NETDATA_DOUBLE)sp->count : 0.0;
@@ -804,12 +804,12 @@ static inline void rrdr_dimension_query_points_statistics(BUFFER *wb, const char
804 }
805 buffer_json_array_close(wb);
806
807 - buffer_json_member_add_array(wb, "ars");
807 + buffer_json_member_add_array(wb, "arc");
808 for(size_t c = 0; c < r->d ; c++) {
809 if (!rrdr_dimension_should_be_exposed(r->od[c], options))
810 continue;
811
812 - buffer_json_add_array_item_uint64(wb, sp[c].anomaly_count * 100 / anomaly_rate_multiplier);
812 + buffer_json_add_array_item_uint64(wb, storage_point_anomaly_rate(sp[c]) / anomaly_rate_multiplier / 100.0 * sp[c].count);
813 }
814 buffer_json_array_close(wb);
815 }
@@ -1384,6 +1384,11 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) {
1384 query_target_summary_labels_v12(wb, qt, "labels", true, &label_key_totals, &label_key_value_totals);
1385 query_target_summary_alerts_v2(wb, qt, "alerts");
1386 }
1387 + if(query_target_aggregatable(qt)) {
1388 + buffer_json_member_add_object(wb, "globals");
1389 + query_target_points_statistics(wb, qt, &qt->query_points);
1390 + buffer_json_object_close(wb); // globals
1391 + }
1392 buffer_json_object_close(wb); // summary
1393
1394 buffer_json_member_add_object(wb, "totals");
web/api/health/health_cmdapi.c
+2 -2
@@ -139,10 +139,10 @@ int web_client_api_request_v1_mgmt_health(RRDHOST *host, struct web_client *w, c
139 ret = HTTP_RESP_FORBIDDEN;
140 } else {
141 while (url) {
142 - char *value = mystrsep(&url, "&");
142 + char *value = strsep_skip_consecutive_separators(&url, "&");
143 if (!value || !*value) continue;
144
145 - char *key = mystrsep(&value, "=");
145 + char *key = strsep_skip_consecutive_separators(&value, "=");
146 if (!key || !*key) continue;
147 if (!value || !*value) continue;
148
web/api/queries/query.c
+3 -2
@@ -798,7 +798,7 @@ RRDR_GROUP_BY group_by_parse(char *s) {
798 RRDR_GROUP_BY group_by = RRDR_GROUP_BY_NONE;
799
800 while(s) {
801 - char *key = mystrsep(&s, ",| ");
801 + char *key = strsep_skip_consecutive_separators(&s, ",| ");
802 if (!key || !*key) continue;
803
804 if (strcmp(key, "selected") == 0)
@@ -2720,6 +2720,7 @@ struct rrdr_group_by_entry {
2720 };
2721
2722 static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
2723 + RRDR *r_tmp = NULL;
2724 RRDR_OPTIONS options = qt->window.options;
2725
2726 if(qt->request.version < 2) {
@@ -3013,7 +3014,7 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {
3014 if(!first_r || !last_r)
3015 goto cleanup;
3016
3016 - RRDR *r_tmp = rrdr_create(owa, qt, 1, qt->window.points);
3017 + r_tmp = rrdr_create(owa, qt, 1, qt->window.points);
3018 if (!r_tmp) {
3019 internal_error(true,
3020 "QUERY: cannot create group by temporary RRDR for %s, after=%ld, before=%ld, dimensions=%d, points=%zu",
web/api/web_api.c
+19 -10
@@ -2,27 +2,32 @@
2
3 #include "web_api.h"
4
5 -int web_client_api_request_vX(RRDHOST *host, struct web_client *w, char *url, struct web_api_command *api_commands) {
6 - if(unlikely(!url || !*url)) {
5 +int web_client_api_request_vX(RRDHOST *host, struct web_client *w, char *url_path_endpoint, struct web_api_command *api_commands) {
6 + if(unlikely(!url_path_endpoint || !*url_path_endpoint)) {
7 buffer_flush(w->response.data);
8 buffer_sprintf(w->response.data, "Which API command?");
9 return HTTP_RESP_BAD_REQUEST;
10 }
11
12 - uint32_t hash = simple_hash(url);
12 + uint32_t hash = simple_hash(url_path_endpoint);
13
14 for(int i = 0; api_commands[i].command ; i++) {
15 - if(unlikely(hash == api_commands[i].hash && !strcmp(url, api_commands[i].command))) {
15 + if(unlikely(hash == api_commands[i].hash && !strcmp(url_path_endpoint, api_commands[i].command))) {
16 if(unlikely(api_commands[i].acl != WEB_CLIENT_ACL_NOCHECK) && !(w->acl & api_commands[i].acl))
17 return web_client_permission_denied(w);
18
19 - return api_commands[i].callback(host, w, (w->decoded_query_string + 1));
19 + char *query_string = (char *)buffer_tostring(w->url_query_string_decoded);
20 +
21 + if(*query_string == '?')
22 + query_string = &query_string[1];
23 +
24 + return api_commands[i].callback(host, w, query_string);
25 }
26 }
27
28 buffer_flush(w->response.data);
29 buffer_strcat(w->response.data, "Unsupported API command: ");
25 - buffer_strcat_htmlescape(w->response.data, url);
30 + buffer_strcat_htmlescape(w->response.data, url_path_endpoint);
31 return HTTP_RESP_NOT_FOUND;
32 }
33
@@ -30,7 +35,7 @@ RRDCONTEXT_TO_JSON_OPTIONS rrdcontext_to_json_parse_options(char *o) {
35 RRDCONTEXT_TO_JSON_OPTIONS options = RRDCONTEXT_OPTION_NONE;
36 char *tok;
37
33 - while(o && *o && (tok = mystrsep(&o, ", |"))) {
38 + while(o && *o && (tok = strsep_skip_consecutive_separators(&o, ", |"))) {
39 if(!*tok) continue;
40
41 if(!strcmp(tok, "full") || !strcmp(tok, "all"))
@@ -78,11 +83,11 @@ int web_client_api_request_weights(RRDHOST *host, struct web_client *w, char *ur
83 };
84
85 while (url) {
81 - char *value = mystrsep(&url, "&");
86 + char *value = strsep_skip_consecutive_separators(&url, "&");
87 if (!value || !*value)
88 continue;
89
85 - char *name = mystrsep(&value, "=");
90 + char *name = strsep_skip_consecutive_separators(&value, "=");
91 if (!name || !*name)
92 continue;
93 if (!value || !*value)
@@ -203,5 +208,9 @@ int web_client_api_request_weights(RRDHOST *host, struct web_client *w, char *ur
208
209 bool web_client_interrupt_callback(void *data) {
210 struct web_client *w = data;
211 +
212 + if(w->interrupt.callback)
213 + return w->interrupt.callback(w, w->interrupt.callback_data);
214 +
215 return sock_has_output_error(w->ofd);
207 -}
\ No newline at end of file
216 +}
web/api/web_api.h
+1 -1
@@ -18,7 +18,7 @@ struct web_api_command {
18
19 struct web_client;
20
21 -int web_client_api_request_vX(RRDHOST *host, struct web_client *w, char *url, struct web_api_command *api_commands);
21 +int web_client_api_request_vX(RRDHOST *host, struct web_client *w, char *url_path_endpoint, struct web_api_command *api_commands);
22
23 static inline void fix_google_param(char *s) {
24 if(unlikely(!s || !*s)) return;
web/api/web_api_v1.c
+44 -51
@@ -178,7 +178,7 @@ inline RRDR_OPTIONS web_client_api_request_v1_data_options(char *o) {
178 RRDR_OPTIONS ret = 0x00000000;
179 char *tok;
180
181 - while(o && *o && (tok = mystrsep(&o, ", |"))) {
181 + while(o && *o && (tok = strsep_skip_consecutive_separators(&o, ", |"))) {
182 if(!*tok) continue;
183
184 uint32_t hash = simple_hash(tok);
@@ -262,7 +262,7 @@ inline uint32_t web_client_api_request_v1_data_google_format(char *name) {
262 int web_client_api_request_v1_alarms_select (char *url) {
263 int all = 0;
264 while(url) {
265 - char *value = mystrsep(&url, "&");
265 + char *value = strsep_skip_consecutive_separators(&url, "&");
266 if (!value || !*value) continue;
267
268 if(!strcmp(value, "all") || !strcmp(value, "all=true")) all = 1;
@@ -300,10 +300,10 @@ inline int web_client_api_request_v1_alarm_count(RRDHOST *host, struct web_clien
300 buffer_sprintf(w->response.data, "[");
301
302 while(url) {
303 - char *value = mystrsep(&url, "&");
303 + char *value = strsep_skip_consecutive_separators(&url, "&");
304 if(!value || !*value) continue;
305
306 - char *name = mystrsep(&value, "=");
306 + char *name = strsep_skip_consecutive_separators(&value, "=");
307 if(!name || !*name) continue;
308 if(!value || !*value) continue;
309
@@ -341,10 +341,10 @@ inline int web_client_api_request_v1_alarm_log(RRDHOST *host, struct web_client
341 char *chart = NULL;
342
343 while(url) {
344 - char *value = mystrsep(&url, "&");
344 + char *value = strsep_skip_consecutive_separators(&url, "&");
345 if (!value || !*value) continue;
346
347 - char *name = mystrsep(&value, "=");
347 + char *name = strsep_skip_consecutive_separators(&value, "=");
348 if(!name || !*name) continue;
349 if(!value || !*value) continue;
350
@@ -365,10 +365,10 @@ inline int web_client_api_request_single_chart(RRDHOST *host, struct web_client
365 buffer_flush(w->response.data);
366
367 while(url) {
368 - char *value = mystrsep(&url, "&");
368 + char *value = strsep_skip_consecutive_separators(&url, "&");
369 if(!value || !*value) continue;
370
371 - char *name = mystrsep(&value, "=");
371 + char *name = strsep_skip_consecutive_separators(&value, "=");
372 if(!name || !*name) continue;
373 if(!value || !*value) continue;
374
@@ -419,10 +419,10 @@ static int web_client_api_request_v1_context(RRDHOST *host, struct web_client *w
419 buffer_flush(w->response.data);
420
421 while(url) {
422 - char *value = mystrsep(&url, "&");
422 + char *value = strsep_skip_consecutive_separators(&url, "&");
423 if(!value || !*value) continue;
424
425 - char *name = mystrsep(&value, "=");
425 + char *name = strsep_skip_consecutive_separators(&value, "=");
426 if(!name || !*name) continue;
427 if(!value || !*value) continue;
428
@@ -483,10 +483,10 @@ static int web_client_api_request_v1_contexts(RRDHOST *host, struct web_client *
483 buffer_flush(w->response.data);
484
485 while(url) {
486 - char *value = mystrsep(&url, "&");
486 + char *value = strsep_skip_consecutive_separators(&url, "&");
487 if(!value || !*value) continue;
488
489 - char *name = mystrsep(&value, "=");
489 + char *name = strsep_skip_consecutive_separators(&value, "=");
490 if(!name || !*name) continue;
491 if(!value || !*value) continue;
492
@@ -579,10 +579,10 @@ static inline int web_client_api_request_v1_data(RRDHOST *host, struct web_clien
579 RRDR_OPTIONS options = 0;
580
581 while(url) {
582 - char *value = mystrsep(&url, "&");
582 + char *value = strsep_skip_consecutive_separators(&url, "&");
583 if(!value || !*value) continue;
584
585 - char *name = mystrsep(&value, "=");
585 + char *name = strsep_skip_consecutive_separators(&value, "=");
586 if(!name || !*name) continue;
587 if(!value || !*value) continue;
588
@@ -628,10 +628,10 @@ static inline int web_client_api_request_v1_data(RRDHOST *host, struct web_clien
628 char *tqx_name, *tqx_value;
629
630 while(value) {
631 - tqx_value = mystrsep(&value, ";");
631 + tqx_value = strsep_skip_consecutive_separators(&value, ";");
632 if(!tqx_value || !*tqx_value) continue;
633
634 - tqx_name = mystrsep(&tqx_value, ":");
634 + tqx_name = strsep_skip_consecutive_separators(&tqx_value, ":");
635 if(!tqx_name || !*tqx_name) continue;
636 if(!tqx_value || !*tqx_value) continue;
637
@@ -722,17 +722,10 @@ static inline int web_client_api_request_v1_data(RRDHOST *host, struct web_clien
722 goto cleanup;
723 }
724
725 - if (timeout) {
726 - struct timeval now;
727 - now_realtime_timeval(&now);
728 - int inqueue = (int)dt_usec(&w->tv_in, &now) / 1000;
729 - timeout -= inqueue;
730 - if (timeout <= 0) {
731 - buffer_flush(w->response.data);
732 - buffer_strcat(w->response.data, "Query timeout exceeded");
733 - ret = HTTP_RESP_BACKEND_FETCH_FAILED;
734 - goto cleanup;
735 - }
725 + web_client_timeout_checkpoint_set(w, timeout);
726 + if(web_client_timeout_checkpoint_and_check(w, NULL)) {
727 + ret = w->response.code;
728 + goto cleanup;
729 }
730
731 if(outFileName && *outFileName) {
@@ -851,10 +844,10 @@ inline int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *
844 buffer_no_cacheable(w->response.data);
845
846 while(url) {
854 - char *value = mystrsep(&url, "&");
847 + char *value = strsep_skip_consecutive_separators(&url, "&");
848 if (!value || !*value) continue;
849
857 - char *name = mystrsep(&value, "=");
850 + char *name = strsep_skip_consecutive_separators(&value, "=");
851 if (!name || !*name) continue;
852 if (!value || !*value) continue;
853
@@ -1287,11 +1280,11 @@ int web_client_api_request_v1_function(RRDHOST *host, struct web_client *w, char
1280 const char *function = NULL;
1281
1282 while (url) {
1290 - char *value = mystrsep(&url, "&");
1283 + char *value = strsep_skip_consecutive_separators(&url, "&");
1284 if (!value || !*value)
1285 continue;
1286
1294 - char *name = mystrsep(&value, "=");
1287 + char *name = strsep_skip_consecutive_separators(&value, "=");
1288 if (!name || !*name)
1289 continue;
1290
@@ -1426,46 +1419,46 @@ int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struc
1419 #endif
1420
1421 static struct web_api_command api_commands_v1[] = {
1429 - { "info", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_info },
1430 - { "data", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_data },
1431 - { "chart", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_chart },
1432 - { "charts", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_charts },
1433 - { "context", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_context },
1434 - { "contexts", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_contexts },
1422 + { "info", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_info },
1423 + { "data", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_data },
1424 + { "chart", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_chart },
1425 + { "charts", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_charts },
1426 + { "context", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_context },
1427 + { "contexts", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_contexts },
1428
1429 // registry checks the ACL by itself, so we allow everything
1430 { "registry", 0, WEB_CLIENT_ACL_NOCHECK, web_client_api_request_v1_registry },
1431
1432 // badges can be fetched with both dashboard and badge permissions
1440 - { "badge.svg", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_BADGE | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_badge },
1433 + { "badge.svg", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC | WEB_CLIENT_ACL_BADGE, web_client_api_request_v1_badge },
1434
1442 - { "alarms", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_alarms },
1443 - { "alarms_values", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_alarms_values },
1444 - { "alarm_log", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_alarm_log },
1445 - { "alarm_variables", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_alarm_variables },
1446 - { "alarm_count", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_alarm_count },
1447 - { "allmetrics", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_allmetrics },
1435 + { "alarms", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarms },
1436 + { "alarms_values", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarms_values },
1437 + { "alarm_log", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_log },
1438 + { "alarm_variables", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_variables },
1439 + { "alarm_count", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_alarm_count },
1440 + { "allmetrics", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_allmetrics },
1441
1442 #if defined(ENABLE_ML)
1450 - { "ml_info", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_ml_info },
1443 + { "ml_info", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_ml_info },
1444 { "ml_models", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_ml_models },
1445 #endif
1446
1447 { "manage/health", 0, WEB_CLIENT_ACL_MGMT | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_mgmt_health },
1455 - { "aclk", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_aclk_state },
1456 - { "metric_correlations", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_metric_correlations },
1457 - { "weights", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_weights },
1448 + { "aclk", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_aclk_state },
1449 + { "metric_correlations", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_metric_correlations },
1450 + { "weights", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_weights },
1451
1452 { "function", 0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_function },
1453 { "functions", 0, WEB_CLIENT_ACL_ACLK | ACL_DEV_OPEN_ACCESS, web_client_api_request_v1_functions },
1454
1462 - { "dbengine_stats", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v1_dbengine_stats },
1455 + { "dbengine_stats", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v1_dbengine_stats },
1456
1457 // terminator
1458 { NULL, 0, WEB_CLIENT_ACL_NONE, NULL },
1459 };
1460
1468 -inline int web_client_api_request_v1(RRDHOST *host, struct web_client *w, char *url) {
1461 +inline int web_client_api_request_v1(RRDHOST *host, struct web_client *w, char *url_path_endpoint) {
1462 static int initialized = 0;
1463
1464 if(unlikely(initialized == 0)) {
@@ -1475,5 +1468,5 @@ inline int web_client_api_request_v1(RRDHOST *host, struct web_client *w, char *
1468 api_commands_v1[i].hash = simple_hash(api_commands_v1[i].command);
1469 }
1470
1478 - return web_client_api_request_vX(host, w, url, api_commands_v1);
1471 + return web_client_api_request_vX(host, w, url_path_endpoint, api_commands_v1);
1472 }
web/api/web_api_v1.h
+1 -1
@@ -24,7 +24,7 @@ int web_client_api_request_v1_charts(RRDHOST *host, struct web_client *w, char *
24 int web_client_api_request_v1_chart(RRDHOST *host, struct web_client *w, char *url);
25 int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *w, char *url);
26 int web_client_api_request_v1_info(RRDHOST *host, struct web_client *w, char *url);
27 -int web_client_api_request_v1(RRDHOST *host, struct web_client *w, char *url);
27 +int web_client_api_request_v1(RRDHOST *host, struct web_client *w, char *url_path_endpoint);
28 int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb);
29
30 void web_client_api_v1_init(void);
web/api/web_api_v2.c
+24 -26
@@ -1,15 +1,16 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "web_api_v2.h"
4 +#include "../rtc/webrtc.h"
5
6 static int web_client_api_request_v2_contexts_internal(RRDHOST *host __maybe_unused, struct web_client *w, char *url, CONTEXTS_V2_OPTIONS options) {
7 struct api_v2_contexts_request req = { 0 };
8
9 while(url) {
9 - char *value = mystrsep(&url, "&");
10 + char *value = strsep_skip_consecutive_separators(&url, "&");
11 if(!value || !*value) continue;
12
12 - char *name = mystrsep(&value, "=");
13 + char *name = strsep_skip_consecutive_separators(&value, "=");
14 if(!name || !*name) continue;
15 if(!value || !*value) continue;
16
@@ -110,10 +111,10 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w
111 size_t group_by_idx = 0, group_by_label_idx = 0, aggregation_idx = 0;
112
113 while(url) {
113 - char *value = mystrsep(&url, "&");
114 + char *value = strsep_skip_consecutive_separators(&url, "&");
115 if(!value || !*value) continue;
116
116 - char *name = mystrsep(&value, "=");
117 + char *name = strsep_skip_consecutive_separators(&value, "=");
118 if(!name || !*name) continue;
119 if(!value || !*value) continue;
120
@@ -161,10 +162,10 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w
162 char *tqx_name, *tqx_value;
163
164 while(value) {
164 - tqx_value = mystrsep(&value, ";");
165 + tqx_value = strsep_skip_consecutive_separators(&value, ";");
166 if(!tqx_value || !*tqx_value) continue;
167
167 - tqx_name = mystrsep(&tqx_value, ":");
168 + tqx_name = strsep_skip_consecutive_separators(&tqx_value, ":");
169 if(!tqx_name || !*tqx_name) continue;
170 if(!tqx_value || !*tqx_value) continue;
171
@@ -235,7 +236,7 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w
236 time_t before = (before_str && *before_str)?str2l(before_str):0;
237 time_t after = (after_str && *after_str) ?str2l(after_str):-600;
238 size_t points = (points_str && *points_str)?str2u(points_str):0;
238 - time_t timeout = (timeout_str && *timeout_str)?str2l(timeout_str): 0;
239 + int timeout = (timeout_str && *timeout_str)?str2i(timeout_str): 0;
240 time_t resampling_time = (resampling_time_str && *resampling_time_str) ? str2l(resampling_time_str) : 0;
241
242 QUERY_TARGET_REQUEST qtr = {
@@ -281,17 +282,10 @@ static int web_client_api_request_v2_data(RRDHOST *host __maybe_unused, struct w
282 goto cleanup;
283 }
284
284 - if (timeout) {
285 - struct timeval now;
286 - now_realtime_timeval(&now);
287 - int inqueue = (int)dt_usec(&w->tv_in, &now) / 1000;
288 - timeout -= inqueue;
289 - if (timeout <= 0) {
290 - buffer_flush(w->response.data);
291 - buffer_strcat(w->response.data, "Query timeout exceeded");
292 - ret = HTTP_RESP_BACKEND_FETCH_FAILED;
293 - goto cleanup;
294 - }
285 + web_client_timeout_checkpoint_set(w, timeout);
286 + if(web_client_timeout_checkpoint_and_check(w, NULL)) {
287 + ret = w->response.code;
288 + goto cleanup;
289 }
290
291 if(outFileName && *outFileName) {
@@ -347,20 +341,24 @@ cleanup:
341 return ret;
342 }
343
350 -
344 +static int web_client_api_request_v2_webrtc(RRDHOST *host __maybe_unused, struct web_client *w, char *url __maybe_unused) {
345 + return webrtc_new_connection(w->post_payload, w->response.data);
346 +}
347
348 static struct web_api_command api_commands_v2[] = {
353 - {"data", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v2_data},
354 - {"nodes", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v2_nodes},
355 - {"contexts", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v2_contexts},
356 - {"weights", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v2_weights},
357 - {"q", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v2_q},
349 + {"data", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_data},
350 + {"nodes", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_nodes},
351 + {"contexts", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_contexts},
352 + {"weights", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_weights},
353 + {"q", 0, WEB_CLIENT_ACL_DASHBOARD_ACLK_WEBRTC, web_client_api_request_v2_q},
354 +
355 + {"rtc_offer", 0, WEB_CLIENT_ACL_DASHBOARD | WEB_CLIENT_ACL_ACLK, web_client_api_request_v2_webrtc},
356
357 // terminator
358 {NULL, 0, WEB_CLIENT_ACL_NONE, NULL},
359 };
360
363 -inline int web_client_api_request_v2(RRDHOST *host, struct web_client *w, char *url) {
361 +inline int web_client_api_request_v2(RRDHOST *host, struct web_client *w, char *url_path_endpoint) {
362 static int initialized = 0;
363
364 if(unlikely(initialized == 0)) {
@@ -370,5 +368,5 @@ inline int web_client_api_request_v2(RRDHOST *host, struct web_client *w, char *
368 api_commands_v2[i].hash = simple_hash(api_commands_v2[i].command);
369 }
370
373 - return web_client_api_request_vX(host, w, url, api_commands_v2);
371 + return web_client_api_request_vX(host, w, url_path_endpoint, api_commands_v2);
372 }
web/api/web_api_v2.h
+1 -1
@@ -7,6 +7,6 @@
7
8 struct web_client;
9
10 -int web_client_api_request_v2(RRDHOST *host, struct web_client *w, char *url);
10 +int web_client_api_request_v2(RRDHOST *host, struct web_client *w, char *url_path_endpoint);
11
12 #endif //NETDATA_WEB_API_V2_H
web/rtc/Makefile.am new
+11
@@ -0,0 +1,11 @@
1 +# SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +AUTOMAKE_OPTIONS = subdir-objects
4 +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5 +
6 +SUBDIRS = \
7 + $(NULL)
8 +
9 +dist_noinst_DATA = \
10 + README.md \
11 + $(NULL)
web/rtc/README.md
web/rtc/webrtc.c new
+740
@@ -0,0 +1,740 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "webrtc.h"
4 +
5 +#include "../server/web_client.h"
6 +#include "../server/web_client_cache.h"
7 +
8 +#ifdef HAVE_LIBDATACHANNEL
9 +
10 +#include "rtc/rtc.h"
11 +
12 +#define WEBRTC_OUR_MAX_MESSAGE_SIZE (5 * 1024 * 1024)
13 +#define WEBRTC_DEFAULT_REMOTE_MAX_MESSAGE_SIZE (65536)
14 +#define WEBRTC_COMPRESSED_HEADER_SIZE 200
15 +
16 +static void webrtc_log(rtcLogLevel level, const char *message) {
17 + switch(level) {
18 + case RTC_LOG_NONE:
19 + break;
20 +
21 + case RTC_LOG_WARNING:
22 + case RTC_LOG_ERROR:
23 + case RTC_LOG_FATAL:
24 + error("WEBRTC: %s", message);
25 + break;
26 +
27 + case RTC_LOG_INFO:
28 + info("WEBRTC: %s", message);
29 + break;
30 +
31 + default:
32 + case RTC_LOG_DEBUG:
33 + case RTC_LOG_VERBOSE:
34 + internal_error(true, "WEBRTC: %s", message);
35 + break;
36 +
37 + }
38 +}
39 +
40 +typedef struct webrtc_datachannel {
41 + int dc;
42 + char *label;
43 + struct webrtc_connection *conn;
44 +
45 + bool open; // atomic
46 +
47 + struct {
48 + struct webrtc_datachannel *prev;
49 + struct webrtc_datachannel *next;
50 + } link;
51 +} WEBRTC_DC;
52 +
53 +typedef struct webrtc_connection {
54 + int pc;
55 + rtcConfiguration config;
56 + rtcState state;
57 + rtcGatheringState gathering_state;
58 +
59 + size_t max_message_size;
60 + size_t local_max_message_size;
61 + size_t remote_max_message_size;
62 +
63 + struct {
64 + SPINLOCK spinlock;
65 + BUFFER *wb;
66 + bool sdp;
67 + bool candidates;
68 + } response;
69 +
70 + struct {
71 + SPINLOCK spinlock;
72 + WEBRTC_DC *head;
73 + } channels;
74 +
75 + struct {
76 + struct webrtc_connection *prev;
77 + struct webrtc_connection *next;
78 + } link;
79 +} WEBRTC_CONN;
80 +
81 +#define WEBRTC_MAX_ICE_SERVERS 100
82 +
83 +static struct {
84 + bool enabled;
85 + char *iceServers[WEBRTC_MAX_ICE_SERVERS];
86 + int iceServersCount;
87 + char *proxyServer;
88 + char *bindAddress;
89 +
90 + struct {
91 + SPINLOCK spinlock;
92 + WEBRTC_CONN *head;
93 + } unsafe;
94 +
95 +} webrtc_base = {
96 +#ifdef NETDATA_INTERNAL_CHECKS
97 + .enabled = true,
98 +#else
99 + .enabled = false,
100 +#endif
101 + .iceServers = {
102 + // Format:
103 + // [("stun"|"turn"|"turns") (":"|"://")][username ":" password "@"]hostname[":" port]["?transport=" ("udp"|"tcp"|"tls")]
104 + //
105 + // Note transports TCP and TLS are only available for a TURN server with libnice as ICE backend and govern only the
106 + // TURN control connection, meaning relaying is always performed over UDP.
107 + //
108 + // If the username or password of a URI contains reserved special characters, they must be percent-encoded.
109 + // In particular, ":" must be encoded as "%3A" and "@" must by encoded as "%40".
110 +
111 + "stun://stun.l.google.com:19302",
112 + NULL, // terminator
113 + },
114 + .iceServersCount = 1,
115 + .proxyServer = NULL, // [("http"|"socks5") (":"|"://")][username ":" password "@"]hostname[" :" port]
116 + .bindAddress = NULL,
117 + .unsafe = {
118 + .spinlock = NETDATA_SPINLOCK_INITIALIZER,
119 + .head = NULL,
120 + },
121 +};
122 +
123 +static inline bool webrtc_dc_is_open(WEBRTC_DC *chan) {
124 + return __atomic_load_n(&chan->open, __ATOMIC_RELAXED);
125 +}
126 +
127 +static void webrtc_config_ice_servers(void) {
128 + BUFFER *wb = buffer_create(0, NULL);
129 +
130 + int i;
131 + for(i = 0; i < WEBRTC_MAX_ICE_SERVERS ;i++) {
132 + if (webrtc_base.iceServers[i]) {
133 + if (buffer_strlen(wb))
134 + buffer_strcat(wb, " ");
135 +
136 + internal_error(true, "WEBRTC: default ice server No %d is: '%s'", i, webrtc_base.iceServers[i]);
137 + buffer_strcat(wb, webrtc_base.iceServers[i]);
138 + }
139 + else
140 + break;
141 + }
142 + webrtc_base.iceServersCount = i;
143 + internal_error(true, "WEBRTC: there are %d default ice servers: '%s'", webrtc_base.iceServersCount, buffer_tostring(wb));
144 +
145 + char *servers = config_get(CONFIG_SECTION_WEBRTC, "ice servers", buffer_tostring(wb));
146 +
147 + webrtc_base.iceServersCount = 0;
148 + char *s = servers, *e;
149 + while(*s) {
150 + if(isspace(*s))
151 + s++;
152 +
153 + e = s;
154 + while(*e && !isspace(*e))
155 + e++;
156 +
157 + if(s != e && webrtc_base.iceServersCount < WEBRTC_MAX_ICE_SERVERS) {
158 + char old = *e;
159 + *e = '\0';
160 + internal_error(true, "WEBRTC: ice server No %d is: '%s'", webrtc_base.iceServersCount, s);
161 + webrtc_base.iceServers[webrtc_base.iceServersCount++] = strdupz(s);
162 + *e = old;
163 + }
164 +
165 + if(*e)
166 + s = e + 1;
167 + else
168 + break;
169 + }
170 +
171 + buffer_free(wb);
172 +}
173 +
174 +void webrtc_initialize() {
175 + webrtc_base.enabled = config_get_boolean(CONFIG_SECTION_WEBRTC, "enabled", webrtc_base.enabled);
176 + internal_error(true, "WEBRTC: is %s", webrtc_base.enabled ? "enabled" : "disabled");
177 +
178 + webrtc_config_ice_servers();
179 +
180 + webrtc_base.proxyServer = config_get(CONFIG_SECTION_WEBRTC, "proxy server", webrtc_base.proxyServer ? webrtc_base.proxyServer : "");
181 + if(!webrtc_base.proxyServer || !*webrtc_base.proxyServer)
182 + webrtc_base.proxyServer = NULL;
183 +
184 + internal_error(true, "WEBRTC: proxy server is: '%s'", webrtc_base.proxyServer ? webrtc_base.proxyServer : "");
185 +
186 + webrtc_base.bindAddress = config_get(CONFIG_SECTION_WEBRTC, "bind address", webrtc_base.bindAddress ? webrtc_base.bindAddress : "");
187 + if(!webrtc_base.bindAddress || !*webrtc_base.bindAddress)
188 + webrtc_base.bindAddress = NULL;
189 +
190 + internal_error(true, "WEBRTC: bind address is: '%s'", webrtc_base.bindAddress ? webrtc_base.bindAddress : "");
191 +
192 + if(!webrtc_base.enabled)
193 + return;
194 +
195 + rtcLogLevel level;
196 +#ifdef NETDATA_INTERNAL_CHECKS
197 + level = RTC_LOG_INFO;
198 +#else
199 + level = RTC_LOG_WARNING;
200 +#endif
201 +
202 + rtcInitLogger(level, webrtc_log);
203 + rtcPreload();
204 +}
205 +
206 +void webrtc_close_all_connections() {
207 + if(!webrtc_base.enabled)
208 + return;
209 +
210 + rtcCleanup();
211 +}
212 +
213 +size_t find_max_message_size_in_sdp(const char *sdp) {
214 + char *s = strstr(sdp, "a=max-message-size:");
215 + if(s)
216 + return str2ul(&s[19]);
217 +
218 + return WEBRTC_DEFAULT_REMOTE_MAX_MESSAGE_SIZE;
219 +}
220 +
221 +// ----------------------------------------------------------------------------
222 +// execute web API requests
223 +
224 +static bool web_client_stop_callback(struct web_client *w __maybe_unused, void *data) {
225 + WEBRTC_DC *chan = data;
226 + return !webrtc_dc_is_open(chan);
227 +}
228 +
229 +static size_t webrtc_send_in_chunks(WEBRTC_DC *chan, const char *data, size_t size, int code, const char *message_type, HTTP_CONTENT_TYPE content_type, size_t max_message_size, bool binary) {
230 + size_t sent_bytes = 0;
231 + size_t chunk = 0;
232 + size_t total_chunks = size / max_message_size;
233 + if(total_chunks * max_message_size < size)
234 + total_chunks++;
235 +
236 + char *send_buffer = mallocz(chan->conn->max_message_size);
237 +
238 + char *s = (char *)data;
239 + size_t remaining = size;
240 + while(remaining > 0) {
241 + chunk++;
242 +
243 + size_t message_size = MIN(remaining, max_message_size);
244 +
245 + int len = snprintfz(send_buffer, WEBRTC_COMPRESSED_HEADER_SIZE, "%d %s %zu %zu %zu %s\r\n",
246 + code,
247 + message_type,
248 + message_size,
249 + chunk,
250 + total_chunks,
251 + web_content_type_to_string(content_type)
252 + );
253 +
254 + internal_fatal((size_t)len != strlen(send_buffer), "WEBRTC compressed header line mismatch");
255 + internal_fatal(len + message_size > chan->conn->max_message_size, "WEBRTC message exceeds max message size");
256 +
257 + memcpy(&send_buffer[len], s, message_size);
258 +
259 + int total_message_size = (int)(len + message_size);
260 + sent_bytes += total_message_size;
261 +
262 + if(!binary)
263 + total_message_size = -total_message_size;
264 +
265 + if(rtcSendMessage(chan->dc, send_buffer, total_message_size) != RTC_ERR_SUCCESS)
266 + error("WEBRTC[%d],DC[%d]: failed to send LZ4 chunk %zu of %zu", chan->conn->pc, chan->dc, chunk, total_chunks);
267 + else
268 + internal_error(true, "WEBRTC[%d],DC[%d]: sent chunk %zu of %zu, size %zu (total %d)",
269 + chan->conn->pc, chan->dc, chunk, total_chunks, message_size, total_message_size);
270 +
271 + s = s + message_size;
272 + remaining -= message_size;
273 + }
274 +
275 + internal_fatal(chunk != total_chunks, "WEBRTC number of compressed chunks mismatch");
276 +
277 + freez(send_buffer);
278 + return sent_bytes;
279 +}
280 +
281 +static void webrtc_execute_api_request(WEBRTC_DC *chan, const char *request, size_t size __maybe_unused, bool binary __maybe_unused) {
282 + struct timeval tv;
283 +
284 + internal_error(true, "WEBRTC[%d],DC[%d]: got request '%s' of size %zu and type %s.",
285 + chan->conn->pc, chan->dc, request, size, binary?"binary":"text");
286 +
287 + struct web_client *w = web_client_get_from_cache();
288 + w->statistics.received_bytes = size;
289 + w->interrupt.callback = web_client_stop_callback;
290 + w->interrupt.callback_data = chan;
291 +
292 + w->acl = WEB_CLIENT_ACL_WEBRTC;
293 +
294 + char *path = (char *)request;
295 + if(strncmp(request, "POST ", 5) == 0) {
296 + w->mode = WEB_CLIENT_MODE_POST;
297 + path += 10;
298 + }
299 + else if(strncmp(request, "GET ", 4) == 0) {
300 + w->mode = WEB_CLIENT_MODE_GET;
301 + path += 4;
302 + }
303 +
304 + web_client_timeout_checkpoint_set(w, 0);
305 + web_client_decode_path_and_query_string(w, path);
306 + path = (char *)buffer_tostring(w->url_path_decoded);
307 + w->response.code = web_client_api_request_with_node_selection(localhost, w, path);
308 + web_client_timeout_checkpoint_response_ready(w, NULL);
309 +
310 + size_t sent_bytes = 0;
311 + size_t response_size = buffer_strlen(w->response.data);
312 +
313 + bool send_plain = true;
314 + int max_message_size = (int)chan->conn->max_message_size - WEBRTC_COMPRESSED_HEADER_SIZE;
315 +
316 + if(!webrtc_dc_is_open(chan)) {
317 + internal_error(true, "WEBRTC[%d],DC[%d]: ignoring API response on closed data channel.", chan->conn->pc, chan->dc);
318 + goto cleanup;
319 + }
320 + else {
321 + internal_error(true, "WEBRTC[%d],DC[%d]: prepared response with code %d, size %zu.",
322 + chan->conn->pc, chan->dc, w->response.code, response_size);
323 + }
324 +
325 +#if defined(ENABLE_COMPRESSION)
326 + int max_compressed_size = LZ4_compressBound((int)response_size);
327 + char *compressed = mallocz(max_compressed_size);
328 +
329 + int compressed_size = LZ4_compress_default(buffer_tostring(w->response.data), compressed,
330 + (int)response_size, max_compressed_size);
331 +
332 + if(compressed_size > 0) {
333 + send_plain = false;
334 + sent_bytes = webrtc_send_in_chunks(chan, compressed, compressed_size,
335 + w->response.code, "LZ4", w->response.data->content_type,
336 + max_message_size, true);
337 + }
338 + freez(compressed);
339 +#endif
340 +
341 + if(send_plain)
342 + sent_bytes = webrtc_send_in_chunks(chan, buffer_tostring(w->response.data), buffer_strlen(w->response.data),
343 + w->response.code, "PLAIN", w->response.data->content_type,
344 + max_message_size, false);
345 +
346 + w->statistics.sent_bytes = sent_bytes;
347 +
348 +cleanup:
349 + now_monotonic_high_precision_timeval(&tv);
350 + log_access("%llu: %d '[RTC]:%d:%d' '%s' (sent/all = %zu/%zu bytes %0.0f%%, prep/sent/total = %0.2f/%0.2f/%0.2f ms) %d '%s'",
351 + w->id
352 + , gettid()
353 + , chan->conn->pc, chan->dc
354 + , "DATA"
355 + , sent_bytes
356 + , response_size
357 + , response_size > sent_bytes ? -(((double)(response_size - sent_bytes) / (double)response_size) * 100.0) : ((response_size > 0) ? (((sent_bytes - response_size) / (double)response_size) * 100.0) : 0.0)
358 + , dt_usec(&w->timings.tv_ready, &w->timings.tv_in) / 1000.0
359 + , dt_usec(&tv, &w->timings.tv_ready) / 1000.0
360 + , dt_usec(&tv, &w->timings.tv_in) / 1000.0
361 + , w->response.code
362 + , strip_control_characters((char *)buffer_tostring(w->url_as_received))
363 + );
364 + web_client_release_to_cache(w);
365 +}
366 +
367 +// ----------------------------------------------------------------------------
368 +// webrtc data channel
369 +
370 +static void myOpenCallback(int id, void *user_ptr) {
371 + webrtc_set_thread_name();
372 +
373 + WEBRTC_DC *chan = user_ptr;
374 + internal_fatal(chan->dc != id, "WEBRTC[%d],DC[%d]: dc mismatch, expected %d, got %d", chan->conn->pc, chan->dc, chan->dc, id);
375 +
376 + log_access("WEBRTC[%d],DC[%d]: %d DATA CHANNEL '%s' OPEN", chan->conn->pc, chan->dc, gettid(), chan->label);
377 + internal_error(true, "WEBRTC[%d],DC[%d]: data channel opened.", chan->conn->pc, chan->dc);
378 + chan->open = true;
379 +}
380 +
381 +static void myClosedCallback(int id, void *user_ptr) {
382 + webrtc_set_thread_name();
383 +
384 + WEBRTC_DC *chan = user_ptr;
385 + internal_fatal(chan->dc != id, "WEBRTC[%d],DC[%d]: dc mismatch, expected %d, got %d", chan->conn->pc, chan->dc, chan->dc, id);
386 +
387 + __atomic_store_n(&chan->open, false, __ATOMIC_RELAXED);
388 + internal_error(true, "WEBRTC[%d],DC[%d]: data channel closed.", chan->conn->pc, chan->dc);
389 +
390 + netdata_spinlock_lock(&chan->conn->channels.spinlock);
391 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(chan->conn->channels.head, chan, link.prev, link.next);
392 + netdata_spinlock_unlock(&chan->conn->channels.spinlock);
393 +
394 + log_access("WEBRTC[%d],DC[%d]: %d DATA CHANNEL '%s' CLOSED", chan->conn->pc, chan->dc, gettid(), chan->label);
395 +
396 + freez(chan->label);
397 + freez(chan);
398 +}
399 +
400 +static void myErrorCallback(int id, const char *error, void *user_ptr) {
401 + webrtc_set_thread_name();
402 +
403 + WEBRTC_DC *chan = user_ptr;
404 + internal_fatal(chan->dc != id, "WEBRTC[%d],DC[%d]: dc mismatch, expected %d, got %d", chan->conn->pc, chan->dc, chan->dc, id);
405 +
406 + error("WEBRTC[%d],DC[%d]: ERROR: '%s'", chan->conn->pc, chan->dc, error);
407 +}
408 +
409 +static void myMessageCallback(int id, const char *message, int size, void *user_ptr) {
410 + webrtc_set_thread_name();
411 +
412 + WEBRTC_DC *chan = user_ptr;
413 + internal_fatal(chan->dc != id, "WEBRTC[%d],DC[%d]: dc mismatch, expected %d, got %d", chan->conn->pc, chan->dc, chan->dc, id);
414 + internal_fatal(!webrtc_dc_is_open(chan), "WEBRTC[%d],DC[%d]: received message on closed channel", chan->conn->pc, chan->dc);
415 +
416 + bool binary = (size >= 0);
417 + if(size < 0)
418 + size = -size;
419 +
420 + webrtc_execute_api_request(chan, message, size, binary);
421 +}
422 +
423 +//#define WEBRTC_MAX_REQUEST_SIZE 65536
424 +//
425 +//static void myAvailableCallback(int id, void *user_ptr) {
426 +// webrtc_set_thread_name();
427 +//
428 +// WEBRTC_DC *chan = user_ptr;
429 +// internal_fatal(chan->dc != id, "WEBRTC[%d],DC[%d]: dc mismatch, expected %d, got %d", chan->conn->pc, chan->dc, chan->dc, id);
430 +//
431 +// internal_fatal(!chan->open, "WEBRTC[%d],DC[%d]: received message on closed channel", chan->conn->pc, chan->dc);
432 +//
433 +// int size = WEBRTC_MAX_REQUEST_SIZE;
434 +// char buffer[WEBRTC_MAX_REQUEST_SIZE];
435 +// while(rtcReceiveMessage(id, buffer, &size) == RTC_ERR_SUCCESS) {
436 +// bool binary = (size >= 0);
437 +// if(size < 0)
438 +// size = -size;
439 +//
440 +// webrtc_execute_api_request(chan, message, size, binary);
441 +// }
442 +//}
443 +
444 +static void myDataChannelCallback(int pc, int dc, void *user_ptr) {
445 + webrtc_set_thread_name();
446 +
447 + WEBRTC_CONN *conn = user_ptr;
448 + internal_fatal(conn->pc != pc, "WEBRTC[%d]: pc mismatch, expected %d, got %d", conn->pc, conn->pc, pc);
449 +
450 + WEBRTC_DC *chan = callocz(1, sizeof(WEBRTC_DC));
451 + chan->dc = dc;
452 + chan->conn = conn;
453 +
454 + netdata_spinlock_lock(&conn->channels.spinlock);
455 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(conn->channels.head, chan, link.prev, link.next);
456 + netdata_spinlock_unlock(&conn->channels.spinlock);
457 +
458 + rtcSetUserPointer(dc, chan);
459 +
460 + char label[1024 + 1];
461 + rtcGetDataChannelLabel(dc, label, 1024);
462 + label[1024] = '\0';
463 +
464 + chan->label = strdupz(label);
465 +
466 + if(rtcSetOpenCallback(dc, myOpenCallback) != RTC_ERR_SUCCESS)
467 + error("WEBRTC[%d],DC[%d]: rtcSetOpenCallback() failed.", conn->pc, chan->dc);
468 +
469 + if(rtcSetClosedCallback(dc, myClosedCallback) != RTC_ERR_SUCCESS)
470 + error("WEBRTC[%d],DC[%d]: rtcSetClosedCallback() failed.", conn->pc, chan->dc);
471 +
472 + if(rtcSetErrorCallback(dc, myErrorCallback) != RTC_ERR_SUCCESS)
473 + error("WEBRTC[%d],DC[%d]: rtcSetErrorCallback() failed.", conn->pc, chan->dc);
474 +
475 + if(rtcSetMessageCallback(dc, myMessageCallback) != RTC_ERR_SUCCESS)
476 + error("WEBRTC[%d],DC[%d]: rtcSetMessageCallback() failed.", conn->pc, chan->dc);
477 +
478 +// if(rtcSetAvailableCallback(dc, myAvailableCallback) != RTC_ERR_SUCCESS)
479 +// error("WEBRTC[%d],DC[%d]: rtcSetAvailableCallback() failed.", conn->pc, chan->dc);
480 +
481 + internal_error(true, "WEBRTC[%d],DC[%d]: new data channel with label '%s'", chan->conn->pc, chan->dc, chan->label);
482 +}
483 +
484 +// ----------------------------------------------------------------------------
485 +// webrtc connection
486 +
487 +static inline void webrtc_destroy_connection_unsafe(WEBRTC_CONN *conn) {
488 + if(conn->state == RTC_CLOSED) {
489 + netdata_spinlock_lock(&conn->channels.spinlock);
490 + WEBRTC_DC *chan = conn->channels.head;
491 + netdata_spinlock_unlock(&conn->channels.spinlock);
492 +
493 + if(!chan) {
494 + internal_error(true, "WEBRTC[%d]: destroying connection", conn->pc);
495 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(webrtc_base.unsafe.head, conn, link.prev, link.next);
496 + freez(conn);
497 + }
498 + else {
499 + internal_error(true, "WEBRTC[%d]: not destroying closed connection because it has data channels running", conn->pc);
500 + }
501 + }
502 +}
503 +
504 +static void cleanupConnections() {
505 + netdata_spinlock_lock(&webrtc_base.unsafe.spinlock);
506 + WEBRTC_CONN *conn = webrtc_base.unsafe.head;
507 + while(conn) {
508 + WEBRTC_CONN *conn_next = conn->link.next;
509 + webrtc_destroy_connection_unsafe(conn);
510 + conn = conn_next;
511 + }
512 + netdata_spinlock_unlock(&webrtc_base.unsafe.spinlock);
513 +}
514 +
515 +static WEBRTC_CONN *webrtc_create_connection(void) {
516 + WEBRTC_CONN *conn = callocz(1, sizeof(WEBRTC_CONN));
517 +
518 + netdata_spinlock_init(&conn->response.spinlock);
519 + netdata_spinlock_init(&conn->channels.spinlock);
520 +
521 + netdata_spinlock_lock(&webrtc_base.unsafe.spinlock);
522 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(webrtc_base.unsafe.head, conn, link.prev, link.next);
523 + netdata_spinlock_unlock(&webrtc_base.unsafe.spinlock);
524 + return conn;
525 +}
526 +
527 +static void myDescriptionCallback(int pc __maybe_unused, const char *sdp, const char *type, void *user_ptr) {
528 + webrtc_set_thread_name();
529 +
530 + WEBRTC_CONN *conn = user_ptr;
531 + internal_fatal(conn->pc != pc, "WEBRTC[%d]: pc mismatch, expected %d, got %d", conn->pc, conn->pc, pc);
532 +
533 + internal_error(true, "WEBRTC[%d]: local description type '%s': %s", conn->pc, type, sdp);
534 + netdata_spinlock_lock(&conn->response.spinlock);
535 + if(!conn->response.candidates) {
536 + buffer_json_member_add_string(conn->response.wb, "sdp", sdp);
537 + buffer_json_member_add_string(conn->response.wb, "type", type);
538 + conn->response.sdp = true;
539 + }
540 + netdata_spinlock_unlock(&conn->response.spinlock);
541 +
542 + conn->local_max_message_size = find_max_message_size_in_sdp(sdp);
543 +}
544 +
545 +static void myCandidateCallback(int pc __maybe_unused, const char *cand, const char *mid __maybe_unused, void *user_ptr) {
546 + webrtc_set_thread_name();
547 +
548 + WEBRTC_CONN *conn = user_ptr;
549 + internal_fatal(conn->pc != pc, "WEBRTC[%d]: pc mismatch, expected %d, got %d", conn->pc, conn->pc, pc);
550 +
551 + netdata_spinlock_lock(&conn->response.spinlock);
552 + if(!conn->response.candidates) {
553 + buffer_json_member_add_array(conn->response.wb, "candidates");
554 + conn->response.candidates = true;
555 + }
556 +
557 + internal_error(true, "WEBRTC[%d]: local candidate '%s', mid '%s'", conn->pc, cand, mid);
558 + buffer_json_add_array_item_string(conn->response.wb, cand);
559 + netdata_spinlock_unlock(&conn->response.spinlock);
560 +}
561 +
562 +static void myStateChangeCallback(int pc __maybe_unused, rtcState state, void *user_ptr) {
563 + webrtc_set_thread_name();
564 +
565 + WEBRTC_CONN *conn = user_ptr;
566 + internal_fatal(conn->pc != pc, "WEBRTC[%d]: pc mismatch, expected %d, got %d", conn->pc, conn->pc, pc);
567 +
568 + conn->state = state;
569 +
570 + switch(state) {
571 + case RTC_NEW:
572 + internal_error(true, "WEBRTC[%d]: new connection...", conn->pc);
573 + break;
574 +
575 + case RTC_CONNECTING:
576 + log_access("WEBRTC[%d]: %d CONNECTING", conn->pc, gettid());
577 + internal_error(true, "WEBRTC[%d]: connecting...", conn->pc);
578 + break;
579 +
580 + case RTC_CONNECTED:
581 + log_access("WEBRTC[%d]: %d CONNECTED", conn->pc, gettid());
582 + internal_error(true, "WEBRTC[%d]: connected!", conn->pc);
583 + break;
584 +
585 + case RTC_DISCONNECTED:
586 + log_access("WEBRTC[%d]: %d DISCONNECTED", conn->pc, gettid());
587 + internal_error(true, "WEBRTC[%d]: disconnected.", conn->pc);
588 + break;
589 +
590 + case RTC_FAILED:
591 + log_access("WEBRTC[%d]: %d CONNECTION FAILED", conn->pc, gettid());
592 + internal_error(true, "WEBRTC[%d]: failed.", conn->pc);
593 + break;
594 +
595 + case RTC_CLOSED:
596 + log_access("WEBRTC[%d]: %d CONNECTION CLOSED", conn->pc, gettid());
597 + internal_error(true, "WEBRTC[%d]: closed.", conn->pc);
598 + netdata_spinlock_lock(&webrtc_base.unsafe.spinlock);
599 + webrtc_destroy_connection_unsafe(conn);
600 + netdata_spinlock_unlock(&webrtc_base.unsafe.spinlock);
601 + break;
602 + }
603 +}
604 +
605 +static void myGatheringStateCallback(int pc __maybe_unused, rtcGatheringState state, void *user_ptr) {
606 + webrtc_set_thread_name();
607 +
608 + WEBRTC_CONN *conn = user_ptr;
609 + internal_fatal(conn->pc != pc, "WEBRTC[%d]: pc mismatch, expected %d, got %d", conn->pc, conn->pc, pc);
610 +
611 + conn->gathering_state = state;
612 +
613 + switch(state) {
614 + case RTC_GATHERING_NEW:
615 + internal_error(true, "WEBRTC[%d]: gathering...", conn->pc);
616 + break;
617 +
618 + case RTC_GATHERING_INPROGRESS:
619 + internal_error(true, "WEBRTC[%d]: gathering in progress...", conn->pc);
620 + break;
621 +
622 + case RTC_GATHERING_COMPLETE:
623 + internal_error(true, "WEBRTC[%d]: gathering complete!", conn->pc);
624 + break;
625 + }
626 +}
627 +
628 +int webrtc_new_connection(const char *sdp, BUFFER *wb) {
629 + if(unlikely(!webrtc_base.enabled)) {
630 + buffer_flush(wb);
631 + buffer_strcat(wb, "WebRTC is not enabled on this agent.");
632 + wb->content_type = CT_TEXT_PLAIN;
633 + return HTTP_RESP_BAD_REQUEST;
634 + }
635 +
636 + cleanupConnections();
637 +
638 + if(unlikely(!sdp || !*sdp)) {
639 + buffer_flush(wb);
640 + buffer_strcat(wb, "No SDP message posted with the request");
641 + wb->content_type = CT_TEXT_PLAIN;
642 + return HTTP_RESP_BAD_REQUEST;
643 + }
644 +
645 + buffer_flush(wb);
646 + buffer_json_initialize(wb, "\"", "\"", 0, true, false);
647 + wb->content_type = CT_APPLICATION_JSON;
648 +
649 + WEBRTC_CONN *conn = webrtc_create_connection();
650 + conn->response.wb = wb;
651 + conn->max_message_size = WEBRTC_DEFAULT_REMOTE_MAX_MESSAGE_SIZE;
652 + conn->local_max_message_size = WEBRTC_OUR_MAX_MESSAGE_SIZE;
653 + conn->remote_max_message_size = find_max_message_size_in_sdp(sdp);
654 +
655 + conn->config.iceServers = (const char **)webrtc_base.iceServers;
656 + conn->config.iceServersCount = webrtc_base.iceServersCount;
657 + conn->config.proxyServer = webrtc_base.proxyServer;
658 + conn->config.bindAddress = webrtc_base.bindAddress;
659 + conn->config.certificateType = RTC_CERTIFICATE_DEFAULT;
660 + conn->config.iceTransportPolicy = RTC_TRANSPORT_POLICY_ALL;
661 + conn->config.enableIceTcp = true; // libnice only
662 + conn->config.enableIceUdpMux = true; // libjuice only
663 + conn->config.disableAutoNegotiation = false;
664 + conn->config.forceMediaTransport = false;
665 + conn->config.portRangeBegin = 0; // 0 means automatic
666 + conn->config.portRangeEnd = 0; // 0 means automatic
667 + conn->config.mtu = 0; // <= 0 means automatic
668 + conn->config.maxMessageSize = WEBRTC_OUR_MAX_MESSAGE_SIZE; // <= 0 means default
669 +
670 + conn->pc = rtcCreatePeerConnection(&conn->config);
671 + rtcSetUserPointer(conn->pc, conn);
672 +
673 + if(rtcSetLocalDescriptionCallback(conn->pc, myDescriptionCallback) != RTC_ERR_SUCCESS)
674 + error("WEBRTC[%d]: rtcSetLocalDescriptionCallback() failed", conn->pc);
675 +
676 + if(rtcSetLocalCandidateCallback(conn->pc, myCandidateCallback) != RTC_ERR_SUCCESS)
677 + error("WEBRTC[%d]: rtcSetLocalCandidateCallback() failed", conn->pc);
678 +
679 + if(rtcSetStateChangeCallback(conn->pc, myStateChangeCallback) != RTC_ERR_SUCCESS)
680 + error("WEBRTC[%d]: rtcSetStateChangeCallback() failed", conn->pc);
681 +
682 + if(rtcSetGatheringStateChangeCallback(conn->pc, myGatheringStateCallback) != RTC_ERR_SUCCESS)
683 + error("WEBRTC[%d]: rtcSetGatheringStateChangeCallback() failed", conn->pc);
684 +
685 + if(rtcSetDataChannelCallback(conn->pc, myDataChannelCallback) != RTC_ERR_SUCCESS)
686 + error("WEBRTC[%d]: rtcSetDataChannelCallback() failed", conn->pc);
687 +
688 + // initialize the handshake
689 + internal_error(true, "WEBRTC[%d]: setting remote sdp: %s", conn->pc, sdp);
690 + if(rtcSetRemoteDescription(conn->pc, sdp, "offer") != RTC_ERR_SUCCESS)
691 + error("WEBRTC[%d]: rtcSetRemoteDescription() failed", conn->pc);
692 +
693 + // initiate the handshake process
694 + if(conn->config.disableAutoNegotiation) {
695 + if(rtcSetLocalDescription(conn->pc, NULL) != RTC_ERR_SUCCESS)
696 + error("WEBRTC[%d]: rtcSetLocalDescription() failed", conn->pc);
697 + }
698 +
699 + bool logged = false;
700 + while(conn->gathering_state != RTC_GATHERING_COMPLETE) {
701 + if(!logged) {
702 + logged = true;
703 + internal_error(true, "WEBRTC[%d]: Waiting for gathering to complete", conn->pc);
704 + }
705 + usleep(1000);
706 + }
707 +
708 + if(logged)
709 + internal_error(true, "WEBRTC[%d]: Gathering finished, our answer is ready", conn->pc);
710 +
711 + internal_fatal(!conn->response.sdp, "WEBRTC[%d]: response does not have an SDP: %s", conn->pc, buffer_tostring(conn->response.wb));
712 + internal_fatal(!conn->response.candidates, "WEBRTC[%d]: response does not have candidates: %s", conn->pc, buffer_tostring(conn->response.wb));
713 +
714 + conn->max_message_size = MIN(conn->local_max_message_size, conn->remote_max_message_size);
715 + if(conn->max_message_size < WEBRTC_COMPRESSED_HEADER_SIZE)
716 + conn->max_message_size = WEBRTC_COMPRESSED_HEADER_SIZE;
717 +
718 + buffer_json_finalize(wb);
719 +
720 + return HTTP_RESP_OK;
721 +}
722 +
723 +#else // ! HAVE_LIBDATACHANNEL
724 +
725 +void webrtc_initialize() {
726 + ;
727 +}
728 +
729 +int webrtc_new_connection(const char *sdp __maybe_unused, BUFFER *wb) {
730 + buffer_flush(wb);
731 + buffer_strcat(wb, "WEBRTC is not available on this server");
732 + wb->content_type = CT_TEXT_PLAIN;
733 + return HTTP_RESP_BAD_REQUEST;
734 +}
735 +
736 +void webrtc_close_all_connections() {
737 + ;
738 +}
739 +
740 +#endif // ! HAVE_LIBDATACHANNEL
web/rtc/webrtc.h new
+12
@@ -0,0 +1,12 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_WEBRTC_H
4 +#define NETDATA_WEBRTC_H
5 +
6 +#include "../../libnetdata/libnetdata.h"
7 +
8 +int webrtc_new_connection(const char *sdp, BUFFER *wb);
9 +void webrtc_close_all_connections();
10 +void webrtc_initialize();
11 +
12 +#endif //NETDATA_WEBRTC_H
web/server/static/static-threaded.c
+22 -7
@@ -28,7 +28,7 @@ long web_client_streaming_rate_t = 0L;
28 static struct web_client *web_client_create_on_fd(POLLINFO *pi) {
29 struct web_client *w;
30
31 - w = web_client_get_from_cache_or_allocate();
31 + w = web_client_get_from_cache();
32 w->ifd = w->ofd = pi->fd;
33
34 strncpyz(w->client_ip, pi->client_ip, sizeof(w->client_ip) - 1);
@@ -39,7 +39,19 @@ static struct web_client *web_client_create_on_fd(POLLINFO *pi) {
39 if(unlikely(!*w->client_port)) strcpy(w->client_port, "-");
40 w->port_acl = pi->port_acl;
41
42 - web_client_initialize_connection(w);
42 + int flag = 1;
43 + if(unlikely(web_client_check_tcp(w) && setsockopt(w->ifd, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int)) != 0))
44 + debug(D_WEB_CLIENT, "%llu: failed to enable TCP_NODELAY on socket fd %d.", w->id, w->ifd);
45 +
46 + flag = 1;
47 + if(unlikely(setsockopt(w->ifd, SOL_SOCKET, SO_KEEPALIVE, (char *) &flag, sizeof(int)) != 0))
48 + debug(D_WEB_CLIENT, "%llu: failed to enable SO_KEEPALIVE on socket fd %d.", w->id, w->ifd);
49 +
50 + web_client_update_acl_matches(w);
51 + web_client_enable_wait_receive(w);
52 +
53 + web_server_log_connection(w, "CONNECTED");
54 +
55 w->pollinfo_slot = pi->slot;
56 return(w);
57 }
@@ -107,7 +119,10 @@ static void web_server_file_del_callback(POLLINFO *pi) {
119
120 if(unlikely(!w->pollinfo_slot)) {
121 debug(D_WEB_CLIENT, "%llu: CROSS WEB CLIENT CLEANUP (iFD %d, oFD %d)", w->id, pi->fd, w->ofd);
110 - web_client_release(w);
122 + web_server_log_connection(w, "DISCONNECTED");
123 + web_client_request_done(w);
124 + web_client_release_to_cache(w);
125 + global_statistics_web_client_disconnected();
126 }
127
128 worker_is_idle();
@@ -278,7 +293,10 @@ static void web_server_del_callback(POLLINFO *pi) {
293 pi->flags |= POLLINFO_FLAG_DONT_CLOSE;
294
295 debug(D_WEB_CLIENT, "%llu: CLOSING CLIENT FD %d", w->id, pi->fd);
281 - web_client_release(w);
296 + web_server_log_connection(w, "DISCONNECTED");
297 + web_client_request_done(w);
298 + web_client_release_to_cache(w);
299 + global_statistics_web_client_disconnected();
300 }
301
302 worker_is_idle();
@@ -402,9 +420,6 @@ cleanup:
420 static void socket_listen_main_static_threaded_worker_cleanup(void *ptr) {
421 worker_private = (struct web_server_static_threaded_worker *)ptr;
422
405 - info("freeing local web clients cache...");
406 - web_client_cache_destroy();
407 -
423 info("stopped after %zu connects, %zu disconnects (max concurrent %zu), %zu receptions and %zu sends",
424 worker_private->connected,
425 worker_private->disconnected,
web/server/web_client.c
+473 -249
@@ -20,19 +20,17 @@ inline int web_client_permission_denied(struct web_client *w) {
20 return HTTP_RESP_FORBIDDEN;
21 }
22
23 -static inline int web_client_crock_socket(struct web_client *w) {
23 +static inline int web_client_crock_socket(struct web_client *w __maybe_unused) {
24 #ifdef TCP_CORK
25 if(likely(web_client_is_corkable(w) && !w->tcp_cork && w->ofd != -1)) {
26 - w->tcp_cork = 1;
26 + w->tcp_cork = true;
27 if(unlikely(setsockopt(w->ofd, IPPROTO_TCP, TCP_CORK, (char *) &w->tcp_cork, sizeof(int)) != 0)) {
28 error("%llu: failed to enable TCP_CORK on socket.", w->id);
29
30 - w->tcp_cork = 0;
30 + w->tcp_cork = false;
31 return -1;
32 }
33 }
34 -#else
35 - (void)w;
34 #endif /* TCP_CORK */
35
36 return 0;
@@ -50,20 +48,18 @@ static inline void web_client_enable_wait_from_ssl(struct web_client *w, int byt
48 }
49 }
50
53 -static inline int web_client_uncrock_socket(struct web_client *w) {
51 +static inline int web_client_uncrock_socket(struct web_client *w __maybe_unused) {
52 #ifdef TCP_CORK
53 if(likely(w->tcp_cork && w->ofd != -1)) {
56 - w->tcp_cork = 0;
54 if(unlikely(setsockopt(w->ofd, IPPROTO_TCP, TCP_CORK, (char *) &w->tcp_cork, sizeof(int)) != 0)) {
55 error("%llu: failed to disable TCP_CORK on socket.", w->id);
59 - w->tcp_cork = 1;
56 + w->tcp_cork = true;
57 return -1;
58 }
59 }
63 -#else
64 - (void)w;
60 #endif /* TCP_CORK */
61
62 + w->tcp_cork = false;
63 return 0;
64 }
65
@@ -79,14 +75,96 @@ char *strip_control_characters(char *url) {
75 return url;
76 }
77
78 +static void web_client_reset_allocations(struct web_client *w, bool free_all) {
79 +
80 + if(free_all) {
81 + // the web client is to be destroyed
82 +
83 + buffer_free(w->url_as_received);
84 + w->url_as_received = NULL;
85 +
86 + buffer_free(w->url_path_decoded);
87 + w->url_path_decoded = NULL;
88 +
89 + buffer_free(w->url_query_string_decoded);
90 + w->url_query_string_decoded = NULL;
91 +
92 + buffer_free(w->response.header_output);
93 + w->response.header_output = NULL;
94 +
95 + buffer_free(w->response.header);
96 + w->response.header = NULL;
97 +
98 + buffer_free(w->response.data);
99 + w->response.data = NULL;
100 +
101 + freez(w->post_payload);
102 + w->post_payload = NULL;
103 + w->post_payload_size = 0;
104 +
105 +#ifdef ENABLE_HTTPS
106 + if ((!web_client_check_unix(w)) && (netdata_ssl_srv_ctx)) {
107 + if (w->ssl.conn) {
108 + SSL_free(w->ssl.conn);
109 + w->ssl.conn = NULL;
110 + }
111 + }
112 +#endif
113 + }
114 + else {
115 + // the web client is to be re-used
116 +
117 + buffer_reset(w->url_as_received);
118 + buffer_reset(w->url_path_decoded);
119 + buffer_reset(w->url_query_string_decoded);
120 +
121 + buffer_reset(w->response.header_output);
122 + buffer_reset(w->response.header);
123 + buffer_reset(w->response.data);
124 +
125 + // leave w->post_payload
126 + // leave w->ssl
127 + }
128 +
129 + freez(w->server_host);
130 + w->server_host = NULL;
131 +
132 + freez(w->forwarded_host);
133 + w->forwarded_host = NULL;
134 +
135 + freez(w->origin);
136 + w->origin = NULL;
137 +
138 + freez(w->user_agent);
139 + w->user_agent = NULL;
140 +
141 + freez(w->auth_bearer_token);
142 + w->auth_bearer_token = NULL;
143 +
144 + // if we had enabled compression, release it
145 +#ifdef NETDATA_WITH_ZLIB
146 + if(w->response.zinitialized) {
147 + deflateEnd(&w->response.zstream);
148 + w->response.zsent = 0;
149 + w->response.zhave = 0;
150 + w->response.zstream.avail_in = 0;
151 + w->response.zstream.avail_out = 0;
152 + w->response.zstream.total_in = 0;
153 + w->response.zstream.total_out = 0;
154 + w->response.zinitialized = false;
155 + w->flags &= ~WEB_CLIENT_CHUNKED_TRANSFER;
156 + }
157 +#endif // NETDATA_WITH_ZLIB
158 +}
159 +
160 void web_client_request_done(struct web_client *w) {
161 web_client_uncrock_socket(w);
162
163 debug(D_WEB_CLIENT, "%llu: Resetting client.", w->id);
164
87 - if(likely(w->last_url[0])) {
165 + if(likely(buffer_strlen(w->url_as_received))) {
166 struct timeval tv;
89 - now_realtime_timeval(&tv);
167 + now_monotonic_high_precision_timeval(&tv);
168
169 size_t size = (w->mode == WEB_CLIENT_MODE_FILECOPY)?w->response.rlen:w->response.data->len;
170 size_t sent = size;
@@ -97,14 +175,14 @@ void web_client_request_done(struct web_client *w) {
175 // --------------------------------------------------------------------
176 // global statistics
177
100 - global_statistics_web_request_completed(dt_usec(&tv, &w->tv_in),
101 - w->stats_received_bytes,
102 - w->stats_sent_bytes,
178 + global_statistics_web_request_completed(dt_usec(&tv, &w->timings.tv_in),
179 + w->statistics.received_bytes,
180 + w->statistics.sent_bytes,
181 size,
182 sent);
183
106 - w->stats_received_bytes = 0;
107 - w->stats_sent_bytes = 0;
184 + w->statistics.received_bytes = 0;
185 + w->statistics.sent_bytes = 0;
186
187
188 // --------------------------------------------------------------------
@@ -123,7 +201,8 @@ void web_client_request_done(struct web_client *w) {
201 mode = "STREAM";
202 break;
203
126 - case WEB_CLIENT_MODE_NORMAL:
204 + case WEB_CLIENT_MODE_POST:
205 + case WEB_CLIENT_MODE_GET:
206 mode = "DATA";
207 break;
208
@@ -142,11 +221,11 @@ void web_client_request_done(struct web_client *w) {
221 , sent
222 , size
223 , -((size > 0) ? ((double)(size - sent) / (double) size * 100.0) : 0.0)
145 - , (double)dt_usec(&w->tv_ready, &w->tv_in) / 1000.0
146 - , (double)dt_usec(&tv, &w->tv_ready) / 1000.0
147 - , (double)dt_usec(&tv, &w->tv_in) / 1000.0
224 + , (double)dt_usec(&w->timings.tv_ready, &w->timings.tv_in) / 1000.0
225 + , (double)dt_usec(&tv, &w->timings.tv_ready) / 1000.0
226 + , (double)dt_usec(&tv, &w->timings.tv_in) / 1000.0
227 , w->response.code
149 - , strip_control_characters(w->last_url)
228 + , strip_control_characters((char *)buffer_tostring(w->url_as_received))
229 );
230 }
231
@@ -164,32 +243,13 @@ void web_client_request_done(struct web_client *w) {
243 }
244 }
245
167 - w->last_url[0] = '\0';
168 - w->cookie1[0] = '\0';
169 - w->cookie2[0] = '\0';
170 - w->origin[0] = '*';
171 - w->origin[1] = '\0';
172 -
173 - freez(w->user_agent); w->user_agent = NULL;
174 - if (w->auth_bearer_token) {
175 - freez(w->auth_bearer_token);
176 - w->auth_bearer_token = NULL;
177 - }
246 + web_client_reset_allocations(w, false);
247
179 - w->mode = WEB_CLIENT_MODE_NORMAL;
248 + w->mode = WEB_CLIENT_MODE_GET;
249
181 - w->tcp_cork = 0;
250 web_client_disable_donottrack(w);
251 web_client_disable_tracking_required(w);
252 web_client_disable_keepalive(w);
185 - w->decoded_url[0] = '\0';
186 -
187 - buffer_reset(w->response.header_output);
188 - buffer_reset(w->response.header);
189 - buffer_reset(w->response.data);
190 - w->response.rlen = 0;
191 - w->response.sent = 0;
192 - w->response.code = 0;
253
254 w->header_parse_tries = 0;
255 w->header_parse_last_size = 0;
@@ -197,23 +257,11 @@ void web_client_request_done(struct web_client *w) {
257 web_client_enable_wait_receive(w);
258 web_client_disable_wait_send(w);
259
200 - w->response.zoutput = 0;
201 -
202 - // if we had enabled compression, release it
203 -#ifdef NETDATA_WITH_ZLIB
204 - if(w->response.zinitialized) {
205 - debug(D_DEFLATE, "%llu: Freeing compression resources.", w->id);
206 - deflateEnd(&w->response.zstream);
207 - w->response.zsent = 0;
208 - w->response.zhave = 0;
209 - w->response.zstream.avail_in = 0;
210 - w->response.zstream.avail_out = 0;
211 - w->response.zstream.total_in = 0;
212 - w->response.zstream.total_out = 0;
213 - w->response.zinitialized = 0;
214 - w->flags &= ~WEB_CLIENT_CHUNKED_TRANSFER;
215 - }
216 -#endif // NETDATA_WITH_ZLIB
260 + w->response.has_cookies = false;
261 + w->response.rlen = 0;
262 + w->response.sent = 0;
263 + w->response.code = 0;
264 + w->response.zoutput = false;
265 }
266
267 static struct {
@@ -438,8 +486,8 @@ void web_client_enable_deflate(struct web_client *w, int gzip) {
486 }
487
488 w->response.zsent = 0;
441 - w->response.zoutput = 1;
442 - w->response.zinitialized = 1;
489 + w->response.zoutput = true;
490 + w->response.zinitialized = true;
491 w->flags |= WEB_CLIENT_CHUNKED_TRANSFER;
492
493 debug(D_DEFLATE, "%llu: Initialized compression.", w->id);
@@ -539,16 +587,16 @@ static inline int UNUSED_FUNCTION(check_host_and_mgmt_acl_and_call)(RRDHOST *hos
587 return check_host_and_call(host, w, url, func);
588 }
589
542 -int web_client_api_request(RRDHOST *host, struct web_client *w, char *url)
590 +int web_client_api_request(RRDHOST *host, struct web_client *w, char *url_path_fragment)
591 {
592 // get the api version
545 - char *tok = mystrsep(&url, "/");
593 + char *tok = strsep_skip_consecutive_separators(&url_path_fragment, "/");
594 if(tok && *tok) {
595 debug(D_WEB_CLIENT, "%llu: Searching for API version '%s'.", w->id, tok);
596 if(strcmp(tok, "v2") == 0)
549 - return web_client_api_request_v2(host, w, url);
597 + return web_client_api_request_v2(host, w, url_path_fragment);
598 else if(strcmp(tok, "v1") == 0)
551 - return web_client_api_request_v1(host, w, url);
599 + return web_client_api_request_v1(host, w, url_path_fragment);
600 else {
601 buffer_flush(w->response.data);
602 w->response.data->content_type = CT_TEXT_HTML;
@@ -729,7 +777,7 @@ static inline char *http_header_parse(struct web_client *w, char *s, int parse_u
777 uint32_t hash = simple_uhash(s);
778
779 if(hash == hash_origin && !strcasecmp(s, "Origin"))
732 - strncpyz(w->origin, v, NETDATA_WEB_REQUEST_ORIGIN_HEADER_SIZE);
780 + w->origin = strdupz(v);
781
782 else if(hash == hash_connection && !strcasecmp(s, "Connection")) {
783 if(strcasestr(v, "keep-alive"))
@@ -741,11 +789,14 @@ static inline char *http_header_parse(struct web_client *w, char *s, int parse_u
789 }
790 else if(parse_useragent && hash == hash_useragent && !strcasecmp(s, "User-Agent")) {
791 w->user_agent = strdupz(v);
744 - } else if(hash == hash_authorization&& !strcasecmp(s, "X-Auth-Token")) {
792 + }
793 + else if(hash == hash_authorization&& !strcasecmp(s, "X-Auth-Token")) {
794 w->auth_bearer_token = strdupz(v);
795 }
747 - else if(hash == hash_host && !strcasecmp(s, "Host")){
748 - strncpyz(w->server_host, v, ((size_t)(ve - v) < sizeof(w->server_host)-1 ? (size_t)(ve - v) : sizeof(w->server_host)-1));
796 + else if(hash == hash_host && !strcasecmp(s, "Host")) {
797 + char buffer[NI_MAXHOST];
798 + strncpyz(buffer, v, ((size_t)(ve - v) < sizeof(buffer) - 1 ? (size_t)(ve - v) : sizeof(buffer) - 1));
799 + w->server_host = strdupz(buffer);
800 }
801 #ifdef NETDATA_WITH_ZLIB
802 else if(hash == hash_accept_encoding && !strcasecmp(s, "Accept-Encoding")) {
@@ -765,8 +816,10 @@ static inline char *http_header_parse(struct web_client *w, char *s, int parse_u
816 w->ssl.flags |= NETDATA_SSL_PROXY_HTTPS;
817 }
818 #endif
768 - else if(hash == hash_forwarded_host && !strcasecmp(s, "X-Forwarded-Host")){
769 - strncpyz(w->forwarded_host, v, ((size_t)(ve - v) < sizeof(w->server_host)-1 ? (size_t)(ve - v) : sizeof(w->server_host)-1));
819 + else if(hash == hash_forwarded_host && !strcasecmp(s, "X-Forwarded-Host")) {
820 + char buffer[NI_MAXHOST];
821 + strncpyz(buffer, v, ((size_t)(ve - v) < sizeof(buffer) - 1 ? (size_t)(ve - v) : sizeof(buffer) - 1));
822 + w->forwarded_host = strdupz(buffer);
823 }
824
825 *e = ':';
@@ -788,12 +841,16 @@ static inline char *web_client_valid_method(struct web_client *w, char *s) {
841 // is is a valid request?
842 if(!strncmp(s, "GET ", 4)) {
843 s = &s[4];
791 - w->mode = WEB_CLIENT_MODE_NORMAL;
844 + w->mode = WEB_CLIENT_MODE_GET;
845 }
846 else if(!strncmp(s, "OPTIONS ", 8)) {
847 s = &s[8];
848 w->mode = WEB_CLIENT_MODE_OPTIONS;
849 }
850 + else if(!strncmp(s, "POST ", 5)) {
851 + s = &s[5];
852 + w->mode = WEB_CLIENT_MODE_POST;
853 + }
854 else if(!strncmp(s, "STREAM ", 7)) {
855 s = &s[7];
856
@@ -836,63 +893,6 @@ static inline char *web_client_valid_method(struct web_client *w, char *s) {
893 return s;
894 }
895
839 -/**
840 - * Set Path Query
841 - *
842 - * Set the pointers to the path and query string according to the input.
843 - *
844 - * @param w is the structure with the client request
845 - * @param s is the first address of the string.
846 - * @param ptr is the address of the separator.
847 - */
848 -static void web_client_set_path_query(struct web_client *w, const char *s, char *ptr) {
849 - w->url_path_length = (size_t)(ptr -s);
850 - w->url_search_path = ptr;
851 -}
852 -
853 -/**
854 - * Split path query
855 - *
856 - * Do the separation between path and query string
857 - *
858 - * @param w is the structure with the client request
859 - * @param s is the string to parse
860 - */
861 -void web_client_split_path_query(struct web_client *w, char *s) {
862 - //I am assuming here that the separator character(?) is not encoded
863 - char *ptr = strchr(s, '?');
864 - if(ptr) {
865 - w->separator = '?';
866 - web_client_set_path_query(w, s, ptr);
867 - return;
868 - }
869 -
870 - //Here I test the second possibility, the URL is completely encoded by the user.
871 - //I am not using the strcasestr, because it is fastest to check %3f and compare
872 - //the next character.
873 - //We executed some tests with "encodeURI(uri);" described in https://www.w3schools.com/jsref/jsref_encodeuri.asp
874 - //on July 1st, 2019, that show us that URLs won't have '?','=' and '&' encoded, but we decided to move in front
875 - //with the next part, because users can develop their own encoded that won't follow this rule.
876 - char *moveme = s;
877 - while (moveme) {
878 - ptr = strchr(moveme, '%');
879 - if(ptr) {
880 - char *test = (ptr+1);
881 - if (!strncmp(test, "3f", 2) || !strncmp(test, "3F", 2)) {
882 - w->separator = *ptr;
883 - web_client_set_path_query(w, s, ptr);
884 - return;
885 - }
886 - ptr++;
887 - }
888 -
889 - moveme = ptr;
890 - }
891 -
892 - w->separator = 0x00;
893 - w->url_path_length = strlen(s);
894 -}
895 -
896 /**
897 * Request validate
898 *
@@ -917,7 +917,7 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
917 if(w->header_parse_last_size < last_pos)
918 last_pos = 0;
919
920 - is_it_valid = url_is_request_complete(s, &s[last_pos], w->header_parse_last_size);
920 + is_it_valid = url_is_request_complete(s, &s[last_pos], w->header_parse_last_size, &w->post_payload, &w->post_payload_size);
921 if(!is_it_valid) {
922 if(w->header_parse_tries > HTTP_REQ_MAX_HEADER_FETCH_TRIES) {
923 info("Disabling slow client after %zu attempts to read the request (%zu bytes received)", w->header_parse_tries, buffer_strlen(w->response.data));
@@ -933,7 +933,7 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
933 is_it_valid = 1;
934 } else {
935 last_pos = w->header_parse_last_size;
936 - is_it_valid = url_is_request_complete(s, &s[last_pos], w->header_parse_last_size);
936 + is_it_valid = url_is_request_complete(s, &s[last_pos], w->header_parse_last_size, &w->post_payload, &w->post_payload_size);
937 }
938
939 s = web_client_valid_method(w, s);
@@ -974,10 +974,6 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
974 // we have the end of encoded_url - remember it
975 char *ue = s;
976
977 - //Variables used to map the variables in the query string case it is present
978 - int total_variables;
979 - char *ptr_variables[WEB_FIELDS_MAX];
980 -
977 // make sure we have complete request
978 // complete requests contain: \r\n\r\n
979 while(*s) {
@@ -994,50 +990,16 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
990 if(unlikely(*s == '\r' && s[1] == '\n')) {
991 // a valid complete HTTP request found
992
993 + char c = *ue;
994 *ue = '\0';
998 - //This is to avoid crash in line
999 - w->url_search_path = NULL;
1000 - if(w->mode != WEB_CLIENT_MODE_NORMAL) {
1001 - if(!url_decode_r(w->decoded_url, encoded_url, NETDATA_WEB_REQUEST_URL_SIZE + 1))
1002 - return HTTP_VALIDATION_MALFORMED_URL;
1003 - } else {
1004 - web_client_split_path_query(w, encoded_url);
1005 -
1006 - if (w->url_search_path && w->separator) {
1007 - *w->url_search_path = '\0';
1008 - }
1009 -
1010 - if(!url_decode_r(w->decoded_url, encoded_url, NETDATA_WEB_REQUEST_URL_SIZE + 1))
1011 - return HTTP_VALIDATION_MALFORMED_URL;
1012 -
1013 - if (w->url_search_path && w->separator) {
1014 - *w->url_search_path = w->separator;
995 + web_client_decode_path_and_query_string(w, encoded_url);
996 + *ue = c;
997
1016 - char *from = (encoded_url + w->url_path_length);
1017 - total_variables = url_map_query_string(ptr_variables, from);
1018 -
1019 - if (url_parse_query_string(w->decoded_query_string, NETDATA_WEB_REQUEST_URL_SIZE + 1, ptr_variables, total_variables)) {
1020 - return HTTP_VALIDATION_MALFORMED_URL;
1021 - }
1022 - } else {
1023 - //make sure there's no leftovers from previous request on the same web client
1024 - w->decoded_query_string[1]='\0';
1025 - }
1026 - }
1027 - *ue = ' ';
1028 -
1029 - // copy the URL - we are going to overwrite parts of it
1030 - // TODO -- ideally we we should avoid copying buffers around
1031 - snprintfz(w->last_url, NETDATA_WEB_REQUEST_URL_SIZE, "%s%s", w->decoded_url, w->decoded_query_string);
998 #ifdef ENABLE_HTTPS
999 if ( (!web_client_check_unix(w)) && (netdata_ssl_srv_ctx) ) {
1000 if ((w->ssl.conn) && ((w->ssl.flags & NETDATA_SSL_NO_HANDSHAKE) && (web_client_is_using_ssl_force(w) || web_client_is_using_ssl_default(w)) && (w->mode != WEB_CLIENT_MODE_STREAM)) ) {
1001 w->header_parse_tries = 0;
1002 w->header_parse_last_size = 0;
1037 - // The client will be redirected for Netdata and we are preserving the original request.
1038 - *ue = '\0';
1039 - strncpyz(w->last_url, encoded_url, NETDATA_WEB_REQUEST_URL_SIZE);
1040 - *ue = ' ';
1003 web_client_disable_wait_receive(w);
1004 return HTTP_VALIDATION_REDIRECT;
1005 }
@@ -1051,9 +1013,7 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
1013 }
1014
1015 // another header line
1054 - s = http_header_parse(w, s,
1055 - (w->mode == WEB_CLIENT_MODE_STREAM) // parse user agent
1056 - );
1016 + s = http_header_parse(w, s, (w->mode == WEB_CLIENT_MODE_STREAM)); // parse user agent
1017 }
1018 }
1019
@@ -1090,9 +1050,9 @@ void web_client_build_http_header(struct web_client *w) {
1050 // set a proper expiration date, if not already set
1051 if(unlikely(!w->response.data->expires)) {
1052 if(w->response.data->options & WB_CONTENT_NO_CACHEABLE)
1093 - w->response.data->expires = w->tv_ready.tv_sec + localhost->rrd_update_every;
1053 + w->response.data->expires = w->timings.tv_ready.tv_sec + localhost->rrd_update_every;
1054 else
1095 - w->response.data->expires = w->tv_ready.tv_sec + 86400;
1055 + w->response.data->expires = w->timings.tv_ready.tv_sec + 86400;
1056 }
1057
1058 // prepare the HTTP response header
@@ -1118,8 +1078,8 @@ void web_client_build_http_header(struct web_client *w) {
1078 "HTTP/1.1 %d %s\r\n"
1079 "Location: https://%s%s\r\n",
1080 w->response.code, code_msg,
1121 - w->server_host,
1122 - w->last_url);
1081 + w->server_host ? w->server_host : "",
1082 + buffer_tostring(w->url_as_received));
1083 }else {
1084 buffer_sprintf(w->response.header_output,
1085 "HTTP/1.1 %d %s\r\n"
@@ -1133,7 +1093,7 @@ void web_client_build_http_header(struct web_client *w) {
1093 code_msg,
1094 web_client_has_keepalive(w)?"keep-alive":"close",
1095 VERSION,
1136 - w->origin,
1096 + w->origin ? w->origin : "*",
1097 content_type_string,
1098 date);
1099 }
@@ -1141,31 +1101,19 @@ void web_client_build_http_header(struct web_client *w) {
1101 if(unlikely(web_x_frame_options))
1102 buffer_sprintf(w->response.header_output, "X-Frame-Options: %s\r\n", web_x_frame_options);
1103
1144 - if(w->cookie1[0] || w->cookie2[0]) {
1145 - if(w->cookie1[0]) {
1146 - buffer_sprintf(w->response.header_output,
1147 - "Set-Cookie: %s\r\n",
1148 - w->cookie1);
1149 - }
1150 -
1151 - if(w->cookie2[0]) {
1152 - buffer_sprintf(w->response.header_output,
1153 - "Set-Cookie: %s\r\n",
1154 - w->cookie2);
1155 - }
1156 -
1104 + if(w->response.has_cookies) {
1105 if(respect_web_browser_do_not_track_policy)
1106 buffer_sprintf(w->response.header_output,
1159 - "Tk: T;cookies\r\n");
1107 + "Tk: T;cookies\r\n");
1108 }
1109 else {
1110 if(respect_web_browser_do_not_track_policy) {
1111 if(web_client_has_tracking_required(w))
1112 buffer_sprintf(w->response.header_output,
1165 - "Tk: T;cookies\r\n");
1113 + "Tk: T;cookies\r\n");
1114 else
1115 buffer_sprintf(w->response.header_output,
1168 - "Tk: N\r\n");
1116 + "Tk: N\r\n");
1117 }
1118 }
1119
@@ -1263,7 +1211,7 @@ static inline void web_client_send_http_header(struct web_client *w) {
1211
1212 if(bytes != (ssize_t) buffer_strlen(w->response.header_output)) {
1213 if(bytes > 0)
1266 - w->stats_sent_bytes += bytes;
1214 + w->statistics.sent_bytes += bytes;
1215
1216 if (bytes < 0) {
1217
@@ -1276,12 +1224,10 @@ static inline void web_client_send_http_header(struct web_client *w) {
1224 }
1225 }
1226 else
1279 - w->stats_sent_bytes += bytes;
1227 + w->statistics.sent_bytes += bytes;
1228 }
1229
1282 -static inline int web_client_process_url(RRDHOST *host, struct web_client *w, char *url);
1283 -
1284 -static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, char *url) {
1230 +static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, char *url, bool nodeid, int (*func)(RRDHOST *, struct web_client *, char *)) {
1231 static uint32_t hash_localhost = 0;
1232
1233 if(unlikely(!hash_localhost)) {
@@ -1294,34 +1240,80 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch
1240 return HTTP_RESP_BAD_REQUEST;
1241 }
1242
1297 - char *tok = mystrsep(&url, "/");
1243 + char *tok = strsep_skip_consecutive_separators(&url, "/");
1244 if(tok && *tok) {
1245 debug(D_WEB_CLIENT, "%llu: Searching for host with name '%s'.", w->id, tok);
1246
1301 - if(!url) { //no delim found
1302 - debug(D_WEB_CLIENT, "%llu: URL doesn't end with / generating redirect.", w->id);
1303 - char *protocol, *url_host;
1247 + if(nodeid) {
1248 + host = find_host_by_node_id(tok);
1249 + if(!host) {
1250 + host = rrdhost_find_by_hostname(tok);
1251 + if (!host)
1252 + host = rrdhost_find_by_guid(tok);
1253 + }
1254 + }
1255 + else {
1256 + host = rrdhost_find_by_hostname(tok);
1257 + if(!host) {
1258 + host = rrdhost_find_by_guid(tok);
1259 + if (!host)
1260 + host = find_host_by_node_id(tok);
1261 + }
1262 + }
1263 +
1264 + if(!host) {
1265 + // we didn't find it, but it may be a uuid case mismatch for MACHINE_GUID
1266 + // so, recreate the machine guid in lower-case.
1267 + uuid_t uuid;
1268 + char txt[UUID_STR_LEN];
1269 + if (uuid_parse(tok, uuid) == 0) {
1270 + uuid_unparse_lower(uuid, txt);
1271 + host = rrdhost_find_by_guid(txt);
1272 + }
1273 + }
1274 +
1275 + if (host) {
1276 + if(!url) { //no delim found
1277 + debug(D_WEB_CLIENT, "%llu: URL doesn't end with / generating redirect.", w->id);
1278 + char *protocol, *url_host;
1279 #ifdef ENABLE_HTTPS
1305 - protocol = ((w->ssl.conn && !w->ssl.flags) || w->ssl.flags & NETDATA_SSL_PROXY_HTTPS) ? "https" : "http";
1280 + protocol = ((w->ssl.conn && !w->ssl.flags) || w->ssl.flags & NETDATA_SSL_PROXY_HTTPS) ? "https" : "http";
1281 #else
1307 - protocol = "http";
1282 + protocol = "http";
1283 #endif
1309 - url_host = (!w->forwarded_host[0])?w->server_host:w->forwarded_host;
1310 - buffer_sprintf(w->response.header, "Location: %s://%s%s/\r\n", protocol, url_host, w->last_url);
1311 - buffer_strcat(w->response.data, "Permanent redirect");
1312 - return HTTP_RESP_REDIR_PERM;
1313 - }
1284
1315 - // copy the URL, we need it to serve files
1316 - w->last_url[0] = '/';
1285 + url_host = w->forwarded_host;
1286 + if(!url_host) {
1287 + url_host = w->server_host;
1288 + if(!url_host) url_host = "";
1289 + }
1290 +
1291 + buffer_sprintf(w->response.header, "Location: %s://%s/%s/%s/%s",
1292 + protocol, url_host, nodeid?"node":"host", tok, buffer_tostring(w->url_path_decoded));
1293 +
1294 + if(buffer_strlen(w->url_query_string_decoded)) {
1295 + const char *query_string = buffer_tostring(w->url_query_string_decoded);
1296 + if(*query_string) {
1297 + if(*query_string != '?')
1298 + buffer_fast_strcat(w->response.header, "?", 1);
1299 + buffer_strcat(w->response.header, query_string);
1300 + }
1301 + }
1302 + buffer_fast_strcat(w->response.header, "\r\n", 2);
1303 + buffer_strcat(w->response.data, "Permanent redirect");
1304 + return HTTP_RESP_REDIR_PERM;
1305 + }
1306
1318 - if(url && *url) strncpyz(&w->last_url[1], url, NETDATA_WEB_REQUEST_URL_SIZE - 1);
1319 - else w->last_url[1] = '\0';
1307 + size_t len = strlen(url) + 2;
1308 + char buf[len];
1309 + buf[0] = '/';
1310 + strcpy(&buf[1], url);
1311 + buf[len - 1] = '\0';
1312
1321 - host = rrdhost_find_by_hostname(tok);
1322 - if (!host)
1323 - host = rrdhost_find_by_guid(tok);
1324 - if (host) return web_client_process_url(host, w, url);
1313 + buffer_flush(w->url_path_decoded);
1314 + buffer_strcat(w->url_path_decoded, buf);
1315 + return func(host, w, buf);
1316 + }
1317 }
1318
1319 buffer_flush(w->response.data);
@@ -1331,14 +1323,49 @@ static inline int web_client_switch_host(RRDHOST *host, struct web_client *w, ch
1323 return HTTP_RESP_NOT_FOUND;
1324 }
1325
1334 -static inline int web_client_process_url(RRDHOST *host, struct web_client *w, char *url) {
1326 +int web_client_api_request_with_node_selection(RRDHOST *host, struct web_client *w, char *decoded_url_path) {
1327 + static uint32_t
1328 + hash_api = 0,
1329 + hash_host = 0,
1330 + hash_node = 0;
1331 +
1332 + if(unlikely(!hash_api)) {
1333 + hash_api = simple_hash("api");
1334 + hash_host = simple_hash("host");
1335 + hash_node = simple_hash("node");
1336 + }
1337 +
1338 + char *tok = strsep_skip_consecutive_separators(&decoded_url_path, "/?");
1339 + if(likely(tok && *tok)) {
1340 + uint32_t hash = simple_hash(tok);
1341 +
1342 + if(unlikely(hash == hash_api && strcmp(tok, "api") == 0)) {
1343 + // current API
1344 + debug(D_WEB_CLIENT_ACCESS, "%llu: API request ...", w->id);
1345 + return check_host_and_call(host, w, decoded_url_path, web_client_api_request);
1346 + }
1347 + else if(unlikely((hash == hash_host && strcmp(tok, "host") == 0) || (hash == hash_node && strcmp(tok, "node") == 0))) {
1348 + // host switching
1349 + debug(D_WEB_CLIENT_ACCESS, "%llu: host switch request ...", w->id);
1350 + return web_client_switch_host(host, w, decoded_url_path, hash == hash_node, web_client_api_request_with_node_selection);
1351 + }
1352 + }
1353 +
1354 + buffer_flush(w->response.data);
1355 + buffer_strcat(w->response.data, "Unknown API endpoint.");
1356 + w->response.data->content_type = CT_TEXT_HTML;
1357 + return HTTP_RESP_NOT_FOUND;
1358 +}
1359 +
1360 +static inline int web_client_process_url(RRDHOST *host, struct web_client *w, char *decoded_url_path) {
1361 if(unlikely(!service_running(ABILITY_WEB_REQUESTS)))
1362 return web_client_permission_denied(w);
1363
1364 static uint32_t
1365 hash_api = 0,
1366 hash_netdata_conf = 0,
1341 - hash_host = 0;
1367 + hash_host = 0,
1368 + hash_node = 0;
1369
1370 #ifdef NETDATA_INTERNAL_CHECKS
1371 static uint32_t hash_exit = 0, hash_debug = 0, hash_mirror = 0;
@@ -1348,6 +1375,7 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch
1375 hash_api = simple_hash("api");
1376 hash_netdata_conf = simple_hash("netdata.conf");
1377 hash_host = simple_hash("host");
1378 + hash_node = simple_hash("node");
1379 #ifdef NETDATA_INTERNAL_CHECKS
1380 hash_exit = simple_hash("exit");
1381 hash_debug = simple_hash("debug");
@@ -1355,18 +1383,22 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch
1383 #endif
1384 }
1385
1358 - char *tok = mystrsep(&url, "/?");
1386 + // keep a copy of the decoded path, in case we need to serve it as a filename
1387 + char filename[FILENAME_MAX + 1];
1388 + strncpyz(filename, buffer_tostring(w->url_path_decoded), FILENAME_MAX);
1389 +
1390 + char *tok = strsep_skip_consecutive_separators(&decoded_url_path, "/?");
1391 if(likely(tok && *tok)) {
1392 uint32_t hash = simple_hash(tok);
1393 debug(D_WEB_CLIENT, "%llu: Processing command '%s'.", w->id, tok);
1394
1395 if(unlikely(hash == hash_api && strcmp(tok, "api") == 0)) { // current API
1396 debug(D_WEB_CLIENT_ACCESS, "%llu: API request ...", w->id);
1365 - return check_host_and_call(host, w, url, web_client_api_request);
1397 + return check_host_and_call(host, w, decoded_url_path, web_client_api_request);
1398 }
1367 - else if(unlikely(hash == hash_host && strcmp(tok, "host") == 0)) { // host switching
1399 + else if(unlikely((hash == hash_host && strcmp(tok, "host") == 0) || (hash == hash_node && strcmp(tok, "node") == 0))) { // host switching
1400 debug(D_WEB_CLIENT_ACCESS, "%llu: host switch request ...", w->id);
1369 - return web_client_switch_host(host, w, url);
1401 + return web_client_switch_host(host, w, decoded_url_path, hash == hash_node, web_client_process_url);
1402 }
1403 else if(unlikely(hash == hash_netdata_conf && strcmp(tok, "netdata.conf") == 0)) { // netdata.conf
1404 if(unlikely(!web_client_can_access_netdataconf(w)))
@@ -1402,7 +1434,7 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch
1434 buffer_flush(w->response.data);
1435
1436 // get the name of the data to show
1405 - tok = mystrsep(&url, "&");
1437 + tok = strsep_skip_consecutive_separators(&decoded_url_path, "&");
1438 if(tok && *tok) {
1439 debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok);
1440
@@ -1452,18 +1484,14 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch
1484 #endif /* NETDATA_INTERNAL_CHECKS */
1485 }
1486
1455 - char filename[FILENAME_MAX+1];
1456 - url = filename;
1457 - strncpyz(filename, w->last_url, FILENAME_MAX);
1458 - tok = mystrsep(&url, "?");
1487 buffer_flush(w->response.data);
1460 - return mysendfile(w, (tok && *tok)?tok:"/");
1488 + return mysendfile(w, filename);
1489 }
1490
1491 void web_client_process_request(struct web_client *w) {
1492
1493 // start timing us
1466 - now_realtime_timeval(&w->tv_in);
1494 + web_client_timeout_checkpoint_init(w);
1495
1496 switch(http_request_validate(w)) {
1497 case HTTP_VALIDATION_OK:
@@ -1474,7 +1502,7 @@ void web_client_process_request(struct web_client *w) {
1502 return;
1503 }
1504
1477 - w->response.code = rrdpush_receiver_thread_spawn(w, w->decoded_url);
1505 + w->response.code = rrdpush_receiver_thread_spawn(w, (char *)buffer_tostring(w->url_query_string_decoded));
1506 return;
1507
1508 case WEB_CLIENT_MODE_OPTIONS:
@@ -1496,7 +1524,8 @@ void web_client_process_request(struct web_client *w) {
1524 break;
1525
1526 case WEB_CLIENT_MODE_FILECOPY:
1499 - case WEB_CLIENT_MODE_NORMAL:
1527 + case WEB_CLIENT_MODE_POST:
1528 + case WEB_CLIENT_MODE_GET:
1529 if(unlikely(
1530 !web_client_can_access_dashboard(w) &&
1531 !web_client_can_access_registry(w) &&
@@ -1508,14 +1537,15 @@ void web_client_process_request(struct web_client *w) {
1537 break;
1538 }
1539
1511 - w->response.code = web_client_process_url(localhost, w, w->decoded_url);
1540 + w->response.code = web_client_process_url(localhost, w, (char *)buffer_tostring(w->url_path_decoded));
1541 break;
1542 }
1543 break;
1544
1545 case HTTP_VALIDATION_INCOMPLETE:
1546 if(w->response.data->len > NETDATA_WEB_REQUEST_MAX_SIZE) {
1518 - strcpy(w->last_url, "too big request");
1547 + buffer_flush(w->url_as_received);
1548 + buffer_strcat(w->url_as_received, "too big request");
1549
1550 debug(D_WEB_CLIENT_ACCESS, "%llu: Received request is too big (%zu bytes).", w->id, w->response.data->len);
1551
@@ -1529,7 +1559,7 @@ void web_client_process_request(struct web_client *w) {
1559 // set to normal to prevent web_server_rcv_callback
1560 // from going into stream mode
1561 if (w->mode == WEB_CLIENT_MODE_STREAM)
1532 - w->mode = WEB_CLIENT_MODE_NORMAL;
1562 + w->mode = WEB_CLIENT_MODE_GET;
1563 return;
1564 }
1565 break;
@@ -1581,13 +1611,13 @@ void web_client_process_request(struct web_client *w) {
1611 }
1612
1613 // keep track of the processing time
1584 - now_realtime_timeval(&w->tv_ready);
1614 + web_client_timeout_checkpoint_response_ready(w, NULL);
1615
1616 w->response.sent = 0;
1617
1618 // set a proper last modified date
1619 if(unlikely(!w->response.data->date))
1590 - w->response.data->date = w->tv_ready.tv_sec;
1620 + w->response.data->date = w->timings.tv_ready.tv_sec;
1621
1622 web_client_send_http_header(w);
1623
@@ -1604,7 +1634,8 @@ void web_client_process_request(struct web_client *w) {
1634 debug(D_WEB_CLIENT, "%llu: Done preparing the OPTIONS response. Sending data (%zu bytes) to client.", w->id, w->response.data->len);
1635 break;
1636
1607 - case WEB_CLIENT_MODE_NORMAL:
1637 + case WEB_CLIENT_MODE_POST:
1638 + case WEB_CLIENT_MODE_GET:
1639 debug(D_WEB_CLIENT, "%llu: Done preparing the response. Sending data (%zu bytes) to client.", w->id, w->response.data->len);
1640 break;
1641
@@ -1647,7 +1678,7 @@ ssize_t web_client_send_chunk_header(struct web_client *w, size_t len)
1678 bytes = web_client_send_data(w,buf,strlen(buf),0);
1679 if(bytes > 0) {
1680 debug(D_DEFLATE, "%llu: Sent chunk header %zd bytes.", w->id, bytes);
1650 - w->stats_sent_bytes += bytes;
1681 + w->statistics.sent_bytes += bytes;
1682 }
1683
1684 else if(bytes == 0) {
@@ -1669,7 +1700,7 @@ ssize_t web_client_send_chunk_close(struct web_client *w)
1700 bytes = web_client_send_data(w,"\r\n",2,0);
1701 if(bytes > 0) {
1702 debug(D_DEFLATE, "%llu: Sent chunk suffix %zd bytes.", w->id, bytes);
1672 - w->stats_sent_bytes += bytes;
1703 + w->statistics.sent_bytes += bytes;
1704 }
1705
1706 else if(bytes == 0) {
@@ -1691,7 +1722,7 @@ ssize_t web_client_send_chunk_finalize(struct web_client *w)
1722 bytes = web_client_send_data(w,"\r\n0\r\n\r\n",7,0);
1723 if(bytes > 0) {
1724 debug(D_DEFLATE, "%llu: Sent chunk suffix %zd bytes.", w->id, bytes);
1694 - w->stats_sent_bytes += bytes;
1725 + w->statistics.sent_bytes += bytes;
1726 }
1727
1728 else if(bytes == 0) {
@@ -1769,7 +1800,7 @@ ssize_t web_client_send_deflate(struct web_client *w)
1800
1801 // ask for FINISH if we have all the input
1802 int flush = Z_SYNC_FLUSH;
1772 - if(w->mode == WEB_CLIENT_MODE_NORMAL
1803 + if((w->mode == WEB_CLIENT_MODE_GET || w->mode == WEB_CLIENT_MODE_POST)
1804 || (w->mode == WEB_CLIENT_MODE_FILECOPY && !web_client_has_wait_receive(w) && w->response.data->len == w->response.rlen)) {
1805 flush = Z_FINISH;
1806 debug(D_DEFLATE, "%llu: Requesting Z_FINISH, if possible.", w->id);
@@ -1803,7 +1834,7 @@ ssize_t web_client_send_deflate(struct web_client *w)
1834
1835 len = web_client_send_data(w,&w->response.zbuffer[w->response.zsent], (size_t) (w->response.zhave - w->response.zsent), MSG_DONTWAIT);
1836 if(len > 0) {
1806 - w->stats_sent_bytes += len;
1837 + w->statistics.sent_bytes += len;
1838 w->response.zsent += len;
1839 len += t;
1840 debug(D_WEB_CLIENT, "%llu: Sent %zd bytes.", w->id, len);
@@ -1858,7 +1889,7 @@ ssize_t web_client_send(struct web_client *w) {
1889
1890 bytes = web_client_send_data(w,&w->response.data->buffer[w->response.sent], w->response.data->len - w->response.sent, MSG_DONTWAIT);
1891 if(likely(bytes > 0)) {
1861 - w->stats_sent_bytes += bytes;
1892 + w->statistics.sent_bytes += bytes;
1893 w->response.sent += bytes;
1894 debug(D_WEB_CLIENT, "%llu: Sent %zd bytes.", w->id, bytes);
1895 }
@@ -1953,7 +1984,7 @@ ssize_t web_client_receive(struct web_client *w)
1984 #endif
1985
1986 if(likely(bytes > 0)) {
1956 - w->stats_received_bytes += bytes;
1987 + w->statistics.received_bytes += bytes;
1988
1989 size_t old = w->response.data->len;
1990 (void)old;
@@ -1993,3 +2024,196 @@ int web_client_socket_is_now_used_for_streaming(struct web_client *w) {
2024
2025 return HTTP_RESP_OK;
2026 }
2027 +
2028 +void web_client_decode_path_and_query_string(struct web_client *w, const char *path_and_query_string) {
2029 + char buffer[NETDATA_WEB_REQUEST_URL_SIZE + 2];
2030 + buffer[0] = '?';
2031 + buffer[0] = '\0';
2032 +
2033 + buffer_flush(w->url_path_decoded);
2034 + buffer_flush(w->url_query_string_decoded);
2035 +
2036 + if(buffer_strlen(w->url_as_received) == 0)
2037 + // do not overwrite this if it is already filled
2038 + buffer_strcat(w->url_as_received, path_and_query_string);
2039 +
2040 + if(w->mode == WEB_CLIENT_MODE_STREAM) {
2041 + // in stream mode, there is no path
2042 +
2043 + url_decode_r(buffer, path_and_query_string, NETDATA_WEB_REQUEST_URL_SIZE + 1);
2044 +
2045 + buffer[NETDATA_WEB_REQUEST_URL_SIZE + 1] = '\0';
2046 + buffer_strcat(w->url_query_string_decoded, buffer);
2047 + }
2048 + else {
2049 + // in non-stream mode, there is a path
2050 +
2051 + // FIXME - the way this is implemented, query string params never accept the symbol &, not even encoded as %26
2052 + // To support the symbol & in query string params, we need to turn the url_query_string_decoded into a
2053 + // dictionary and decode each of the parameters individually.
2054 + // OR: in url_query_string_decoded use as separator a control character that cannot appear in the URL.
2055 +
2056 + char *question_mark_start = strchr(path_and_query_string, '?');
2057 + if (question_mark_start)
2058 + url_decode_r(buffer, question_mark_start, NETDATA_WEB_REQUEST_URL_SIZE + 1);
2059 +
2060 + buffer[NETDATA_WEB_REQUEST_URL_SIZE + 1] = '\0';
2061 + buffer_strcat(w->url_query_string_decoded, buffer);
2062 +
2063 + if (question_mark_start) {
2064 + char c = *question_mark_start;
2065 + *question_mark_start = '\0';
2066 + url_decode_r(buffer, path_and_query_string, NETDATA_WEB_REQUEST_URL_SIZE + 1);
2067 + *question_mark_start = c;
2068 + } else
2069 + url_decode_r(buffer, path_and_query_string, NETDATA_WEB_REQUEST_URL_SIZE + 1);
2070 +
2071 + buffer[NETDATA_WEB_REQUEST_URL_SIZE + 1] = '\0';
2072 + buffer_strcat(w->url_path_decoded, buffer);
2073 + }
2074 +}
2075 +
2076 +#ifdef ENABLE_HTTPS
2077 +void web_client_reuse_ssl(struct web_client *w) {
2078 + if (netdata_ssl_srv_ctx) {
2079 + if (w->ssl.conn) {
2080 + SSL_SESSION *session = SSL_get_session(w->ssl.conn);
2081 + SSL *old = w->ssl.conn;
2082 + w->ssl.conn = SSL_new(netdata_ssl_srv_ctx);
2083 + if (session) {
2084 +#if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_111
2085 + if (SSL_SESSION_is_resumable(session))
2086 +#endif
2087 + SSL_set_session(w->ssl.conn, session);
2088 + }
2089 + SSL_free(old);
2090 + }
2091 + }
2092 +}
2093 +#endif
2094 +
2095 +void web_client_zero(struct web_client *w) {
2096 + // zero everything about it - but keep the buffers
2097 +
2098 + web_client_reset_allocations(w, false);
2099 +
2100 + // remember the pointers to the buffers
2101 + BUFFER *b1 = w->response.data;
2102 + BUFFER *b2 = w->response.header;
2103 + BUFFER *b3 = w->response.header_output;
2104 + BUFFER *b4 = w->url_path_decoded;
2105 + BUFFER *b5 = w->url_as_received;
2106 + BUFFER *b6 = w->url_query_string_decoded;
2107 +
2108 +#ifdef ENABLE_HTTPS
2109 + web_client_reuse_ssl(w);
2110 + SSL *ssl = w->ssl.conn;
2111 +#endif
2112 +
2113 + size_t use_count = w->use_count;
2114 + size_t *statistics_memory_accounting = w->statistics.memory_accounting;
2115 +
2116 + // zero everything
2117 + memset(w, 0, sizeof(struct web_client));
2118 +
2119 + w->ifd = w->ofd = -1;
2120 + w->statistics.memory_accounting = statistics_memory_accounting;
2121 + w->use_count = use_count;
2122 +
2123 +#ifdef ENABLE_HTTPS
2124 + w->ssl.conn = ssl;
2125 + w->ssl.flags = NETDATA_SSL_START;
2126 + debug(D_WEB_CLIENT_ACCESS,"Reusing SSL structure with (w->ssl = NULL, w->accepted = %u)", w->ssl.flags);
2127 +#endif
2128 +
2129 + // restore the pointers of the buffers
2130 + w->response.data = b1;
2131 + w->response.header = b2;
2132 + w->response.header_output = b3;
2133 + w->url_path_decoded = b4;
2134 + w->url_as_received = b5;
2135 + w->url_query_string_decoded = b6;
2136 +}
2137 +
2138 +struct web_client *web_client_create(size_t *statistics_memory_accounting) {
2139 + struct web_client *w = (struct web_client *)callocz(1, sizeof(struct web_client));
2140 + w->use_count = 1;
2141 + w->statistics.memory_accounting = statistics_memory_accounting;
2142 +
2143 + w->url_as_received = buffer_create(NETDATA_WEB_DECODED_URL_INITIAL_SIZE, w->statistics.memory_accounting);
2144 + w->url_path_decoded = buffer_create(NETDATA_WEB_DECODED_URL_INITIAL_SIZE, w->statistics.memory_accounting);
2145 + w->url_query_string_decoded = buffer_create(NETDATA_WEB_DECODED_URL_INITIAL_SIZE, w->statistics.memory_accounting);
2146 + w->response.data = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, w->statistics.memory_accounting);
2147 + w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_INITIAL_SIZE, w->statistics.memory_accounting);
2148 + w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_INITIAL_SIZE, w->statistics.memory_accounting);
2149 +
2150 + __atomic_add_fetch(w->statistics.memory_accounting, sizeof(struct web_client), __ATOMIC_RELAXED);
2151 +
2152 + return w;
2153 +}
2154 +
2155 +void web_client_free(struct web_client *w) {
2156 + web_client_reset_allocations(w, true);
2157 +
2158 + __atomic_sub_fetch(w->statistics.memory_accounting, sizeof(struct web_client), __ATOMIC_RELAXED);
2159 + freez(w);
2160 +}
2161 +
2162 +inline void web_client_timeout_checkpoint_init(struct web_client *w) {
2163 + now_monotonic_high_precision_timeval(&w->timings.tv_in);
2164 +}
2165 +
2166 +inline void web_client_timeout_checkpoint_set(struct web_client *w, int timeout_ms) {
2167 + w->timings.timeout_ut = timeout_ms * USEC_PER_MS;
2168 +
2169 + if(!w->timings.tv_in.tv_sec)
2170 + web_client_timeout_checkpoint_init(w);
2171 +
2172 + if(!w->timings.tv_timeout_last_checkpoint.tv_sec)
2173 + w->timings.tv_timeout_last_checkpoint = w->timings.tv_in;
2174 +}
2175 +
2176 +inline usec_t web_client_timeout_checkpoint(struct web_client *w) {
2177 + struct timeval now;
2178 + now_monotonic_high_precision_timeval(&now);
2179 +
2180 + if (!w->timings.tv_timeout_last_checkpoint.tv_sec)
2181 + w->timings.tv_timeout_last_checkpoint = w->timings.tv_in;
2182 +
2183 + usec_t since_last_check_ut = dt_usec(&w->timings.tv_timeout_last_checkpoint, &now);
2184 +
2185 + w->timings.tv_timeout_last_checkpoint = now;
2186 +
2187 + return since_last_check_ut;
2188 +}
2189 +
2190 +inline usec_t web_client_timeout_checkpoint_response_ready(struct web_client *w, usec_t *usec_since_last_checkpoint) {
2191 + usec_t since_last_check_ut = web_client_timeout_checkpoint(w);
2192 + if(usec_since_last_checkpoint)
2193 + *usec_since_last_checkpoint = since_last_check_ut;
2194 +
2195 + w->timings.tv_ready = w->timings.tv_timeout_last_checkpoint;
2196 +
2197 + // return the total time of the query
2198 + return dt_usec(&w->timings.tv_in, &w->timings.tv_ready);
2199 +}
2200 +
2201 +inline bool web_client_timeout_checkpoint_and_check(struct web_client *w, usec_t *usec_since_last_checkpoint) {
2202 +
2203 + usec_t since_last_check_ut = web_client_timeout_checkpoint(w);
2204 + if(usec_since_last_checkpoint)
2205 + *usec_since_last_checkpoint = since_last_check_ut;
2206 +
2207 + if(!w->timings.timeout_ut)
2208 + return false;
2209 +
2210 + usec_t since_reception_ut = dt_usec(&w->timings.tv_in, &w->timings.tv_timeout_last_checkpoint);
2211 + if (since_reception_ut >= w->timings.timeout_ut) {
2212 + buffer_flush(w->response.data);
2213 + buffer_strcat(w->response.data, "Query timeout exceeded");
2214 + w->response.code = HTTP_RESP_BACKEND_FETCH_FAILED;
2215 + return true;
2216 + }
2217 +
2218 + return false;
2219 +}
web/server/web_client.h
+85 -49
@@ -39,10 +39,11 @@ extern int respect_web_browser_do_not_track_policy;
39 extern char *web_x_frame_options;
40
41 typedef enum web_client_mode {
42 - WEB_CLIENT_MODE_NORMAL = 0,
43 - WEB_CLIENT_MODE_FILECOPY = 1,
44 - WEB_CLIENT_MODE_OPTIONS = 2,
45 - WEB_CLIENT_MODE_STREAM = 3
42 + WEB_CLIENT_MODE_GET = 0,
43 + WEB_CLIENT_MODE_POST = 1,
44 + WEB_CLIENT_MODE_FILECOPY = 2,
45 + WEB_CLIENT_MODE_OPTIONS = 3,
46 + WEB_CLIENT_MODE_STREAM = 4,
47 } WEB_CLIENT_MODE;
48
49 typedef enum {
@@ -122,8 +123,6 @@ typedef enum web_client_flags {
123 #define web_client_is_corkable(w) web_client_flag_check(w, WEB_CLIENT_FLAG_TCP_CLIENT)
124
125 #define NETDATA_WEB_REQUEST_URL_SIZE 65536 // static allocation
125 -#define NETDATA_WEB_REQUEST_COOKIE_SIZE 1024 // static allocation
126 -#define NETDATA_WEB_REQUEST_ORIGIN_HEADER_SIZE 1024 // static allocation
126
127 #define NETDATA_WEB_RESPONSE_ZLIB_CHUNK_SIZE 16384
128
@@ -131,82 +130,99 @@ typedef enum web_client_flags {
130 #define NETDATA_WEB_RESPONSE_INITIAL_SIZE 8192
131 #define NETDATA_WEB_REQUEST_INITIAL_SIZE 8192
132 #define NETDATA_WEB_REQUEST_MAX_SIZE 65536
133 +#define NETDATA_WEB_DECODED_URL_INITIAL_SIZE 512
134
135 struct response {
136 - BUFFER *header; // our response header
137 - BUFFER *header_output; // internal use
138 - BUFFER *data; // our response data buffer
136 + BUFFER *header; // our response header
137 + BUFFER *header_output; // internal use
138 + BUFFER *data; // our response data buffer
139
140 - int code; // the HTTP response code
140 + short int code; // the HTTP response code
141 + bool has_cookies;
142
143 size_t rlen; // if non-zero, the excepted size of ifd (input of firecopy)
144 size_t sent; // current data length sent to output
145
145 - int zoutput; // if set to 1, web_client_send() will send compressed data
146 + bool zoutput; // if set to 1, web_client_send() will send compressed data
147 +
148 #ifdef NETDATA_WITH_ZLIB
149 + bool zinitialized;
150 z_stream zstream; // zlib stream for sending compressed output to client
148 - Bytef zbuffer[NETDATA_WEB_RESPONSE_ZLIB_CHUNK_SIZE]; // temporary buffer for storing compressed output
151 size_t zsent; // the compressed bytes we have sent to the client
152 size_t zhave; // the compressed bytes that we have received from zlib
151 - unsigned int zinitialized : 1;
153 + Bytef zbuffer[NETDATA_WEB_RESPONSE_ZLIB_CHUNK_SIZE]; // temporary buffer for storing compressed output
154 #endif /* NETDATA_WITH_ZLIB */
155 };
156
157 +struct web_client;
158 +typedef bool (*web_client_interrupt_t)(struct web_client *, void *data);
159 +
160 struct web_client {
161 unsigned long long id;
162 + size_t use_count;
163
158 - WEB_CLIENT_FLAGS flags; // status flags for the client
159 - WEB_CLIENT_MODE mode; // the operational mode of the client
160 - WEB_CLIENT_ACL acl; // the access list of the client
161 - int port_acl; // the operations permitted on the port the client connected to
162 - char *auth_bearer_token; // the Bearer auth token (if sent)
164 + WEB_CLIENT_FLAGS flags; // status flags for the client
165 + WEB_CLIENT_MODE mode; // the operational mode of the client
166 + WEB_CLIENT_ACL acl; // the access list of the client
167 + int port_acl; // the operations permitted on the port the client connected to
168 size_t header_parse_tries;
169 size_t header_parse_last_size;
170
166 - int tcp_cork; // 1 = we have a cork on the socket
167 -
171 + bool tcp_cork;
172 int ifd;
173 int ofd;
174
171 - char client_ip[INET6_ADDRSTRLEN]; // Defined buffer sizes include null-terminators
175 + char client_ip[INET6_ADDRSTRLEN]; // Defined buffer sizes include null-terminators
176 char client_port[NI_MAXSERV];
173 - char server_host[NI_MAXHOST];
177 char client_host[NI_MAXHOST];
175 - char forwarded_host[NI_MAXHOST]; //Used with proxy
176 -
177 - char decoded_url[NETDATA_WEB_REQUEST_URL_SIZE + 1]; // we decode the URL in this buffer
178 - char decoded_query_string[NETDATA_WEB_REQUEST_URL_SIZE + 1]; // we decode the Query String in this buffer
179 - char last_url[NETDATA_WEB_REQUEST_URL_SIZE + 1]; // we keep a copy of the decoded URL here
180 - size_t url_path_length;
181 - char separator; // This value can be either '?' or 'f'
182 - char *url_search_path; //A pointer to the search path sent by the client
183 -
184 - struct timeval tv_in, tv_ready;
185 -
186 - char cookie1[NETDATA_WEB_REQUEST_COOKIE_SIZE + 1];
187 - char cookie2[NETDATA_WEB_REQUEST_COOKIE_SIZE + 1];
188 - char origin[NETDATA_WEB_REQUEST_ORIGIN_HEADER_SIZE + 1];
189 - char *user_agent;
190 -
191 - struct response response;
178
193 - size_t stats_received_bytes;
194 - size_t stats_sent_bytes;
179 + BUFFER *url_as_received; // the entire URL as received, used for logging - DO NOT MODIFY
180 + BUFFER *url_path_decoded; // the path, decoded - it is incrementally parsed and altered
181 + BUFFER *url_query_string_decoded; // the query string, decoded - it is incrementally parsed and altered
182
196 - // cache of web_client allocations
197 - struct web_client *prev; // maintain a linked list of web clients
198 - struct web_client *next; // for the web servers that need it
183 + // THESE NEED TO BE FREED
184 + char *auth_bearer_token; // the Bearer auth token (if sent)
185 + char *server_host; // the Host: header
186 + char *forwarded_host; // the X-Forwarded-For: header
187 + char *origin; // the Origin: header
188 + char *user_agent; // the User-Agent: header
189
200 - // MULTI-THREADED WEB SERVER MEMBERS
201 - netdata_thread_t thread; // the thread servicing this client
202 - volatile int running; // 1 when the thread runs, 0 otherwise
190 + char *post_payload; // when this request is a POST, this has the payload
191 + size_t post_payload_size; // the size of the buffer allocated for the payload
192 + // the actual contents may be less than the size
193
194 // STATIC-THREADED WEB SERVER MEMBERS
205 - size_t pollinfo_slot; // POLLINFO slot of the web client
206 - size_t pollinfo_filecopy_slot; // POLLINFO slot of the file read
195 + size_t pollinfo_slot; // POLLINFO slot of the web client
196 + size_t pollinfo_filecopy_slot; // POLLINFO slot of the file read
197 +
198 #ifdef ENABLE_HTTPS
199 struct netdata_ssl ssl;
200 #endif
201 +
202 + struct { // A callback to check if the query should be interrupted / stopped
203 + web_client_interrupt_t callback;
204 + void *callback_data;
205 + } interrupt;
206 +
207 + struct {
208 + size_t received_bytes;
209 + size_t sent_bytes;
210 + size_t *memory_accounting; // temporary pointer for constructor to use
211 + } statistics;
212 +
213 + struct {
214 + usec_t timeout_ut; // timeout if set, or zero
215 + struct timeval tv_in; // request received
216 + struct timeval tv_ready; // request processed - response ready
217 + struct timeval tv_timeout_last_checkpoint; // last checkpoint
218 + } timings;
219 +
220 + struct {
221 + struct web_client *prev;
222 + struct web_client *next;
223 + } cache;
224 +
225 + struct response response;
226 };
227
228 int web_client_permission_denied(struct web_client *w);
@@ -227,8 +243,28 @@ char *strip_control_characters(char *url);
243
244 int web_client_socket_is_now_used_for_streaming(struct web_client *w);
245
246 +void web_client_zero(struct web_client *w);
247 +struct web_client *web_client_create(size_t *statistics_memory_accounting);
248 +void web_client_free(struct web_client *w);
249 +
250 +#ifdef ENABLE_HTTPS
251 +void web_client_reuse_ssl(struct web_client *w);
252 +#endif
253 +
254 #include "web/api/web_api_v1.h"
255 #include "web/api/web_api_v2.h"
256 #include "daemon/common.h"
257
258 +void web_client_decode_path_and_query_string(struct web_client *w, const char *path_and_query_string);
259 +int web_client_api_request(RRDHOST *host, struct web_client *w, char *url_path_fragment);
260 +const char *web_content_type_to_string(HTTP_CONTENT_TYPE content_type);
261 +void web_client_enable_deflate(struct web_client *w, int gzip);
262 +int web_client_api_request_with_node_selection(RRDHOST *host, struct web_client *w, char *decoded_url_path);
263 +
264 +void web_client_timeout_checkpoint_init(struct web_client *w);
265 +void web_client_timeout_checkpoint_set(struct web_client *w, int timeout_ms);
266 +usec_t web_client_timeout_checkpoint(struct web_client *w);
267 +bool web_client_timeout_checkpoint_and_check(struct web_client *w, usec_t *usec_since_last_checkpoint);
268 +usec_t web_client_timeout_checkpoint_response_ready(struct web_client *w, usec_t *usec_since_last_checkpoint);
269 +
270 #endif
web/server/web_client_cache.c
+89 -220
@@ -6,77 +6,6 @@
6 // ----------------------------------------------------------------------------
7 // allocate and free web_clients
8
9 -#ifdef ENABLE_HTTPS
10 -
11 -static void web_client_reuse_ssl(struct web_client *w) {
12 - if (netdata_ssl_srv_ctx) {
13 - if (w->ssl.conn) {
14 - SSL_SESSION *session = SSL_get_session(w->ssl.conn);
15 - SSL *old = w->ssl.conn;
16 - w->ssl.conn = SSL_new(netdata_ssl_srv_ctx);
17 - if (session) {
18 -#if OPENSSL_VERSION_NUMBER >= OPENSSL_VERSION_111
19 - if (SSL_SESSION_is_resumable(session))
20 -#endif
21 - SSL_set_session(w->ssl.conn, session);
22 - }
23 - SSL_free(old);
24 - }
25 - }
26 -}
27 -#endif
28 -
29 -
30 -static void web_client_zero(struct web_client *w) {
31 - // zero everything about it - but keep the buffers
32 -
33 - // remember the pointers to the buffers
34 - BUFFER *b1 = w->response.data;
35 - BUFFER *b2 = w->response.header;
36 - BUFFER *b3 = w->response.header_output;
37 -
38 - // empty the buffers
39 - buffer_flush(b1);
40 - buffer_flush(b2);
41 - buffer_flush(b3);
42 -
43 - freez(w->user_agent);
44 -
45 - // zero everything
46 - memset(w, 0, sizeof(struct web_client));
47 -
48 - // restore the pointers of the buffers
49 - w->response.data = b1;
50 - w->response.header = b2;
51 - w->response.header_output = b3;
52 -}
53 -
54 -static void web_client_free(struct web_client *w) {
55 - buffer_free(w->response.header_output);
56 - buffer_free(w->response.header);
57 - buffer_free(w->response.data);
58 - freez(w->user_agent);
59 -#ifdef ENABLE_HTTPS
60 - if ((!web_client_check_unix(w)) && (netdata_ssl_srv_ctx)) {
61 - if (w->ssl.conn) {
62 - SSL_free(w->ssl.conn);
63 - w->ssl.conn = NULL;
64 - }
65 - }
66 -#endif
67 - freez(w);
68 - __atomic_sub_fetch(&netdata_buffers_statistics.buffers_web, sizeof(struct web_client), __ATOMIC_RELAXED);
69 -}
70 -
71 -static struct web_client *web_client_alloc(void) {
72 - struct web_client *w = callocz(1, sizeof(struct web_client));
73 - __atomic_add_fetch(&netdata_buffers_statistics.buffers_web, sizeof(struct web_client), __ATOMIC_RELAXED);
74 - w->response.data = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_web);
75 - w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_INITIAL_SIZE, &netdata_buffers_statistics.buffers_web);
76 - w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_INITIAL_SIZE, &netdata_buffers_statistics.buffers_web);
77 - return w;
78 -}
79 -
9 // ----------------------------------------------------------------------------
10 // web clients caching
11
@@ -87,194 +16,134 @@ static struct web_client *web_client_alloc(void) {
16 // The size of the cache is adaptive. It caches the structures of 2x
17 // the number of currently connected clients.
18
90 -// Comments per server:
91 -// SINGLE-THREADED : 1 cache is maintained
92 -// MULTI-THREADED : 1 cache is maintained
93 -// STATIC-THREADED : 1 cache for each thread of the web server
94 -
95 -__thread struct clients_cache web_clients_cache = {
96 - .pid = 0,
97 - .used = NULL,
98 - .used_count = 0,
99 - .avail = NULL,
100 - .avail_count = 0,
101 - .allocated = 0,
102 - .reused = 0
19 +static struct clients_cache {
20 + struct {
21 + SPINLOCK spinlock;
22 + struct web_client *head; // the structures of the currently connected clients
23 + size_t count; // the count the currently connected clients
24 +
25 + size_t allocated; // the number of allocations
26 + size_t reused; // the number of re-uses
27 + } used;
28 +
29 + struct {
30 + SPINLOCK spinlock;
31 + struct web_client *head; // the cached structures, available for future clients
32 + size_t count; // the number of cached structures
33 + } avail;
34 +} web_clients_cache = {
35 + .used = {
36 + .spinlock = NETDATA_SPINLOCK_INITIALIZER,
37 + .head = NULL,
38 + .count = 0,
39 + .reused = 0,
40 + .allocated = 0,
41 + },
42 + .avail = {
43 + .spinlock = NETDATA_SPINLOCK_INITIALIZER,
44 + .head = NULL,
45 + .count = 0,
46 + },
47 };
48
105 -inline void web_client_cache_verify(int force) {
106 -#ifdef NETDATA_INTERNAL_CHECKS
107 - static __thread size_t count = 0;
108 - count++;
109 -
110 - if(unlikely(force || count > 1000)) {
111 - count = 0;
112 -
113 - struct web_client *w;
114 - size_t used = 0, avail = 0;
115 - for(w = web_clients_cache.used; w ; w = w->next) used++;
116 - for(w = web_clients_cache.avail; w ; w = w->next) avail++;
117 -
118 - info("web_client_cache has %zu (%zu) used and %zu (%zu) available clients, allocated %zu, reused %zu (hit %zu%%)."
119 - , used, web_clients_cache.used_count
120 - , avail, web_clients_cache.avail_count
121 - , web_clients_cache.allocated
122 - , web_clients_cache.reused
123 - , (web_clients_cache.allocated + web_clients_cache.reused)?(web_clients_cache.reused * 100 / (web_clients_cache.allocated + web_clients_cache.reused)):0
124 - );
125 - }
126 -#else
127 - if(unlikely(force)) {
128 - info("web_client_cache has %zu used and %zu available clients, allocated %zu, reused %zu (hit %zu%%)."
129 - , web_clients_cache.used_count
130 - , web_clients_cache.avail_count
131 - , web_clients_cache.allocated
132 - , web_clients_cache.reused
133 - , (web_clients_cache.allocated + web_clients_cache.reused)?(web_clients_cache.reused * 100 / (web_clients_cache.allocated + web_clients_cache.reused)):0
134 - );
135 - }
136 -#endif
137 -}
138 -
49 // destroy the cache and free all the memory it uses
50 void web_client_cache_destroy(void) {
141 -#ifdef NETDATA_INTERNAL_CHECKS
142 - if(unlikely(web_clients_cache.pid != 0 && web_clients_cache.pid != gettid()))
143 - error("Oops! wrong thread accessing the cache. Expected %d, found %d", (int)web_clients_cache.pid, (int)gettid());
144 -
145 - web_client_cache_verify(1);
146 -#endif
147 -
148 - netdata_thread_disable_cancelability();
51 + internal_error(true, "web_client_cache has %zu used and %zu available clients, allocated %zu, reused %zu (hit %zu%%)."
52 + , web_clients_cache.used.count
53 + , web_clients_cache.avail.count
54 + , web_clients_cache.used.allocated
55 + , web_clients_cache.used.reused
56 + , (web_clients_cache.used.allocated + web_clients_cache.used.reused)?(web_clients_cache.used.reused * 100 / (web_clients_cache.used.allocated + web_clients_cache.used.reused)):0
57 + );
58
59 struct web_client *w, *t;
60
152 - w = web_clients_cache.used;
61 + netdata_spinlock_lock(&web_clients_cache.avail.spinlock);
62 + w = web_clients_cache.avail.head;
63 while(w) {
64 t = w;
155 - w = w->next;
65 + w = w->cache.next;
66 web_client_free(t);
67 }
158 - web_clients_cache.used = NULL;
159 - web_clients_cache.used_count = 0;
160 -
161 - w = web_clients_cache.avail;
162 - while(w) {
163 - t = w;
164 - w = w->next;
165 - web_client_free(t);
166 - }
167 - web_clients_cache.avail = NULL;
168 - web_clients_cache.avail_count = 0;
169 -
170 - netdata_thread_enable_cancelability();
68 + web_clients_cache.avail.head = NULL;
69 + web_clients_cache.avail.count = 0;
70 + netdata_spinlock_unlock(&web_clients_cache.avail.spinlock);
71 +
72 +// DO NOT FREE THEM IF THEY ARE USED
73 +// netdata_spinlock_lock(&web_clients_cache.used.spinlock);
74 +// w = web_clients_cache.used.head;
75 +// while(w) {
76 +// t = w;
77 +// w = w->next;
78 +// web_client_free(t);
79 +// }
80 +// web_clients_cache.used.head = NULL;
81 +// web_clients_cache.used.count = 0;
82 +// web_clients_cache.used.reused = 0;
83 +// web_clients_cache.used.allocated = 0;
84 +// netdata_spinlock_unlock(&web_clients_cache.used.spinlock);
85 }
86
173 -struct web_client *web_client_get_from_cache_or_allocate() {
174 -
175 -#ifdef NETDATA_INTERNAL_CHECKS
176 - if(unlikely(web_clients_cache.pid == 0))
177 - web_clients_cache.pid = gettid();
178 -
179 - if(unlikely(web_clients_cache.pid != 0 && web_clients_cache.pid != gettid()))
180 - error("Oops! wrong thread accessing the cache. Expected %d, found %d", (int)web_clients_cache.pid, (int)gettid());
181 -#endif
182 -
183 - netdata_thread_disable_cancelability();
184 -
185 - struct web_client *w = web_clients_cache.avail;
186 -
87 +struct web_client *web_client_get_from_cache(void) {
88 + netdata_spinlock_lock(&web_clients_cache.avail.spinlock);
89 + struct web_client *w = web_clients_cache.avail.head;
90 if(w) {
91 // get it from avail
189 - if (w == web_clients_cache.avail) web_clients_cache.avail = w->next;
190 - if(w->prev) w->prev->next = w->next;
191 - if(w->next) w->next->prev = w->prev;
192 - web_clients_cache.avail_count--;
193 -#ifdef ENABLE_HTTPS
194 - web_client_reuse_ssl(w);
195 - SSL *ssl = w->ssl.conn;
196 -#endif
92 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(web_clients_cache.avail.head, w, cache.prev, cache.next);
93 + web_clients_cache.avail.count--;
94 + netdata_spinlock_unlock(&web_clients_cache.avail.spinlock);
95 +
96 web_client_zero(w);
198 - web_clients_cache.reused++;
199 -#ifdef ENABLE_HTTPS
200 - w->ssl.conn = ssl;
201 - w->ssl.flags = NETDATA_SSL_START;
202 - debug(D_WEB_CLIENT_ACCESS,"Reusing SSL structure with (w->ssl = NULL, w->accepted = %u)", w->ssl.flags);
203 -#endif
97 +
98 + netdata_spinlock_lock(&web_clients_cache.used.spinlock);
99 + web_clients_cache.used.reused++;
100 }
101 else {
102 + netdata_spinlock_unlock(&web_clients_cache.avail.spinlock);
103 +
104 // allocate it
207 - w = web_client_alloc();
105 + w = web_client_create(&netdata_buffers_statistics.buffers_web);
106 +
107 #ifdef ENABLE_HTTPS
108 w->ssl.flags = NETDATA_SSL_START;
109 debug(D_WEB_CLIENT_ACCESS,"Starting SSL structure with (w->ssl = NULL, w->accepted = %u)", w->ssl.flags);
110 #endif
212 - web_clients_cache.allocated++;
111 +
112 + netdata_spinlock_lock(&web_clients_cache.used.spinlock);
113 + web_clients_cache.used.allocated++;
114 }
115
116 // link it to used web clients
216 - if (web_clients_cache.used) web_clients_cache.used->prev = w;
217 - w->next = web_clients_cache.used;
218 - w->prev = NULL;
219 - web_clients_cache.used = w;
220 - web_clients_cache.used_count++;
117 + DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(web_clients_cache.used.head, w, cache.prev, cache.next);
118 + web_clients_cache.used.count++;
119 + netdata_spinlock_unlock(&web_clients_cache.used.spinlock);
120
121 // initialize it
122 + w->use_count++;
123 w->id = global_statistics_web_client_connected();
224 - w->mode = WEB_CLIENT_MODE_NORMAL;
225 -
226 - netdata_thread_enable_cancelability();
124 + w->mode = WEB_CLIENT_MODE_GET;
125
126 return w;
127 }
128
231 -void web_client_release(struct web_client *w) {
232 -#ifdef NETDATA_INTERNAL_CHECKS
233 - if(unlikely(web_clients_cache.pid != 0 && web_clients_cache.pid != gettid()))
234 - error("Oops! wrong thread accessing the cache. Expected %d, found %d", (int)web_clients_cache.pid, (int)gettid());
235 -
236 - if(unlikely(w->running))
237 - error("%llu: releasing web client from %s port %s, but it still running.", w->id, w->client_ip, w->client_port);
238 -#endif
239 -
240 - debug(D_WEB_CLIENT_ACCESS, "%llu: Closing web client from %s port %s.", w->id, w->client_ip, w->client_port);
241 -
242 - web_server_log_connection(w, "DISCONNECTED");
243 - web_client_request_done(w);
244 - global_statistics_web_client_disconnected();
245 -
246 - netdata_thread_disable_cancelability();
247 -
248 - if(web_server_mode != WEB_SERVER_MODE_STATIC_THREADED) {
249 - if (w->ifd != -1) close(w->ifd);
250 - if (w->ofd != -1 && w->ofd != w->ifd) close(w->ofd);
251 - w->ifd = w->ofd = -1;
252 -#ifdef ENABLE_HTTPS
253 - web_client_reuse_ssl(w);
254 - w->ssl.flags = NETDATA_SSL_START;
255 -#endif
256 -
257 - }
258 -
129 +void web_client_release_to_cache(struct web_client *w) {
130 // unlink it from the used
260 - if (w == web_clients_cache.used) web_clients_cache.used = w->next;
261 - if(w->prev) w->prev->next = w->next;
262 - if(w->next) w->next->prev = w->prev;
263 - web_clients_cache.used_count--;
131 + netdata_spinlock_lock(&web_clients_cache.used.spinlock);
132 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(web_clients_cache.used.head, w, cache.prev, cache.next);
133 + ssize_t used_count = (ssize_t)--web_clients_cache.used.count;
134 + netdata_spinlock_unlock(&web_clients_cache.used.spinlock);
135 +
136 + netdata_spinlock_lock(&web_clients_cache.avail.spinlock);
137 + if(w->use_count > 100 || (used_count > 0 && web_clients_cache.avail.count >= 2 * (size_t)used_count) || (used_count <= 10 && web_clients_cache.avail.count >= 20)) {
138 + netdata_spinlock_unlock(&web_clients_cache.avail.spinlock);
139
265 - if(web_clients_cache.avail_count >= 2 * web_clients_cache.used_count) {
140 // we have too many of them - free it
141 web_client_free(w);
142 }
143 else {
144 // link it to the avail
271 - if (web_clients_cache.avail) web_clients_cache.avail->prev = w;
272 - w->next = web_clients_cache.avail;
273 - w->prev = NULL;
274 - web_clients_cache.avail = w;
275 - web_clients_cache.avail_count++;
145 + DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(web_clients_cache.avail.head, w, cache.prev, cache.next);
146 + web_clients_cache.avail.count++;
147 + netdata_spinlock_unlock(&web_clients_cache.avail.spinlock);
148 }
277 -
278 - netdata_thread_enable_cancelability();
149 }
280 -
web/server/web_client_cache.h
+2 -18
@@ -6,25 +6,9 @@
6 #include "libnetdata/libnetdata.h"
7 #include "web_client.h"
8
9 -struct clients_cache {
10 - pid_t pid;
11 -
12 - struct web_client *used; // the structures of the currently connected clients
13 - size_t used_count; // the count the currently connected clients
14 -
15 - struct web_client *avail; // the cached structures, available for future clients
16 - size_t avail_count; // the number of cached structures
17 -
18 - size_t reused; // the number of re-uses
19 - size_t allocated; // the number of allocations
20 -};
21 -
22 -extern __thread struct clients_cache web_clients_cache;
23 -
24 -void web_client_release(struct web_client *w);
25 -struct web_client *web_client_get_from_cache_or_allocate();
9 +void web_client_release_to_cache(struct web_client *w);
10 +struct web_client *web_client_get_from_cache(void);
11 void web_client_cache_destroy(void);
27 -void web_client_cache_verify(int force);
12
13 #include "web_server.h"
14
web/server/web_server.c
-25
@@ -132,28 +132,3 @@ void web_client_update_acl_matches(struct web_client *w) {
132 void web_server_log_connection(struct web_client *w, const char *msg) {
133 log_access("%llu: %d '[%s]:%s' '%s'", w->id, gettid(), w->client_ip, w->client_port, msg);
134 }
135 -
136 -// --------------------------------------------------------------------------------------
137 -
138 -void web_client_initialize_connection(struct web_client *w) {
139 - int flag = 1;
140 -
141 - if(unlikely(web_client_check_tcp(w) && setsockopt(w->ifd, IPPROTO_TCP, TCP_NODELAY, (char *) &flag, sizeof(int)) != 0))
142 - debug(D_WEB_CLIENT, "%llu: failed to enable TCP_NODELAY on socket fd %d.", w->id, w->ifd);
143 -
144 - flag = 1;
145 - if(unlikely(setsockopt(w->ifd, SOL_SOCKET, SO_KEEPALIVE, (char *) &flag, sizeof(int)) != 0))
146 - debug(D_WEB_CLIENT, "%llu: failed to enable SO_KEEPALIVE on socket fd %d.", w->id, w->ifd);
147 -
148 - web_client_update_acl_matches(w);
149 -
150 - w->origin[0] = '*'; w->origin[1] = '\0';
151 - w->cookie1[0] = '\0'; w->cookie2[0] = '\0';
152 - freez(w->user_agent); w->user_agent = NULL;
153 -
154 - web_client_enable_wait_receive(w);
155 -
156 - web_server_log_connection(w, "CONNECTED");
157 -
158 - web_client_cache_verify(0);
159 -}
web/server/web_server.h
-1
@@ -51,7 +51,6 @@ extern long web_client_streaming_rate_t;
51 extern LISTEN_SOCKETS api_sockets;
52 void web_client_update_acl_matches(struct web_client *w);
53 void web_server_log_connection(struct web_client *w, const char *msg);
54 -void web_client_initialize_connection(struct web_client *w);
54 struct web_client *web_client_create_on_listenfd(int listener);
55
56 #include "web_client_cache.h"